letro/letro-authentication-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge remote-tracking branch 'origin/main' into ref-merge/077df809a751dac03c94bb21e1def43ee4f1ae13

Browse Source
This commit is contained in:
Quentin Gliech
2025-02-04 16:21:10 +01:00
parent 077df809a7 9c19c54896
commit 930855f2cb
115 changed files with 5339 additions and 1197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
.github/workflows/build.yaml vendored
View File

@@ -97,7 +97,7 @@ jobs:
tool: cargo-zigbuild
- name: Install frontend Node
uses: actions/setup-node@v4.1.0
uses: actions/setup-node@v4.2.0
with:
node-version: 20
@@ -240,7 +240,7 @@ jobs:
# For pull-requests, only read from the cache, do not try to push to the
# cache or the image itself
- name: Build
uses: docker/bake-action@v6.2.0
uses: docker/bake-action@v6.3.0
if: github.event_name == 'pull_request'
with:
files: |
@@ -253,7 +253,7 @@ jobs:
- name: Build and push
id: bake
uses: docker/bake-action@v6.2.0
uses: docker/bake-action@v6.3.0
if: github.event_name != 'pull_request'
with:
files: |
@@ -306,7 +306,7 @@ jobs:
uses: actions/checkout@v4.2.2
- name: Install Node
uses: actions/setup-node@v4.1.0
uses: actions/setup-node@v4.2.0
with:
node-version-file: ./tools/syn2mas/.nvmrc

12
.github/workflows/ci.yaml vendored
View File

@@ -61,7 +61,7 @@ jobs:
uses: actions/checkout@v4.2.2
- name: Install Node
uses: actions/setup-node@v4.1.0
uses: actions/setup-node@v4.2.0
with:
node-version: 20
@@ -86,7 +86,7 @@ jobs:
uses: actions/checkout@v4.2.2
- name: Install Node
uses: actions/setup-node@v4.1.0
uses: actions/setup-node@v4.2.0
with:
node-version: 20
@@ -111,7 +111,7 @@ jobs:
uses: actions/checkout@v4.2.2
- name: Install Node
uses: actions/setup-node@v4.1.0
uses: actions/setup-node@v4.2.0
with:
node-version: 20
@@ -184,7 +184,7 @@ jobs:
uses: mozilla-actions/sccache-action@v0.0.7
- name: Install Node
uses: actions/setup-node@v4.1.0
uses: actions/setup-node@v4.2.0
with:
node-version: 20
@@ -321,7 +321,7 @@ jobs:
tool: cargo-nextest
- name: Install Node
uses: actions/setup-node@v4.1.0
uses: actions/setup-node@v4.2.0
with:
node-version: 20
@@ -367,7 +367,7 @@ jobs:
uses: actions/checkout@v4.2.2
- name: Install Node
uses: actions/setup-node@v4.1.0
uses: actions/setup-node@v4.2.0
with:
node-version-file: ./tools/syn2mas/.nvmrc

10
.github/workflows/coverage.yaml vendored
View File

@@ -36,7 +36,7 @@ jobs:
run: make coverage
- name: Upload to codecov.io
uses: codecov/codecov-action@v5.2.0
uses: codecov/codecov-action@v5.3.1
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: policies/coverage.json
@@ -55,7 +55,7 @@ jobs:
uses: actions/checkout@v4.2.2
- name: Install Node
uses: actions/setup-node@v4.1.0
uses: actions/setup-node@v4.2.0
with:
node-version: 20
@@ -74,7 +74,7 @@ jobs:
run: npm run coverage
- name: Upload to codecov.io
uses: codecov/codecov-action@v5.2.0
uses: codecov/codecov-action@v5.3.1
with:
token: ${{ secrets.CODECOV_TOKEN }}
directory: frontend/coverage/
@@ -117,7 +117,7 @@ jobs:
rustup component add llvm-tools-preview
- name: Install Node
uses: actions/setup-node@v4.1.0
uses: actions/setup-node@v4.2.0
with:
node-version: 20
@@ -161,7 +161,7 @@ jobs:
grcov . --binary-path ./target/debug/deps/ -s . -t lcov --branch --ignore-not-existing --ignore '../*' --ignore "/*" -o target/coverage/tests.lcov
- name: Upload to codecov.io
uses: codecov/codecov-action@v5.2.0
uses: codecov/codecov-action@v5.3.1
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: target/coverage/*.lcov

2
.github/workflows/docs.yaml vendored
View File

@@ -36,7 +36,7 @@ jobs:
tool: mdbook
- name: Install Node
uses: actions/setup-node@v4.1.0
uses: actions/setup-node@v4.2.0
with:
node-version: 20

2
.github/workflows/release-branch.yaml vendored
View File

@@ -50,7 +50,7 @@ jobs:
uses: actions/checkout@v4.2.2
- name: Install Node
uses: actions/setup-node@v4.1.0
uses: actions/setup-node@v4.2.0
with:
node-version: 20

2
.github/workflows/translations-download.yaml vendored
View File

@@ -17,7 +17,7 @@ jobs:
uses: actions/checkout@v4.2.2
- name: Install Node
uses: actions/setup-node@v4.1.0
uses: actions/setup-node@v4.2.0
with:
node-version: 20

2
.github/workflows/translations-upload.yaml vendored
View File

@@ -16,7 +16,7 @@ jobs:
uses: actions/checkout@v4.2.2
- name: Install Node
uses: actions/setup-node@v4.1.0
uses: actions/setup-node@v4.2.0
with:
node-version: 20

225
Cargo.lock generated
View File

@@ -85,9 +85,9 @@ dependencies = [
[[package]]
name = "aide"
version = "0.13.5"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5678d2978845ddb4bd736a026f467dd652d831e9e6254b0e41b07f7ee7523309"
checksum = "0d4787eac8d785c99d76058a086b151006f5a872bc8adea8364108a39a518ac2"
dependencies = [
"aide-macros",
"axum",
@@ -99,8 +99,7 @@ dependencies = [
"schemars",
"serde",
"serde_json",
"serde_qs",
"thiserror 1.0.69",
"thiserror 2.0.11",
"tower-layer",
"tower-service",
"tracing",
@@ -108,12 +107,11 @@ dependencies = [
[[package]]
name = "aide-macros"
version = "0.7.0"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0487f8598afe49e6bc950a613a678bd962c4a6f431022ded62643c8b990301a"
checksum = "be8e0d4af7cc08353807aaf80722125a229bf2d67be7fe0b89163c648db3d223"
dependencies = [
"darling",
"proc-macro2",
"quote",
"syn",
]
@@ -571,13 +569,13 @@ dependencies = [
[[package]]
name = "axum"
version = "0.7.9"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f"
checksum = "6d6fd624c75e18b3b4c6b9caf42b1afe24437daaee904069137d8bab077be8b8"
dependencies = [
"async-trait",
"axum-core",
"bytes",
"form_urlencoded",
"futures-util",
"http",
"http-body",
@@ -605,11 +603,10 @@ dependencies = [
[[package]]
name = "axum-core"
version = "0.4.5"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199"
checksum = "df1362f362fd16024ae199c1970ce98f9661bf5ef94b9808fee734bc3698b733"
dependencies = [
"async-trait",
"bytes",
"futures-util",
"http",
@@ -626,22 +623,20 @@ dependencies = [
[[package]]
name = "axum-extra"
version = "0.9.6"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c794b30c904f0a1c2fb7740f7df7f7972dfaa14ef6f57cb6178dc63e5dca2f04"
checksum = "460fc6f625a1f7705c6cf62d0d070794e94668988b1c38111baeec177c715f7b"
dependencies = [
"axum",
"axum-core",
"bytes",
"cookie",
"fastrand",
"futures-util",
"headers",
"http",
"http-body",
"http-body-util",
"mime",
"multer",
"pin-project-lite",
"serde",
"tower",
@@ -651,9 +646,9 @@ dependencies = [
[[package]]
name = "axum-macros"
version = "0.4.2"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57d123550fa8d071b7255cb0cc04dc302baa6c8c4a79f55701552684d8399bce"
checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c"
dependencies = [
"proc-macro2",
"quote",
@@ -861,6 +856,15 @@ dependencies = [
"serde",
]
[[package]]
name = "castaway"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0abae9be0aaf9ea96a3b1b8b1b55c602ca751eba1b1500220cea4ecbafe7c0d5"
dependencies = [
"rustversion",
]
[[package]]
name = "cbc"
version = "0.1.2"
@@ -1081,6 +1085,20 @@ dependencies = [
"memchr",
]
[[package]]
name = "compact_str"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b79c4069c6cad78e2e0cdfcbd26275770669fb39fd308a752dc110e83b9af32"
dependencies = [
"castaway",
"cfg-if",
"itoa",
"rustversion",
"ryu",
"static_assertions",
]
[[package]]
name = "concurrent-queue"
version = "2.5.0"
@@ -2329,9 +2347,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
[[package]]
name = "hyper"
version = "1.5.2"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0"
checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80"
dependencies = [
"bytes",
"futures-channel",
@@ -2787,13 +2805,14 @@ dependencies = [
[[package]]
name = "insta"
version = "1.42.0"
version = "1.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6513e4067e16e69ed1db5ab56048ed65db32d10ba5fc1217f5393f8f17d8b5a5"
checksum = "71c1b125e30d93896b365e156c33dadfffab45ee8400afcbba4752f59de08a86"
dependencies = [
"console",
"linked-hash-map",
"once_cell",
"pin-project",
"serde",
"similar",
]
@@ -3143,6 +3162,7 @@ dependencies = [
"dialoguer",
"dotenvy",
"figment",
"futures-util",
"http-body-util",
"hyper",
"ipnetwork",
@@ -3165,6 +3185,7 @@ dependencies = [
"mas-tasks",
"mas-templates",
"mas-tower",
"oauth2-types",
"opentelemetry",
"opentelemetry-http",
"opentelemetry-jaeger-propagator",
@@ -3179,13 +3200,16 @@ dependencies = [
"rand_chacha",
"reqwest",
"rustls",
"sd-notify",
"sentry",
"sentry-tower",
"sentry-tracing",
"serde_json",
"serde_yaml",
"sqlx",
"syn2mas",
"tokio",
"tokio-stream",
"tokio-util",
"tower",
"tower-http",
@@ -3505,6 +3529,7 @@ version = "0.13.0"
dependencies = [
"anyhow",
"async-trait",
"ruma-common",
"tokio",
]
@@ -3731,9 +3756,9 @@ dependencies = [
[[package]]
name = "matchit"
version = "0.7.3"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
[[package]]
name = "md-5"
@@ -3784,9 +3809,9 @@ dependencies = [
[[package]]
name = "minijinja"
version = "2.6.0"
version = "2.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "212b4cab3aad057bc6e611814472905546c533295723b9e26a31c7feb19a8e65"
checksum = "cff7b8df5e85e30b87c2b0b3f58ba3a87b68e133738bf512a7713769326dbca9"
dependencies = [
"memo-map",
"self_cell",
@@ -3797,9 +3822,9 @@ dependencies = [
[[package]]
name = "minijinja-contrib"
version = "2.6.0"
version = "2.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22c4652069ecd6ce1e7724229d859f11f8a0804512da4c274e67d937b833e47c"
checksum = "7ac3e47a9006ed0500425a092c9f8b2e56d10f8aeec8ce870c5e8a7c6ef2d7c3"
dependencies = [
"minijinja",
"serde",
@@ -4622,9 +4647,9 @@ dependencies = [
[[package]]
name = "proc-macro2"
version = "1.0.92"
version = "1.0.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0"
checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99"
dependencies = [
"unicode-ident",
]
@@ -4688,9 +4713,9 @@ checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94"
[[package]]
name = "psl"
version = "2.1.80"
version = "2.1.81"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05ff66fe75e86ef6bb57a5e7c1af22cc3ff5368ec610559609ea08e304d7c772"
checksum = "5871e872678223987b84739333bf13e42f0c1fb102e30bba8dcdf1340d0bbcc9"
dependencies = [
"psl-types",
]
@@ -4790,9 +4815,9 @@ dependencies = [
[[package]]
name = "quote"
version = "1.0.37"
version = "1.0.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc"
dependencies = [
"proc-macro2",
]
@@ -5133,9 +5158,9 @@ dependencies = [
[[package]]
name = "rustls"
version = "0.23.21"
version = "0.23.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f287924602bf649d949c63dc8ac8b235fa5387d394020705b80c4eb597ce5b8"
checksum = "9fb9263ab4eb695e42321db096e3b8fbd715a59b154d5c88d82db2175b681ba7"
dependencies = [
"aws-lc-rs",
"log",
@@ -5171,9 +5196,9 @@ dependencies = [
[[package]]
name = "rustls-pki-types"
version = "1.10.1"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37"
checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c"
dependencies = [
"web-time",
]
@@ -5301,6 +5326,15 @@ dependencies = [
"sha2",
]
[[package]]
name = "sd-notify"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b943eadf71d8b69e661330cb0e2656e31040acf21ee7708e2c238a0ec6af2bf4"
dependencies = [
"libc",
]
[[package]]
name = "sea-query"
version = "0.32.1"
@@ -5391,9 +5425,9 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b"
[[package]]
name = "sentry"
version = "0.34.0"
version = "0.36.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5484316556650182f03b43d4c746ce0e3e48074a21e2f51244b648b6542e1066"
checksum = "3a7332159e544e34db06b251b1eda5e546bd90285c3f58d9c8ff8450b484e0da"
dependencies = [
"httpdate",
"reqwest",
@@ -5408,9 +5442,9 @@ dependencies = [
[[package]]
name = "sentry-backtrace"
version = "0.34.0"
version = "0.36.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40aa225bb41e2ec9d7c90886834367f560efc1af028f1c5478a6cce6a59c463a"
checksum = "565ec31ad37bab8e6d9f289f34913ed8768347b133706192f10606dabd5c6bc4"
dependencies = [
"backtrace",
"once_cell",
@@ -5420,9 +5454,9 @@ dependencies = [
[[package]]
name = "sentry-contexts"
version = "0.34.0"
version = "0.36.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a8dd746da3d16cb8c39751619cefd4fcdbd6df9610f3310fd646b55f6e39910"
checksum = "e860275f25f27e8c0c7726ce116c7d5c928c5bba2ee73306e52b20a752298ea6"
dependencies = [
"hostname",
"libc",
@@ -5434,9 +5468,9 @@ dependencies = [
[[package]]
name = "sentry-core"
version = "0.34.0"
version = "0.36.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "161283cfe8e99c8f6f236a402b9ccf726b201f365988b5bb637ebca0abbd4a30"
checksum = "653942e6141f16651273159f4b8b1eaeedf37a7554c00cd798953e64b8a9bf72"
dependencies = [
"once_cell",
"rand",
@@ -5447,9 +5481,9 @@ dependencies = [
[[package]]
name = "sentry-panic"
version = "0.34.0"
version = "0.36.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc74f229c7186dd971a9491ffcbe7883544aa064d1589bd30b83fb856cd22d63"
checksum = "105e3a956c8aa9dab1e4087b1657b03271bfc49d838c6ae9bfc7c58c802fd0ef"
dependencies = [
"sentry-backtrace",
"sentry-core",
@@ -5457,9 +5491,9 @@ dependencies = [
[[package]]
name = "sentry-tower"
version = "0.34.0"
version = "0.36.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c90802b38c899a2c9e557dff25ad186362eddf755d5f5244001b172dd03bead"
checksum = "082f781dfc504d984e16d99f8dbf94d6ee4762dd0fc28de25713d0f900a8164d"
dependencies = [
"http",
"pin-project",
@@ -5471,9 +5505,9 @@ dependencies = [
[[package]]
name = "sentry-tracing"
version = "0.34.0"
version = "0.36.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd3c5faf2103cd01eeda779ea439b68c4ee15adcdb16600836e97feafab362ec"
checksum = "64e75c831b4d8b34a5aec1f65f67c5d46a26c7c5d3c7abd8b5ef430796900cf8"
dependencies = [
"sentry-backtrace",
"sentry-core",
@@ -5483,9 +5517,9 @@ dependencies = [
[[package]]
name = "sentry-types"
version = "0.34.0"
version = "0.36.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d68cdf6bc41b8ff3ae2a9c4671e97426dcdd154cc1d4b6b72813f285d6b163f"
checksum = "2d4203359e60724aa05cf2385aaf5d4f147e837185d7dd2b9ccf1ee77f4420c8"
dependencies = [
"debugid",
"hex",
@@ -5544,9 +5578,9 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.137"
version = "1.0.138"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "930cfb6e6abf99298aaad7d29abbef7a9999a9a8806a40088f55f0dcec03146b"
checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949"
dependencies = [
"indexmap 2.7.1",
"itoa",
@@ -5565,19 +5599,6 @@ dependencies = [
"serde",
]
[[package]]
name = "serde_qs"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd34f36fe4c5ba9654417139a9b3a20d2e1de6012ee678ad14d240c22c78d8d6"
dependencies = [
"axum",
"futures",
"percent-encoding",
"serde",
"thiserror 1.0.69",
]
[[package]]
name = "serde_spanned"
version = "0.6.8"
@@ -6012,6 +6033,12 @@ dependencies = [
"windows-sys 0.59.0",
]
[[package]]
name = "static_assertions"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]]
name = "static_assertions_next"
version = "1.1.2"
@@ -6065,15 +6092,39 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "syn"
version = "2.0.90"
version = "2.0.96"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31"
checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "syn2mas"
version = "0.13.0"
dependencies = [
"anyhow",
"camino",
"chrono",
"compact_str",
"figment",
"futures-util",
"insta",
"mas-config",
"mas-storage-pg",
"rand",
"serde",
"sqlx",
"thiserror 2.0.11",
"thiserror-ext",
"tokio",
"tracing",
"ulid",
"uuid",
]
[[package]]
name = "sync_wrapper"
version = "1.0.1"
@@ -6140,6 +6191,28 @@ dependencies = [
"thiserror-impl 2.0.11",
]
[[package]]
name = "thiserror-ext"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef4323942237f7cc071061f2c5f0db919e6053c2cdf58c6bc974883073429737"
dependencies = [
"thiserror 1.0.69",
"thiserror-ext-derive",
]
[[package]]
name = "thiserror-ext-derive"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96541747c50e6c73e094737938f4f5dfaf50c48a31adff4197a3e2a481371674"
dependencies = [
"either",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "thiserror-impl"
version = "1.0.69"
@@ -6283,9 +6356,9 @@ dependencies = [
[[package]]
name = "tokio-stream"
version = "0.1.16"
version = "0.1.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1"
checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047"
dependencies = [
"futures-core",
"pin-project-lite",
@@ -6592,9 +6665,9 @@ checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893"
[[package]]
name = "unicode-ident"
version = "1.0.14"
version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83"
checksum = "11cd88e12b17c6494200a9c1b683a04fcac9573ed74cd1b62aeb2727c5592243"
[[package]]
name = "unicode-normalization"

44
Cargo.toml
View File

@@ -54,17 +54,21 @@ mas-tasks = { path = "./crates/tasks/", version = "=0.13.0" }
mas-templates = { path = "./crates/templates/", version = "=0.13.0" }
mas-tower = { path = "./crates/tower/", version = "=0.13.0" }
oauth2-types = { path = "./crates/oauth2-types/", version = "=0.13.0" }
syn2mas = { path = "./crates/syn2mas", version = "=0.13.0" }
# OpenAPI schema generation and validation
[workspace.dependencies.aide]
version = "0.13.5"
features = ["axum", "axum-headers", "macros"]
version = "0.14.0"
features = ["axum", "axum-extra", "axum-json", "axum-query", "macros"]
# GraphQL server
[workspace.dependencies.async-graphql]
version = "7.0.14"
features = ["chrono", "url", "tracing"]
[workspace.dependencies.async-stream]
version = "0.3.6"
# Utility to write and implement async traits
[workspace.dependencies.async-trait]
version = "0.1.85"
@@ -75,11 +79,11 @@ version = "1.0.95"
# HTTP router
[workspace.dependencies.axum]
version = "0.7.9"
version = "0.8.1"
# Extra utilities for Axum
[workspace.dependencies.axum-extra]
version = "0.9.6"
version = "0.10.0"
features = ["cookie-private", "cookie-key-expansion", "typed-header"]
# Constant-time base64
@@ -94,6 +98,10 @@ version = "1.9.0"
[workspace.dependencies.camino]
version = "1.1.9"
# Memory optimisation for short strings
[workspace.dependencies.compact_str]
version = "0.8.1"
# Time utilities
[workspace.dependencies.chrono]
version = "0.4.39"
@@ -145,7 +153,7 @@ version = "0.1.2"
# HTTP client and server
[workspace.dependencies.hyper]
version = "1.5.2"
version = "1.6.0"
features = ["client", "http1", "http2"]
# Additional Hyper utilties
@@ -169,7 +177,7 @@ default-features = false
# Snapshot testing
[workspace.dependencies.insta]
version = "1.42.0"
version = "1.42.1"
features = ["yaml", "json"]
# Email sending
@@ -188,12 +196,12 @@ features = [
# Templates
[workspace.dependencies.minijinja]
version = "2.6.0"
version = "2.7.0"
features = ["loader", "json", "speedups", "unstable_machinery"]
# Additional filters for minijinja
[workspace.dependencies.minijinja-contrib]
version = "2.6.0"
version = "2.7.0"
features = ["pycompat"]
# Utilities to deal with non-zero values
@@ -240,9 +248,13 @@ version = "0.12.12"
default-features = false
features = ["http2", "rustls-tls-manual-roots", "charset", "json", "socks"]
# Matrix-related types
[workspace.dependencies.ruma-common]
version = "0.15.0"
# TLS stack
[workspace.dependencies.rustls]
version = "0.23.21"
version = "0.23.22"
# Use platform-specific verifier for TLS
[workspace.dependencies.rustls-platform-verifier]
@@ -271,18 +283,18 @@ features = [
# Sentry error tracking
[workspace.dependencies.sentry]
version = "0.34.0"
version = "0.36.0"
default-features = false
features = ["backtrace", "contexts", "panic", "tower", "reqwest"]
# Sentry tower layer
[workspace.dependencies.sentry-tower]
version = "0.34.0"
version = "0.36.0"
features = ["http"]
# Sentry tracing integration
[workspace.dependencies.sentry-tracing]
version = "0.34.0"
version = "0.36.0"
# Serialization and deserialization
[workspace.dependencies.serde]
@@ -291,7 +303,7 @@ features = ["derive"] # Most of the time, if we need serde, we need derive
# JSON serialization and deserialization
[workspace.dependencies.serde_json]
version = "1.0.137"
version = "1.0.138"
features = ["preserve_order"]
# SQL database support
@@ -312,11 +324,17 @@ features = [
[workspace.dependencies.thiserror]
version = "2.0.11"
[workspace.dependencies.thiserror-ext]
version = "0.2.1"
# Async runtime
[workspace.dependencies.tokio]
version = "1.43.0"
features = ["full"]
[workspace.dependencies.tokio-stream]
version = "0.1.17"
# Useful async utilities
[workspace.dependencies.tokio-util]
version = "0.7.13"

2
clippy.toml
View File

@@ -1,4 +1,4 @@
doc-valid-idents = ["OpenID", "OAuth", "..", "PostgreSQL"]
doc-valid-idents = ["OpenID", "OAuth", "..", "PostgreSQL", "SQLite"]
disallowed-methods = [
{ path = "rand::thread_rng", reason = "do not create rngs on the fly, pass them as parameters" },

4
crates/axum-utils/src/client_authorization.rs
View File

@@ -1,4 +1,4 @@
// Copyright 2024 New Vector Ltd.
// Copyright 2024, 2025 New Vector Ltd.
// Copyright 2022-2024 The Matrix.org Foundation C.I.C.
//
// SPDX-License-Identifier: AGPL-3.0-only
@@ -6,7 +6,6 @@
use std::collections::HashMap;
use async_trait::async_trait;
use axum::{
extract::{
rejection::{FailedToDeserializeForm, FormRejection},
@@ -321,7 +320,6 @@ impl IntoResponse for ClientAuthorizationError {
}
}
#[async_trait]
impl<S, F> FromRequest<S> for ClientAuthorization<F>
where
F: DeserializeOwned,

4
crates/axum-utils/src/cookies.rs
View File

@@ -1,4 +1,4 @@
// Copyright 2024 New Vector Ltd.
// Copyright 2024, 2025 New Vector Ltd.
// Copyright 2022-2024 The Matrix.org Foundation C.I.C.
//
// SPDX-License-Identifier: AGPL-3.0-only
@@ -8,7 +8,6 @@
use std::convert::Infallible;
use async_trait::async_trait;
use axum::{
extract::{FromRef, FromRequestParts},
response::{IntoResponseParts, ResponseParts},
@@ -65,7 +64,6 @@ impl CookieManager {
}
}
#[async_trait]
impl<S> FromRequestParts<S> for CookieJar
where
CookieManager: FromRef<S>,

4
crates/axum-utils/src/user_authorization.rs
View File

@@ -1,4 +1,4 @@
// Copyright 2024 New Vector Ltd.
// Copyright 2024, 2025 New Vector Ltd.
// Copyright 2022-2024 The Matrix.org Foundation C.I.C.
//
// SPDX-License-Identifier: AGPL-3.0-only
@@ -6,7 +6,6 @@
use std::{collections::HashMap, error::Error};
use async_trait::async_trait;
use axum::{
extract::{
rejection::{FailedToDeserializeForm, FormRejection},
@@ -284,7 +283,6 @@ where
}
}
#[async_trait]
impl<S, F> FromRequest<S> for UserAuthorization<F>
where
F: DeserializeOwned,

6
crates/cli/Cargo.toml
View File

@@ -23,6 +23,7 @@ console = "0.15.10"
dialoguer = { version = "0.11.0", features = ["fuzzy-select"] }
dotenvy = "0.15.7"
figment.workspace = true
futures-util.workspace = true
http-body-util.workspace = true
hyper.workspace = true
ipnetwork = "0.20.0"
@@ -32,11 +33,13 @@ rand.workspace = true
rand_chacha = "0.3.1"
reqwest.workspace = true
rustls.workspace = true
sd-notify = "0.4.5"
serde_json.workspace = true
serde_yaml = "0.9.34"
sqlx.workspace = true
tokio.workspace = true
tokio-util.workspace = true
tokio-stream.workspace = true
tower.workspace = true
tower-http.workspace = true
url.workspace = true
@@ -78,6 +81,9 @@ mas-tasks.workspace = true
mas-templates.workspace = true
mas-tower.workspace = true
oauth2-types.workspace = true
syn2mas.workspace = true
[build-dependencies]
anyhow.workspace = true
vergen-gitcl = { version = "1.0.5", features = ["rustc"] }

69
crates/cli/src/app_state.rs
View File

@@ -6,10 +6,7 @@
use std::{convert::Infallible, net::IpAddr, sync::Arc, time::Instant};
use axum::{
async_trait,
extract::{FromRef, FromRequestParts},
};
use axum::extract::{FromRef, FromRequestParts};
use ipnetwork::IpNetwork;
use mas_data_model::SiteConfig;
use mas_handlers::{
@@ -28,6 +25,7 @@ use mas_templates::Templates;
use opentelemetry::{metrics::Histogram, KeyValue};
use rand::SeedableRng;
use sqlx::PgPool;
use tracing::Instrument;
use crate::telemetry::METER;
@@ -88,29 +86,43 @@ impl AppState {
self.conn_acquisition_histogram = Some(histogram);
}
/// Init the metadata cache.
///
/// # Panics
///
/// Panics if the metadata cache could not be initialized.
pub async fn init_metadata_cache(&self) {
// XXX: this panics because the error is annoying to propagate
let conn = self
.pool
.acquire()
.await
.expect("Failed to acquire a database connection");
/// Init the metadata cache in the background
pub fn init_metadata_cache(&self) {
let pool = self.pool.clone();
let metadata_cache = self.metadata_cache.clone();
let http_client = self.http_client.clone();
let mut repo = PgRepository::from_conn(conn);
tokio::spawn(
async move {
let conn = match pool.acquire().await {
Ok(conn) => conn,
Err(e) => {
tracing::error!(
error = &e as &dyn std::error::Error,
"Failed to acquire a database connection"
);
return;
}
};
self.metadata_cache
.warm_up_and_run(
&self.http_client,
std::time::Duration::from_secs(60 * 15),
&mut repo,
)
.await
.expect("Failed to warm up the metadata cache");
let mut repo = PgRepository::from_conn(conn);
if let Err(e) = metadata_cache
.warm_up_and_run(
&http_client,
std::time::Duration::from_secs(60 * 15),
&mut repo,
)
.await
{
tracing::error!(
error = &e as &dyn std::error::Error,
"Failed to warm up the metadata cache"
);
}
}
.instrument(tracing::info_span!("metadata_cache.background_warmup")),
);
}
}
@@ -198,7 +210,6 @@ impl FromRef<AppState> for BoxHomeserverConnection {
}
}
#[async_trait]
impl FromRequestParts<AppState> for BoxClock {
type Rejection = Infallible;
@@ -211,7 +222,6 @@ impl FromRequestParts<AppState> for BoxClock {
}
}
#[async_trait]
impl FromRequestParts<AppState> for BoxRng {
type Rejection = Infallible;
@@ -228,7 +238,6 @@ impl FromRequestParts<AppState> for BoxRng {
}
}
#[async_trait]
impl FromRequestParts<AppState> for Policy {
type Rejection = ErrorWrapper<mas_policy::InstantiateError>;
@@ -241,7 +250,6 @@ impl FromRequestParts<AppState> for Policy {
}
}
#[async_trait]
impl FromRequestParts<AppState> for ActivityTracker {
type Rejection = Infallible;
@@ -300,7 +308,6 @@ fn infer_client_ip(
client_ip.or(fallback)
}
#[async_trait]
impl FromRequestParts<AppState> for BoundActivityTracker {
type Rejection = Infallible;
@@ -315,7 +322,6 @@ impl FromRequestParts<AppState> for BoundActivityTracker {
}
}
#[async_trait]
impl FromRequestParts<AppState> for RequesterFingerprint {
type Rejection = Infallible;
@@ -337,7 +343,6 @@ impl FromRequestParts<AppState> for RequesterFingerprint {
}
}
#[async_trait]
impl FromRequestParts<AppState> for BoxRepository {
type Rejection = ErrorWrapper<mas_storage_pg::DatabaseError>;

2
crates/cli/src/commands/manage.rs
View File

@@ -310,7 +310,7 @@ impl Options {
info!(
%compat_access_token.id,
%compat_session.id,
%compat_session.device,
compat_session.device = compat_session.device.map(tracing::field::display),
%user.id,
%user.username,
"Compatibility token issued: {}", compat_access_token.token

7
crates/cli/src/commands/mod.rs
View File

@@ -19,6 +19,7 @@ mod debug;
mod doctor;
mod manage;
mod server;
mod syn2mas;
mod templates;
mod worker;
@@ -48,6 +49,11 @@ enum Subcommand {
/// Run diagnostics on the deployment
Doctor(self::doctor::Options),
/// Migrate from Synapse's built-in auth system to MAS.
#[clap(name = "syn2mas")]
// Box<> is to work around a 'large size difference between variants' lint
Syn2Mas(Box<self::syn2mas::Options>),
}
#[derive(Parser, Debug)]
@@ -75,6 +81,7 @@ impl Options {
Some(S::Templates(c)) => Box::pin(c.run(figment)).await,
Some(S::Debug(c)) => Box::pin(c.run(figment)).await,
Some(S::Doctor(c)) => Box::pin(c.run(figment)).await,
Some(S::Syn2Mas(c)) => Box::pin(c.run(figment)).await,
None => Box::pin(self::server::Options::default().run(figment)).await,
}
}

20
crates/cli/src/commands/server.rs
View File

@@ -24,11 +24,11 @@ use tracing::{info, info_span, warn, Instrument};
use crate::{
app_state::AppState,
shutdown::ShutdownManager,
lifecycle::LifecycleManager,
util::{
database_pool_from_config, mailer_from_config, password_manager_from_config,
policy_factory_from_config, register_sighup, site_config_from_config,
templates_from_config,
policy_factory_from_config, site_config_from_config, templates_from_config,
test_mailer_in_background,
},
};
@@ -57,7 +57,7 @@ impl Options {
#[allow(clippy::too_many_lines)]
pub async fn run(self, figment: &Figment) -> anyhow::Result<ExitCode> {
let span = info_span!("cli.run.init").entered();
let shutdown = ShutdownManager::new()?;
let mut shutdown = LifecycleManager::new()?;
let config = AppConfig::extract(figment)?;
info!(version = crate::VERSION, "Starting up");
@@ -145,6 +145,7 @@ impl Options {
// Load and compile the templates
let templates =
templates_from_config(&config.templates, &site_config, &url_builder).await?;
shutdown.register_reloadable(&templates);
let http_client = mas_http::reqwest_client();
@@ -157,7 +158,7 @@ impl Options {
if !self.no_worker {
let mailer = mailer_from_config(&config.email, &templates)?;
mailer.test_connection().await?;
test_mailer_in_background(&mailer, Duration::from_secs(30));
info!("Starting task worker");
mas_tasks::init(
@@ -186,6 +187,9 @@ impl Options {
shutdown.task_tracker(),
shutdown.soft_shutdown_token(),
);
shutdown.register_reloadable(&activity_tracker);
let trusted_proxies = config.http.trusted_proxies.clone();
// Build a rate limiter.
@@ -197,9 +201,6 @@ impl Options {
// Explicitly the config to properly zeroize secret keys
drop(config);
// Listen for SIGHUP
register_sighup(&templates, &activity_tracker)?;
limiter.start();
let graphql_schema = mas_handlers::graphql_schema(
@@ -233,8 +234,7 @@ impl Options {
conn_acquisition_histogram: None,
};
s.init_metrics();
// XXX: this might panic
s.init_metadata_cache().await;
s.init_metadata_cache();
s
};

248
crates/cli/src/commands/syn2mas.rs Normal file
View File

@@ -0,0 +1,248 @@
use std::{collections::HashMap, process::ExitCode};
use anyhow::Context;
use camino::Utf8PathBuf;
use clap::Parser;
use figment::Figment;
use mas_config::{
ConfigurationSection, ConfigurationSectionExt, DatabaseConfig, MatrixConfig, SyncConfig,
UpstreamOAuth2Config,
};
use mas_storage::SystemClock;
use mas_storage_pg::MIGRATOR;
use rand::thread_rng;
use sqlx::{postgres::PgConnectOptions, types::Uuid, Connection, Either, PgConnection};
use syn2mas::{synapse_config, LockedMasDatabase, MasWriter, SynapseReader};
use tracing::{error, info_span, warn, Instrument};
use crate::util::database_connection_from_config;
/// The exit code used by `syn2mas check` and `syn2mas migrate` when there are
/// errors preventing migration.
const EXIT_CODE_CHECK_ERRORS: u8 = 10;
/// The exit code used by `syn2mas check` when there are warnings which should
/// be considered prior to migration.
const EXIT_CODE_CHECK_WARNINGS: u8 = 11;
#[derive(Parser, Debug)]
pub(super) struct Options {
#[command(subcommand)]
subcommand: Subcommand,
/// This version of the syn2mas tool is EXPERIMENTAL and INCOMPLETE. It is
/// only suitable for TESTING. If you want to use this tool anyway,
/// please pass this argument.
///
/// If you want to migrate from Synapse to MAS today, please use the
/// Node.js-based tool in the MAS repository.
#[clap(long = "i-swear-i-am-just-testing-in-a-staging-environment")]
experimental_accepted: bool,
/// Path to the Synapse configuration (in YAML format).
/// May be specified multiple times if multiple Synapse configuration files
/// are in use.
#[clap(long = "synapse-config")]
synapse_configuration_files: Vec<Utf8PathBuf>,
/// Override the Synapse database URI.
/// syn2mas normally loads the Synapse database connection details from the
/// Synapse configuration. However, it may sometimes be necessary to
/// override the database URI and in that case this flag can be used.
///
/// Should be a connection URI of the following general form:
/// ```text
/// postgresql://[user[:password]@][host][:port][/dbname][?param1=value1&...]
/// ```
/// To use a UNIX socket at a custom path, the host should be a path to a
/// socket, but in the URI string it must be URI-encoded by replacing
/// `/` with `%2F`.
///
/// Finally, any missing values will be loaded from the libpq-compatible
/// environment variables `PGHOST`, `PGPORT`, `PGUSER`, `PGDATABASE`,
/// `PGPASSWORD`, etc. It is valid to specify the URL `postgresql:` and
/// configure all values through those environment variables.
#[clap(long = "synapse-database-uri")]
synapse_database_uri: Option<PgConnectOptions>,
}
#[derive(Parser, Debug)]
enum Subcommand {
/// Check the setup for potential problems before running a migration.
///
/// It is OK for Synapse to be online during these checks.
Check,
/// Perform a migration. Synapse must be offline during this process.
Migrate,
}
/// The number of parallel writing transactions active against the MAS database.
const NUM_WRITER_CONNECTIONS: usize = 8;
impl Options {
#[allow(clippy::too_many_lines)]
pub async fn run(self, figment: &Figment) -> anyhow::Result<ExitCode> {
warn!("This version of the syn2mas tool is EXPERIMENTAL and INCOMPLETE. Do not use it, except for TESTING.");
if !self.experimental_accepted {
error!("Please agree that you can only use this tool for testing.");
return Ok(ExitCode::FAILURE);
}
if self.synapse_configuration_files.is_empty() {
error!("Please specify the path to the Synapse configuration file(s).");
return Ok(ExitCode::FAILURE);
}
let synapse_config = synapse_config::Config::load(&self.synapse_configuration_files)
.context("Failed to load Synapse configuration")?;
// Establish a connection to Synapse's Postgres database
let syn_connection_options = if let Some(db_override) = self.synapse_database_uri {
db_override
} else {
synapse_config
.database
.to_sqlx_postgres()
.context("Synapse configuration does not use Postgres, cannot migrate.")?
};
let mut syn_conn = PgConnection::connect_with(&syn_connection_options)
.await
.context("could not connect to Synapse Postgres database")?;
let config = DatabaseConfig::extract_or_default(figment)?;
let mut mas_connection = database_connection_from_config(&config).await?;
MIGRATOR
.run(&mut mas_connection)
.instrument(info_span!("db.migrate"))
.await
.context("could not run migrations")?;
if matches!(&self.subcommand, Subcommand::Migrate { .. }) {
// First perform a config sync
// This is crucial to ensure we register upstream OAuth providers
// in the MAS database
//
let config = SyncConfig::extract(figment)?;
let clock = SystemClock::default();
let encrypter = config.secrets.encrypter();
crate::sync::config_sync(
config.upstream_oauth2,
config.clients,
&mut mas_connection,
&encrypter,
&clock,
// Don't prune — we don't want to be unnecessarily destructive
false,
// Not a dry run — we do want to create the providers in the database
false,
)
.await?;
}
let Either::Left(mut mas_connection) = LockedMasDatabase::try_new(&mut mas_connection)
.await
.context("failed to issue query to lock database")?
else {
error!("Failed to acquire syn2mas lock on the database.");
error!("This likely means that another syn2mas instance is already running!");
return Ok(ExitCode::FAILURE);
};
// Check configuration
let (mut check_warnings, mut check_errors) = syn2mas::synapse_config_check(&synapse_config);
{
let (extra_warnings, extra_errors) =
syn2mas::synapse_config_check_against_mas_config(&synapse_config, figment).await?;
check_warnings.extend(extra_warnings);
check_errors.extend(extra_errors);
}
// Check databases
syn2mas::mas_pre_migration_checks(&mut mas_connection).await?;
{
let (extra_warnings, extra_errors) =
syn2mas::synapse_database_check(&mut syn_conn, &synapse_config, figment).await?;
check_warnings.extend(extra_warnings);
check_errors.extend(extra_errors);
}
// Display errors and warnings
if !check_errors.is_empty() {
eprintln!("===== Errors =====");
eprintln!("These issues prevent migrating from Synapse to MAS right now:\n");
for error in &check_errors {
eprintln!("• {error}\n");
}
}
if !check_warnings.is_empty() {
eprintln!("===== Warnings =====");
eprintln!("These potential issues should be considered before migrating from Synapse to MAS right now:\n");
for warning in &check_warnings {
eprintln!("• {warning}\n");
}
}
// Do not proceed if there are any errors
if !check_errors.is_empty() {
return Ok(ExitCode::from(EXIT_CODE_CHECK_ERRORS));
}
match self.subcommand {
Subcommand::Check => {
if !check_warnings.is_empty() {
return Ok(ExitCode::from(EXIT_CODE_CHECK_WARNINGS));
}
println!("Check completed successfully with no errors or warnings.");
Ok(ExitCode::SUCCESS)
}
Subcommand::Migrate => {
let provider_id_mappings: HashMap<String, Uuid> = {
let mas_oauth2 = UpstreamOAuth2Config::extract_or_default(figment)?;
mas_oauth2
.providers
.iter()
.filter_map(|provider| {
let synapse_idp_id = provider.synapse_idp_id.clone()?;
Some((synapse_idp_id, Uuid::from(provider.id)))
})
.collect()
};
// TODO how should we handle warnings at this stage?
let mut reader = SynapseReader::new(&mut syn_conn, true).await?;
let mut writer_mas_connections = Vec::with_capacity(NUM_WRITER_CONNECTIONS);
for _ in 0..NUM_WRITER_CONNECTIONS {
writer_mas_connections.push(database_connection_from_config(&config).await?);
}
let mut writer = MasWriter::new(mas_connection, writer_mas_connections).await?;
// TODO is this rng ok?
#[allow(clippy::disallowed_methods)]
let mut rng = thread_rng();
// TODO progress reporting
let mas_matrix = MatrixConfig::extract(figment)?;
syn2mas::migrate(
&mut reader,
&mut writer,
&mas_matrix.homeserver,
&mut rng,
&provider_id_mappings,
)
.await?;
reader.finish().await?;
writer.finish().await?;
Ok(ExitCode::SUCCESS)
}
}
}
}

10
crates/cli/src/commands/worker.rs
View File

@@ -4,7 +4,7 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Please see LICENSE in the repository root for full details.
use std::process::ExitCode;
use std::{process::ExitCode, time::Duration};
use clap::Parser;
use figment::Figment;
@@ -14,10 +14,10 @@ use mas_router::UrlBuilder;
use tracing::{info, info_span};
use crate::{
shutdown::ShutdownManager,
lifecycle::LifecycleManager,
util::{
database_pool_from_config, mailer_from_config, site_config_from_config,
templates_from_config,
templates_from_config, test_mailer_in_background,
},
};
@@ -26,7 +26,7 @@ pub(super) struct Options {}
impl Options {
pub async fn run(self, figment: &Figment) -> anyhow::Result<ExitCode> {
let shutdown = ShutdownManager::new()?;
let shutdown = LifecycleManager::new()?;
let span = info_span!("cli.worker.init").entered();
let config = AppConfig::extract(figment)?;
@@ -55,7 +55,7 @@ impl Options {
templates_from_config(&config.templates, &site_config, &url_builder).await?;
let mailer = mailer_from_config(&config.email, &templates)?;
mailer.test_connection().await?;
test_mailer_in_background(&mailer, Duration::from_secs(30));
let http_client = mas_http::reqwest_client();
let conn = SynapseConnection::new(

239
crates/cli/src/lifecycle.rs Normal file
View File

@@ -0,0 +1,239 @@
// Copyright 2024, 2025 New Vector Ltd.
//
// SPDX-License-Identifier: AGPL-3.0-only
// Please see LICENSE in the repository root for full details.
use std::{future::Future, process::ExitCode, time::Duration};
use futures_util::future::{BoxFuture, Either};
use mas_handlers::ActivityTracker;
use mas_templates::Templates;
use tokio::signal::unix::{Signal, SignalKind};
use tokio_util::{sync::CancellationToken, task::TaskTracker};
/// A helper to manage the lifecycle of the service, inclusing handling graceful
/// shutdowns and configuration reloads.
///
/// It will listen for SIGTERM and SIGINT signals, and will trigger a soft
/// shutdown on the first signal, and a hard shutdown on the second signal or
/// after a timeout.
///
/// Users of this manager should use the `soft_shutdown_token` to react to a
/// soft shutdown, which should gracefully finish requests and close
/// connections, and the `hard_shutdown_token` to react to a hard shutdown,
/// which should drop all connections and finish all requests.
///
/// They should also use the `task_tracker` to make it track things running, so
/// that it knows when the soft shutdown is over and worked.
///
/// It also integrates with [`sd_notify`] to notify the service manager of the
/// state of the service.
pub struct LifecycleManager {
hard_shutdown_token: CancellationToken,
soft_shutdown_token: CancellationToken,
task_tracker: TaskTracker,
sigterm: Signal,
sigint: Signal,
sighup: Signal,
timeout: Duration,
reload_handlers: Vec<Box<dyn Fn() -> BoxFuture<'static, ()>>>,
}
/// Represents a thing that can be reloaded with a SIGHUP
pub trait Reloadable: Clone + Send {
fn reload(&self) -> impl Future<Output = ()> + Send;
}
impl Reloadable for ActivityTracker {
async fn reload(&self) {
self.flush().await;
}
}
impl Reloadable for Templates {
async fn reload(&self) {
if let Err(err) = self.reload().await {
tracing::error!(
error = &err as &dyn std::error::Error,
"Failed to reload templates"
);
}
}
}
/// A wrapper around [`sd_notify::notify`] that logs any errors
fn notify(states: &[sd_notify::NotifyState]) {
if let Err(e) = sd_notify::notify(false, states) {
tracing::error!(
error = &e as &dyn std::error::Error,
"Failed to notify service manager"
);
}
}
impl LifecycleManager {
/// Create a new shutdown manager, installing the signal handlers
///
/// # Errors
///
/// Returns an error if the signal handler could not be installed
pub fn new() -> Result<Self, std::io::Error> {
let hard_shutdown_token = CancellationToken::new();
let soft_shutdown_token = hard_shutdown_token.child_token();
let sigterm = tokio::signal::unix::signal(SignalKind::terminate())?;
let sigint = tokio::signal::unix::signal(SignalKind::interrupt())?;
let sighup = tokio::signal::unix::signal(SignalKind::hangup())?;
let timeout = Duration::from_secs(60);
let task_tracker = TaskTracker::new();
notify(&[sd_notify::NotifyState::MainPid(std::process::id())]);
Ok(Self {
hard_shutdown_token,
soft_shutdown_token,
task_tracker,
sigterm,
sigint,
sighup,
timeout,
reload_handlers: Vec::new(),
})
}
/// Add a handler to be called when the server gets a SIGHUP
pub fn register_reloadable(&mut self, reloadable: &(impl Reloadable + 'static)) {
let reloadable = reloadable.clone();
self.reload_handlers.push(Box::new(move || {
let reloadable = reloadable.clone();
Box::pin(async move { reloadable.reload().await })
}));
}
/// Get a reference to the task tracker
#[must_use]
pub fn task_tracker(&self) -> &TaskTracker {
&self.task_tracker
}
/// Get a cancellation token that can be used to react to a hard shutdown
#[must_use]
pub fn hard_shutdown_token(&self) -> CancellationToken {
self.hard_shutdown_token.clone()
}
/// Get a cancellation token that can be used to react to a soft shutdown
#[must_use]
pub fn soft_shutdown_token(&self) -> CancellationToken {
self.soft_shutdown_token.clone()
}
/// Run until we finish completely shutting down.
pub async fn run(mut self) -> ExitCode {
notify(&[sd_notify::NotifyState::Ready]);
// This will be `Some` if we have the watchdog enabled, and `None` if not
let mut watchdog_interval = {
let mut watchdog_usec = 0;
if sd_notify::watchdog_enabled(false, &mut watchdog_usec) {
Some(tokio::time::interval(Duration::from_micros(
watchdog_usec / 2,
)))
} else {
None
}
};
// Wait for a first shutdown signal and trigger the soft shutdown
let likely_crashed = loop {
// This makes a Future that will either yield the watchdog tick if enabled, or a
// pending Future if not
let watchdog_tick = if let Some(watchdog_interval) = &mut watchdog_interval {
Either::Left(watchdog_interval.tick())
} else {
Either::Right(futures_util::future::pending())
};
tokio::select! {
() = self.soft_shutdown_token.cancelled() => {
tracing::warn!("Another task triggered a shutdown, it likely crashed! Shutting down");
break true;
},
_ = self.sigterm.recv() => {
tracing::info!("Shutdown signal received (SIGTERM), shutting down");
break false;
},
_ = self.sigint.recv() => {
tracing::info!("Shutdown signal received (SIGINT), shutting down");
break false;
},
_ = watchdog_tick => {
notify(&[
sd_notify::NotifyState::Watchdog,
]);
},
_ = self.sighup.recv() => {
tracing::info!("Reload signal received (SIGHUP), reloading");
notify(&[
sd_notify::NotifyState::Reloading,
sd_notify::NotifyState::monotonic_usec_now()
.expect("Failed to read monotonic clock")
]);
// XXX: if one handler takes a long time, it will block the
// rest of the shutdown process, which is not ideal. We
// should probably have a timeout here
futures_util::future::join_all(
self.reload_handlers
.iter()
.map(|handler| handler())
).await;
notify(&[sd_notify::NotifyState::Ready]);
tracing::info!("Reloading done");
},
}
};
notify(&[sd_notify::NotifyState::Stopping]);
self.soft_shutdown_token.cancel();
self.task_tracker.close();
// Start the timeout
let timeout = tokio::time::sleep(self.timeout);
tokio::select! {
_ = self.sigterm.recv() => {
tracing::warn!("Second shutdown signal received (SIGTERM), abort");
},
_ = self.sigint.recv() => {
tracing::warn!("Second shutdown signal received (SIGINT), abort");
},
() = timeout => {
tracing::warn!("Shutdown timeout reached, abort");
},
() = self.task_tracker.wait() => {
// This is the "happy path", we have gracefully shutdown
},
}
self.hard_shutdown_token().cancel();
// TODO: we may want to have a time out on the task tracker, in case we have
// really stuck tasks on it
self.task_tracker().wait().await;
tracing::info!("All tasks are done, exitting");
if likely_crashed {
ExitCode::FAILURE
} else {
ExitCode::SUCCESS
}
}
}

2
crates/cli/src/main.rs
View File

@@ -18,8 +18,8 @@ use tracing_subscriber::{
mod app_state;
mod commands;
mod lifecycle;
mod server;
mod shutdown;
mod sync;
mod telemetry;
mod util;

130
crates/cli/src/shutdown.rs
View File

@@ -1,130 +0,0 @@
// Copyright 2024 New Vector Ltd.
//
// SPDX-License-Identifier: AGPL-3.0-only
// Please see LICENSE in the repository root for full details.
use std::{process::ExitCode, time::Duration};
use tokio::signal::unix::{Signal, SignalKind};
use tokio_util::{sync::CancellationToken, task::TaskTracker};
/// A helper to manage graceful shutdowns and track tasks that gracefully
/// shutdown.
///
/// It will listen for SIGTERM and SIGINT signals, and will trigger a soft
/// shutdown on the first signal, and a hard shutdown on the second signal or
/// after a timeout.
///
/// Users of this manager should use the `soft_shutdown_token` to react to a
/// soft shutdown, which should gracefully finish requests and close
/// connections, and the `hard_shutdown_token` to react to a hard shutdown,
/// which should drop all connections and finish all requests.
///
/// They should also use the `task_tracker` to make it track things running, so
/// that it knows when the soft shutdown is over and worked.
pub struct ShutdownManager {
hard_shutdown_token: CancellationToken,
soft_shutdown_token: CancellationToken,
task_tracker: TaskTracker,
sigterm: Signal,
sigint: Signal,
timeout: Duration,
}
impl ShutdownManager {
/// Create a new shutdown manager, installing the signal handlers
///
/// # Errors
///
/// Returns an error if the signal handler could not be installed
pub fn new() -> Result<Self, std::io::Error> {
let hard_shutdown_token = CancellationToken::new();
let soft_shutdown_token = hard_shutdown_token.child_token();
let sigterm = tokio::signal::unix::signal(SignalKind::terminate())?;
let sigint = tokio::signal::unix::signal(SignalKind::interrupt())?;
let timeout = Duration::from_secs(60);
let task_tracker = TaskTracker::new();
Ok(Self {
hard_shutdown_token,
soft_shutdown_token,
task_tracker,
sigterm,
sigint,
timeout,
})
}
/// Get a reference to the task tracker
#[must_use]
pub fn task_tracker(&self) -> &TaskTracker {
&self.task_tracker
}
/// Get a cancellation token that can be used to react to a hard shutdown
#[must_use]
pub fn hard_shutdown_token(&self) -> CancellationToken {
self.hard_shutdown_token.clone()
}
/// Get a cancellation token that can be used to react to a soft shutdown
#[must_use]
pub fn soft_shutdown_token(&self) -> CancellationToken {
self.soft_shutdown_token.clone()
}
/// Run until we finish completely shutting down.
pub async fn run(mut self) -> ExitCode {
// Wait for a first signal and trigger the soft shutdown
let likely_crashed = tokio::select! {
() = self.soft_shutdown_token.cancelled() => {
tracing::warn!("Another task triggered a shutdown, it likely crashed! Shutting down");
true
},
_ = self.sigterm.recv() => {
tracing::info!("Shutdown signal received (SIGTERM), shutting down");
false
},
_ = self.sigint.recv() => {
tracing::info!("Shutdown signal received (SIGINT), shutting down");
false
},
};
self.soft_shutdown_token.cancel();
self.task_tracker.close();
// Start the timeout
let timeout = tokio::time::sleep(self.timeout);
tokio::select! {
_ = self.sigterm.recv() => {
tracing::warn!("Second shutdown signal received (SIGTERM), abort");
},
_ = self.sigint.recv() => {
tracing::warn!("Second shutdown signal received (SIGINT), abort");
},
() = timeout => {
tracing::warn!("Shutdown timeout reached, abort");
},
() = self.task_tracker.wait() => {
// This is the "happy path", we have gracefully shutdown
},
}
self.hard_shutdown_token().cancel();
// TODO: we may want to have a time out on the task tracker, in case we have
// really stuck tasks on it
self.task_tracker().wait().await;
tracing::info!("All tasks are done, exitting");
if likely_crashed {
ExitCode::FAILURE
} else {
ExitCode::SUCCESS
}
}
}

56
crates/cli/src/util.rs
View File

@@ -14,7 +14,7 @@ use mas_config::{
};
use mas_data_model::SiteConfig;
use mas_email::{MailTransport, Mailer};
use mas_handlers::{passwords::PasswordManager, ActivityTracker};
use mas_handlers::passwords::PasswordManager;
use mas_policy::PolicyFactory;
use mas_router::UrlBuilder;
use mas_templates::{SiteConfigExt, TemplateLoadingError, Templates};
@@ -22,7 +22,7 @@ use sqlx::{
postgres::{PgConnectOptions, PgPoolOptions},
ConnectOptions, PgConnection, PgPool,
};
use tracing::{error, info, log::LevelFilter};
use tracing::{log::LevelFilter, Instrument};
pub async fn password_manager_from_config(
config: &PasswordsConfig,
@@ -99,6 +99,27 @@ pub fn mailer_from_config(
Ok(Mailer::new(templates.clone(), transport, from, reply_to))
}
/// Test the connection to the mailer in a background task
pub fn test_mailer_in_background(mailer: &Mailer, timeout: Duration) {
let mailer = mailer.clone();
let span = tracing::info_span!("cli.test_mailer");
tokio::spawn(async move {
match tokio::time::timeout(timeout, mailer.test_connection()).await {
Ok(Ok(())) => {}
Ok(Err(err)) => {
tracing::warn!(
error = &err as &dyn std::error::Error,
"Could not connect to the mail backend, tasks sending mails may fail!"
);
}
Err(_) => {
tracing::warn!("Timed out while testing the mail backend connection, tasks sending mails may fail!");
}
}
}.instrument(span));
}
pub async fn policy_factory_from_config(
config: &PolicyConfig,
matrix_config: &MatrixConfig,
@@ -313,37 +334,6 @@ pub async fn database_connection_from_config(
.context("could not connect to the database")
}
/// Reload templates on SIGHUP
pub fn register_sighup(
templates: &Templates,
activity_tracker: &ActivityTracker,
) -> anyhow::Result<()> {
#[cfg(unix)]
{
let mut signal = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::hangup())?;
let templates = templates.clone();
let activity_tracker = activity_tracker.clone();
tokio::spawn(async move {
loop {
if signal.recv().await.is_none() {
// No more signals will be received, breaking
break;
};
info!("SIGHUP received, reloading templates & flushing activity tracker");
activity_tracker.flush().await;
templates.clone().reload().await.unwrap_or_else(|err| {
error!(?err, "Error while reloading templates");
});
}
});
}
Ok(())
}
#[cfg(test)]
mod tests {
use rand::SeedableRng;

2
crates/config/Cargo.toml
View File

@@ -30,7 +30,7 @@ serde_with = { version = "3.12.0", features = ["hex", "chrono"] }
serde_json.workspace = true
pem-rfc7468 = "0.7.0"
rustls-pki-types = "1.10.1"
rustls-pki-types = "1.11.0"
rustls-pemfile = "2.2.0"
rand.workspace = true
rand_chacha = "0.3.1"

2
crates/config/src/sections/mod.rs
View File

@@ -51,7 +51,7 @@ pub use self::{
ClaimsImports as UpstreamOAuth2ClaimsImports, DiscoveryMode as UpstreamOAuth2DiscoveryMode,
EmailImportPreference as UpstreamOAuth2EmailImportPreference,
ImportAction as UpstreamOAuth2ImportAction, PkceMethod as UpstreamOAuth2PkceMethod,
ResponseMode as UpstreamOAuth2ResponseMode,
Provider as UpstreamOAuth2Provider, ResponseMode as UpstreamOAuth2ResponseMode,
TokenAuthMethod as UpstreamOAuth2TokenAuthMethod, UpstreamOAuth2Config,
},
};

2
crates/config/src/sections/passwords.rs
View File

@@ -179,7 +179,7 @@ fn default_bcrypt_cost() -> Option<u32> {
}
/// A hashing algorithm
#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema)]
#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum Algorithm {
/// bcrypt

18
crates/config/src/sections/upstream_oauth2.rs
View File

@@ -391,6 +391,7 @@ pub struct SignInWithApple {
pub key_id: String,
}
/// Configuration for one upstream OAuth 2 provider.
#[skip_serializing_none]
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
pub struct Provider {
@@ -537,4 +538,21 @@ pub struct Provider {
/// Orders of the keys are not preserved.
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub additional_authorization_parameters: BTreeMap<String, String>,
/// The ID of the provider that was used by Synapse.
/// In order to perform a Synapse-to-MAS migration, this must be specified.
///
/// ## For providers that used OAuth 2.0 or OpenID Connect in Synapse
///
/// ### For `oidc_providers`:
/// This should be specified as `oidc-` followed by the ID that was
/// configured as `idp_id` in one of the `oidc_providers` in the Synapse
/// configuration.
/// For example, if Synapse's configuration contained `idp_id: wombat` for
/// this provider, then specify `oidc-wombat` here.
///
/// ### For `oidc_config` (legacy):
/// Specify `oidc` here.
#[serde(skip_serializing_if = "Option::is_none")]
pub synapse_idp_id: Option<String>,
}

2
crates/data-model/Cargo.toml
View File

@@ -24,7 +24,7 @@ rand.workspace = true
rand_chacha = "0.3.1"
regex = "1.11.1"
woothee = "0.13.0"
ruma-common = "0.15.0"
ruma-common.workspace = true
mas-iana.workspace = true
mas-jose.workspace = true

2
crates/data-model/src/compat/session.rs
View File

@@ -71,7 +71,7 @@ pub struct CompatSession {
pub id: Ulid,
pub state: CompatSessionState,
pub user_id: Ulid,
pub device: Device,
pub device: Option<Device>,
pub user_session_id: Option<Ulid>,
pub created_at: DateTime<Utc>,
pub is_synapse_admin: bool,

8
crates/data-model/src/oauth2/authorization_grant.rs
View File

@@ -17,7 +17,7 @@ use rand::{
distributions::{Alphanumeric, DistString},
RngCore,
};
use ruma_common::{OwnedUserId, UserId};
use ruma_common::UserId;
use serde::Serialize;
use ulid::Ulid;
use url::Url;
@@ -142,8 +142,8 @@ impl AuthorizationGrantStage {
}
}
pub enum LoginHint {
MXID(OwnedUserId),
pub enum LoginHint<'a> {
MXID(&'a UserId),
None,
}
@@ -200,7 +200,7 @@ impl AuthorizationGrant {
match prefix {
"mxid" => {
// Instead of erroring just return none
let Ok(mxid) = UserId::parse(value) else {
let Ok(mxid) = <&UserId>::try_from(value) else {
return LoginHint::None;
};

4
crates/handlers/Cargo.toml
View File

@@ -33,7 +33,7 @@ hyper.workspace = true
tower.workspace = true
tower-http.workspace = true
axum.workspace = true
axum-macros = "0.4.2"
axum-macros = "0.5.0"
axum-extra.workspace = true
rustls.workspace = true
@@ -75,7 +75,7 @@ elliptic-curve.workspace = true
governor.workspace = true
indexmap = "2.7.1"
pkcs8.workspace = true
psl = "2.1.80"
psl = "2.1.81"
time = "0.3.37"
url.workspace = true
mime = "0.3.17"

3
crates/handlers/src/admin/call_context.rs
View File

@@ -1,4 +1,4 @@
// Copyright 2024 New Vector Ltd.
// Copyright 2024, 2025 New Vector Ltd.
// Copyright 2024 The Matrix.org Foundation C.I.C.
//
// SPDX-License-Identifier: AGPL-3.0-only
@@ -107,7 +107,6 @@ pub struct CallContext {
pub session: Session,
}
#[async_trait::async_trait]
impl<S> FromRequestParts<S> for CallContext
where
S: Send + Sync,

2
crates/handlers/src/admin/mod.rs
View File

@@ -47,7 +47,7 @@ where
Templates: FromRef<S>,
UrlBuilder: FromRef<S>,
{
aide::gen::in_context(|ctx| {
aide::generate::in_context(|ctx| {
ctx.schema = schemars::gen::SchemaGenerator::new(schemars::gen::SchemaSettings::openapi3());
});

2
crates/handlers/src/admin/params.rs
View File

@@ -10,7 +10,6 @@
use std::num::NonZeroUsize;
use aide::OperationIo;
use async_trait::async_trait;
use axum::{
extract::{
rejection::{PathRejection, QueryRejection},
@@ -110,7 +109,6 @@ impl IntoResponse for PaginationRejection {
#[aide(input_with = "Query<PaginationParams>")]
pub struct Pagination(pub mas_storage::Pagination);
#[async_trait]
impl<S: Send + Sync> FromRequestParts<S> for Pagination {
type Rejection = PaginationRejection;

16
crates/handlers/src/admin/v1/mod.rs
View File

@@ -32,7 +32,7 @@ where
get_with(self::oauth2_sessions::list, self::oauth2_sessions::list_doc),
)
.api_route(
"/oauth2-sessions/:id",
"/oauth2-sessions/{id}",
get_with(self::oauth2_sessions::get, self::oauth2_sessions::get_doc),
)
.api_route(
@@ -41,31 +41,31 @@ where
.post_with(self::users::add, self::users::add_doc),
)
.api_route(
"/users/:id",
"/users/{id}",
get_with(self::users::get, self::users::get_doc),
)
.api_route(
"/users/:id/set-password",
"/users/{id}/set-password",
post_with(self::users::set_password, self::users::set_password_doc),
)
.api_route(
"/users/by-username/:username",
"/users/by-username/{username}",
get_with(self::users::by_username, self::users::by_username_doc),
)
.api_route(
"/users/:id/set-admin",
"/users/{id}/set-admin",
post_with(self::users::set_admin, self::users::set_admin_doc),
)
.api_route(
"/users/:id/deactivate",
"/users/{id}/deactivate",
post_with(self::users::deactivate, self::users::deactivate_doc),
)
.api_route(
"/users/:id/lock",
"/users/{id}/lock",
post_with(self::users::lock, self::users::lock_doc),
)
.api_route(
"/users/:id/unlock",
"/users/{id}/unlock",
post_with(self::users::unlock, self::users::unlock_doc),
)
}

3
crates/handlers/src/bin/api-schema.rs
View File

@@ -1,4 +1,4 @@
// Copyright 2024 New Vector Ltd.
// Copyright 2024, 2025 New Vector Ltd.
// Copyright 2024 The Matrix.org Foundation C.I.C.
//
// SPDX-License-Identifier: AGPL-3.0-only
@@ -26,7 +26,6 @@ struct DummyState;
macro_rules! impl_from_request_parts {
($type:ty) => {
#[axum::async_trait]
impl axum::extract::FromRequestParts<DummyState> for $type {
type Rejection = std::convert::Infallible;

110
crates/handlers/src/compat/login.rs
View File

@@ -130,7 +130,7 @@ pub enum Identifier {
#[derive(Debug, Serialize, Deserialize)]
pub struct ResponseBody {
access_token: String,
device_id: Device,
device_id: Option<Device>,
user_id: String,
refresh_token: Option<String>,
#[serde_as(as = "Option<DurationMilliSeconds<i64>>")]
@@ -386,10 +386,13 @@ async fn user_password_login(
username: String,
password: String,
) -> Result<(CompatSession, User), RouteError> {
// Try getting the localpart out of the MXID
let username = homeserver.localpart(&username).unwrap_or(&username);
// Find the user
let user = repo
.user()
.find_by_username(&username)
.find_by_username(username)
.await?
.filter(mas_data_model::User::is_valid)
.ok_or(RouteError::UserNotFound)?;
@@ -539,23 +542,25 @@ mod tests {
assert_eq!(body["errcode"], "M_UNRECOGNIZED");
}
/// Test that a user can login with a password using the Matrix
/// compatibility API.
#[sqlx::test(migrator = "mas_storage_pg::MIGRATOR")]
async fn test_user_password_login(pool: PgPool) {
setup();
let state = TestState::from_pool(pool).await.unwrap();
// Let's provision a user and add a password to it. This part is hard to test
// with just HTTP requests, so we'll use the repository directly.
async fn user_with_password(state: &TestState, username: &str, password: &str) {
let mut rng = state.rng();
let mut repo = state.repository().await.unwrap();
let user = repo
.user()
.add(&mut state.rng(), &state.clock, "alice".to_owned())
.add(&mut rng, &state.clock, username.to_owned())
.await
.unwrap();
let (version, hash) = state
.password_manager
.hash(&mut rng, Zeroizing::new(password.as_bytes().to_vec()))
.await
.unwrap();
repo.user_password()
.add(&mut rng, &state.clock, &user, version, hash, None)
.await
.unwrap();
let mxid = state.homeserver_connection.mxid(&user.username);
state
.homeserver_connection
@@ -563,28 +568,17 @@ mod tests {
.await
.unwrap();
let (version, hashed_password) = state
.password_manager
.hash(
&mut state.rng(),
Zeroizing::new("password".to_owned().into_bytes()),
)
.await
.unwrap();
repo.user_password()
.add(
&mut state.rng(),
&state.clock,
&user,
version,
hashed_password,
None,
)
.await
.unwrap();
repo.save().await.unwrap();
}
/// Test that a user can login with a password using the Matrix
/// compatibility API.
#[sqlx::test(migrator = "mas_storage_pg::MIGRATOR")]
async fn test_user_password_login(pool: PgPool) {
setup();
let state = TestState::from_pool(pool).await.unwrap();
user_with_password(&state, "alice", "password").await;
// Now let's try to login with the password, without asking for a refresh token.
let request = Request::post("/_matrix/client/v3/login").json(serde_json::json!({
@@ -601,7 +595,7 @@ mod tests {
let body: ResponseBody = response.json();
assert!(!body.access_token.is_empty());
assert_eq!(body.device_id.as_str().len(), 10);
assert_eq!(body.device_id.as_ref().unwrap().as_str().len(), 10);
assert_eq!(body.user_id, "@alice:example.com");
assert_eq!(body.refresh_token, None);
assert_eq!(body.expires_in_ms, None);
@@ -622,7 +616,7 @@ mod tests {
let body: ResponseBody = response.json();
assert!(!body.access_token.is_empty());
assert_eq!(body.device_id.as_str().len(), 10);
assert_eq!(body.device_id.as_ref().unwrap().as_str().len(), 10);
assert_eq!(body.user_id, "@alice:example.com");
assert!(body.refresh_token.is_some());
assert!(body.expires_in_ms.is_some());
@@ -662,6 +656,50 @@ mod tests {
assert_eq!(body, old_body);
}
/// Test that a user can login with a password using the Matrix
/// compatibility API, using a MXID as identifier
#[sqlx::test(migrator = "mas_storage_pg::MIGRATOR")]
async fn test_user_password_login_mxid(pool: PgPool) {
setup();
let state = TestState::from_pool(pool).await.unwrap();
user_with_password(&state, "alice", "password").await;
// Login with a full MXID as identifier
let request = Request::post("/_matrix/client/v3/login").json(serde_json::json!({
"type": "m.login.password",
"identifier": {
"type": "m.id.user",
"user": "@alice:example.com",
},
"password": "password",
}));
let response = state.request(request).await;
response.assert_status(StatusCode::OK);
let body: ResponseBody = response.json();
assert!(!body.access_token.is_empty());
assert_eq!(body.device_id.as_ref().unwrap().as_str().len(), 10);
assert_eq!(body.user_id, "@alice:example.com");
assert_eq!(body.refresh_token, None);
assert_eq!(body.expires_in_ms, None);
// With a MXID, but with the wrong server name
let request = Request::post("/_matrix/client/v3/login").json(serde_json::json!({
"type": "m.login.password",
"identifier": {
"type": "m.id.user",
"user": "@alice:something.corp",
},
"password": "password",
}));
let response = state.request(request).await;
response.assert_status(StatusCode::FORBIDDEN);
let body: serde_json::Value = response.json();
assert_eq!(body["errcode"], "M_FORBIDDEN");
}
/// Test that password logins are rate limited.
#[sqlx::test(migrator = "mas_storage_pg::MIGRATOR")]
async fn test_password_login_rate_limit(pool: PgPool) {
@@ -776,7 +814,7 @@ mod tests {
let body: ResponseBody = response.json();
assert!(!body.access_token.is_empty());
assert_eq!(body.device_id, device);
assert_eq!(body.device_id, Some(device));
assert_eq!(body.user_id, "@alice:example.com");
assert_eq!(body.refresh_token, None);
assert_eq!(body.expires_in_ms, None);

3
crates/handlers/src/graphql/mod.rs
View File

@@ -14,7 +14,6 @@ use async_graphql::{
EmptySubscription, InputObject,
};
use axum::{
async_trait,
body::Body,
extract::{RawQuery, State as AxumState},
http::StatusCode,
@@ -78,7 +77,7 @@ struct GraphQLState {
limiter: Limiter,
}
#[async_trait]
#[async_trait::async_trait]
impl state::State for GraphQLState {
async fn repository(&self) -> Result<BoxRepository, RepositoryError> {
let repo = PgRepository::from_pool(&self.pool)

5
crates/handlers/src/graphql/model/compat_sessions.rs
View File

@@ -7,6 +7,7 @@
use anyhow::Context as _;
use async_graphql::{Context, Description, Enum, Object, ID};
use chrono::{DateTime, Utc};
use mas_data_model::Device;
use mas_storage::{compat::CompatSessionRepository, user::UserRepository};
use url::Url;
@@ -81,8 +82,8 @@ impl CompatSession {
}
/// The Matrix Device ID of this session.
async fn device_id(&self) -> &str {
self.session.device.as_str()
async fn device_id(&self) -> Option<&str> {
self.session.device.as_ref().map(Device::as_str)
}
/// When the object was created.

3
crates/handlers/src/oauth2/device/consent.rs
View File

@@ -7,10 +7,9 @@
use anyhow::Context;
use axum::{
extract::{Path, State},
response::{IntoResponse, Response},
response::{Html, IntoResponse, Response},
Form,
};
use axum_extra::response::Html;
use mas_axum_utils::{
cookies::CookieJar,
csrf::{CsrfExt, ProtectedForm},

17
crates/handlers/src/oauth2/device/link.rs
View File

@@ -6,9 +6,8 @@
use axum::{
extract::{Query, State},
response::IntoResponse,
response::{Html, IntoResponse},
};
use axum_extra::response::Html;
use mas_axum_utils::{cookies::CookieJar, FancyError};
use mas_router::UrlBuilder;
use mas_storage::{BoxClock, BoxRepository};
@@ -21,7 +20,8 @@ use crate::PreferredLanguage;
#[derive(Serialize, Deserialize)]
pub struct Params {
code: String,
#[serde(default)]
code: Option<String>,
}
#[tracing::instrument(name = "handlers.oauth2.device.link.get", skip_all, err)]
@@ -32,17 +32,14 @@ pub(crate) async fn get(
State(templates): State<Templates>,
State(url_builder): State<UrlBuilder>,
cookie_jar: CookieJar,
query: Option<Query<Params>>,
Query(query): Query<Params>,
) -> Result<impl IntoResponse, FancyError> {
let mut form_state = FormState::default();
let mut form_state = FormState::from_form(&query);
// If we have a code in query, find it in the database
if let Some(Query(params)) = query {
// Save the form state so that we echo back the code
form_state = FormState::from_form(&params);
if let Some(code) = &query.code {
// Find the code in the database
let code = params.code.to_uppercase();
let code = code.to_uppercase();
let grant = repo
.oauth2_device_code_grant()
.find_by_user_code(&code)

20
crates/handlers/src/oauth2/introspection.rs
View File

@@ -10,7 +10,7 @@ use mas_axum_utils::{
client_authorization::{ClientAuthorization, CredentialsVerificationError},
sentry::SentryEventID,
};
use mas_data_model::{TokenFormatError, TokenType};
use mas_data_model::{Device, TokenFormatError, TokenType};
use mas_iana::oauth::{OAuthClientAuthenticationMethod, OAuthTokenTypeHint};
use mas_keystore::Encrypter;
use mas_storage::{
@@ -364,11 +364,12 @@ pub(crate) async fn post(
}
// Grant the synapse admin scope if the session has the admin flag set.
let synapse_admin = session.is_synapse_admin.then_some(SYNAPSE_ADMIN_SCOPE);
let device_scope = session.device.to_scope_token();
let scope = [API_SCOPE, device_scope]
let synapse_admin_scope_opt = session.is_synapse_admin.then_some(SYNAPSE_ADMIN_SCOPE);
let device_scope_opt = session.device.as_ref().map(Device::to_scope_token);
let scope = [API_SCOPE]
.into_iter()
.chain(synapse_admin)
.chain(device_scope_opt)
.chain(synapse_admin_scope_opt)
.collect();
activity_tracker
@@ -423,11 +424,12 @@ pub(crate) async fn post(
}
// Grant the synapse admin scope if the session has the admin flag set.
let synapse_admin = session.is_synapse_admin.then_some(SYNAPSE_ADMIN_SCOPE);
let device_scope = session.device.to_scope_token();
let scope = [API_SCOPE, device_scope]
let synapse_admin_scope_opt = session.is_synapse_admin.then_some(SYNAPSE_ADMIN_SCOPE);
let device_scope_opt = session.device.as_ref().map(Device::to_scope_token);
let scope = [API_SCOPE]
.into_iter()
.chain(synapse_admin)
.chain(device_scope_opt)
.chain(synapse_admin_scope_opt)
.collect();
activity_tracker

11
crates/handlers/src/preferred_language.rs
View File

@@ -1,4 +1,4 @@
// Copyright 2024 New Vector Ltd.
// Copyright 2024, 2025 New Vector Ltd.
// Copyright 2023, 2024 The Matrix.org Foundation C.I.C.
//
// SPDX-License-Identifier: AGPL-3.0-only
@@ -7,17 +7,15 @@
use std::{convert::Infallible, sync::Arc};
use axum::{
async_trait,
extract::{FromRef, FromRequestParts},
http::request::Parts,
};
use axum_extra::typed_header::TypedHeader;
use headers::HeaderMapExt as _;
use mas_axum_utils::language_detection::AcceptLanguage;
use mas_i18n::{locale, DataLocale, Translator};
pub struct PreferredLanguage(pub DataLocale);
#[async_trait]
impl<S> FromRequestParts<S> for PreferredLanguage
where
S: Send + Sync,
@@ -27,12 +25,11 @@ where
async fn from_request_parts(parts: &mut Parts, state: &S) -> Result<Self, Self::Rejection> {
let translator: Arc<Translator> = FromRef::from_ref(state);
let accept_language: Option<TypedHeader<AcceptLanguage>> =
FromRequestParts::from_request_parts(parts, state).await?;
let accept_language = parts.headers.typed_get::<AcceptLanguage>();
let iter = accept_language
.iter()
.flat_map(|TypedHeader(accept_language)| accept_language.iter())
.flat_map(AcceptLanguage::iter)
.flat_map(|lang| {
let lang = DataLocale::from(lang);
// XXX: this is hacky as we may want to actually maintain proper language

10
crates/handlers/src/test_utils.rs
View File

@@ -11,7 +11,6 @@ use std::{
};
use axum::{
async_trait,
body::{Bytes, HttpBody},
extract::{FromRef, FromRequestParts},
response::{IntoResponse, IntoResponseParts},
@@ -383,7 +382,7 @@ struct TestGraphQLState {
limiter: Limiter,
}
#[async_trait]
#[async_trait::async_trait]
impl graphql::State for TestGraphQLState {
async fn repository(&self) -> Result<BoxRepository, mas_storage::RepositoryError> {
let repo = PgRepository::from_pool(&self.pool)
@@ -512,7 +511,6 @@ impl FromRef<TestState> for reqwest::Client {
}
}
#[async_trait]
impl FromRequestParts<TestState> for ActivityTracker {
type Rejection = Infallible;
@@ -524,7 +522,6 @@ impl FromRequestParts<TestState> for ActivityTracker {
}
}
#[async_trait]
impl FromRequestParts<TestState> for BoundActivityTracker {
type Rejection = Infallible;
@@ -537,7 +534,6 @@ impl FromRequestParts<TestState> for BoundActivityTracker {
}
}
#[async_trait]
impl FromRequestParts<TestState> for RequesterFingerprint {
type Rejection = Infallible;
@@ -549,7 +545,6 @@ impl FromRequestParts<TestState> for RequesterFingerprint {
}
}
#[async_trait]
impl FromRequestParts<TestState> for BoxClock {
type Rejection = Infallible;
@@ -561,7 +556,6 @@ impl FromRequestParts<TestState> for BoxClock {
}
}
#[async_trait]
impl FromRequestParts<TestState> for BoxRng {
type Rejection = Infallible;
@@ -575,7 +569,6 @@ impl FromRequestParts<TestState> for BoxRng {
}
}
#[async_trait]
impl FromRequestParts<TestState> for BoxRepository {
type Rejection = ErrorWrapper<mas_storage_pg::DatabaseError>;
@@ -588,7 +581,6 @@ impl FromRequestParts<TestState> for BoxRepository {
}
}
#[async_trait]
impl FromRequestParts<TestState> for Policy {
type Rejection = ErrorWrapper<mas_policy::InstantiateError>;

7
crates/handlers/src/upstream_oauth2/callback.rs
View File

@@ -7,10 +7,9 @@
use axum::{
extract::{Path, State},
http::Method,
response::{IntoResponse, Response},
response::{Html, IntoResponse, Response},
Form,
};
use axum_extra::response::Html;
use hyper::StatusCode;
use mas_axum_utils::{cookies::CookieJar, sentry::SentryEventID};
use mas_data_model::{UpstreamOAuthProvider, UpstreamOAuthProviderResponseMode};
@@ -162,7 +161,7 @@ pub(crate) async fn handler(
PreferredLanguage(locale): PreferredLanguage,
cookie_jar: CookieJar,
Path(provider_id): Path<Ulid>,
params: Option<Form<Params>>,
Form(params): Form<Option<Params>>,
) -> Result<Response, RouteError> {
let provider = repo
.upstream_oauth_provider()
@@ -173,7 +172,7 @@ pub(crate) async fn handler(
let sessions_cookie = UpstreamSessionsCookie::load(&cookie_jar);
let Some(Form(params)) = params else {
let Some(params) = params else {
if let Method::GET = method {
return Err(RouteError::MissingQueryParams);
}

10
crates/handlers/src/views/app.rs
View File

@@ -12,23 +12,29 @@ use mas_axum_utils::{cookies::CookieJar, FancyError, SessionInfoExt};
use mas_router::{PostAuthAction, UrlBuilder};
use mas_storage::{BoxClock, BoxRepository};
use mas_templates::{AppContext, TemplateContext, Templates};
use serde::Deserialize;
use crate::{BoundActivityTracker, PreferredLanguage};
#[derive(Deserialize)]
pub struct Params {
#[serde(default, flatten)]
action: Option<mas_router::AccountAction>,
}
#[tracing::instrument(name = "handlers.views.app.get", skip_all, err)]
pub async fn get(
PreferredLanguage(locale): PreferredLanguage,
State(templates): State<Templates>,
activity_tracker: BoundActivityTracker,
State(url_builder): State<UrlBuilder>,
action: Option<Query<mas_router::AccountAction>>,
Query(Params { action }): Query<Params>,
mut repo: BoxRepository,
clock: BoxClock,
cookie_jar: CookieJar,
) -> Result<impl IntoResponse, FancyError> {
let (session_info, cookie_jar) = cookie_jar.session_info();
let session = session_info.load_session(&mut repo).await?;
let action = action.map(|Query(a)| a);
// TODO: keep the full path, not just the action
let Some(session) = session else {

116
crates/handlers/src/views/login.rs
View File

@@ -168,6 +168,11 @@ pub(crate) async fn post(
return Ok((cookie_jar, Html(content)).into_response());
}
// Extract the localpart of the MXID, fallback to the bare username
let username = homeserver
.localpart(&form.username)
.unwrap_or(&form.username);
match login(
password_manager,
&mut repo,
@@ -175,7 +180,7 @@ pub(crate) async fn post(
&clock,
limiter,
requester,
&form.username,
username,
&form.password,
user_agent,
)
@@ -479,23 +484,17 @@ mod test {
.contains(&escape_html(&second_provider_login.path_and_query())));
}
#[sqlx::test(migrator = "mas_storage_pg::MIGRATOR")]
async fn test_password_login(pool: PgPool) {
setup();
let state = TestState::from_pool(pool).await.unwrap();
async fn user_with_password(state: &TestState, username: &str, password: &str) {
let mut rng = state.rng();
let cookies = CookieHelper::new();
// Provision a user with a password
let mut repo = state.repository().await.unwrap();
let user = repo
.user()
.add(&mut rng, &state.clock, "john".to_owned())
.add(&mut rng, &state.clock, username.to_owned())
.await
.unwrap();
let (version, hash) = state
.password_manager
.hash(&mut rng, Zeroizing::new("hunter2".as_bytes().to_vec()))
.hash(&mut rng, Zeroizing::new(password.as_bytes().to_vec()))
.await
.unwrap();
repo.user_password()
@@ -503,6 +502,16 @@ mod test {
.await
.unwrap();
repo.save().await.unwrap();
}
#[sqlx::test(migrator = "mas_storage_pg::MIGRATOR")]
async fn test_password_login(pool: PgPool) {
setup();
let state = TestState::from_pool(pool).await.unwrap();
let cookies = CookieHelper::new();
// Provision a user with a password
user_with_password(&state, "john", "hunter2").await;
// Render the login page to get a CSRF token
let request = Request::get("/login").empty();
@@ -542,6 +551,93 @@ mod test {
assert!(response.body().contains("john"));
}
#[sqlx::test(migrator = "mas_storage_pg::MIGRATOR")]
async fn test_password_login_with_mxid(pool: PgPool) {
setup();
let state = TestState::from_pool(pool).await.unwrap();
let cookies = CookieHelper::new();
// Provision a user with a password
user_with_password(&state, "john", "hunter2").await;
// Render the login page to get a CSRF token
let request = Request::get("/login").empty();
let request = cookies.with_cookies(request);
let response = state.request(request).await;
cookies.save_cookies(&response);
response.assert_status(StatusCode::OK);
response.assert_header_value(CONTENT_TYPE, "text/html; charset=utf-8");
// Extract the CSRF token from the response body
let csrf_token = response
.body()
.split("name=\"csrf\" value=\"")
.nth(1)
.unwrap()
.split('\"')
.next()
.unwrap();
// Submit the login form
let request = Request::post("/login").form(serde_json::json!({
"csrf": csrf_token,
"username": "@john:example.com",
"password": "hunter2",
}));
let request = cookies.with_cookies(request);
let response = state.request(request).await;
cookies.save_cookies(&response);
response.assert_status(StatusCode::SEE_OTHER);
// Now if we get to the home page, we should see the user's username
let request = Request::get("/").empty();
let request = cookies.with_cookies(request);
let response = state.request(request).await;
cookies.save_cookies(&response);
response.assert_status(StatusCode::OK);
response.assert_header_value(CONTENT_TYPE, "text/html; charset=utf-8");
assert!(response.body().contains("john"));
}
#[sqlx::test(migrator = "mas_storage_pg::MIGRATOR")]
async fn test_password_login_with_mxid_wrong_server(pool: PgPool) {
setup();
let state = TestState::from_pool(pool).await.unwrap();
let cookies = CookieHelper::new();
// Provision a user with a password
user_with_password(&state, "john", "hunter2").await;
// Render the login page to get a CSRF token
let request = Request::get("/login").empty();
let request = cookies.with_cookies(request);
let response = state.request(request).await;
cookies.save_cookies(&response);
response.assert_status(StatusCode::OK);
response.assert_header_value(CONTENT_TYPE, "text/html; charset=utf-8");
// Extract the CSRF token from the response body
let csrf_token = response
.body()
.split("name=\"csrf\" value=\"")
.nth(1)
.unwrap()
.split('\"')
.next()
.unwrap();
// Submit the login form
let request = Request::post("/login").form(serde_json::json!({
"csrf": csrf_token,
"username": "@john:something.corp",
"password": "hunter2",
}));
let request = cookies.with_cookies(request);
let response = state.request(request).await;
// This shouldn't have worked, we're back on the login page
response.assert_status(StatusCode::OK);
assert!(response.body().contains("Invalid credentials"));
}
#[sqlx::test(migrator = "mas_storage_pg::MIGRATOR")]
async fn test_password_login_rate_limit(pool: PgPool) {
setup();

3
crates/handlers/src/views/register/mod.rs
View File

@@ -5,9 +5,8 @@
use axum::{
extract::{Query, State},
response::{IntoResponse, Response},
response::{Html, IntoResponse, Response},
};
use axum_extra::response::Html;
use mas_axum_utils::{cookies::CookieJar, csrf::CsrfExt as _, FancyError, SessionInfoExt};
use mas_data_model::SiteConfig;
use mas_router::{PasswordRegister, UpstreamOAuth2Authorize, UrlBuilder};

7
crates/i18n-scan/src/main.rs
View File

@@ -8,7 +8,7 @@
// which is annoying with this clippy lint
#![allow(clippy::default_constructed_unit_structs)]
use std::fs::File;
use std::{fs::File, io::BufReader};
use ::minijinja::{machinery::WhitespaceConfig, syntax::SyntaxConfig};
use camino::Utf8PathBuf;
@@ -50,8 +50,9 @@ fn main() {
// Open the existing translation file if one was provided
let mut tree = if let Some(path) = &options.existing {
let mut file = File::open(path).expect("Failed to open existing translation file");
serde_json::from_reader(&mut file).expect("Failed to parse existing translation file")
let file = File::open(path).expect("Failed to open existing translation file");
let mut reader = BufReader::new(file);
serde_json::from_reader(&mut reader).expect("Failed to parse existing translation file")
} else {
TranslationTree::default()
};

8
crates/i18n/src/translator.rs
View File

@@ -4,7 +4,7 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Please see LICENSE in the repository root for full details.
use std::{collections::HashMap, fs::File, str::FromStr};
use std::{collections::HashMap, fs::File, io::BufReader, str::FromStr};
use camino::{Utf8Path, Utf8PathBuf};
use icu_list::{ListError, ListFormatter, ListLength};
@@ -135,12 +135,14 @@ impl Translator {
Err(source) => return Err(LoadError::InvalidLocale { path, source }),
};
let mut file = match File::open(&path) {
let file = match File::open(&path) {
Ok(file) => file,
Err(source) => return Err(LoadError::ReadFile { path, source }),
};
let content = match serde_json::from_reader(&mut file) {
let mut reader = BufReader::new(file);
let content = match serde_json::from_reader(&mut reader) {
Ok(content) => content,
Err(source) => return Err(LoadError::Deserialize { path, source }),
};

1
crates/matrix/Cargo.toml
View File

@@ -15,3 +15,4 @@ workspace = true
anyhow.workspace = true
async-trait.workspace = true
tokio.workspace = true
ruma-common.workspace = true

18
crates/matrix/src/lib.rs
View File

@@ -8,6 +8,8 @@ mod mock;
use std::{collections::HashSet, sync::Arc};
use ruma_common::UserId;
pub use self::mock::HomeserverConnection as MockHomeserverConnection;
// TODO: this should probably be another error type by default
@@ -193,6 +195,22 @@ pub trait HomeserverConnection: Send + Sync {
format!("@{}:{}", localpart, self.homeserver())
}
/// Get the localpart of a Matrix ID if it has the right server name
///
/// Returns [`None`] if the input isn't a valid MXID, or if the server name
/// doesn't match
///
/// # Parameters
///
/// * `mxid` - The MXID of the user
fn localpart<'a>(&self, mxid: &'a str) -> Option<&'a str> {
let mxid = <&UserId>::try_from(mxid).ok()?;
if mxid.server_name() != self.homeserver() {
return None;
}
Some(mxid.localpart())
}
/// Query the state of a user on the homeserver.
///
/// # Parameters

60
crates/router/src/endpoints.rs
View File

@@ -444,7 +444,7 @@ impl From<Option<PostAuthAction>> for PasswordRegister {
}
}
/// `GET|POST /register/steps/:id/display-name`
/// `GET|POST /register/steps/{id}/display-name`
#[derive(Debug, Clone)]
pub struct RegisterDisplayName {
id: Ulid,
@@ -460,7 +460,7 @@ impl RegisterDisplayName {
impl Route for RegisterDisplayName {
type Query = ();
fn route() -> &'static str {
"/register/steps/:id/display-name"
"/register/steps/{id}/display-name"
}
fn path(&self) -> std::borrow::Cow<'static, str> {
@@ -468,7 +468,7 @@ impl Route for RegisterDisplayName {
}
}
/// `GET|POST /register/steps/:id/verify-email`
/// `GET|POST /register/steps/{id}/verify-email`
#[derive(Debug, Clone)]
pub struct RegisterVerifyEmail {
id: Ulid,
@@ -484,7 +484,7 @@ impl RegisterVerifyEmail {
impl Route for RegisterVerifyEmail {
type Query = ();
fn route() -> &'static str {
"/register/steps/:id/verify-email"
"/register/steps/{id}/verify-email"
}
fn path(&self) -> std::borrow::Cow<'static, str> {
@@ -492,7 +492,7 @@ impl Route for RegisterVerifyEmail {
}
}
/// `GET /register/steps/:id/finish`
/// `GET /register/steps/{id}/finish`
#[derive(Debug, Clone)]
pub struct RegisterFinish {
id: Ulid,
@@ -508,7 +508,7 @@ impl RegisterFinish {
impl Route for RegisterFinish {
type Query = ();
fn route() -> &'static str {
"/register/steps/:id/finish"
"/register/steps/{id}/finish"
}
fn path(&self) -> std::borrow::Cow<'static, str> {
@@ -567,7 +567,7 @@ impl Route for Account {
pub struct AccountWildcard;
impl SimpleRoute for AccountWildcard {
const PATH: &'static str = "/account/*rest";
const PATH: &'static str = "/account/{*rest}";
}
/// `GET /account/password/change`
@@ -581,14 +581,14 @@ impl SimpleRoute for AccountPasswordChange {
const PATH: &'static str = "/account/password/change";
}
/// `GET /authorize/:grant_id`
/// `GET /authorize/{grant_id}`
#[derive(Debug, Clone)]
pub struct ContinueAuthorizationGrant(pub Ulid);
impl Route for ContinueAuthorizationGrant {
type Query = ();
fn route() -> &'static str {
"/authorize/:grant_id"
"/authorize/{grant_id}"
}
fn path(&self) -> std::borrow::Cow<'static, str> {
@@ -596,14 +596,14 @@ impl Route for ContinueAuthorizationGrant {
}
}
/// `GET /consent/:grant_id`
/// `GET /consent/{grant_id}`
#[derive(Debug, Clone)]
pub struct Consent(pub Ulid);
impl Route for Consent {
type Query = ();
fn route() -> &'static str {
"/consent/:grant_id"
"/consent/{grant_id}"
}
fn path(&self) -> std::borrow::Cow<'static, str> {
@@ -615,28 +615,28 @@ impl Route for Consent {
pub struct CompatLogin;
impl SimpleRoute for CompatLogin {
const PATH: &'static str = "/_matrix/client/:version/login";
const PATH: &'static str = "/_matrix/client/{version}/login";
}
/// `POST /_matrix/client/v3/logout`
pub struct CompatLogout;
impl SimpleRoute for CompatLogout {
const PATH: &'static str = "/_matrix/client/:version/logout";
const PATH: &'static str = "/_matrix/client/{version}/logout";
}
/// `POST /_matrix/client/v3/refresh`
pub struct CompatRefresh;
impl SimpleRoute for CompatRefresh {
const PATH: &'static str = "/_matrix/client/:version/refresh";
const PATH: &'static str = "/_matrix/client/{version}/refresh";
}
/// `GET /_matrix/client/v3/login/sso/redirect`
pub struct CompatLoginSsoRedirect;
impl SimpleRoute for CompatLoginSsoRedirect {
const PATH: &'static str = "/_matrix/client/:version/login/sso/redirect";
const PATH: &'static str = "/_matrix/client/{version}/login/sso/redirect";
}
/// `GET /_matrix/client/v3/login/sso/redirect/`
@@ -646,14 +646,14 @@ impl SimpleRoute for CompatLoginSsoRedirect {
pub struct CompatLoginSsoRedirectSlash;
impl SimpleRoute for CompatLoginSsoRedirectSlash {
const PATH: &'static str = "/_matrix/client/:version/login/sso/redirect/";
const PATH: &'static str = "/_matrix/client/{version}/login/sso/redirect/";
}
/// `GET /_matrix/client/v3/login/sso/redirect/:idp`
/// `GET /_matrix/client/v3/login/sso/redirect/{idp}`
pub struct CompatLoginSsoRedirectIdp;
impl SimpleRoute for CompatLoginSsoRedirectIdp {
const PATH: &'static str = "/_matrix/client/:version/login/sso/redirect/:idp";
const PATH: &'static str = "/_matrix/client/{version}/login/sso/redirect/{idp}";
}
#[derive(Debug, Serialize, Deserialize, Clone, Copy)]
@@ -669,7 +669,7 @@ pub struct CompatLoginSsoActionParams {
action: CompatLoginSsoAction,
}
/// `GET|POST /complete-compat-sso/:id`
/// `GET|POST /complete-compat-sso/{id}`
pub struct CompatLoginSsoComplete {
id: Ulid,
query: Option<CompatLoginSsoActionParams>,
@@ -693,7 +693,7 @@ impl Route for CompatLoginSsoComplete {
}
fn route() -> &'static str {
"/complete-compat-sso/:grant_id"
"/complete-compat-sso/{grant_id}"
}
fn path(&self) -> std::borrow::Cow<'static, str> {
@@ -701,7 +701,7 @@ impl Route for CompatLoginSsoComplete {
}
}
/// `GET /upstream/authorize/:id`
/// `GET /upstream/authorize/{id}`
pub struct UpstreamOAuth2Authorize {
id: Ulid,
post_auth_action: Option<PostAuthAction>,
@@ -726,7 +726,7 @@ impl UpstreamOAuth2Authorize {
impl Route for UpstreamOAuth2Authorize {
type Query = PostAuthAction;
fn route() -> &'static str {
"/upstream/authorize/:provider_id"
"/upstream/authorize/{provider_id}"
}
fn path(&self) -> std::borrow::Cow<'static, str> {
@@ -738,7 +738,7 @@ impl Route for UpstreamOAuth2Authorize {
}
}
/// `GET /upstream/callback/:id`
/// `GET /upstream/callback/{id}`
pub struct UpstreamOAuth2Callback {
id: Ulid,
}
@@ -753,7 +753,7 @@ impl UpstreamOAuth2Callback {
impl Route for UpstreamOAuth2Callback {
type Query = ();
fn route() -> &'static str {
"/upstream/callback/:provider_id"
"/upstream/callback/{provider_id}"
}
fn path(&self) -> std::borrow::Cow<'static, str> {
@@ -761,7 +761,7 @@ impl Route for UpstreamOAuth2Callback {
}
}
/// `GET /upstream/link/:id`
/// `GET /upstream/link/{id}`
pub struct UpstreamOAuth2Link {
id: Ulid,
}
@@ -776,7 +776,7 @@ impl UpstreamOAuth2Link {
impl Route for UpstreamOAuth2Link {
type Query = ();
fn route() -> &'static str {
"/upstream/link/:link_id"
"/upstream/link/{link_id}"
}
fn path(&self) -> std::borrow::Cow<'static, str> {
@@ -808,7 +808,7 @@ impl Route for DeviceCodeLink {
}
}
/// `GET|POST /device/:device_code_id`
/// `GET|POST /device/{device_code_id}`
#[derive(Default, Serialize, Deserialize, Debug, Clone)]
pub struct DeviceCodeConsent {
id: Ulid,
@@ -817,7 +817,7 @@ pub struct DeviceCodeConsent {
impl Route for DeviceCodeConsent {
type Query = ();
fn route() -> &'static str {
"/device/:device_code_id"
"/device/{device_code_id}"
}
fn path(&self) -> std::borrow::Cow<'static, str> {
@@ -848,7 +848,7 @@ impl SimpleRoute for AccountRecoveryStart {
const PATH: &'static str = "/recover";
}
/// `GET|POST /recover/progress/:session_id`
/// `GET|POST /recover/progress/{session_id}`
#[derive(Default, Serialize, Deserialize, Debug, Clone)]
pub struct AccountRecoveryProgress {
session_id: Ulid,
@@ -864,7 +864,7 @@ impl AccountRecoveryProgress {
impl Route for AccountRecoveryProgress {
type Query = ();
fn route() -> &'static str {
"/recover/progress/:session_id"
"/recover/progress/{session_id}"
}
fn path(&self) -> std::borrow::Cow<'static, str> {

2
crates/storage-pg/.sqlx/query-bb6f55a4cc10bec8ec0fc138485f6b4d308302bb1fa3accb12932d1e5ce457e9.json generated
View File

@@ -61,7 +61,7 @@
},
"nullable": [
false,
false,
true,
false,
true,
false,

7
crates/storage-pg/migrations/20250114135939_allow_deviceless_compat_sessions.sql Normal file
View File

@@ -0,0 +1,7 @@
-- Copyright 2025 New Vector Ltd.
--
-- SPDX-License-Identifier: AGPL-3.0-only
-- Please see LICENSE in the repository root for full details.
-- Drop the `NOT NULL` requirement on compat sessions, so we can import device-less access tokens from Synapse.
ALTER TABLE compat_sessions ALTER COLUMN device_id DROP NOT NULL;

30
crates/storage-pg/migrations/20250124151529_unsupported_threepids_table.sql Normal file
View File

@@ -0,0 +1,30 @@
-- Copyright 2025 New Vector Ltd.
--
-- SPDX-License-Identifier: AGPL-3.0-only
-- Please see LICENSE in the repository root for full details.
-- Tracks third-party ID associations that have been verified but are
-- not currently supported by MAS.
-- This is currently used when importing third-party IDs from Synapse,
-- which historically could verify at least phone numbers.
-- E-mail associations will not be stored in this table because those are natively
-- supported by MAS; see the `user_emails` table.
CREATE TABLE user_unsupported_third_party_ids(
-- The owner of the third-party ID assocation
user_id UUID NOT NULL
REFERENCES users(user_id) ON DELETE CASCADE,
-- What type of association is this?
medium TEXT NOT NULL,
-- The address of the associated ID, e.g. a phone number or other identifier.
address TEXT NOT NULL,
-- When the association was created
created_at TIMESTAMP WITH TIME ZONE NOT NULL,
PRIMARY KEY (user_id, medium, address)
);

17
crates/storage-pg/src/app_session.rs
View File

@@ -117,16 +117,19 @@ impl TryFrom<AppSessionLookup> for AppSession {
None,
Some(user_id),
None,
Some(device_id),
device_id_opt,
Some(is_synapse_admin),
) => {
let id = compat_session_id.into();
let device = Device::try_from(device_id).map_err(|e| {
DatabaseInconsistencyError::on("compat_sessions")
.column("device_id")
.row(id)
.source(e)
})?;
let device = device_id_opt
.map(Device::try_from)
.transpose()
.map_err(|e| {
DatabaseInconsistencyError::on("compat_sessions")
.column("device_id")
.row(id)
.source(e)
})?;
let state = match finished_at {
None => CompatSessionState::Valid,

6
crates/storage-pg/src/compat/mod.rs
View File

@@ -83,7 +83,7 @@ mod tests {
.await
.unwrap();
assert_eq!(session.user_id, user.id);
assert_eq!(session.device.as_str(), device_str);
assert_eq!(session.device.as_ref().unwrap().as_str(), device_str);
assert!(session.is_valid());
assert!(!session.is_finished());
@@ -117,7 +117,7 @@ mod tests {
.expect("compat session not found");
assert_eq!(session_lookup.id, session.id);
assert_eq!(session_lookup.user_id, user.id);
assert_eq!(session_lookup.device.as_str(), device_str);
assert_eq!(session.device.as_ref().unwrap().as_str(), device_str);
assert!(session_lookup.is_valid());
assert!(!session_lookup.is_finished());
@@ -154,7 +154,7 @@ mod tests {
let session_lookup = &list.edges[0].0;
assert_eq!(session_lookup.id, session.id);
assert_eq!(session_lookup.user_id, user.id);
assert_eq!(session_lookup.device.as_str(), device_str);
assert_eq!(session.device.as_ref().unwrap().as_str(), device_str);
assert!(session_lookup.is_valid());
assert!(!session_lookup.is_finished());

40
crates/storage-pg/src/compat/session.rs
View File

@@ -47,7 +47,7 @@ impl<'c> PgCompatSessionRepository<'c> {
struct CompatSessionLookup {
compat_session_id: Uuid,
device_id: String,
device_id: Option<String>,
user_id: Uuid,
user_session_id: Option<Uuid>,
created_at: DateTime<Utc>,
@@ -63,12 +63,16 @@ impl TryFrom<CompatSessionLookup> for CompatSession {
fn try_from(value: CompatSessionLookup) -> Result<Self, Self::Error> {
let id = value.compat_session_id.into();
let device = Device::try_from(value.device_id).map_err(|e| {
DatabaseInconsistencyError::on("compat_sessions")
.column("device_id")
.row(id)
.source(e)
})?;
let device = value
.device_id
.map(Device::try_from)
.transpose()
.map_err(|e| {
DatabaseInconsistencyError::on("compat_sessions")
.column("device_id")
.row(id)
.source(e)
})?;
let state = match value.finished_at {
None => CompatSessionState::Valid,
@@ -96,7 +100,7 @@ impl TryFrom<CompatSessionLookup> for CompatSession {
#[enum_def]
struct CompatSessionAndSsoLoginLookup {
compat_session_id: Uuid,
device_id: String,
device_id: Option<String>,
user_id: Uuid,
user_session_id: Option<Uuid>,
created_at: DateTime<Utc>,
@@ -118,12 +122,16 @@ impl TryFrom<CompatSessionAndSsoLoginLookup> for (CompatSession, Option<CompatSs
fn try_from(value: CompatSessionAndSsoLoginLookup) -> Result<Self, Self::Error> {
let id = value.compat_session_id.into();
let device = Device::try_from(value.device_id).map_err(|e| {
DatabaseInconsistencyError::on("compat_sessions")
.column("device_id")
.row(id)
.source(e)
})?;
let device = value
.device_id
.map(Device::try_from)
.transpose()
.map_err(|e| {
DatabaseInconsistencyError::on("compat_sessions")
.column("device_id")
.row(id)
.source(e)
})?;
let state = match value.finished_at {
None => CompatSessionState::Valid,
@@ -347,7 +355,7 @@ impl CompatSessionRepository for PgCompatSessionRepository<'_> {
id,
state: CompatSessionState::default(),
user_id: user.id,
device,
device: Some(device),
user_session_id: browser_session.map(|s| s.id),
created_at,
is_synapse_admin,
@@ -364,7 +372,7 @@ impl CompatSessionRepository for PgCompatSessionRepository<'_> {
db.query.text,
%compat_session.id,
user.id = %compat_session.user_id,
compat_session.device.id = compat_session.device.as_str(),
compat_session.device.id = compat_session.device.as_ref().map(mas_data_model::Device::as_str),
),
err,
)]

2
crates/storage-pg/src/compat/sso_login.rs
View File

@@ -293,7 +293,7 @@ impl CompatSsoLoginRepository for PgCompatSsoLoginRepository<'_> {
db.query.text,
%compat_sso_login.id,
%compat_session.id,
compat_session.device.id = compat_session.device.as_str(),
compat_session.device.id = compat_session.device.as_ref().map(mas_data_model::Device::as_str),
user.id = %compat_session.user_id,
),
err,

16
crates/syn2mas/.sqlx/query-07ec66733b67a9990cc9d483b564c8d05c577cf8f049d8822746c7d1dbd23752.json generated Normal file
View File

@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO syn2mas_restore_indices (name, table_name, definition)\n VALUES ($1, $2, $3)\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "07ec66733b67a9990cc9d483b564c8d05c577cf8f049d8822746c7d1dbd23752"
}

34
crates/syn2mas/.sqlx/query-12112011318abc0bdd7f722ed8c5d4a86bf5758f8c32d9d41a22999b2f0698ca.json generated Normal file
View File

@@ -0,0 +1,34 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT conrelid::regclass::text AS \"table_name!\", conname AS \"name!\", pg_get_constraintdef(c.oid) AS \"definition!\"\n FROM pg_constraint c\n JOIN pg_namespace n ON n.oid = c.connamespace\n WHERE contype IN ('f', 'p', 'u') AND conrelid::regclass::text = $1\n AND n.nspname = current_schema;\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "table_name!",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "name!",
"type_info": "Name"
},
{
"ordinal": 2,
"name": "definition!",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null,
false,
null
]
},
"hash": "12112011318abc0bdd7f722ed8c5d4a86bf5758f8c32d9d41a22999b2f0698ca"
}

34
crates/syn2mas/.sqlx/query-486f3177dcf6117c6b966954a44d9f96a754eba64912566e81a90bd4cbd186f0.json generated Normal file
View File

@@ -0,0 +1,34 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT indexname AS \"name!\", indexdef AS \"definition!\", schemaname AS \"table_name!\"\n FROM pg_indexes\n WHERE schemaname = current_schema AND tablename = $1 AND indexname IS NOT NULL AND indexdef IS NOT NULL\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "name!",
"type_info": "Name"
},
{
"ordinal": 1,
"name": "definition!",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "table_name!",
"type_info": "Name"
}
],
"parameters": {
"Left": [
"Name"
]
},
"nullable": [
true,
true,
true
]
},
"hash": "486f3177dcf6117c6b966954a44d9f96a754eba64912566e81a90bd4cbd186f0"
}

34
crates/syn2mas/.sqlx/query-5b4840f42ae00c5dc9f59f2745d664b16ebd813dfa0aa32a6d39dd5c393af299.json generated Normal file
View File

@@ -0,0 +1,34 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT conrelid::regclass::text AS \"table_name!\", conname AS \"name!\", pg_get_constraintdef(c.oid) AS \"definition!\"\n FROM pg_constraint c\n JOIN pg_namespace n ON n.oid = c.connamespace\n WHERE contype = 'f' AND confrelid::regclass::text = $1\n AND n.nspname = current_schema;\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "table_name!",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "name!",
"type_info": "Name"
},
{
"ordinal": 2,
"name": "definition!",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null,
false,
null
]
},
"hash": "5b4840f42ae00c5dc9f59f2745d664b16ebd813dfa0aa32a6d39dd5c393af299"
}

16
crates/syn2mas/.sqlx/query-69aa96208513c3ea64a446c7739747fcb5e79d7e8c1212b2a679c3bde908ce93.json generated Normal file
View File

@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO syn2mas_restore_constraints (name, table_name, definition)\n VALUES ($1, $2, $3)\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "69aa96208513c3ea64a446c7739747fcb5e79d7e8c1212b2a679c3bde908ce93"
}

32
crates/syn2mas/.sqlx/query-78ed3bf1032cd678b42230d68fb2b8e3d74161c8b6c5fe1a746b6958ccd2fd84.json generated Normal file
View File

@@ -0,0 +1,32 @@
{
"db_name": "PostgreSQL",
"query": "SELECT table_name, name, definition FROM syn2mas_restore_constraints ORDER BY order_key",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "table_name",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "name",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "definition",
"type_info": "Text"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
false,
false
]
},
"hash": "78ed3bf1032cd678b42230d68fb2b8e3d74161c8b6c5fe1a746b6958ccd2fd84"
}

32
crates/syn2mas/.sqlx/query-979bedd942b4f71c58f3672f2917cee05ac1a628e51fe61ba6dfed253e0c63c2.json generated Normal file
View File

@@ -0,0 +1,32 @@
{
"db_name": "PostgreSQL",
"query": "SELECT table_name, name, definition FROM syn2mas_restore_indices ORDER BY order_key",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "table_name",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "name",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "definition",
"type_info": "Text"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
false,
false
]
},
"hash": "979bedd942b4f71c58f3672f2917cee05ac1a628e51fe61ba6dfed253e0c63c2"
}

17
crates/syn2mas/.sqlx/query-b11590549fdd4cdcd36c937a353b5b37ab50db3505712c35610b822cda322b5b.json generated Normal file
View File

@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO syn2mas__user_unsupported_third_party_ids\n (user_id, medium, address, created_at)\n SELECT * FROM UNNEST($1::UUID[], $2::TEXT[], $3::TEXT[], $4::TIMESTAMP WITH TIME ZONE[])\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"UuidArray",
"TextArray",
"TextArray",
"TimestamptzArray"
]
},
"nullable": []
},
"hash": "b11590549fdd4cdcd36c937a353b5b37ab50db3505712c35610b822cda322b5b"
}

22
crates/syn2mas/.sqlx/query-b27828d7510d52456b50b4c4b9712878ee329ca72070d849eb61ac9c8f9d1c76.json generated Normal file
View File

@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT 1 AS _dummy FROM pg_tables WHERE schemaname = current_schema\n AND tablename = ANY($1)\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "_dummy",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"NameArray"
]
},
"nullable": [
null
]
},
"hash": "b27828d7510d52456b50b4c4b9712878ee329ca72070d849eb61ac9c8f9d1c76"
}

18
crates/syn2mas/.sqlx/query-c6c7db1d578efc45b9e8c8bfea47cafe3f85d639452fd0593b2773997dfc7425.json generated Normal file
View File

@@ -0,0 +1,18 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO syn2mas__user_passwords\n (user_password_id, user_id, hashed_password, created_at, version)\n SELECT * FROM UNNEST($1::UUID[], $2::UUID[], $3::TEXT[], $4::TIMESTAMP WITH TIME ZONE[], $5::INTEGER[])\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"UuidArray",
"UuidArray",
"TextArray",
"TimestamptzArray",
"Int4Array"
]
},
"nullable": []
},
"hash": "c6c7db1d578efc45b9e8c8bfea47cafe3f85d639452fd0593b2773997dfc7425"
}

18
crates/syn2mas/.sqlx/query-c7d2277606b4b326b0c375a056cd57488c930fe431311e53e5e1af6fb1d4e56f.json generated Normal file
View File

@@ -0,0 +1,18 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO syn2mas__users\n (user_id, username, created_at, locked_at, can_request_admin)\n SELECT * FROM UNNEST($1::UUID[], $2::TEXT[], $3::TIMESTAMP WITH TIME ZONE[], $4::TIMESTAMP WITH TIME ZONE[], $5::BOOL[])\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"UuidArray",
"TextArray",
"TimestamptzArray",
"TimestamptzArray",
"BoolArray"
]
},
"nullable": []
},
"hash": "c7d2277606b4b326b0c375a056cd57488c930fe431311e53e5e1af6fb1d4e56f"
}

18
crates/syn2mas/.sqlx/query-d79fd99ebed9033711f96113005096c848ae87c43b6430246ef3b6a1dc6a7a32.json generated Normal file
View File

@@ -0,0 +1,18 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO syn2mas__upstream_oauth_links\n (upstream_oauth_link_id, user_id, upstream_oauth_provider_id, subject, created_at)\n SELECT * FROM UNNEST($1::UUID[], $2::UUID[], $3::UUID[], $4::TEXT[], $5::TIMESTAMP WITH TIME ZONE[])\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"UuidArray",
"UuidArray",
"UuidArray",
"TextArray",
"TimestamptzArray"
]
},
"nullable": []
},
"hash": "d79fd99ebed9033711f96113005096c848ae87c43b6430246ef3b6a1dc6a7a32"
}

17
crates/syn2mas/.sqlx/query-dfbd462f7874d3dae551f2a0328a853a8a7efccdc20b968d99d8c18deda8dd00.json generated Normal file
View File

@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO syn2mas__user_emails\n (user_email_id, user_id, email, created_at, confirmed_at)\n SELECT * FROM UNNEST($1::UUID[], $2::UUID[], $3::TEXT[], $4::TIMESTAMP WITH TIME ZONE[], $4::TIMESTAMP WITH TIME ZONE[])\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"UuidArray",
"UuidArray",
"TextArray",
"TimestamptzArray"
]
},
"nullable": []
},
"hash": "dfbd462f7874d3dae551f2a0328a853a8a7efccdc20b968d99d8c18deda8dd00"
}

40
crates/syn2mas/Cargo.toml Normal file
View File

@@ -0,0 +1,40 @@
[package]
name = "syn2mas"
version.workspace = true
license.workspace = true
authors.workspace = true
edition.workspace = true
homepage.workspace = true
repository.workspace = true
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
anyhow.workspace = true
camino.workspace = true
figment.workspace = true
serde.workspace = true
thiserror.workspace = true
thiserror-ext.workspace = true
tokio.workspace = true
sqlx.workspace = true
chrono.workspace = true
compact_str.workspace = true
tracing.workspace = true
futures-util = "0.3.30"
rand.workspace = true
uuid = "1.10.0"
ulid = { workspace = true, features = ["uuid"] }
mas-config.workspace = true
[dev-dependencies]
mas-storage-pg.workspace = true
anyhow.workspace = true
insta.workspace = true
serde.workspace = true
[lints]
workspace = true

20
crates/syn2mas/src/lib.rs Normal file
View File

@@ -0,0 +1,20 @@
// Copyright 2024 New Vector Ltd.
//
// SPDX-License-Identifier: AGPL-3.0-only
// Please see LICENSE in the repository root for full details.
mod mas_writer;
mod synapse_reader;
mod migration;
pub use self::{
mas_writer::{checks::mas_pre_migration_checks, locking::LockedMasDatabase, MasWriter},
migration::migrate,
synapse_reader::{
checks::{
synapse_config_check, synapse_config_check_against_mas_config, synapse_database_check,
},
config as synapse_config, SynapseReader,
},
};

77
crates/syn2mas/src/mas_writer/checks.rs Normal file
View File

@@ -0,0 +1,77 @@
// Copyright 2024 New Vector Ltd.
//
// SPDX-License-Identifier: AGPL-3.0-only
// Please see LICENSE in the repository root for full details.
//! # MAS Database Checks
//!
//! This module provides safety checks to run against a MAS database before
//! running the Synapse-to-MAS migration.
use thiserror::Error;
use thiserror_ext::ContextInto;
use super::{is_syn2mas_in_progress, locking::LockedMasDatabase, MAS_TABLES_AFFECTED_BY_MIGRATION};
#[derive(Debug, Error, ContextInto)]
pub enum Error {
#[error("the MAS database is not empty: rows found in at least `{table}`")]
MasDatabaseNotEmpty { table: &'static str },
#[error("query against {table} failed — is this actually a MAS database?")]
MaybeNotMas {
#[source]
source: sqlx::Error,
table: &'static str,
},
#[error(transparent)]
Sqlx(#[from] sqlx::Error),
#[error("unable to check if syn2mas is already in progress")]
UnableToCheckInProgress(#[source] super::Error),
}
/// Check that a MAS database is ready for being migrated to.
///
/// Concretely, this checks that the database is empty.
///
/// If syn2mas is already in progress on this database, the checks are skipped.
///
/// # Errors
///
/// Errors are returned under the following circumstances:
///
/// - If any database access error occurs.
/// - If any MAS tables involved in the migration are not empty.
/// - If we can't check whether syn2mas is already in progress on this database
/// or not.
#[tracing::instrument(skip_all)]
pub async fn mas_pre_migration_checks<'a>(
mas_connection: &mut LockedMasDatabase<'a>,
) -> Result<(), Error> {
if is_syn2mas_in_progress(mas_connection.as_mut())
.await
.map_err(Error::UnableToCheckInProgress)?
{
// syn2mas already in progress, so we already performed the checks
return Ok(());
}
// Check that the database looks like a MAS database and that it is also an
// empty database.
for &table in MAS_TABLES_AFFECTED_BY_MIGRATION {
let row_present = sqlx::query(&format!("SELECT 1 AS dummy FROM {table} LIMIT 1"))
.fetch_optional(mas_connection.as_mut())
.await
.into_maybe_not_mas(table)?
.is_some();
if row_present {
return Err(Error::MasDatabaseNotEmpty { table });
}
}
Ok(())
}

151
crates/syn2mas/src/mas_writer/constraint_pausing.rs Normal file
View File

@@ -0,0 +1,151 @@
// Copyright 2024 New Vector Ltd.
//
// SPDX-License-Identifier: AGPL-3.0-only
// Please see LICENSE in the repository root for full details.
use sqlx::PgConnection;
use tracing::debug;
use super::{Error, IntoDatabase};
/// Description of a constraint, which allows recreating it later.
pub struct ConstraintDescription {
pub name: String,
pub table_name: String,
pub definition: String,
}
pub struct IndexDescription {
pub name: String,
pub table_name: String,
pub definition: String,
}
/// Look up and return the definition of a constraint.
pub async fn describe_constraints_on_table(
conn: &mut PgConnection,
table_name: &str,
) -> Result<Vec<ConstraintDescription>, Error> {
sqlx::query_as!(
ConstraintDescription,
r#"
SELECT conrelid::regclass::text AS "table_name!", conname AS "name!", pg_get_constraintdef(c.oid) AS "definition!"
FROM pg_constraint c
JOIN pg_namespace n ON n.oid = c.connamespace
WHERE contype IN ('f', 'p', 'u') AND conrelid::regclass::text = $1
AND n.nspname = current_schema;
"#,
table_name
).fetch_all(&mut *conn).await.into_database_with(|| format!("could not read constraint definitions of {table_name}"))
}
/// Look up and return the definitions of foreign-key constraints whose
/// target table is the one specified.
pub async fn describe_foreign_key_constraints_to_table(
conn: &mut PgConnection,
target_table_name: &str,
) -> Result<Vec<ConstraintDescription>, Error> {
sqlx::query_as!(
ConstraintDescription,
r#"
SELECT conrelid::regclass::text AS "table_name!", conname AS "name!", pg_get_constraintdef(c.oid) AS "definition!"
FROM pg_constraint c
JOIN pg_namespace n ON n.oid = c.connamespace
WHERE contype = 'f' AND confrelid::regclass::text = $1
AND n.nspname = current_schema;
"#,
target_table_name
).fetch_all(&mut *conn).await.into_database_with(|| format!("could not read FK constraint definitions targetting {target_table_name}"))
}
/// Look up and return the definitions of all indices on a given table.
pub async fn describe_indices_on_table(
conn: &mut PgConnection,
table_name: &str,
) -> Result<Vec<IndexDescription>, Error> {
sqlx::query_as!(
IndexDescription,
r#"
SELECT indexname AS "name!", indexdef AS "definition!", schemaname AS "table_name!"
FROM pg_indexes
WHERE schemaname = current_schema AND tablename = $1 AND indexname IS NOT NULL AND indexdef IS NOT NULL
"#,
table_name
).fetch_all(&mut *conn).await.into_database("cannot search for indices")
}
/// Drops a constraint from the database.
///
/// The constraint must exist prior to this call.
pub async fn drop_constraint(
conn: &mut PgConnection,
constraint: &ConstraintDescription,
) -> Result<(), Error> {
let name = &constraint.name;
let table_name = &constraint.table_name;
debug!("dropping constraint {name} on table {table_name}");
sqlx::query(&format!("ALTER TABLE {table_name} DROP CONSTRAINT {name};"))
.execute(&mut *conn)
.await
.into_database_with(|| format!("failed to drop constraint {name} on {table_name}"))?;
Ok(())
}
/// Drops an index from the database.
///
/// The index must exist prior to this call.
pub async fn drop_index(conn: &mut PgConnection, index: &IndexDescription) -> Result<(), Error> {
let index_name = &index.name;
debug!("dropping index {index_name}");
sqlx::query(&format!("DROP INDEX {index_name};"))
.execute(&mut *conn)
.await
.into_database_with(|| format!("failed to temporarily drop {index_name}"))?;
Ok(())
}
/// Restores (recreates) a constraint.
///
/// The constraint must not exist prior to this call.
pub async fn restore_constraint(
conn: &mut PgConnection,
constraint: &ConstraintDescription,
) -> Result<(), Error> {
let ConstraintDescription {
name,
table_name,
definition,
} = &constraint;
sqlx::query(&format!(
"ALTER TABLE {table_name} ADD CONSTRAINT {name} {definition};"
))
.execute(conn)
.await
.into_database_with(|| {
format!("failed to recreate constraint {name} on {table_name} with {definition}")
})?;
Ok(())
}
/// Restores (recreates) a index.
///
/// The index must not exist prior to this call.
pub async fn restore_index(conn: &mut PgConnection, index: &IndexDescription) -> Result<(), Error> {
let IndexDescription {
name,
table_name,
definition,
} = &index;
sqlx::query(&format!("{definition};"))
.execute(conn)
.await
.into_database_with(|| {
format!("failed to recreate index {name} on {table_name} with {definition}")
})?;
Ok(())
}

16
crates/syn2mas/src/mas_writer/fixtures/upstream_provider.sql Normal file
View File

@@ -0,0 +1,16 @@
INSERT INTO upstream_oauth_providers
(
upstream_oauth_provider_id,
scope,
client_id,
token_endpoint_auth_method,
created_at
)
VALUES
(
'00000000-0000-0000-0000-000000000004',
'openid',
'someClientId',
'client_secret_basic',
'2011-12-13 14:15:16Z'
);

60
crates/syn2mas/src/mas_writer/locking.rs Normal file
View File

@@ -0,0 +1,60 @@
// Copyright 2024 New Vector Ltd.
//
// SPDX-License-Identifier: AGPL-3.0-only
// Please see LICENSE in the repository root for full details.
use std::sync::LazyLock;
use sqlx::{
postgres::{PgAdvisoryLock, PgAdvisoryLockGuard},
Either, PgConnection,
};
static SYN2MAS_ADVISORY_LOCK: LazyLock<PgAdvisoryLock> =
LazyLock::new(|| PgAdvisoryLock::new("syn2mas-maswriter"));
/// A wrapper around a Postgres connection which holds a session-wide advisory
/// lock preventing concurrent access by other syn2mas instances.
pub struct LockedMasDatabase<'conn> {
inner: PgAdvisoryLockGuard<'static, &'conn mut PgConnection>,
}
impl<'conn> LockedMasDatabase<'conn> {
/// Attempts to lock the MAS database against concurrent access by other
/// syn2mas instances.
///
/// If the lock can be acquired, returns a `LockedMasDatabase`.
/// If the lock cannot be acquired, returns the connection back to the
/// caller wrapped in `Either::Right`.
///
/// # Errors
///
/// Errors are returned for underlying database errors.
pub async fn try_new(
mas_connection: &'conn mut PgConnection,
) -> Result<Either<Self, &'conn mut PgConnection>, sqlx::Error> {
SYN2MAS_ADVISORY_LOCK
.try_acquire(mas_connection)
.await
.map(|either| match either {
Either::Left(inner) => Either::Left(LockedMasDatabase { inner }),
Either::Right(unlocked) => Either::Right(unlocked),
})
}
/// Releases the advisory lock on the MAS database, returning the underlying
/// connection.
///
/// # Errors
///
/// Errors are returned for underlying database errors.
pub async fn unlock(self) -> Result<&'conn mut PgConnection, sqlx::Error> {
self.inner.release_now().await
}
}
impl AsMut<PgConnection> for LockedMasDatabase<'_> {
fn as_mut(&mut self) -> &mut PgConnection {
self.inner.as_mut()
}
}

1108
crates/syn2mas/src/mas_writer/mod.rs Normal file
View File

File diff suppressed because it is too large Load Diff

11
crates/syn2mas/src/mas_writer/snapshots/syn2mas__mas_writer__test__write_user.snap Normal file
View File

@@ -0,0 +1,11 @@
---
source: crates/syn2mas/src/mas_writer/mod.rs
expression: db_snapshot
---
users:
- can_request_admin: "false"
created_at: "1970-01-01 00:00:00+00"
locked_at: ~
primary_user_email_id: ~
user_id: 00000000-0000-0000-0000-000000000001
username: alice

17
crates/syn2mas/src/mas_writer/snapshots/syn2mas__mas_writer__test__write_user_with_email.snap Normal file
View File

@@ -0,0 +1,17 @@
---
source: crates/syn2mas/src/mas_writer/mod.rs
expression: db_snapshot
---
user_emails:
- confirmed_at: "1970-01-01 00:00:00+00"
created_at: "1970-01-01 00:00:00+00"
email: alice@example.org
user_email_id: 00000000-0000-0000-0000-000000000002
user_id: 00000000-0000-0000-0000-000000000001
users:
- can_request_admin: "false"
created_at: "1970-01-01 00:00:00+00"
locked_at: ~
primary_user_email_id: ~
user_id: 00000000-0000-0000-0000-000000000001
username: alice

18
crates/syn2mas/src/mas_writer/snapshots/syn2mas__mas_writer__test__write_user_with_password.snap Normal file
View File

@@ -0,0 +1,18 @@
---
source: crates/syn2mas/src/mas_writer/mod.rs
expression: db_snapshot
---
user_passwords:
- created_at: "1970-01-01 00:00:00+00"
hashed_password: $bcrypt$aaaaaaaaaaa
upgraded_from_id: ~
user_id: 00000000-0000-0000-0000-000000000001
user_password_id: 00000000-0000-0000-0000-00000000002a
version: "1"
users:
- can_request_admin: "false"
created_at: "1970-01-01 00:00:00+00"
locked_at: ~
primary_user_email_id: ~
user_id: 00000000-0000-0000-0000-000000000001
username: alice

16
crates/syn2mas/src/mas_writer/snapshots/syn2mas__mas_writer__test__write_user_with_unsupported_threepid.snap Normal file
View File

@@ -0,0 +1,16 @@
---
source: crates/syn2mas/src/mas_writer/mod.rs
expression: db_snapshot
---
user_unsupported_third_party_ids:
- address: "441189998819991197253"
created_at: "1970-01-01 00:00:00+00"
medium: msisdn
user_id: 00000000-0000-0000-0000-000000000001
users:
- can_request_admin: "false"
created_at: "1970-01-01 00:00:00+00"
locked_at: ~
primary_user_email_id: ~
user_id: 00000000-0000-0000-0000-000000000001
username: alice

42
crates/syn2mas/src/mas_writer/snapshots/syn2mas__mas_writer__test__write_user_with_upstream_provider_link.snap Normal file
View File

@@ -0,0 +1,42 @@
---
source: crates/syn2mas/src/mas_writer/mod.rs
expression: db_snapshot
---
upstream_oauth_links:
- created_at: "1970-01-01 00:00:00+00"
human_account_name: ~
subject: "12345.67890"
upstream_oauth_link_id: 00000000-0000-0000-0000-000000000003
upstream_oauth_provider_id: 00000000-0000-0000-0000-000000000004
user_id: 00000000-0000-0000-0000-000000000001
upstream_oauth_providers:
- additional_parameters: ~
authorization_endpoint_override: ~
brand_name: ~
claims_imports: "{}"
client_id: someClientId
created_at: "2011-12-13 14:15:16+00"
disabled_at: ~
discovery_mode: oidc
encrypted_client_secret: ~
fetch_userinfo: "false"
human_name: ~
id_token_signed_response_alg: RS256
issuer: ~
jwks_uri_override: ~
pkce_mode: auto
response_mode: query
scope: openid
token_endpoint_auth_method: client_secret_basic
token_endpoint_override: ~
token_endpoint_signing_alg: ~
upstream_oauth_provider_id: 00000000-0000-0000-0000-000000000004
userinfo_endpoint_override: ~
userinfo_signed_response_alg: ~
users:
- can_request_admin: "false"
created_at: "1970-01-01 00:00:00+00"
locked_at: ~
primary_user_email_id: ~
user_id: 00000000-0000-0000-0000-000000000001
username: alice

15
crates/syn2mas/src/mas_writer/syn2mas_revert_temporary_tables.sql Normal file
View File

@@ -0,0 +1,15 @@
-- Copyright 2024 New Vector Ltd.
--
-- SPDX-License-Identifier: AGPL-3.0-only
-- Please see LICENSE in the repository root for full details.
-- This script should revert what `syn2mas_temporary_tables.sql` does.
DROP TABLE syn2mas_restore_constraints;
DROP TABLE syn2mas_restore_indices;
ALTER TABLE syn2mas__users RENAME TO users;
ALTER TABLE syn2mas__user_passwords RENAME TO user_passwords;
ALTER TABLE syn2mas__user_emails RENAME TO user_emails;
ALTER TABLE syn2mas__user_unsupported_third_party_ids RENAME TO user_unsupported_third_party_ids;
ALTER TABLE syn2mas__upstream_oauth_links RENAME TO upstream_oauth_links;

44
crates/syn2mas/src/mas_writer/syn2mas_temporary_tables.sql Normal file
View File

@@ -0,0 +1,44 @@
-- Copyright 2024 New Vector Ltd.
--
-- SPDX-License-Identifier: AGPL-3.0-only
-- Please see LICENSE in the repository root for full details.
-- # syn2mas Temporary Tables
-- This file takes a MAS database and:
--
-- 1. creates temporary tables used by syn2mas for storing restore data
-- 2. renames important tables with the `syn2mas__` prefix, to prevent
-- running MAS instances from having any opportunity to see or modify
-- the partial data in the database, especially whilst it is not protected
-- by constraints.
--
-- All changes in this file must be reverted by `syn2mas_revert_temporary_tables.sql`
-- in the same directory.
-- corresponds to `ConstraintDescription`
CREATE TABLE syn2mas_restore_constraints (
-- synthetic auto-incrementing ID so we can load these in order
order_key SERIAL NOT NULL PRIMARY KEY,
table_name TEXT NOT NULL,
name TEXT NOT NULL,
definition TEXT NOT NULL
);
-- corresponds to `IndexDescription`
CREATE TABLE syn2mas_restore_indices (
-- synthetic auto-incrementing ID so we can load these in order
order_key SERIAL NOT NULL PRIMARY KEY,
table_name TEXT NOT NULL,
name TEXT NOT NULL,
definition TEXT NOT NULL
);
-- Now we rename all tables that we touch during the migration.
ALTER TABLE users RENAME TO syn2mas__users;
ALTER TABLE user_passwords RENAME TO syn2mas__user_passwords;
ALTER TABLE user_emails RENAME TO syn2mas__user_emails;
ALTER TABLE user_unsupported_third_party_ids RENAME TO syn2mas__user_unsupported_third_party_ids;
ALTER TABLE upstream_oauth_links RENAME TO syn2mas__upstream_oauth_links;

352
crates/syn2mas/src/migration.rs Normal file
View File

@@ -0,0 +1,352 @@
// Copyright 2024 New Vector Ltd.
//
// SPDX-License-Identifier: AGPL-3.0-only
// Please see LICENSE in the repository root for full details.
//! # Migration
//!
//! This module provides the high-level logic for performing the Synapse-to-MAS
//! database migration.
//!
//! This module does not implement any of the safety checks that should be run
//! *before* the migration.
use std::{collections::HashMap, pin::pin};
use chrono::{DateTime, Utc};
use compact_str::CompactString;
use futures_util::StreamExt as _;
use rand::RngCore;
use thiserror::Error;
use thiserror_ext::ContextInto;
use tracing::Level;
use ulid::Ulid;
use uuid::Uuid;
use crate::{
mas_writer::{
self, MasNewEmailThreepid, MasNewUnsupportedThreepid, MasNewUpstreamOauthLink, MasNewUser,
MasNewUserPassword, MasWriteBuffer, MasWriter,
},
synapse_reader::{
self, ExtractLocalpartError, FullUserId, SynapseExternalId, SynapseThreepid, SynapseUser,
},
SynapseReader,
};
#[derive(Debug, Error, ContextInto)]
pub enum Error {
#[error("error when reading synapse DB ({context}): {source}")]
Synapse {
source: synapse_reader::Error,
context: String,
},
#[error("error when writing to MAS DB ({context}): {source}")]
Mas {
source: mas_writer::Error,
context: String,
},
#[error("failed to extract localpart of {user:?}: {source}")]
ExtractLocalpart {
source: ExtractLocalpartError,
user: FullUserId,
},
#[error("user {user} was not found for migration but a row in {table} was found for them")]
MissingUserFromDependentTable { table: String, user: FullUserId },
#[error("missing a mapping for the auth provider with ID {synapse_id:?} (used by {user} and maybe other users)")]
MissingAuthProviderMapping {
/// `auth_provider` ID of the provider in Synapse, for which we have no
/// mapping
synapse_id: String,
/// a user that is using this auth provider
user: FullUserId,
},
}
struct UsersMigrated {
/// Lookup table from user localpart to that user's UUID in MAS.
user_localparts_to_uuid: HashMap<CompactString, Uuid>,
}
/// Performs a migration from Synapse's database to MAS' database.
///
/// # Panics
///
/// - If there are more than `usize::MAX` users
///
/// # Errors
///
/// Errors are returned under the following circumstances:
///
/// - An underlying database access error, either to MAS or to Synapse.
/// - Invalid data in the Synapse database.
#[allow(clippy::implicit_hasher)]
pub async fn migrate(
synapse: &mut SynapseReader<'_>,
mas: &mut MasWriter<'_>,
server_name: &str,
rng: &mut impl RngCore,
provider_id_mapping: &HashMap<String, Uuid>,
) -> Result<(), Error> {
let counts = synapse.count_rows().await.into_synapse("counting users")?;
let migrated_users = migrate_users(
synapse,
mas,
counts
.users
.try_into()
.expect("More than usize::MAX users — wow!"),
server_name,
rng,
)
.await?;
migrate_threepids(
synapse,
mas,
server_name,
rng,
&migrated_users.user_localparts_to_uuid,
)
.await?;
migrate_external_ids(
synapse,
mas,
server_name,
rng,
&migrated_users.user_localparts_to_uuid,
provider_id_mapping,
)
.await?;
Ok(())
}
#[tracing::instrument(skip_all, level = Level::INFO)]
async fn migrate_users(
synapse: &mut SynapseReader<'_>,
mas: &mut MasWriter<'_>,
user_count_hint: usize,
server_name: &str,
rng: &mut impl RngCore,
) -> Result<UsersMigrated, Error> {
let mut user_buffer = MasWriteBuffer::new(MasWriter::write_users);
let mut password_buffer = MasWriteBuffer::new(MasWriter::write_passwords);
let mut users_stream = pin!(synapse.read_users());
// TODO is 1:1 capacity enough for a hashmap?
let mut user_localparts_to_uuid = HashMap::with_capacity(user_count_hint);
while let Some(user_res) = users_stream.next().await {
let user = user_res.into_synapse("reading user")?;
let (mas_user, mas_password_opt) = transform_user(&user, server_name, rng)?;
user_localparts_to_uuid.insert(CompactString::new(&mas_user.username), mas_user.user_id);
user_buffer
.write(mas, mas_user)
.await
.into_mas("writing user")?;
if let Some(mas_password) = mas_password_opt {
password_buffer
.write(mas, mas_password)
.await
.into_mas("writing password")?;
}
}
user_buffer.finish(mas).await.into_mas("writing users")?;
password_buffer
.finish(mas)
.await
.into_mas("writing passwords")?;
Ok(UsersMigrated {
user_localparts_to_uuid,
})
}
#[tracing::instrument(skip_all, level = Level::INFO)]
async fn migrate_threepids(
synapse: &mut SynapseReader<'_>,
mas: &mut MasWriter<'_>,
server_name: &str,
rng: &mut impl RngCore,
user_localparts_to_uuid: &HashMap<CompactString, Uuid>,
) -> Result<(), Error> {
let mut email_buffer = MasWriteBuffer::new(MasWriter::write_email_threepids);
let mut unsupported_buffer = MasWriteBuffer::new(MasWriter::write_unsupported_threepids);
let mut users_stream = pin!(synapse.read_threepids());
while let Some(threepid_res) = users_stream.next().await {
let SynapseThreepid {
user_id: synapse_user_id,
medium,
address,
added_at,
} = threepid_res.into_synapse("reading threepid")?;
let created_at: DateTime<Utc> = added_at.into();
let username = synapse_user_id
.extract_localpart(server_name)
.into_extract_localpart(synapse_user_id.clone())?
.to_owned();
let Some(user_id) = user_localparts_to_uuid.get(username.as_str()).copied() else {
return Err(Error::MissingUserFromDependentTable {
table: "user_threepids".to_owned(),
user: synapse_user_id,
});
};
if medium == "email" {
email_buffer
.write(
mas,
MasNewEmailThreepid {
user_id,
user_email_id: Uuid::from(Ulid::from_datetime_with_source(
created_at.into(),
rng,
)),
email: address,
created_at,
},
)
.await
.into_mas("writing email")?;
} else {
unsupported_buffer
.write(
mas,
MasNewUnsupportedThreepid {
user_id,
medium,
address,
created_at,
},
)
.await
.into_mas("writing unsupported threepid")?;
}
}
email_buffer
.finish(mas)
.await
.into_mas("writing email threepids")?;
unsupported_buffer
.finish(mas)
.await
.into_mas("writing unsupported threepids")?;
Ok(())
}
/// # Parameters
///
/// - `provider_id_mapping`: mapping from Synapse `auth_provider` ID to UUID of
/// the upstream provider in MAS.
#[tracing::instrument(skip_all, level = Level::INFO)]
async fn migrate_external_ids(
synapse: &mut SynapseReader<'_>,
mas: &mut MasWriter<'_>,
server_name: &str,
rng: &mut impl RngCore,
user_localparts_to_uuid: &HashMap<CompactString, Uuid>,
provider_id_mapping: &HashMap<String, Uuid>,
) -> Result<(), Error> {
let mut write_buffer = MasWriteBuffer::new(MasWriter::write_upstream_oauth_links);
let mut extids_stream = pin!(synapse.read_user_external_ids());
while let Some(extid_res) = extids_stream.next().await {
let SynapseExternalId {
user_id: synapse_user_id,
auth_provider,
external_id: subject,
} = extid_res.into_synapse("reading external ID")?;
let username = synapse_user_id
.extract_localpart(server_name)
.into_extract_localpart(synapse_user_id.clone())?
.to_owned();
let Some(user_id) = user_localparts_to_uuid.get(username.as_str()).copied() else {
return Err(Error::MissingUserFromDependentTable {
table: "user_external_ids".to_owned(),
user: synapse_user_id,
});
};
let Some(&upstream_provider_id) = provider_id_mapping.get(&auth_provider) else {
return Err(Error::MissingAuthProviderMapping {
synapse_id: auth_provider,
user: synapse_user_id,
});
};
// To save having to store user creation times, extract it from the ULID
// This gives millisecond precision — good enough.
let user_created_ts = Ulid::from(user_id).datetime();
let link_id: Uuid = Ulid::from_datetime_with_source(user_created_ts, rng).into();
write_buffer
.write(
mas,
MasNewUpstreamOauthLink {
link_id,
user_id,
upstream_provider_id,
subject,
created_at: user_created_ts.into(),
},
)
.await
.into_mas("failed to write upstream link")?;
}
write_buffer
.finish(mas)
.await
.into_mas("writing threepids")?;
Ok(())
}
fn transform_user(
user: &SynapseUser,
server_name: &str,
rng: &mut impl RngCore,
) -> Result<(MasNewUser, Option<MasNewUserPassword>), Error> {
let username = user
.name
.extract_localpart(server_name)
.into_extract_localpart(user.name.clone())?
.to_owned();
let new_user = MasNewUser {
user_id: Uuid::from(Ulid::from_datetime_with_source(
DateTime::<Utc>::from(user.creation_ts).into(),
rng,
)),
username,
created_at: user.creation_ts.into(),
locked_at: bool::from(user.deactivated).then_some(user.creation_ts.into()),
can_request_admin: bool::from(user.admin),
};
let mas_password = user
.password_hash
.clone()
.map(|password_hash| MasNewUserPassword {
user_password_id: Uuid::from(Ulid::from_datetime_with_source(
DateTime::<Utc>::from(user.creation_ts).into(),
rng,
)),
user_id: new_user.user_id,
hashed_password: password_hash,
created_at: new_user.created_at,
});
Ok((new_user, mas_password))
}

306
crates/syn2mas/src/synapse_reader/checks.rs Normal file
View File

@@ -0,0 +1,306 @@
// Copyright 2025 New Vector Ltd.
//
// SPDX-License-Identifier: AGPL-3.0-only
// Please see LICENSE in the repository root for full details.
//! # Synapse Checks
//!
//! This module provides safety checks to run against a Synapse database before
//! running the Synapse-to-MAS migration.
use figment::Figment;
use mas_config::{
BrandingConfig, CaptchaConfig, ConfigurationSection, ConfigurationSectionExt, MatrixConfig,
PasswordAlgorithm, PasswordsConfig, UpstreamOAuth2Config,
};
use sqlx::{prelude::FromRow, query_as, query_scalar, PgConnection};
use thiserror::Error;
use super::config::Config;
use crate::mas_writer::MIGRATED_PASSWORD_VERSION;
#[derive(Debug, Error)]
pub enum Error {
#[error("query failed: {0}")]
Sqlx(#[from] sqlx::Error),
#[error("failed to load MAS config: {0}")]
MasConfig(#[from] figment::Error),
#[error("failed to load MAS password config: {0}")]
MasPasswordConfig(#[source] anyhow::Error),
}
/// An error found whilst checking the Synapse database, that should block a
/// migration.
#[derive(Debug, Error)]
pub enum CheckError {
#[error("MAS config is missing a password hashing scheme with version '1'")]
MissingPasswordScheme,
#[error("Password scheme version '1' in the MAS config must use the Bcrypt algorithm, so that Synapse passwords can be imported and will be compatible.")]
PasswordSchemeNotBcrypt,
#[error("Password scheme version '1' in the MAS config must have the same secret as the `pepper` value from Synapse, so that Synapse passwords can be imported and will be compatible.")]
PasswordSchemeWrongPepper,
#[error("Synapse database contains {num_guests} guests which aren't supported by MAS. See https://github.com/element-hq/matrix-authentication-service/issues/1445")]
GuestsInDatabase { num_guests: i64 },
#[error("Guest support is enabled in the Synapse configuration. Guests aren't supported by MAS, but if you don't have any then you could disable the option. See https://github.com/element-hq/matrix-authentication-service/issues/1445")]
GuestsEnabled,
#[error("Synapse database contains {num_non_email_3pids} non-email 3PIDs (probably phone numbers), which are not supported by MAS.")]
NonEmailThreepidsInDatabase { num_non_email_3pids: i64 },
#[error(
"Synapse config has `enable_3pid_changes` explicitly enabled, which must be disabled or removed."
)]
ThreepidChangesEnabled,
#[error("Synapse config has `login_via_existing_session.enabled` set to true, which must be disabled.")]
LoginViaExistingSessionEnabled,
#[error("MAS configuration has the wrong `matrix.homeserver` set ({mas:?}), it should match Synapse's `server_name` ({synapse:?})")]
ServerNameMismatch { synapse: String, mas: String },
#[error("Synapse database contains {num_users} users associated to the OpenID Connect or OAuth2 provider '{provider}' but the Synapse configuration does not contain this provider.")]
SynapseMissingOAuthProvider { provider: String, num_users: i64 },
#[error("Synapse config contains an OpenID Connect or OAuth2 provider '{provider}' (issuer: {issuer:?}) used by {num_users} users which must also be configured in the MAS configuration as an upstream provider.")]
MasMissingOAuthProvider {
provider: String,
issuer: String,
num_users: i64,
},
}
/// A potential hazard found whilst checking the Synapse database, that should
/// be presented to the operator to check they are aware of a caveat before
/// proceeding with the migration.
#[derive(Debug, Error)]
pub enum CheckWarning {
#[error("Synapse config contains OIDC auth configuration (issuer: {issuer:?}) which will need to be manually mapped to an upstream OpenID Connect Provider during migration.")]
UpstreamOidcProvider { issuer: String },
#[error("Synapse config contains {0} auth configuration which will need to be manually mapped as an upstream OAuth 2.0 provider during migration.")]
ExternalAuthSystem(&'static str),
#[error("Synapse config has registration enabled. This must be disabled after migration before bringing Synapse back online.")]
DisableRegistrationAfterMigration,
#[error("Synapse config has `user_consent` enabled. This should be disabled after migration.")]
DisableUserConsentAfterMigration,
#[error("Synapse config has `user_consent` enabled but MAS has not been configured with terms of service. You may wish to set up a `tos_uri` in your MAS branding configuration to replace the user consent.")]
ShouldPortUserConsentAsTerms,
#[error("Synapse config has a registration CAPTCHA enabled, but no CAPTCHA has been configured in MAS. You may wish to manually configure this.")]
ShouldPortRegistrationCaptcha,
}
/// Check that the Synapse configuration is sane for migration.
#[must_use]
pub fn synapse_config_check(synapse_config: &Config) -> (Vec<CheckWarning>, Vec<CheckError>) {
let mut errors = Vec::new();
let mut warnings = Vec::new();
if synapse_config.enable_registration {
warnings.push(CheckWarning::DisableRegistrationAfterMigration);
}
if synapse_config.user_consent {
warnings.push(CheckWarning::DisableUserConsentAfterMigration);
}
// TODO check the settings directly against the MAS settings
for provider in synapse_config.all_oidc_providers().values() {
if let Some(ref issuer) = provider.issuer {
warnings.push(CheckWarning::UpstreamOidcProvider {
issuer: issuer.clone(),
});
}
}
// TODO provide guidance on migrating these
if synapse_config.cas_config.enabled {
warnings.push(CheckWarning::ExternalAuthSystem("CAS"));
}
if synapse_config.saml2_config.enabled {
warnings.push(CheckWarning::ExternalAuthSystem("SAML2"));
}
if synapse_config.jwt_config.enabled {
warnings.push(CheckWarning::ExternalAuthSystem("JWT"));
}
// TODO provide guidance on migrating these
if synapse_config.password_config.enabled && !synapse_config.password_config.localdb_enabled {
warnings.push(CheckWarning::ExternalAuthSystem(
"non-standard password provider plugin",
));
}
if synapse_config.enable_3pid_changes {
errors.push(CheckError::ThreepidChangesEnabled);
}
if synapse_config.login_via_existing_session.enabled {
errors.push(CheckError::LoginViaExistingSessionEnabled);
}
(warnings, errors)
}
/// Check that the given Synapse configuration is sane for migration to a MAS
/// with the given MAS configuration.
///
/// # Errors
///
/// - If any necessary section of MAS config cannot be parsed.
/// - If the MAS password configuration (including any necessary secrets) can't
/// be loaded.
pub async fn synapse_config_check_against_mas_config(
synapse: &Config,
mas: &Figment,
) -> Result<(Vec<CheckWarning>, Vec<CheckError>), Error> {
let mut errors = Vec::new();
let mut warnings = Vec::new();
let mas_passwords = PasswordsConfig::extract_or_default(mas)?;
let mas_password_schemes = mas_passwords
.load()
.await
.map_err(Error::MasPasswordConfig)?;
let mas_matrix = MatrixConfig::extract(mas)?;
// Look for the MAS password hashing scheme that will be used for imported
// Synapse passwords, then check the configuration matches so that Synapse
// passwords will be compatible with MAS.
if let Some((_, algorithm, _, secret)) = mas_password_schemes
.iter()
.find(|(version, _, _, _)| *version == MIGRATED_PASSWORD_VERSION)
{
if algorithm != &PasswordAlgorithm::Bcrypt {
errors.push(CheckError::PasswordSchemeNotBcrypt);
}
let synapse_pepper = synapse
.password_config
.pepper
.as_ref()
.map(String::as_bytes);
if secret.as_deref() != synapse_pepper {
errors.push(CheckError::PasswordSchemeWrongPepper);
}
} else {
errors.push(CheckError::MissingPasswordScheme);
}
if synapse.allow_guest_access {
errors.push(CheckError::GuestsEnabled);
}
if synapse.server_name != mas_matrix.homeserver {
errors.push(CheckError::ServerNameMismatch {
synapse: synapse.server_name.clone(),
mas: mas_matrix.homeserver.clone(),
});
}
let mas_captcha = CaptchaConfig::extract_or_default(mas)?;
if synapse.enable_registration_captcha && mas_captcha.service.is_none() {
warnings.push(CheckWarning::ShouldPortRegistrationCaptcha);
}
let mas_branding = BrandingConfig::extract_or_default(mas)?;
if synapse.user_consent && mas_branding.tos_uri.is_none() {
warnings.push(CheckWarning::ShouldPortUserConsentAsTerms);
}
Ok((warnings, errors))
}
/// Check that the Synapse database is sane for migration. Returns a list of
/// warnings and errors.
///
/// # Errors
///
/// - If there is some database connection error, or the given database is not a
/// Synapse database.
/// - If the OAuth2 section of the MAS configuration could not be parsed.
#[tracing::instrument(skip_all)]
pub async fn synapse_database_check(
synapse_connection: &mut PgConnection,
synapse: &Config,
mas: &Figment,
) -> Result<(Vec<CheckWarning>, Vec<CheckError>), Error> {
#[derive(FromRow)]
struct UpstreamOAuthProvider {
auth_provider: String,
num_users: i64,
}
let mut errors = Vec::new();
let warnings = Vec::new();
let num_guests: i64 = query_scalar("SELECT COUNT(1) FROM users WHERE is_guest <> 0")
.fetch_one(&mut *synapse_connection)
.await?;
if num_guests > 0 {
errors.push(CheckError::GuestsInDatabase { num_guests });
}
let num_non_email_3pids: i64 =
query_scalar("SELECT COUNT(1) FROM user_threepids WHERE medium <> 'email'")
.fetch_one(&mut *synapse_connection)
.await?;
if num_non_email_3pids > 0 {
errors.push(CheckError::NonEmailThreepidsInDatabase {
num_non_email_3pids,
});
}
let oauth_provider_user_counts = query_as::<_, UpstreamOAuthProvider>(
"
SELECT auth_provider, COUNT(*) AS num_users
FROM user_external_ids
GROUP BY auth_provider
ORDER BY auth_provider
",
)
.fetch_all(&mut *synapse_connection)
.await?;
if !oauth_provider_user_counts.is_empty() {
let syn_oauth2 = synapse.all_oidc_providers();
let mas_oauth2 = UpstreamOAuth2Config::extract_or_default(mas)?;
for row in oauth_provider_user_counts {
let matching_syn = syn_oauth2.get(&row.auth_provider);
let Some(matching_syn) = matching_syn else {
errors.push(CheckError::SynapseMissingOAuthProvider {
provider: row.auth_provider,
num_users: row.num_users,
});
continue;
};
// Matching by `synapse_idp_id` is the same as what we'll do for the migration
let matching_mas = mas_oauth2.providers.iter().find(|mas_provider| {
mas_provider.synapse_idp_id.as_ref() == Some(&row.auth_provider)
});
if matching_mas.is_none() {
errors.push(CheckError::MasMissingOAuthProvider {
provider: row.auth_provider,
issuer: matching_syn
.issuer
.clone()
.unwrap_or("<unspecified>".to_owned()),
num_users: row.num_users,
});
}
}
}
Ok((warnings, errors))
}

297
crates/syn2mas/src/synapse_reader/config.rs Normal file
View File

@@ -0,0 +1,297 @@
// Copyright 2025 New Vector Ltd.
//
// SPDX-License-Identifier: AGPL-3.0-only
// Please see LICENSE in the repository root for full details.
use std::collections::BTreeMap;
use camino::Utf8PathBuf;
use figment::providers::{Format, Yaml};
use serde::Deserialize;
use sqlx::postgres::PgConnectOptions;
/// The root of a Synapse configuration.
/// This struct only includes fields which the Synapse-to-MAS migration is
/// interested in.
///
/// See: <https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html>
#[derive(Deserialize)]
#[allow(clippy::struct_excessive_bools)]
pub struct Config {
pub database: DatabaseSection,
pub password_config: PasswordSection,
#[serde(default)]
pub allow_guest_access: bool,
#[serde(default)]
pub enable_registration: bool,
#[serde(default)]
pub enable_registration_captcha: bool,
/// Normally this defaults to true, but when MAS integration is enabled in
/// Synapse it defaults to false.
#[serde(default)]
pub enable_3pid_changes: bool,
#[serde(default)]
pub user_consent: bool,
#[serde(default)]
pub registrations_require_3pid: Vec<String>,
#[serde(default)]
pub registration_requires_token: bool,
pub registration_shared_secret: Option<String>,
#[serde(default)]
pub login_via_existing_session: EnableableSection,
#[serde(default)]
pub cas_config: EnableableSection,
#[serde(default)]
pub saml2_config: EnableableSection,
#[serde(default)]
pub jwt_config: EnableableSection,
#[serde(default)]
pub oidc_config: Option<OidcProvider>,
#[serde(default)]
pub oidc_providers: Vec<OidcProvider>,
pub server_name: String,
}
impl Config {
/// Load a Synapse configuration from the given list of configuration files.
///
/// # Errors
///
/// - If there is a problem reading any of the files.
/// - If the configuration is not valid.
pub fn load(files: &[Utf8PathBuf]) -> Result<Config, figment::Error> {
let mut figment = figment::Figment::new();
for file in files {
// TODO this is not exactly correct behaviour — Synapse does not merge anything
// other than the top level dict.
// https://github.com/element-hq/matrix-authentication-service/pull/3805#discussion_r1922680825
// https://github.com/element-hq/synapse/blob/develop/synapse/config/_base.py?rgh-link-date=2025-01-20T17%3A02%3A56Z#L870
figment = figment.merge(Yaml::file(file));
}
figment.extract::<Config>()
}
/// Returns a map of all OIDC providers from the Synapse configuration.
///
/// The keys are the `auth_provider` IDs as they would have been stored in
/// Synapse's database.
///
/// These are compatible with the `synapse_idp_id` field of
/// [`mas_config::UpstreamOAuth2Provider`].
#[must_use]
pub fn all_oidc_providers(&self) -> BTreeMap<String, OidcProvider> {
let mut out = BTreeMap::new();
if let Some(provider) = &self.oidc_config {
if provider.issuer.is_some() {
// The legacy configuration has an implied IdP ID of `oidc`.
out.insert("oidc".to_owned(), provider.clone());
}
}
for provider in &self.oidc_providers {
if let Some(idp_id) = &provider.idp_id {
// Synapse internally prefixes the IdP IDs with `oidc-`.
out.insert(format!("oidc-{idp_id}"), provider.clone());
}
}
out
}
}
/// The `database` section of the Synapse configuration.
///
/// See: <https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#database>
#[derive(Deserialize)]
pub struct DatabaseSection {
/// Expecting `psycopg2` for Postgres or `sqlite3` for `SQLite3`, but may be
/// an arbitrary string and future versions of Synapse may support other
/// database drivers, e.g. psycopg3.
pub name: String,
#[serde(default)]
pub args: DatabaseArgsSuboption,
}
/// The database driver name for Synapse when it is using Postgres via psycopg2.
pub const SYNAPSE_DATABASE_DRIVER_NAME_PSYCOPG2: &str = "psycopg2";
/// The database driver name for Synapse when it is using SQLite 3.
pub const SYNAPSE_DATABASE_DRIVER_NAME_SQLITE3: &str = "sqlite3";
impl DatabaseSection {
/// Process the configuration into Postgres connection options.
///
/// Environment variables and libpq defaults will be used as fallback for
/// any missing values; this should match what Synapse does.
/// But note that if syn2mas is not run in the same context (host, user,
/// environment variables) as Synapse normally runs, then the connection
/// options may not be valid.
///
/// Returns `None` if this database configuration is not configured for
/// Postgres.
#[must_use]
pub fn to_sqlx_postgres(&self) -> Option<PgConnectOptions> {
if self.name != SYNAPSE_DATABASE_DRIVER_NAME_PSYCOPG2 {
return None;
}
let mut opts = PgConnectOptions::new().application_name("syn2mas-synapse");
if let Some(host) = &self.args.host {
opts = opts.host(host);
}
if let Some(port) = self.args.port {
opts = opts.port(port);
}
if let Some(dbname) = &self.args.dbname {
opts = opts.database(dbname);
}
if let Some(user) = &self.args.user {
opts = opts.username(user);
}
if let Some(password) = &self.args.password {
opts = opts.password(password);
}
Some(opts)
}
}
/// The `args` suboption of the `database` section of the Synapse configuration.
/// This struct assumes Postgres is in use and does not represent fields used by
/// SQLite.
#[derive(Deserialize, Default)]
pub struct DatabaseArgsSuboption {
pub user: Option<String>,
pub password: Option<String>,
pub dbname: Option<String>,
pub host: Option<String>,
pub port: Option<u16>,
}
/// The `password_config` section of the Synapse configuration.
///
/// See: <https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#password_config>
#[derive(Deserialize)]
pub struct PasswordSection {
#[serde(default = "default_true")]
pub enabled: bool,
#[serde(default = "default_true")]
pub localdb_enabled: bool,
pub pepper: Option<String>,
}
impl Default for PasswordSection {
fn default() -> Self {
PasswordSection {
enabled: true,
localdb_enabled: true,
pepper: None,
}
}
}
/// A section that we only care about whether it's enabled or not, but is not
/// enabled by default.
#[derive(Default, Deserialize)]
pub struct EnableableSection {
#[serde(default)]
pub enabled: bool,
}
#[derive(Clone, Deserialize)]
pub struct OidcProvider {
/// At least for `oidc_config`, if the dict is present but left empty then
/// the config should be ignored, so this field must be optional.
pub issuer: Option<String>,
/// Required, except for the old `oidc_config` where this is implied to be
/// "oidc".
pub idp_id: Option<String>,
}
fn default_true() -> bool {
true
}
#[cfg(test)]
mod test {
use sqlx::postgres::PgConnectOptions;
use super::{DatabaseArgsSuboption, DatabaseSection};
#[test]
fn test_to_sqlx_postgres() {
#[track_caller]
#[allow(clippy::needless_pass_by_value)]
fn assert_eq_options(config: DatabaseSection, uri: &str) {
let config_connect_options = config
.to_sqlx_postgres()
.expect("no connection options generated by config");
let uri_connect_options: PgConnectOptions = uri
.parse()
.expect("example URI did not parse as PgConnectionOptions");
assert_eq!(
config_connect_options.get_host(),
uri_connect_options.get_host()
);
assert_eq!(
config_connect_options.get_port(),
uri_connect_options.get_port()
);
assert_eq!(
config_connect_options.get_username(),
uri_connect_options.get_username()
);
// The password is not public so we can't assert it. But that's hopefully fine.
assert_eq!(
config_connect_options.get_database(),
uri_connect_options.get_database()
);
}
// SQLite configs are not accepted
assert!(DatabaseSection {
name: "sqlite3".to_owned(),
args: DatabaseArgsSuboption::default(),
}
.to_sqlx_postgres()
.is_none());
assert_eq_options(
DatabaseSection {
name: "psycopg2".to_owned(),
args: DatabaseArgsSuboption::default(),
},
"postgresql:///",
);
assert_eq_options(
DatabaseSection {
name: "psycopg2".to_owned(),
args: DatabaseArgsSuboption {
user: Some("synapse_user".to_owned()),
password: Some("verysecret".to_owned()),
dbname: Some("synapse_db".to_owned()),
host: Some("synapse-db.example.com".to_owned()),
port: Some(42),
},
},
"postgresql://synapse_user:verysecret@synapse-db.example.com:42/synapse_db",
);
}
}

12
crates/syn2mas/src/synapse_reader/fixtures/external_ids_alice.sql Normal file
View File

@@ -0,0 +1,12 @@
INSERT INTO user_external_ids
(
user_id,
auth_provider,
external_id
)
VALUES
(
'@alice:example.com',
'oidc-raasu',
'871.syn30'
);

23
crates/syn2mas/src/synapse_reader/fixtures/threepids_alice.sql Normal file
View File

@@ -0,0 +1,23 @@
INSERT INTO user_threepids
(
user_id,
medium,
address,
validated_at,
added_at
)
VALUES
(
'@alice:example.com',
'email',
'alice@example.com',
1554228492026,
1554228549014
),
(
'@alice:example.com',
'msisdn',
'441189998819991197253',
1555228492026,
1555228549014
);

40
crates/syn2mas/src/synapse_reader/fixtures/user_alice.sql Normal file
View File

@@ -0,0 +1,40 @@
--
INSERT INTO users
(
name,
password_hash,
creation_ts,
admin,
upgrade_ts,
is_guest,
appservice_id,
consent_version,
consent_server_notice_sent,
user_type,
deactivated,
shadow_banned,
consent_ts,
approved,
locked,
suspended
)
VALUES
(
'@alice:example.com',
'$2b$12$aaa/aaaaaaaaaa.aaaaaaaaaaaaaaa./aaaaaaaaaaaaaaaaaaa/A',
1530393962,
0,
NULL,
0,
NULL,
'1.0',
'1.0',
NULL,
0,
NULL,
NULL,
NULL,
false,
false
);

418
crates/syn2mas/src/synapse_reader/mod.rs Normal file
View File

@@ -0,0 +1,418 @@
// Copyright 2024 New Vector Ltd.
//
// SPDX-License-Identifier: AGPL-3.0-only
// Please see LICENSE in the repository root for full details.
//! # Synapse Database Reader
//!
//! This module provides facilities for streaming relevant types of database
//! records from a Synapse database.
use std::fmt::Display;
use chrono::{DateTime, Utc};
use futures_util::{Stream, TryStreamExt};
use sqlx::{query, Acquire, FromRow, PgConnection, Postgres, Row, Transaction, Type};
use thiserror::Error;
use thiserror_ext::ContextInto;
pub mod checks;
pub mod config;
#[derive(Debug, Error, ContextInto)]
pub enum Error {
#[error("database error whilst {context}")]
Database {
#[source]
source: sqlx::Error,
context: String,
},
}
#[derive(Clone, Debug, sqlx::Decode, PartialEq, Eq, PartialOrd, Ord)]
pub struct FullUserId(pub String);
impl Display for FullUserId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.0.fmt(f)
}
}
impl Type<Postgres> for FullUserId {
fn type_info() -> <sqlx::Postgres as sqlx::Database>::TypeInfo {
<String as Type<Postgres>>::type_info()
}
}
#[derive(Debug, Error)]
pub enum ExtractLocalpartError {
#[error("user ID does not start with `@` sigil")]
NoAtSigil,
#[error("user ID does not have a `:` separator")]
NoSeparator,
#[error("wrong server name: expected {expected:?}, got {found:?}")]
WrongServerName { expected: String, found: String },
}
impl FullUserId {
/// Extract the localpart from the User ID, asserting that the User ID has
/// the correct server name.
///
/// # Errors
///
/// A handful of basic validity checks are performed and an error may be
/// returned if the User ID is not valid.
/// However, the User ID grammar is not checked fully.
///
/// If the wrong server name is asserted, returns an error.
pub fn extract_localpart(
&self,
expected_server_name: &str,
) -> Result<&str, ExtractLocalpartError> {
let Some(without_sigil) = self.0.strip_prefix('@') else {
return Err(ExtractLocalpartError::NoAtSigil);
};
let Some((localpart, server_name)) = without_sigil.split_once(':') else {
return Err(ExtractLocalpartError::NoSeparator);
};
if server_name != expected_server_name {
return Err(ExtractLocalpartError::WrongServerName {
expected: expected_server_name.to_owned(),
found: server_name.to_owned(),
});
};
Ok(localpart)
}
}
/// A Synapse boolean.
/// Synapse stores booleans as 0 or 1, due to compatibility with old SQLite
/// versions that did not have native boolean support.
#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub struct SynapseBool(bool);
impl<'r> sqlx::Decode<'r, Postgres> for SynapseBool {
fn decode(
value: <Postgres as sqlx::Database>::ValueRef<'r>,
) -> Result<Self, sqlx::error::BoxDynError> {
<i16 as sqlx::Decode<Postgres>>::decode(value)
.map(|boolean_int| SynapseBool(boolean_int != 0))
}
}
impl sqlx::Type<Postgres> for SynapseBool {
fn type_info() -> <Postgres as sqlx::Database>::TypeInfo {
<i16 as sqlx::Type<Postgres>>::type_info()
}
}
impl From<SynapseBool> for bool {
fn from(SynapseBool(value): SynapseBool) -> Self {
value
}
}
/// A timestamp stored as the number of seconds since the Unix epoch.
/// Note that Synapse stores MOST timestamps as numbers of **milliseconds**
/// since the Unix epoch. But some timestamps are still stored in seconds.
#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub struct SecondsTimestamp(DateTime<Utc>);
impl From<SecondsTimestamp> for DateTime<Utc> {
fn from(SecondsTimestamp(value): SecondsTimestamp) -> Self {
value
}
}
impl<'r> sqlx::Decode<'r, Postgres> for SecondsTimestamp {
fn decode(
value: <Postgres as sqlx::Database>::ValueRef<'r>,
) -> Result<Self, sqlx::error::BoxDynError> {
<i64 as sqlx::Decode<Postgres>>::decode(value).map(|seconds_since_epoch| {
SecondsTimestamp(DateTime::from_timestamp_nanos(
seconds_since_epoch * 1_000_000_000,
))
})
}
}
impl sqlx::Type<Postgres> for SecondsTimestamp {
fn type_info() -> <Postgres as sqlx::Database>::TypeInfo {
<i64 as sqlx::Type<Postgres>>::type_info()
}
}
/// A timestamp stored as the number of milliseconds since the Unix epoch.
/// Note that Synapse stores some timestamps in seconds.
#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub struct MillisecondsTimestamp(DateTime<Utc>);
impl From<MillisecondsTimestamp> for DateTime<Utc> {
fn from(MillisecondsTimestamp(value): MillisecondsTimestamp) -> Self {
value
}
}
impl<'r> sqlx::Decode<'r, Postgres> for MillisecondsTimestamp {
fn decode(
value: <Postgres as sqlx::Database>::ValueRef<'r>,
) -> Result<Self, sqlx::error::BoxDynError> {
<i64 as sqlx::Decode<Postgres>>::decode(value).map(|milliseconds_since_epoch| {
MillisecondsTimestamp(DateTime::from_timestamp_nanos(
milliseconds_since_epoch * 1_000_000,
))
})
}
}
impl sqlx::Type<Postgres> for MillisecondsTimestamp {
fn type_info() -> <Postgres as sqlx::Database>::TypeInfo {
<i64 as sqlx::Type<Postgres>>::type_info()
}
}
#[derive(Clone, Debug, FromRow, PartialEq, Eq, PartialOrd, Ord)]
pub struct SynapseUser {
/// Full User ID of the user
pub name: FullUserId,
/// Password hash string for the user. Optional (null if no password is
/// set).
pub password_hash: Option<String>,
/// Whether the user is a Synapse Admin
pub admin: SynapseBool,
/// Whether the user is deactivated
pub deactivated: SynapseBool,
/// When the user was created
pub creation_ts: SecondsTimestamp,
// TODO ...
// TODO is_guest
// TODO do we care about upgrade_ts (users who upgraded from guest accounts to real accounts)
}
/// Row of the `user_threepids` table in Synapse.
#[derive(Clone, Debug, FromRow, PartialEq, Eq, PartialOrd, Ord)]
pub struct SynapseThreepid {
pub user_id: FullUserId,
pub medium: String,
pub address: String,
pub added_at: MillisecondsTimestamp,
}
/// Row of the `user_external_ids` table in Synapse.
#[derive(Clone, Debug, FromRow, PartialEq, Eq, PartialOrd, Ord)]
pub struct SynapseExternalId {
pub user_id: FullUserId,
pub auth_provider: String,
pub external_id: String,
}
/// List of Synapse tables that we should acquire an `EXCLUSIVE` lock on.
///
/// This is a safety measure against other processes changing the data
/// underneath our feet. It's still not a good idea to run Synapse at the same
/// time as the migration.
// TODO not complete!
const TABLES_TO_LOCK: &[&str] = &["users", "user_threepids", "user_external_ids"];
/// Number of migratable rows in various Synapse tables.
/// Used to estimate progress.
#[derive(Clone, Debug)]
pub struct SynapseRowCounts {
pub users: i64,
}
pub struct SynapseReader<'c> {
txn: Transaction<'c, Postgres>,
}
impl<'conn> SynapseReader<'conn> {
/// Create a new Synapse reader, which entails creating a transaction and
/// locking Synapse tables.
///
/// # Errors
///
/// Errors are returned under the following circumstances:
///
/// - An underlying database error
/// - If we can't lock the Synapse tables (pointing to the fact that Synapse
/// may still be running)
pub async fn new(
synapse_connection: &'conn mut PgConnection,
dry_run: bool,
) -> Result<Self, Error> {
let mut txn = synapse_connection
.begin()
.await
.into_database("begin transaction")?;
query("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE READ ONLY DEFERRABLE;")
.execute(&mut *txn)
.await
.into_database("set transaction")?;
let lock_type = if dry_run {
// We expect dry runs to be done alongside Synapse running, so we don't want to
// interfere with Synapse's database access in that case.
"ACCESS SHARE"
} else {
"EXCLUSIVE"
};
for table in TABLES_TO_LOCK {
query(&format!("LOCK TABLE {table} IN {lock_type} MODE NOWAIT;"))
.execute(&mut *txn)
.await
.into_database_with(|| format!("locking Synapse table `{table}`"))?;
}
Ok(Self { txn })
}
/// Finishes the Synapse reader, committing the transaction.
///
/// # Errors
///
/// Errors are returned under the following circumstances:
///
/// - An underlying database error whilst committing the transaction.
pub async fn finish(self) -> Result<(), Error> {
// TODO enforce that this is called somehow.
self.txn.commit().await.into_database("end transaction")?;
Ok(())
}
/// Counts the rows in the Synapse database to get an estimate of how large
/// the migration is going to be.
///
/// # Errors
///
/// Errors are returned under the following circumstances:
///
/// - An underlying database error
pub async fn count_rows(&mut self) -> Result<SynapseRowCounts, Error> {
let users = sqlx::query(
"
SELECT COUNT(1) FROM users
WHERE appservice_id IS NULL AND is_guest = 0
",
)
.fetch_one(&mut *self.txn)
.await
.into_database("counting Synapse users")?
.try_get::<i64, _>(0)
.into_database("couldn't decode count of Synapse users table")?;
Ok(SynapseRowCounts { users })
}
/// Reads Synapse users, excluding application service users (which do not
/// need to be migrated), from the database.
pub fn read_users(&mut self) -> impl Stream<Item = Result<SynapseUser, Error>> + '_ {
sqlx::query_as(
"
SELECT
name, password_hash, admin, deactivated, creation_ts
FROM users
WHERE appservice_id IS NULL AND is_guest = 0
",
)
.fetch(&mut *self.txn)
.map_err(|err| err.into_database("reading Synapse users"))
}
/// Reads threepids (such as e-mail and phone number associations) from
/// Synapse.
pub fn read_threepids(&mut self) -> impl Stream<Item = Result<SynapseThreepid, Error>> + '_ {
sqlx::query_as(
"
SELECT
user_id, medium, address, added_at
FROM user_threepids
",
)
.fetch(&mut *self.txn)
.map_err(|err| err.into_database("reading Synapse threepids"))
}
/// Read associations between Synapse users and external identity providers
pub fn read_user_external_ids(
&mut self,
) -> impl Stream<Item = Result<SynapseExternalId, Error>> + '_ {
sqlx::query_as(
"
SELECT
user_id, auth_provider, external_id
FROM user_external_ids
",
)
.fetch(&mut *self.txn)
.map_err(|err| err.into_database("reading Synapse user external IDs"))
}
}
#[cfg(test)]
mod test {
use std::collections::BTreeSet;
use futures_util::TryStreamExt;
use insta::assert_debug_snapshot;
use sqlx::{migrate::Migrator, PgPool};
use crate::{
synapse_reader::{SynapseExternalId, SynapseThreepid, SynapseUser},
SynapseReader,
};
// TODO test me
static MIGRATOR: Migrator = sqlx::migrate!("./test_synapse_migrations");
#[sqlx::test(migrator = "MIGRATOR", fixtures("user_alice"))]
async fn test_read_users(pool: PgPool) {
let mut conn = pool.acquire().await.expect("failed to get connection");
let mut reader = SynapseReader::new(&mut conn, false)
.await
.expect("failed to make SynapseReader");
let users: BTreeSet<SynapseUser> = reader
.read_users()
.try_collect()
.await
.expect("failed to read Synapse users");
assert_debug_snapshot!(users);
}
#[sqlx::test(migrator = "MIGRATOR", fixtures("user_alice", "threepids_alice"))]
async fn test_read_threepids(pool: PgPool) {
let mut conn = pool.acquire().await.expect("failed to get connection");
let mut reader = SynapseReader::new(&mut conn, false)
.await
.expect("failed to make SynapseReader");
let threepids: BTreeSet<SynapseThreepid> = reader
.read_threepids()
.try_collect()
.await
.expect("failed to read Synapse threepids");
assert_debug_snapshot!(threepids);
}
#[sqlx::test(migrator = "MIGRATOR", fixtures("user_alice", "external_ids_alice"))]
async fn test_read_external_ids(pool: PgPool) {
let mut conn = pool.acquire().await.expect("failed to get connection");
let mut reader = SynapseReader::new(&mut conn, false)
.await
.expect("failed to make SynapseReader");
let external_ids: BTreeSet<SynapseExternalId> = reader
.read_user_external_ids()
.try_collect()
.await
.expect("failed to read Synapse external user IDs");
assert_debug_snapshot!(external_ids);
}
}

13
crates/syn2mas/src/synapse_reader/snapshots/syn2mas__synapse_reader__test__read_external_ids.snap Normal file
View File

@@ -0,0 +1,13 @@
---
source: crates/syn2mas/src/synapse_reader/mod.rs
expression: external_ids
---
{
SynapseExternalId {
user_id: FullUserId(
"@alice:example.com",
),
auth_provider: "oidc-raasu",
external_id: "871.syn30",
},
}

26
crates/syn2mas/src/synapse_reader/snapshots/syn2mas__synapse_reader__test__read_threepids.snap Normal file
View File

@@ -0,0 +1,26 @@
---
source: crates/syn2mas/src/synapse_reader/mod.rs
expression: threepids
---
{
SynapseThreepid {
user_id: FullUserId(
"@alice:example.com",
),
medium: "email",
address: "alice@example.com",
added_at: MillisecondsTimestamp(
2019-04-02T18:09:09.014Z,
),
},
SynapseThreepid {
user_id: FullUserId(
"@alice:example.com",
),
medium: "msisdn",
address: "441189998819991197253",
added_at: MillisecondsTimestamp(
2019-04-14T07:55:49.014Z,
),
},
}

23
crates/syn2mas/src/synapse_reader/snapshots/syn2mas__synapse_reader__test__read_users.snap Normal file
View File

@@ -0,0 +1,23 @@
---
source: crates/syn2mas/src/synapse_reader/mod.rs
expression: users
---
{
SynapseUser {
name: FullUserId(
"@alice:example.com",
),
password_hash: Some(
"$2b$12$aaa/aaaaaaaaaa.aaaaaaaaaaaaaaa./aaaaaaaaaaaaaaaaaaa/A",
),
admin: SynapseBool(
false,
),
deactivated: SynapseBool(
false,
),
creation_ts: SecondsTimestamp(
2018-06-30T21:26:02Z,
),
},
}

26
crates/syn2mas/test_synapse_migrations/20250117064958_users.sql Normal file
View File

@@ -0,0 +1,26 @@
-- Copyright 2025 New Vector Ltd.
--
-- SPDX-License-Identifier: AGPL-3.0-only
-- Please see LICENSE in the repository root for full details.
-- Brings in the `users` table from Synapse
CREATE TABLE users (
name text,
password_hash text,
creation_ts bigint,
admin smallint DEFAULT 0 NOT NULL,
upgrade_ts bigint,
is_guest smallint DEFAULT 0 NOT NULL,
appservice_id text,
consent_version text,
consent_server_notice_sent text,
user_type text,
deactivated smallint DEFAULT 0 NOT NULL,
shadow_banned boolean,
consent_ts bigint,
approved boolean,
locked boolean DEFAULT false NOT NULL,
suspended boolean DEFAULT false NOT NULL
);

14
crates/syn2mas/test_synapse_migrations/20250128141011_threepids.sql Normal file
View File

@@ -0,0 +1,14 @@
-- Copyright 2025 New Vector Ltd.
--
-- SPDX-License-Identifier: AGPL-3.0-only
-- Please see LICENSE in the repository root for full details.
-- Brings in the `user_threepids` table from Synapse
CREATE TABLE user_threepids (
user_id text NOT NULL,
medium text NOT NULL,
address text NOT NULL,
validated_at bigint NOT NULL,
added_at bigint NOT NULL
);

Some files were not shown because too many files have changed in this diff Show More