From e1631f6dfd7a3c5776e2cff12fc8fac102b3e7e1 Mon Sep 17 00:00:00 2001 From: "Kai A. Hiller" Date: Wed, 16 Apr 2025 15:05:36 +0200 Subject: [PATCH 01/35] Fix headings in config doc Signed-off-by: Kai A. Hiller --- docs/reference/configuration.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 30dbbfca9..122ae1460 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -560,7 +560,7 @@ telemetry: dsn: https://public@host:port/1 ``` -### `email` +## `email` Settings related to sending emails @@ -589,13 +589,13 @@ email: #transport: aws_ses ``` -### `upstream_oauth2` +## `upstream_oauth2` Settings related to upstream OAuth 2.0/OIDC providers. Additions and modifications within this section are synced with the database on server startup. Removed entries are only removed with the [`config sync --prune`](./cli/config.md#config-sync---prune---dry-run) command. -#### `upstream_oauth2.providers` +### `upstream_oauth2.providers` A list of upstream OAuth 2.0/OIDC providers to use to authenticate users. From 35cd982e6f7b50ec08c22fe9067df4683aa77649 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Fri, 18 Apr 2025 11:32:26 +0200 Subject: [PATCH 02/35] syn2mas: refactor the metrics logic in the progress module We don't need to carry around the various meters. Just make them global. --- crates/syn2mas/src/migration.rs | 175 ++++---------------------------- crates/syn2mas/src/progress.rs | 108 ++++++++++++++++++-- crates/syn2mas/src/telemetry.rs | 23 +---- 3 files changed, 126 insertions(+), 180 deletions(-) diff --git a/crates/syn2mas/src/migration.rs b/crates/syn2mas/src/migration.rs index efefc25d7..37612d1d3 100644 --- a/crates/syn2mas/src/migration.rs +++ b/crates/syn2mas/src/migration.rs @@ -17,7 +17,6 @@ use chrono::{DateTime, Utc}; use compact_str::CompactString; use futures_util::{SinkExt, StreamExt as _, TryFutureExt, TryStreamExt as _}; use mas_storage::Clock; -use opentelemetry::{KeyValue, metrics::Counter}; use rand::{RngCore, SeedableRng}; use thiserror::Error; use thiserror_ext::ContextInto; @@ -33,16 +32,11 @@ use crate::{ MasNewEmailThreepid, MasNewUnsupportedThreepid, MasNewUpstreamOauthLink, MasNewUser, MasNewUserPassword, MasWriteBuffer, MasWriter, }, - progress::Progress, + progress::{EntityType, Progress}, synapse_reader::{ self, ExtractLocalpartError, FullUserId, SynapseAccessToken, SynapseDevice, SynapseExternalId, SynapseRefreshableTokenPair, SynapseThreepid, SynapseUser, }, - telemetry::{ - K_ENTITY, METER, V_ENTITY_DEVICES, V_ENTITY_EXTERNAL_IDS, - V_ENTITY_NONREFRESHABLE_ACCESS_TOKENS, V_ENTITY_REFRESHABLE_TOKEN_PAIRS, - V_ENTITY_THREEPIDS, V_ENTITY_USERS, - }, }; #[derive(Debug, Error, ContextInto)] @@ -146,7 +140,7 @@ struct MigrationState { /// /// - An underlying database access error, either to MAS or to Synapse. /// - Invalid data in the Synapse database. -#[allow(clippy::implicit_hasher, clippy::too_many_lines)] +#[allow(clippy::implicit_hasher)] pub async fn migrate( mut synapse: SynapseReader<'_>, mas: MasWriter, @@ -158,49 +152,6 @@ pub async fn migrate( ) -> Result<(), Error> { let counts = synapse.count_rows().await.into_synapse("counting users")?; - let approx_total_counter = METER - .u64_counter("syn2mas.entity.approx_total") - .with_description("Approximate number of entities of this type to be migrated") - .build(); - let migrated_otel_counter = METER - .u64_counter("syn2mas.entity.migrated") - .with_description("Number of entities of this type that have been migrated so far") - .build(); - let skipped_otel_counter = METER - .u64_counter("syn2mas.entity.skipped") - .with_description("Number of entities of this type that have been skipped so far") - .build(); - - approx_total_counter.add( - counts.users as u64, - &[KeyValue::new(K_ENTITY, V_ENTITY_USERS)], - ); - approx_total_counter.add( - counts.devices as u64, - &[KeyValue::new(K_ENTITY, V_ENTITY_DEVICES)], - ); - approx_total_counter.add( - counts.threepids as u64, - &[KeyValue::new(K_ENTITY, V_ENTITY_THREEPIDS)], - ); - approx_total_counter.add( - counts.external_ids as u64, - &[KeyValue::new(K_ENTITY, V_ENTITY_EXTERNAL_IDS)], - ); - // assume 1 refreshable access token per refresh token. - let approx_nonrefreshable_access_tokens = counts.access_tokens - counts.refresh_tokens; - approx_total_counter.add( - approx_nonrefreshable_access_tokens as u64, - &[KeyValue::new( - K_ENTITY, - V_ENTITY_NONREFRESHABLE_ACCESS_TOKENS, - )], - ); - approx_total_counter.add( - counts.refresh_tokens as u64, - &[KeyValue::new(K_ENTITY, V_ENTITY_REFRESHABLE_TOKEN_PAIRS)], - ); - let state = MigrationState { server_name, // We oversize the hashmaps, as the estimates are innaccurate, and we would like to avoid @@ -213,83 +164,32 @@ pub async fn migrate( provider_id_mapping, }; - let progress_counter = progress.migrating_data(V_ENTITY_USERS, counts.users); - let (mas, state) = migrate_users( - &mut synapse, - mas, - state, - rng, - progress_counter, - migrated_otel_counter.clone(), - skipped_otel_counter.clone(), - ) - .await?; + let progress_counter = progress.migrating_data(EntityType::Users, counts.users); + let (mas, state) = migrate_users(&mut synapse, mas, state, rng, progress_counter).await?; - let progress_counter = progress.migrating_data(V_ENTITY_THREEPIDS, counts.threepids); - let (mas, state) = migrate_threepids( - &mut synapse, - mas, - rng, - state, - progress_counter, - migrated_otel_counter.clone(), - skipped_otel_counter.clone(), - ) - .await?; + let progress_counter = progress.migrating_data(EntityType::ThreePids, counts.threepids); + let (mas, state) = migrate_threepids(&mut synapse, mas, rng, state, progress_counter).await?; - let progress_counter = progress.migrating_data(V_ENTITY_EXTERNAL_IDS, counts.external_ids); - let (mas, state) = migrate_external_ids( - &mut synapse, - mas, - rng, - state, - progress_counter, - migrated_otel_counter.clone(), - skipped_otel_counter.clone(), - ) - .await?; + let progress_counter = progress.migrating_data(EntityType::ExternalIds, counts.external_ids); + let (mas, state) = + migrate_external_ids(&mut synapse, mas, rng, state, progress_counter).await?; let progress_counter = progress.migrating_data( - V_ENTITY_NONREFRESHABLE_ACCESS_TOKENS, + EntityType::NonRefreshableAccessTokens, counts.access_tokens - counts.refresh_tokens, ); - let (mas, state) = migrate_unrefreshable_access_tokens( - &mut synapse, - mas, - clock, - rng, - state, - progress_counter, - migrated_otel_counter.clone(), - skipped_otel_counter.clone(), - ) - .await?; + let (mas, state) = + migrate_unrefreshable_access_tokens(&mut synapse, mas, clock, rng, state, progress_counter) + .await?; let progress_counter = - progress.migrating_data(V_ENTITY_REFRESHABLE_TOKEN_PAIRS, counts.refresh_tokens); - let (mas, state) = migrate_refreshable_token_pairs( - &mut synapse, - mas, - clock, - rng, - state, - progress_counter, - migrated_otel_counter.clone(), - skipped_otel_counter.clone(), - ) - .await?; + progress.migrating_data(EntityType::RefreshableTokens, counts.refresh_tokens); + let (mas, state) = + migrate_refreshable_token_pairs(&mut synapse, mas, clock, rng, state, progress_counter) + .await?; - let progress_counter = progress.migrating_data("devices", counts.devices); - let (mas, _state) = migrate_devices( - &mut synapse, - mas, - rng, - state, - progress_counter, - migrated_otel_counter.clone(), - skipped_otel_counter.clone(), - ) - .await?; + let progress_counter = progress.migrating_data(EntityType::Devices, counts.devices); + let (mas, _state) = migrate_devices(&mut synapse, mas, rng, state, progress_counter).await?; synapse .finish() @@ -310,11 +210,8 @@ async fn migrate_users( mut state: MigrationState, rng: &mut impl RngCore, progress_counter: ProgressCounter, - migrated_otel_counter: Counter, - skipped_otel_counter: Counter, ) -> Result<(MasWriter, MigrationState), Error> { let start = Instant::now(); - let otel_kv = [KeyValue::new(K_ENTITY, V_ENTITY_USERS)]; let (tx, mut rx) = tokio::sync::mpsc::channel::(10 * 1024 * 1024); @@ -356,7 +253,6 @@ async fn migrate_users( if user.appservice_id.is_some() { flags |= UserFlags::IS_APPSERVICE; - skipped_otel_counter.add(1, &otel_kv); progress_counter.increment_skipped(); // Special case for appservice users: we don't insert them into the database @@ -391,7 +287,6 @@ async fn migrate_users( .into_mas("writing password")?; } - migrated_otel_counter.add(1, &otel_kv); progress_counter.increment_migrated(); } @@ -437,11 +332,8 @@ async fn migrate_threepids( rng: &mut impl RngCore, state: MigrationState, progress_counter: ProgressCounter, - migrated_otel_counter: Counter, - skipped_otel_counter: Counter, ) -> Result<(MasWriter, MigrationState), Error> { let start = Instant::now(); - let otel_kv = [KeyValue::new(K_ENTITY, V_ENTITY_THREEPIDS)]; let mut email_buffer = MasWriteBuffer::new(&mas, MasWriter::write_email_threepids); let mut unsupported_buffer = MasWriteBuffer::new(&mas, MasWriter::write_unsupported_threepids); @@ -469,7 +361,6 @@ async fn migrate_threepids( let Some(mas_user_id) = user_infos.mas_user_id else { progress_counter.increment_skipped(); - skipped_otel_counter.add(1, &otel_kv); continue; }; @@ -504,7 +395,6 @@ async fn migrate_threepids( .into_mas("writing unsupported threepid")?; } - migrated_otel_counter.add(1, &otel_kv); progress_counter.increment_migrated(); } @@ -536,11 +426,8 @@ async fn migrate_external_ids( rng: &mut impl RngCore, state: MigrationState, progress_counter: ProgressCounter, - migrated_otel_counter: Counter, - skipped_otel_counter: Counter, ) -> Result<(MasWriter, MigrationState), Error> { let start = Instant::now(); - let otel_kv = [KeyValue::new(K_ENTITY, V_ENTITY_EXTERNAL_IDS)]; let mut write_buffer = MasWriteBuffer::new(&mas, MasWriter::write_upstream_oauth_links); let mut extids_stream = pin!(synapse.read_user_external_ids()); @@ -564,7 +451,6 @@ async fn migrate_external_ids( let Some(mas_user_id) = user_infos.mas_user_id else { progress_counter.increment_skipped(); - skipped_otel_counter.add(1, &otel_kv); continue; }; @@ -595,7 +481,6 @@ async fn migrate_external_ids( .await .into_mas("failed to write upstream link")?; - migrated_otel_counter.add(1, &otel_kv); progress_counter.increment_migrated(); } @@ -627,11 +512,8 @@ async fn migrate_devices( rng: &mut impl RngCore, mut state: MigrationState, progress_counter: ProgressCounter, - migrated_otel_counter: Counter, - skipped_otel_counter: Counter, ) -> Result<(MasWriter, MigrationState), Error> { let start = Instant::now(); - let otel_kv = [KeyValue::new(K_ENTITY, V_ENTITY_DEVICES)]; let (tx, mut rx) = tokio::sync::mpsc::channel(10 * 1024 * 1024); @@ -664,7 +546,6 @@ async fn migrate_devices( let Some(mas_user_id) = user_infos.mas_user_id else { progress_counter.increment_skipped(); - skipped_otel_counter.add(1, &otel_kv); continue; }; @@ -721,7 +602,6 @@ async fn migrate_devices( .await .into_mas("writing compat sessions")?; - migrated_otel_counter.add(1, &otel_kv); progress_counter.increment_migrated(); } @@ -759,7 +639,6 @@ async fn migrate_devices( /// Migrates unrefreshable access tokens (those without an associated refresh /// token). Some of these may be deviceless. #[tracing::instrument(skip_all, level = Level::INFO)] -#[allow(clippy::too_many_arguments)] async fn migrate_unrefreshable_access_tokens( synapse: &mut SynapseReader<'_>, mut mas: MasWriter, @@ -767,14 +646,8 @@ async fn migrate_unrefreshable_access_tokens( rng: &mut impl RngCore, mut state: MigrationState, progress_counter: ProgressCounter, - migrated_otel_counter: Counter, - skipped_otel_counter: Counter, ) -> Result<(MasWriter, MigrationState), Error> { let start = Instant::now(); - let otel_kv = [KeyValue::new( - K_ENTITY, - V_ENTITY_NONREFRESHABLE_ACCESS_TOKENS, - )]; let (tx, mut rx) = tokio::sync::mpsc::channel(10 * 1024 * 1024); @@ -809,7 +682,6 @@ async fn migrate_unrefreshable_access_tokens( let Some(mas_user_id) = user_infos.mas_user_id else { progress_counter.increment_skipped(); - skipped_otel_counter.add(1, &otel_kv); continue; }; @@ -818,7 +690,6 @@ async fn migrate_unrefreshable_access_tokens( || user_infos.flags.is_appservice() { progress_counter.increment_skipped(); - skipped_otel_counter.add(1, &otel_kv); continue; } @@ -879,7 +750,6 @@ async fn migrate_unrefreshable_access_tokens( .await .into_mas("writing compat access tokens")?; - migrated_otel_counter.add(1, &otel_kv); progress_counter.increment_migrated(); } write_buffer @@ -920,7 +790,6 @@ async fn migrate_unrefreshable_access_tokens( /// Migrates (access token, refresh token) pairs. /// Does not migrate non-refreshable access tokens. #[tracing::instrument(skip_all, level = Level::INFO)] -#[allow(clippy::too_many_arguments)] async fn migrate_refreshable_token_pairs( synapse: &mut SynapseReader<'_>, mut mas: MasWriter, @@ -928,11 +797,8 @@ async fn migrate_refreshable_token_pairs( rng: &mut impl RngCore, mut state: MigrationState, progress_counter: ProgressCounter, - migrated_otel_counter: Counter, - skipped_otel_counter: Counter, ) -> Result<(MasWriter, MigrationState), Error> { let start = Instant::now(); - let otel_kv = [KeyValue::new(K_ENTITY, V_ENTITY_REFRESHABLE_TOKEN_PAIRS)]; let mut token_stream = pin!(synapse.read_refreshable_token_pairs()); let mut access_token_write_buffer = @@ -963,7 +829,6 @@ async fn migrate_refreshable_token_pairs( let Some(mas_user_id) = user_infos.mas_user_id else { progress_counter.increment_skipped(); - skipped_otel_counter.add(1, &otel_kv); continue; }; @@ -972,7 +837,6 @@ async fn migrate_refreshable_token_pairs( || user_infos.flags.is_appservice() { progress_counter.increment_skipped(); - skipped_otel_counter.add(1, &otel_kv); continue; } @@ -1017,7 +881,6 @@ async fn migrate_refreshable_token_pairs( .await .into_mas("writing compat refresh tokens")?; - migrated_otel_counter.add(1, &otel_kv); progress_counter.increment_migrated(); } diff --git a/crates/syn2mas/src/progress.rs b/crates/syn2mas/src/progress.rs index e5f61d292..3c67825ce 100644 --- a/crates/syn2mas/src/progress.rs +++ b/crates/syn2mas/src/progress.rs @@ -1,6 +1,89 @@ -use std::sync::{Arc, atomic::AtomicU32}; +// Copyright 2025 New Vector Ltd. +// +// SPDX-License-Identifier: AGPL-3.0-only +// Please see LICENSE in the repository root for full details. + +use std::sync::{Arc, LazyLock, atomic::AtomicU32}; use arc_swap::ArcSwap; +use opentelemetry::{ + KeyValue, + metrics::{Counter, Gauge}, +}; + +use crate::telemetry::METER; + +/// A gauge that tracks the approximate number of entities of a given type +/// that will be migrated. +pub static APPROX_TOTAL_GAUGE: LazyLock> = LazyLock::new(|| { + METER + .u64_gauge("syn2mas.entity.approx_total") + .with_description("Approximate number of entities of this type to be migrated") + .build() +}); + +/// A counter that tracks the number of entities of a given type that have +/// been migrated so far. +pub static MIGRATED_COUNTER: LazyLock> = LazyLock::new(|| { + METER + .u64_counter("syn2mas.entity.migrated") + .with_description("Number of entities of this type that have been migrated so far") + .build() +}); + +/// A counter that tracks the number of entities of a given type that have +/// been skipped so far. +pub static SKIPPED_COUNTER: LazyLock> = LazyLock::new(|| { + METER + .u64_counter("syn2mas.entity.skipped") + .with_description("Number of entities of this type that have been skipped so far") + .build() +}); + +/// Enum representing the different types of entities that syn2mas can migrate. +#[derive(Debug, Clone, Copy)] +pub enum EntityType { + /// Represents users + Users, + + /// Represents devices + Devices, + + /// Represents third-party IDs + ThreePids, + + /// Represents external IDs + ExternalIds, + + /// Represents non-refreshable access tokens + NonRefreshableAccessTokens, + + /// Represents refreshable access tokens + RefreshableTokens, +} + +impl std::fmt::Display for EntityType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.name()) + } +} + +impl EntityType { + pub const fn name(self) -> &'static str { + match self { + Self::Users => "users", + Self::Devices => "devices", + Self::ThreePids => "threepids", + Self::ExternalIds => "external_ids", + Self::NonRefreshableAccessTokens => "nonrefreshable_access_tokens", + Self::RefreshableTokens => "refreshable_tokens", + } + } + + pub fn as_kv(self) -> KeyValue { + KeyValue::new("entity", self.name()) + } +} /// Tracker for the progress of the migration /// @@ -11,25 +94,37 @@ pub struct Progress { current_stage: Arc>, } -#[derive(Clone, Default)] +#[derive(Clone)] pub struct ProgressCounter { inner: Arc, } -#[derive(Default)] struct ProgressCounterInner { + kv: [KeyValue; 1], migrated: AtomicU32, skipped: AtomicU32, } impl ProgressCounter { + fn new(entity: EntityType) -> Self { + Self { + inner: Arc::new(ProgressCounterInner { + kv: [entity.as_kv()], + migrated: AtomicU32::new(0), + skipped: AtomicU32::new(0), + }), + } + } + pub fn increment_migrated(&self) { + MIGRATED_COUNTER.add(1, &self.inner.kv); self.inner .migrated .fetch_add(1, std::sync::atomic::Ordering::Relaxed); } pub fn increment_skipped(&self) { + SKIPPED_COUNTER.add(1, &self.inner.kv); self.inner .skipped .fetch_add(1, std::sync::atomic::Ordering::Relaxed); @@ -52,8 +147,9 @@ impl ProgressCounter { impl Progress { #[must_use] - pub fn migrating_data(&self, entity: &'static str, approx_count: usize) -> ProgressCounter { - let counter = ProgressCounter::default(); + pub fn migrating_data(&self, entity: EntityType, approx_count: usize) -> ProgressCounter { + let counter = ProgressCounter::new(entity); + APPROX_TOTAL_GAUGE.record(approx_count as u64, &[entity.as_kv()]); self.set_current_stage(ProgressStage::MigratingData { entity, counter: counter.clone(), @@ -99,7 +195,7 @@ impl Default for Progress { pub enum ProgressStage { SettingUp, MigratingData { - entity: &'static str, + entity: EntityType, counter: ProgressCounter, approx_count: u64, }, diff --git a/crates/syn2mas/src/telemetry.rs b/crates/syn2mas/src/telemetry.rs index 5c1c0a54a..e9a3385fb 100644 --- a/crates/syn2mas/src/telemetry.rs +++ b/crates/syn2mas/src/telemetry.rs @@ -1,3 +1,8 @@ +// Copyright 2025 New Vector Ltd. +// +// SPDX-License-Identifier: AGPL-3.0-only +// Please see LICENSE in the repository root for full details. + use std::sync::LazyLock; use opentelemetry::{InstrumentationScope, metrics::Meter}; @@ -12,21 +17,3 @@ static SCOPE: LazyLock = LazyLock::new(|| { pub static METER: LazyLock = LazyLock::new(|| opentelemetry::global::meter_with_scope(SCOPE.clone())); - -/// Attribute key for syn2mas.entity metrics representing what entity. -pub const K_ENTITY: &str = "entity"; - -/// Attribute value for syn2mas.entity metrics representing users. -pub const V_ENTITY_USERS: &str = "users"; -/// Attribute value for syn2mas.entity metrics representing devices. -pub const V_ENTITY_DEVICES: &str = "devices"; -/// Attribute value for syn2mas.entity metrics representing threepids. -pub const V_ENTITY_THREEPIDS: &str = "threepids"; -/// Attribute value for syn2mas.entity metrics representing external IDs. -pub const V_ENTITY_EXTERNAL_IDS: &str = "external_ids"; -/// Attribute value for syn2mas.entity metrics representing non-refreshable -/// access token entities. -pub const V_ENTITY_NONREFRESHABLE_ACCESS_TOKENS: &str = "nonrefreshable_access_tokens"; -/// Attribute value for syn2mas.entity metrics representing refreshable -/// access/refresh token pairs. -pub const V_ENTITY_REFRESHABLE_TOKEN_PAIRS: &str = "refreshable_token_pairs"; From b4a1e5f6913251b3241b326c515f121224c41786 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Fri, 18 Apr 2025 11:34:27 +0200 Subject: [PATCH 03/35] syn2mas: replace #[allow] annotations with #[expect] Also removes unused #[allow] annotations. --- crates/syn2mas/src/mas_writer/mod.rs | 5 +---- crates/syn2mas/src/migration.rs | 2 +- crates/syn2mas/src/synapse_reader/config.rs | 4 ++-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/crates/syn2mas/src/mas_writer/mod.rs b/crates/syn2mas/src/mas_writer/mod.rs index 865bf02fe..fe52e0eed 100644 --- a/crates/syn2mas/src/mas_writer/mod.rs +++ b/crates/syn2mas/src/mas_writer/mod.rs @@ -46,7 +46,7 @@ pub enum Error { }, #[error("writer connection pool shut down due to error")] - #[allow(clippy::enum_variant_names)] + #[expect(clippy::enum_variant_names)] WriterConnectionPoolError, #[error("inconsistent database: {0}")] @@ -390,7 +390,6 @@ impl MasWriter { /// Errors are returned in the following conditions: /// /// - If the database connection experiences an error. - #[allow(clippy::missing_panics_doc)] // not real #[tracing::instrument(name = "syn2mas.mas_writer.new", skip_all)] pub async fn new( mut conn: LockedMasDatabase, @@ -632,7 +631,6 @@ impl MasWriter { /// Errors are returned in the following conditions: /// /// - If the database writer connection pool had an error. - #[allow(clippy::missing_panics_doc)] // not a real panic #[tracing::instrument(skip_all, level = Level::DEBUG)] pub fn write_users(&mut self, users: Vec) -> BoxFuture<'_, Result<(), Error>> { self.writer_pool @@ -711,7 +709,6 @@ impl MasWriter { /// Errors are returned in the following conditions: /// /// - If the database writer connection pool had an error. - #[allow(clippy::missing_panics_doc)] // not a real panic #[tracing::instrument(skip_all, level = Level::DEBUG)] pub fn write_passwords( &mut self, diff --git a/crates/syn2mas/src/migration.rs b/crates/syn2mas/src/migration.rs index 37612d1d3..6d0420077 100644 --- a/crates/syn2mas/src/migration.rs +++ b/crates/syn2mas/src/migration.rs @@ -140,7 +140,7 @@ struct MigrationState { /// /// - An underlying database access error, either to MAS or to Synapse. /// - Invalid data in the Synapse database. -#[allow(clippy::implicit_hasher)] +#[expect(clippy::implicit_hasher)] pub async fn migrate( mut synapse: SynapseReader<'_>, mas: MasWriter, diff --git a/crates/syn2mas/src/synapse_reader/config.rs b/crates/syn2mas/src/synapse_reader/config.rs index 2c413a1b9..789be6845 100644 --- a/crates/syn2mas/src/synapse_reader/config.rs +++ b/crates/syn2mas/src/synapse_reader/config.rs @@ -16,7 +16,7 @@ use sqlx::postgres::PgConnectOptions; /// /// See: #[derive(Deserialize)] -#[allow(clippy::struct_excessive_bools)] +#[expect(clippy::struct_excessive_bools)] pub struct Config { pub database: DatabaseSection, @@ -239,7 +239,7 @@ mod test { #[test] fn test_to_sqlx_postgres() { #[track_caller] - #[allow(clippy::needless_pass_by_value)] + #[expect(clippy::needless_pass_by_value)] fn assert_eq_options(config: DatabaseSection, uri: &str) { let config_connect_options = config .to_sqlx_postgres() From a061db35d7c45758ebd2cd5cda2ed2efdb0c8328 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Fri, 18 Apr 2025 18:24:35 +0200 Subject: [PATCH 04/35] Make a few password-related options public in the config crate It also adds docs to a few of those options --- crates/config/src/sections/mod.rs | 4 +++- crates/config/src/sections/passwords.rs | 30 +++++++++++++++++-------- docs/config.schema.json | 12 +++++++++- 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/crates/config/src/sections/mod.rs b/crates/config/src/sections/mod.rs index d415f646a..9a9fc9de8 100644 --- a/crates/config/src/sections/mod.rs +++ b/crates/config/src/sections/mod.rs @@ -38,7 +38,9 @@ pub use self::{ Resource as HttpResource, TlsConfig as HttpTlsConfig, UnixOrTcp, }, matrix::{HomeserverKind, MatrixConfig}, - passwords::{Algorithm as PasswordAlgorithm, PasswordsConfig}, + passwords::{ + Algorithm as PasswordAlgorithm, HashingScheme as PasswordHashingScheme, PasswordsConfig, + }, policy::PolicyConfig, rate_limiting::RateLimitingConfig, secrets::SecretsConfig, diff --git a/crates/config/src/sections/passwords.rs b/crates/config/src/sections/passwords.rs index 455dbfd61..07ea71b0e 100644 --- a/crates/config/src/sections/passwords.rs +++ b/crates/config/src/sections/passwords.rs @@ -16,7 +16,7 @@ use crate::ConfigurationSection; fn default_schemes() -> Vec { vec![HashingScheme { version: 1, - algorithm: Algorithm::Argon2id, + algorithm: Algorithm::default(), cost: None, secret: None, secret_file: None, @@ -36,10 +36,14 @@ fn default_minimum_complexity() -> u8 { pub struct PasswordsConfig { /// Whether password-based authentication is enabled #[serde(default = "default_enabled")] - enabled: bool, + pub enabled: bool, + /// The hashing schemes to use for hashing and validating passwords + /// + /// The hashing scheme with the highest version number will be used for + /// hashing new passwords. #[serde(default = "default_schemes")] - schemes: Vec, + pub schemes: Vec, /// Score between 0 and 4 determining the minimum allowed password /// complexity. Scores are based on the ESTIMATED number of guesses @@ -154,23 +158,30 @@ impl PasswordsConfig { } } +/// Parameters for a password hashing scheme #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct HashingScheme { - version: u16, + /// The version of the hashing scheme. They must be unique, and the highest + /// version will be used for hashing new passwords. + pub version: u16, - algorithm: Algorithm, + /// The hashing algorithm to use + pub algorithm: Algorithm, /// Cost for the bcrypt algorithm #[serde(skip_serializing_if = "Option::is_none")] #[schemars(default = "default_bcrypt_cost")] - cost: Option, + pub cost: Option, + /// An optional secret to use when hashing passwords. This makes it harder + /// to brute-force the passwords in case of a database leak. #[serde(skip_serializing_if = "Option::is_none")] - secret: Option, + pub secret: Option, + /// Same as `secret`, but read from a file. #[serde(skip_serializing_if = "Option::is_none")] #[schemars(with = "Option")] - secret_file: Option, + pub secret_file: Option, } #[allow(clippy::unnecessary_wraps)] @@ -179,13 +190,14 @@ fn default_bcrypt_cost() -> Option { } /// A hashing algorithm -#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq, Default)] #[serde(rename_all = "lowercase")] pub enum Algorithm { /// bcrypt Bcrypt, /// argon2id + #[default] Argon2id, /// PBKDF2 diff --git a/docs/config.schema.json b/docs/config.schema.json index 165cf947d..9a1184bfd 100644 --- a/docs/config.schema.json +++ b/docs/config.schema.json @@ -1566,6 +1566,7 @@ "type": "boolean" }, "schemes": { + "description": "The hashing schemes to use for hashing and validating passwords\n\nThe hashing scheme with the highest version number will be used for hashing new passwords.", "default": [ { "version": 1, @@ -1587,6 +1588,7 @@ } }, "HashingScheme": { + "description": "Parameters for a password hashing scheme", "type": "object", "required": [ "algorithm", @@ -1594,12 +1596,18 @@ ], "properties": { "version": { + "description": "The version of the hashing scheme. They must be unique, and the highest version will be used for hashing new passwords.", "type": "integer", "format": "uint16", "minimum": 0.0 }, "algorithm": { - "$ref": "#/definitions/Algorithm" + "description": "The hashing algorithm to use", + "allOf": [ + { + "$ref": "#/definitions/Algorithm" + } + ] }, "cost": { "description": "Cost for the bcrypt algorithm", @@ -1609,9 +1617,11 @@ "minimum": 0.0 }, "secret": { + "description": "An optional secret to use when hashing passwords. This makes it harder to brute-force the passwords in case of a database leak.", "type": "string" }, "secret_file": { + "description": "Same as `secret`, but read from a file.", "type": "string" } } From 0792171f91bd5b18e74f7034d07aba2741705969 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Fri, 18 Apr 2025 18:25:46 +0200 Subject: [PATCH 05/35] Move the synapse_idp_id field to the top of the provider section This means that when serializing those, it will be at a more obvious place. --- crates/config/src/sections/upstream_oauth2.rs | 34 +++++++++---------- docs/config.schema.json | 8 ++--- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/crates/config/src/sections/upstream_oauth2.rs b/crates/config/src/sections/upstream_oauth2.rs index 623a97c14..1183a6421 100644 --- a/crates/config/src/sections/upstream_oauth2.rs +++ b/crates/config/src/sections/upstream_oauth2.rs @@ -418,6 +418,23 @@ pub struct Provider { )] pub id: Ulid, + /// The ID of the provider that was used by Synapse. + /// In order to perform a Synapse-to-MAS migration, this must be specified. + /// + /// ## For providers that used OAuth 2.0 or OpenID Connect in Synapse + /// + /// ### For `oidc_providers`: + /// This should be specified as `oidc-` followed by the ID that was + /// configured as `idp_id` in one of the `oidc_providers` in the Synapse + /// configuration. + /// For example, if Synapse's configuration contained `idp_id: wombat` for + /// this provider, then specify `oidc-wombat` here. + /// + /// ### For `oidc_config` (legacy): + /// Specify `oidc` here. + #[serde(skip_serializing_if = "Option::is_none")] + pub synapse_idp_id: Option, + /// The OIDC issuer URL /// /// This is required if OIDC discovery is enabled (which is the default) @@ -548,21 +565,4 @@ pub struct Provider { /// Orders of the keys are not preserved. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub additional_authorization_parameters: BTreeMap, - - /// The ID of the provider that was used by Synapse. - /// In order to perform a Synapse-to-MAS migration, this must be specified. - /// - /// ## For providers that used OAuth 2.0 or OpenID Connect in Synapse - /// - /// ### For `oidc_providers`: - /// This should be specified as `oidc-` followed by the ID that was - /// configured as `idp_id` in one of the `oidc_providers` in the Synapse - /// configuration. - /// For example, if Synapse's configuration contained `idp_id: wombat` for - /// this provider, then specify `oidc-wombat` here. - /// - /// ### For `oidc_config` (legacy): - /// Specify `oidc` here. - #[serde(skip_serializing_if = "Option::is_none")] - pub synapse_idp_id: Option, } diff --git a/docs/config.schema.json b/docs/config.schema.json index 9a1184bfd..7906e5378 100644 --- a/docs/config.schema.json +++ b/docs/config.schema.json @@ -1983,6 +1983,10 @@ "type": "string", "pattern": "^[0123456789ABCDEFGHJKMNPQRSTVWXYZ]{26}$" }, + "synapse_idp_id": { + "description": "The ID of the provider that was used by Synapse. In order to perform a Synapse-to-MAS migration, this must be specified.\n\n## For providers that used OAuth 2.0 or OpenID Connect in Synapse\n\n### For `oidc_providers`: This should be specified as `oidc-` followed by the ID that was configured as `idp_id` in one of the `oidc_providers` in the Synapse configuration. For example, if Synapse's configuration contained `idp_id: wombat` for this provider, then specify `oidc-wombat` here.\n\n### For `oidc_config` (legacy): Specify `oidc` here.", + "type": "string" + }, "issuer": { "description": "The OIDC issuer URL\n\nThis is required if OIDC discovery is enabled (which is the default)", "type": "string" @@ -2110,10 +2114,6 @@ "additionalProperties": { "type": "string" } - }, - "synapse_idp_id": { - "description": "The ID of the provider that was used by Synapse. In order to perform a Synapse-to-MAS migration, this must be specified.\n\n## For providers that used OAuth 2.0 or OpenID Connect in Synapse\n\n### For `oidc_providers`: This should be specified as `oidc-` followed by the ID that was configured as `idp_id` in one of the `oidc_providers` in the Synapse configuration. For example, if Synapse's configuration contained `idp_id: wombat` for this provider, then specify `oidc-wombat` here.\n\n### For `oidc_config` (legacy): Specify `oidc` here.", - "type": "string" } } }, From 1fcf6503221239a7093baf680506a5aac4fb2cfc Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Fri, 18 Apr 2025 18:26:29 +0200 Subject: [PATCH 06/35] Option to generate a MAS config from an existing Synapse config This is a best-effort conversion, which will warn about unsupported options. --- Cargo.lock | 4 + crates/cli/src/commands/config.rs | 24 +- crates/syn2mas/Cargo.toml | 4 + crates/syn2mas/src/synapse_reader/checks.rs | 2 +- .../{config.rs => config/mod.rs} | 117 +++++- .../syn2mas/src/synapse_reader/config/oidc.rs | 347 ++++++++++++++++++ 6 files changed, 475 insertions(+), 23 deletions(-) rename crates/syn2mas/src/synapse_reader/{config.rs => config/mod.rs} (70%) create mode 100644 crates/syn2mas/src/synapse_reader/config/oidc.rs diff --git a/Cargo.lock b/Cargo.lock index e4a179e53..a3170d80a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6161,14 +6161,17 @@ dependencies = [ "futures-util", "insta", "mas-config", + "mas-iana", "mas-storage", "mas-storage-pg", + "oauth2-types", "opentelemetry", "opentelemetry-semantic-conventions", "rand 0.8.5", "rand_chacha 0.3.1", "rustc-hash 2.1.1", "serde", + "serde_json", "sqlx", "thiserror 2.0.12", "thiserror-ext", @@ -6176,6 +6179,7 @@ dependencies = [ "tokio-util", "tracing", "ulid", + "url", "uuid", ] diff --git a/crates/cli/src/commands/config.rs b/crates/cli/src/commands/config.rs index 0a246d86c..26b944016 100644 --- a/crates/cli/src/commands/config.rs +++ b/crates/cli/src/commands/config.rs @@ -11,7 +11,7 @@ use camino::Utf8PathBuf; use clap::Parser; use figment::Figment; use mas_config::{ConfigurationSection, RootConfig, SyncConfig}; -use mas_storage::SystemClock; +use mas_storage::{Clock as _, SystemClock}; use mas_storage_pg::MIGRATOR; use rand::SeedableRng; use tokio::io::AsyncWriteExt; @@ -46,6 +46,10 @@ enum Subcommand { /// If not specified, the config will be written to stdout #[clap(short, long)] output: Option, + + /// Existing Synapse configuration used to generate the MAS config + #[arg(short, long, action = clap::ArgAction::Append)] + synapse_config: Vec, }, /// Sync the clients and providers from the config file to the database @@ -88,14 +92,24 @@ impl Options { info!("Configuration file looks good"); } - SC::Generate { output } => { + SC::Generate { + output, + synapse_config, + } => { let _span = info_span!("cli.config.generate").entered(); + let clock = SystemClock::default(); // XXX: we should disallow SeedableRng::from_entropy - let rng = rand_chacha::ChaChaRng::from_entropy(); - let config = RootConfig::generate(rng).await?; - let config = serde_yaml::to_string(&config)?; + let mut rng = rand_chacha::ChaChaRng::from_entropy(); + let mut config = RootConfig::generate(&mut rng).await?; + if !synapse_config.is_empty() { + info!("Adjusting MAS config to match Synapse config from {synapse_config:?}"); + let synapse_config = syn2mas::synapse_config::Config::load(&synapse_config)?; + config = synapse_config.adjust_mas_config(config, &mut rng, clock.now()); + } + + let config = serde_yaml::to_string(&config)?; if let Some(output) = output { info!("Writing configuration to {output:?}"); let mut file = tokio::fs::File::create(output).await?; diff --git a/crates/syn2mas/Cargo.toml b/crates/syn2mas/Cargo.toml index 0e82867ce..61e7ac2d5 100644 --- a/crates/syn2mas/Cargo.toml +++ b/crates/syn2mas/Cargo.toml @@ -16,6 +16,7 @@ bitflags.workspace = true camino.workspace = true figment.workspace = true serde.workspace = true +serde_json.workspace = true thiserror.workspace = true thiserror-ext.workspace = true tokio.workspace = true @@ -26,6 +27,7 @@ compact_str.workspace = true tracing.workspace = true futures-util = "0.3.31" rustc-hash = "2.1.1" +url.workspace = true rand.workspace = true rand_chacha = "0.3.1" @@ -33,7 +35,9 @@ uuid = "1.16.0" ulid = { workspace = true, features = ["uuid"] } mas-config.workspace = true +mas-iana.workspace = true mas-storage.workspace = true +oauth2-types.workspace = true opentelemetry.workspace = true opentelemetry-semantic-conventions.workspace = true diff --git a/crates/syn2mas/src/synapse_reader/checks.rs b/crates/syn2mas/src/synapse_reader/checks.rs index 360e6d38d..4dca03029 100644 --- a/crates/syn2mas/src/synapse_reader/checks.rs +++ b/crates/syn2mas/src/synapse_reader/checks.rs @@ -157,7 +157,7 @@ pub fn synapse_config_check(synapse_config: &Config) -> (Vec, Vec< )); } - if synapse_config.enable_3pid_changes { + if synapse_config.enable_3pid_changes == Some(true) { errors.push(CheckError::ThreepidChangesEnabled); } diff --git a/crates/syn2mas/src/synapse_reader/config.rs b/crates/syn2mas/src/synapse_reader/config/mod.rs similarity index 70% rename from crates/syn2mas/src/synapse_reader/config.rs rename to crates/syn2mas/src/synapse_reader/config/mod.rs index 789be6845..390dacaa8 100644 --- a/crates/syn2mas/src/synapse_reader/config.rs +++ b/crates/syn2mas/src/synapse_reader/config/mod.rs @@ -3,12 +3,21 @@ // SPDX-License-Identifier: AGPL-3.0-only // Please see LICENSE in the repository root for full details. +mod oidc; + use std::collections::BTreeMap; use camino::Utf8PathBuf; +use chrono::{DateTime, Utc}; use figment::providers::{Format, Yaml}; +use mas_config::{PasswordAlgorithm, PasswordHashingScheme}; +use rand::Rng; use serde::Deserialize; use sqlx::postgres::PgConnectOptions; +use tracing::warn; +use url::Url; + +pub use self::oidc::OidcProvider; /// The root of a Synapse configuration. /// This struct only includes fields which the Synapse-to-MAS migration is @@ -23,6 +32,8 @@ pub struct Config { #[serde(default)] pub password_config: PasswordSection, + pub bcrypt_rounds: Option, + #[serde(default)] pub allow_guest_access: bool, @@ -31,11 +42,16 @@ pub struct Config { #[serde(default)] pub enable_registration_captcha: bool, + pub recaptcha_public_key: Option, + pub recaptcha_private_key: Option, /// Normally this defaults to true, but when MAS integration is enabled in /// Synapse it defaults to false. #[serde(default)] - pub enable_3pid_changes: bool, + pub enable_3pid_changes: Option, + + #[serde(default = "default_true")] + enable_set_display_name: bool, #[serde(default)] pub user_consent: Option, @@ -67,6 +83,8 @@ pub struct Config { pub oidc_providers: Vec, pub server_name: String, + + pub public_baseurl: Option, } impl Config { @@ -100,21 +118,97 @@ impl Config { let mut out = BTreeMap::new(); if let Some(provider) = &self.oidc_config { - if provider.issuer.is_some() { + if provider.has_required_fields() { + let mut provider = provider.clone(); // The legacy configuration has an implied IdP ID of `oidc`. - out.insert("oidc".to_owned(), provider.clone()); + let idp_id = provider.idp_id.take().unwrap_or("oidc".to_owned()); + provider.idp_id = Some(idp_id.clone()); + out.insert(idp_id, provider); } } for provider in &self.oidc_providers { - if let Some(idp_id) = &provider.idp_id { + let mut provider = provider.clone(); + let idp_id = match provider.idp_id.take() { + None => "oidc".to_owned(), + Some(idp_id) if idp_id == "oidc" => idp_id, // Synapse internally prefixes the IdP IDs with `oidc-`. - out.insert(format!("oidc-{idp_id}"), provider.clone()); - } + Some(idp_id) => format!("oidc-{idp_id}"), + }; + provider.idp_id = Some(idp_id.clone()); + out.insert(idp_id, provider); } out } + + /// Adjust a MAS configuration to match this Synapse configuration. + #[must_use] + pub fn adjust_mas_config( + self, + mut mas_config: mas_config::RootConfig, + rng: &mut impl Rng, + now: DateTime, + ) -> mas_config::RootConfig { + let providers = self.all_oidc_providers(); + for provider in providers.into_values() { + let Some(mas_provider_config) = provider.into_mas_config(rng, now) else { + // TODO: better log message + warn!("Could not convert OIDC provider to MAS config"); + continue; + }; + + mas_config + .upstream_oauth2 + .providers + .push(mas_provider_config); + } + + // TODO: manage when the option is not set + if let Some(enable_3pid_changes) = self.enable_3pid_changes { + mas_config.account.email_change_allowed = enable_3pid_changes; + } + mas_config.account.displayname_change_allowed = self.enable_set_display_name; + if self.password_config.enabled { + mas_config.passwords.enabled = true; + mas_config.passwords.schemes = vec![ + // This is the password hashing scheme synapse uses + PasswordHashingScheme { + version: 1, + algorithm: PasswordAlgorithm::Bcrypt, + cost: self.bcrypt_rounds, + secret: self.password_config.pepper, + secret_file: None, + }, + // Use the default algorithm MAS uses as a second hashing scheme, so that users + // will get their password hash upgraded to a more modern algorithm over time + PasswordHashingScheme { + version: 2, + algorithm: PasswordAlgorithm::default(), + cost: None, + secret: None, + secret_file: None, + }, + ]; + + mas_config.account.password_registration_enabled = self.enable_registration; + } else { + mas_config.passwords.enabled = false; + } + + if self.enable_registration_captcha { + mas_config.captcha.service = Some(mas_config::CaptchaServiceKind::RecaptchaV2); + mas_config.captcha.site_key = self.recaptcha_public_key; + mas_config.captcha.secret_key = self.recaptcha_private_key; + } + + mas_config.matrix.homeserver = self.server_name; + if let Some(public_baseurl) = self.public_baseurl { + mas_config.matrix.endpoint = public_baseurl; + } + + mas_config + } } /// The `database` section of the Synapse configuration. @@ -215,17 +309,6 @@ pub struct EnableableSection { pub enabled: bool, } -#[derive(Clone, Deserialize)] -pub struct OidcProvider { - /// At least for `oidc_config`, if the dict is present but left empty then - /// the config should be ignored, so this field must be optional. - pub issuer: Option, - - /// Required, except for the old `oidc_config` where this is implied to be - /// "oidc". - pub idp_id: Option, -} - fn default_true() -> bool { true } diff --git a/crates/syn2mas/src/synapse_reader/config/oidc.rs b/crates/syn2mas/src/synapse_reader/config/oidc.rs new file mode 100644 index 000000000..5a9321ce2 --- /dev/null +++ b/crates/syn2mas/src/synapse_reader/config/oidc.rs @@ -0,0 +1,347 @@ +// Copyright 2025 New Vector Ltd. +// +// SPDX-License-Identifier: AGPL-3.0-only +// Please see LICENSE in the repository root for full details. + +use std::{collections::BTreeMap, str::FromStr as _}; + +use chrono::{DateTime, Utc}; +use mas_config::{ + UpstreamOAuth2ClaimsImports, UpstreamOAuth2DiscoveryMode, UpstreamOAuth2ImportAction, + UpstreamOAuth2PkceMethod, UpstreamOAuth2ResponseMode, UpstreamOAuth2TokenAuthMethod, +}; +use mas_iana::jose::JsonWebSignatureAlg; +use oauth2_types::scope::{OPENID, Scope, ScopeToken}; +use rand::Rng; +use serde::Deserialize; +use tracing::warn; +use ulid::Ulid; +use url::Url; + +#[derive(Clone, Deserialize, Default)] +enum UserMappingProviderModule { + #[default] + #[serde(rename = "synapse.handlers.oidc.JinjaOidcMappingProvider")] + Jinja, + + #[serde(rename = "synapse.handlers.oidc_handler.JinjaOidcMappingProvider")] + JinjaLegacy, + + #[serde(other)] + Other, +} + +#[derive(Clone, Deserialize, Default)] +struct UserMappingProviderConfig { + subject_template: Option, + subject_claim: Option, + localpart_template: Option, + display_name_template: Option, + email_template: Option, + + #[serde(default)] + confirm_localpart: bool, +} + +impl UserMappingProviderConfig { + fn into_mas_config(self) -> UpstreamOAuth2ClaimsImports { + let mut config = UpstreamOAuth2ClaimsImports::default(); + + match (self.subject_claim, self.subject_template) { + (Some(_), Some(subject_template)) => { + warn!( + "Both `subject_claim` and `subject_template` options are set, using `subject_template`." + ); + config.subject.template = Some(subject_template); + } + (None, Some(subject_template)) => { + config.subject.template = Some(subject_template); + } + (Some(subject_claim), None) => { + config.subject.template = Some(format!("{{{{ user.{subject_claim} }}}}")); + } + (None, None) => {} + } + + if let Some(localpart_template) = self.localpart_template { + config.localpart.template = Some(localpart_template); + config.localpart.action = if self.confirm_localpart { + UpstreamOAuth2ImportAction::Suggest + } else { + UpstreamOAuth2ImportAction::Require + }; + } + + if let Some(displayname_template) = self.display_name_template { + config.displayname.template = Some(displayname_template); + config.displayname.action = if self.confirm_localpart { + UpstreamOAuth2ImportAction::Suggest + } else { + UpstreamOAuth2ImportAction::Force + }; + } + + if let Some(email_template) = self.email_template { + config.email.template = Some(email_template); + config.email.action = if self.confirm_localpart { + UpstreamOAuth2ImportAction::Suggest + } else { + UpstreamOAuth2ImportAction::Force + }; + } + + config + } +} + +#[derive(Clone, Deserialize, Default)] +struct UserMappingProvider { + #[serde(default)] + module: UserMappingProviderModule, + #[serde(default)] + config: UserMappingProviderConfig, +} + +#[derive(Clone, Deserialize, Default)] +#[serde(rename_all = "lowercase")] +enum PkceMethod { + #[default] + Auto, + Always, + Never, + #[serde(other)] + Other, +} + +#[derive(Clone, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +enum UserProfileMethod { + #[default] + Auto, + UserinfoEndpoint, + #[serde(other)] + Other, +} + +#[derive(Clone, Deserialize)] +#[expect(clippy::struct_excessive_bools)] +pub struct OidcProvider { + pub issuer: Option, + + /// Required, except for the old `oidc_config` where this is implied to be + /// "oidc". + pub idp_id: Option, + + idp_name: Option, + idp_brand: Option, + + #[serde(default = "default_true")] + discover: bool, + + client_id: Option, + client_secret: Option, + + // Unsupported, we want to shout about it + client_secret_path: Option, + + // Unsupported, we want to shout about it + client_secret_jwt_key: Option, + client_auth_method: Option, + #[serde(default)] + pkce_method: PkceMethod, + // Unsupported, we want to shout about it + id_token_signing_alg_values_supported: Option>, + scopes: Option>, + authorization_endpoint: Option, + token_endpoint: Option, + userinfo_endpoint: Option, + jwks_uri: Option, + #[serde(default)] + skip_verification: bool, + + // Unsupported, we want to shout about it + #[serde(default)] + backchannel_logout_enabled: bool, + + #[serde(default)] + user_profile_method: UserProfileMethod, + + // Unsupported, we want to shout about it + attribute_requirements: Option, + + // Unsupported, we want to shout about it + #[serde(default = "default_true")] + enable_registration: bool, + #[serde(default)] + additional_authorization_parameters: BTreeMap, + #[serde(default)] + user_mapping_provider: UserMappingProvider, +} + +fn default_true() -> bool { + true +} + +impl OidcProvider { + /// Returns true if the two 'required' fields are set. This is used to + /// ignore an empty dict on the `oidc_config` section. + #[must_use] + pub(crate) fn has_required_fields(&self) -> bool { + self.issuer.is_some() && self.client_id.is_some() + } + + /// Map this Synapse OIDC provider config to a MAS upstream provider config. + #[expect(clippy::too_many_lines)] + pub(crate) fn into_mas_config( + self, + rng: &mut impl Rng, + now: DateTime, + ) -> Option { + let client_id = self.client_id?; + + if self.client_secret_path.is_some() { + warn!( + "The `client_secret_path` option is not supported, ignoring. You *will* need to include the secret in the `client_secret` field." + ); + } + + if self.client_secret_jwt_key.is_some() { + warn!("The `client_secret_jwt_key` option is not supported, ignoring."); + } + + if self.attribute_requirements.is_some() { + warn!("The `attribute_requirements` option is not supported, ignoring."); + } + + if self.id_token_signing_alg_values_supported.is_some() { + warn!("The `id_token_signing_alg_values_supported` option is not supported, ignoring."); + } + + if self.backchannel_logout_enabled { + warn!("The `backchannel_logout_enabled` option is not supported, ignoring."); + } + + if !self.enable_registration { + warn!( + "Setting the `enable_registration` option to `false` is not supported, ignoring." + ); + } + + let scope: Scope = match self.scopes { + None => [OPENID].into_iter().collect(), // Synapse defaults to the 'openid' scope + Some(scopes) => scopes + .into_iter() + .filter_map(|scope| match ScopeToken::from_str(&scope) { + Ok(scope) => Some(scope), + Err(err) => { + warn!("OIDC provider scope '{scope}' is invalid: {err}"); + None + } + }) + .collect(), + }; + + let id = Ulid::from_datetime_with_source(now.into(), rng); + + let token_endpoint_auth_method = self.client_auth_method.unwrap_or_else(|| { + // The token auth method defaults to 'none' if no client_secret is set and + // 'client_secret_basic' otherwise + if self.client_secret.is_some() { + UpstreamOAuth2TokenAuthMethod::ClientSecretBasic + } else { + UpstreamOAuth2TokenAuthMethod::None + } + }); + + let discovery_mode = match (self.discover, self.skip_verification) { + (true, false) => UpstreamOAuth2DiscoveryMode::Oidc, + (true, true) => UpstreamOAuth2DiscoveryMode::Insecure, + (false, _) => UpstreamOAuth2DiscoveryMode::Disabled, + }; + + let pkce_method = match self.pkce_method { + PkceMethod::Auto => UpstreamOAuth2PkceMethod::Auto, + PkceMethod::Always => UpstreamOAuth2PkceMethod::Always, + PkceMethod::Never => UpstreamOAuth2PkceMethod::Never, + PkceMethod::Other => { + warn!( + "The `pkce_method` option is not supported, expected 'auto', 'always', or 'never'; assuming 'auto'." + ); + UpstreamOAuth2PkceMethod::default() + } + }; + + // "auto" doesn't mean the same thing depending on whether we request the openid + // scope or not + let has_openid_scope = scope.contains(&OPENID); + let fetch_userinfo = match self.user_profile_method { + UserProfileMethod::Auto => has_openid_scope, + UserProfileMethod::UserinfoEndpoint => true, + UserProfileMethod::Other => { + warn!( + "The `user_profile_method` option is not supported, expected 'auto' or 'userinfo_endpoint'; assuming 'auto'." + ); + has_openid_scope + } + }; + + // Check if there is a `response_mode` set in the additional authorization + // parameters + let mut additional_authorization_parameters = self.additional_authorization_parameters; + let response_mode = if let Some(response_mode) = + additional_authorization_parameters.remove("response_mode") + { + match response_mode.to_ascii_lowercase().as_str() { + "query" => Some(UpstreamOAuth2ResponseMode::Query), + "form_post" => Some(UpstreamOAuth2ResponseMode::FormPost), + _ => { + warn!( + "Invalid `response_mode` in the `additional_authorization_parameters` option, expected 'query' or 'form_post'; ignoring." + ); + None + } + } + } else { + None + }; + + let claims_imports = if matches!( + self.user_mapping_provider.module, + UserMappingProviderModule::Other + ) { + warn!( + "The `user_mapping_provider` module specified is not supported, ignoring. Please adjust the `claims_imports` to match the mapping provider behaviour." + ); + UpstreamOAuth2ClaimsImports::default() + } else { + self.user_mapping_provider.config.into_mas_config() + }; + + Some(mas_config::UpstreamOAuth2Provider { + enabled: true, + id, + synapse_idp_id: self.idp_id, + issuer: self.issuer, + human_name: self.idp_name, + brand_name: self.idp_brand, + client_id, + client_secret: self.client_secret, + token_endpoint_auth_method, + sign_in_with_apple: None, + token_endpoint_auth_signing_alg: None, + id_token_signed_response_alg: JsonWebSignatureAlg::Rs256, + scope: scope.to_string(), + discovery_mode, + pkce_method, + fetch_userinfo, + userinfo_signed_response_alg: None, + authorization_endpoint: self.authorization_endpoint, + userinfo_endpoint: self.userinfo_endpoint, + token_endpoint: self.token_endpoint, + jwks_uri: self.jwks_uri, + response_mode, + claims_imports, + additional_authorization_parameters, + }) + } +} From ef81b3ce4f8c721884a96c9c7bbc2b33d954fe77 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 22 Apr 2025 13:40:14 +0200 Subject: [PATCH 07/35] syn2mas: add a buffered channel for writing threepids --- crates/syn2mas/src/migration.rs | 145 +++++++++++++++++++------------- 1 file changed, 85 insertions(+), 60 deletions(-) diff --git a/crates/syn2mas/src/migration.rs b/crates/syn2mas/src/migration.rs index 6d0420077..f5efaa722 100644 --- a/crates/syn2mas/src/migration.rs +++ b/crates/syn2mas/src/migration.rs @@ -335,77 +335,102 @@ async fn migrate_threepids( ) -> Result<(MasWriter, MigrationState), Error> { let start = Instant::now(); - let mut email_buffer = MasWriteBuffer::new(&mas, MasWriter::write_email_threepids); - let mut unsupported_buffer = MasWriteBuffer::new(&mas, MasWriter::write_unsupported_threepids); - let mut users_stream = pin!(synapse.read_threepids()); + let (tx, mut rx) = tokio::sync::mpsc::channel::(10 * 1024 * 1024); - while let Some(threepid_res) = users_stream.next().await { - let SynapseThreepid { - user_id: synapse_user_id, - medium, - address, - added_at, - } = threepid_res.into_synapse("reading threepid")?; - let created_at: DateTime = added_at.into(); + // create a new RNG seeded from the passed RNG so that we can move it into the + // spawned task + let mut rng = rand_chacha::ChaChaRng::from_rng(rng).expect("failed to seed rng"); + let task = tokio::spawn( + async move { + let mut email_buffer = MasWriteBuffer::new(&mas, MasWriter::write_email_threepids); + let mut unsupported_buffer = + MasWriteBuffer::new(&mas, MasWriter::write_unsupported_threepids); - let username = synapse_user_id - .extract_localpart(&state.server_name) - .into_extract_localpart(synapse_user_id.clone())? - .to_owned(); - let Some(user_infos) = state.users.get(username.as_str()).copied() else { - return Err(Error::MissingUserFromDependentTable { - table: "user_threepids".to_owned(), - user: synapse_user_id, - }); - }; + while let Some(threepid) = rx.recv().await { + let SynapseThreepid { + user_id: synapse_user_id, + medium, + address, + added_at, + } = threepid; + let created_at: DateTime = added_at.into(); - let Some(mas_user_id) = user_infos.mas_user_id else { - progress_counter.increment_skipped(); - continue; - }; + let username = synapse_user_id + .extract_localpart(&state.server_name) + .into_extract_localpart(synapse_user_id.clone())? + .to_owned(); + let Some(user_infos) = state.users.get(username.as_str()).copied() else { + return Err(Error::MissingUserFromDependentTable { + table: "user_threepids".to_owned(), + user: synapse_user_id, + }); + }; + + let Some(mas_user_id) = user_infos.mas_user_id else { + progress_counter.increment_skipped(); + continue; + }; + + if medium == "email" { + email_buffer + .write( + &mut mas, + MasNewEmailThreepid { + user_id: mas_user_id, + user_email_id: Uuid::from(Ulid::from_datetime_with_source( + created_at.into(), + &mut rng, + )), + email: address, + created_at, + }, + ) + .await + .into_mas("writing email")?; + } else { + unsupported_buffer + .write( + &mut mas, + MasNewUnsupportedThreepid { + user_id: mas_user_id, + medium, + address, + created_at, + }, + ) + .await + .into_mas("writing unsupported threepid")?; + } + + progress_counter.increment_migrated(); + } - if medium == "email" { email_buffer - .write( - &mut mas, - MasNewEmailThreepid { - user_id: mas_user_id, - user_email_id: Uuid::from(Ulid::from_datetime_with_source( - created_at.into(), - rng, - )), - email: address, - created_at, - }, - ) + .finish(&mut mas) .await - .into_mas("writing email")?; - } else { + .into_mas("writing email threepids")?; unsupported_buffer - .write( - &mut mas, - MasNewUnsupportedThreepid { - user_id: mas_user_id, - medium, - address, - created_at, - }, - ) + .finish(&mut mas) .await - .into_mas("writing unsupported threepid")?; + .into_mas("writing unsupported threepids")?; + + Ok((mas, state)) } + .instrument(tracing::info_span!("ingest_task")), + ); - progress_counter.increment_migrated(); - } + // In case this has an error, we still want to join the task, so we look at the + // error later + let res = synapse + .read_threepids() + .map_err(|e| e.into_synapse("reading threepids")) + .forward(PollSender::new(tx).sink_map_err(|_| Error::ChannelClosed)) + .inspect_err(|e| tracing::error!(error = e as &dyn std::error::Error)) + .await; - email_buffer - .finish(&mut mas) - .await - .into_mas("writing email threepids")?; - unsupported_buffer - .finish(&mut mas) - .await - .into_mas("writing unsupported threepids")?; + let (mas, state) = task.await.into_join("threepid write task")??; + + res?; info!( "third-party IDs migrated in {:.1}s", From c292da7ac99604498c904a7c73c362c977077c27 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 22 Apr 2025 13:44:52 +0200 Subject: [PATCH 08/35] syn2mas: add a buffered channel for writing external IDs --- crates/syn2mas/src/migration.rs | 132 +++++++++++++++++++------------- 1 file changed, 77 insertions(+), 55 deletions(-) diff --git a/crates/syn2mas/src/migration.rs b/crates/syn2mas/src/migration.rs index f5efaa722..fb2ea5487 100644 --- a/crates/syn2mas/src/migration.rs +++ b/crates/syn2mas/src/migration.rs @@ -440,10 +440,6 @@ async fn migrate_threepids( Ok((mas, state)) } -/// # Parameters -/// -/// - `provider_id_mapping`: mapping from Synapse `auth_provider` ID to UUID of -/// the upstream provider in MAS. #[tracing::instrument(skip_all, level = Level::INFO)] async fn migrate_external_ids( synapse: &mut SynapseReader<'_>, @@ -454,65 +450,91 @@ async fn migrate_external_ids( ) -> Result<(MasWriter, MigrationState), Error> { let start = Instant::now(); - let mut write_buffer = MasWriteBuffer::new(&mas, MasWriter::write_upstream_oauth_links); - let mut extids_stream = pin!(synapse.read_user_external_ids()); + let (tx, mut rx) = tokio::sync::mpsc::channel::(10 * 1024 * 1024); - while let Some(extid_res) = extids_stream.next().await { - let SynapseExternalId { - user_id: synapse_user_id, - auth_provider, - external_id: subject, - } = extid_res.into_synapse("reading external ID")?; - let username = synapse_user_id - .extract_localpart(&state.server_name) - .into_extract_localpart(synapse_user_id.clone())? - .to_owned(); - let Some(user_infos) = state.users.get(username.as_str()).copied() else { - return Err(Error::MissingUserFromDependentTable { - table: "user_external_ids".to_owned(), - user: synapse_user_id, - }); - }; + // create a new RNG seeded from the passed RNG so that we can move it into the + // spawned task + let mut rng = rand_chacha::ChaChaRng::from_rng(rng).expect("failed to seed rng"); + let task = tokio::spawn( + async move { + let mut write_buffer = MasWriteBuffer::new(&mas, MasWriter::write_upstream_oauth_links); - let Some(mas_user_id) = user_infos.mas_user_id else { - progress_counter.increment_skipped(); - continue; - }; + while let Some(extid) = rx.recv().await { + let SynapseExternalId { + user_id: synapse_user_id, + auth_provider, + external_id: subject, + } = extid; + let username = synapse_user_id + .extract_localpart(&state.server_name) + .into_extract_localpart(synapse_user_id.clone())? + .to_owned(); + let Some(user_infos) = state.users.get(username.as_str()).copied() else { + return Err(Error::MissingUserFromDependentTable { + table: "user_external_ids".to_owned(), + user: synapse_user_id, + }); + }; - let Some(&upstream_provider_id) = state.provider_id_mapping.get(&auth_provider) else { - return Err(Error::MissingAuthProviderMapping { - synapse_id: auth_provider, - user: synapse_user_id, - }); - }; + let Some(mas_user_id) = user_infos.mas_user_id else { + progress_counter.increment_skipped(); + continue; + }; - // To save having to store user creation times, extract it from the ULID - // This gives millisecond precision — good enough. - let user_created_ts = Ulid::from(mas_user_id.get()).datetime(); + let Some(&upstream_provider_id) = state.provider_id_mapping.get(&auth_provider) + else { + return Err(Error::MissingAuthProviderMapping { + synapse_id: auth_provider, + user: synapse_user_id, + }); + }; - let link_id: Uuid = Ulid::from_datetime_with_source(user_created_ts, rng).into(); + // To save having to store user creation times, extract it from the ULID + // This gives millisecond precision — good enough. + let user_created_ts = Ulid::from(mas_user_id.get()).datetime(); - write_buffer - .write( - &mut mas, - MasNewUpstreamOauthLink { - link_id, - user_id: mas_user_id, - upstream_provider_id, - subject, - created_at: user_created_ts.into(), - }, - ) - .await - .into_mas("failed to write upstream link")?; + let link_id: Uuid = + Ulid::from_datetime_with_source(user_created_ts, &mut rng).into(); - progress_counter.increment_migrated(); - } + write_buffer + .write( + &mut mas, + MasNewUpstreamOauthLink { + link_id, + user_id: mas_user_id, + upstream_provider_id, + subject, + created_at: user_created_ts.into(), + }, + ) + .await + .into_mas("failed to write upstream link")?; - write_buffer - .finish(&mut mas) - .await - .into_mas("writing upstream links")?; + progress_counter.increment_migrated(); + } + + write_buffer + .finish(&mut mas) + .await + .into_mas("writing upstream links")?; + + Ok((mas, state)) + } + .instrument(tracing::info_span!("ingest_task")), + ); + + // In case this has an error, we still want to join the task, so we look at the + // error later + let res = synapse + .read_user_external_ids() + .map_err(|e| e.into_synapse("reading external ID")) + .forward(PollSender::new(tx).sink_map_err(|_| Error::ChannelClosed)) + .inspect_err(|e| tracing::error!(error = e as &dyn std::error::Error)) + .await; + + let (mas, state) = task.await.into_join("external IDs write task")??; + + res?; info!( "upstream links (external IDs) migrated in {:.1}s", From b21748c2bd6284569faf0f71dd96dbb240c89bc9 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 22 Apr 2025 13:48:47 +0200 Subject: [PATCH 09/35] syn2mas: add a buffered channel for writing refreshable tokens --- crates/syn2mas/src/migration.rs | 202 ++++++++++++++++++-------------- 1 file changed, 115 insertions(+), 87 deletions(-) diff --git a/crates/syn2mas/src/migration.rs b/crates/syn2mas/src/migration.rs index fb2ea5487..3388ff387 100644 --- a/crates/syn2mas/src/migration.rs +++ b/crates/syn2mas/src/migration.rs @@ -11,7 +11,7 @@ //! This module does not implement any of the safety checks that should be run //! *before* the migration. -use std::{pin::pin, time::Instant}; +use std::time::Instant; use chrono::{DateTime, Utc}; use compact_str::CompactString; @@ -847,99 +847,127 @@ async fn migrate_refreshable_token_pairs( ) -> Result<(MasWriter, MigrationState), Error> { let start = Instant::now(); - let mut token_stream = pin!(synapse.read_refreshable_token_pairs()); - let mut access_token_write_buffer = - MasWriteBuffer::new(&mas, MasWriter::write_compat_access_tokens); - let mut refresh_token_write_buffer = - MasWriteBuffer::new(&mas, MasWriter::write_compat_refresh_tokens); + let (tx, mut rx) = tokio::sync::mpsc::channel::(10 * 1024 * 1024); - while let Some(token_res) = token_stream.next().await { - let SynapseRefreshableTokenPair { - user_id: synapse_user_id, - device_id, - access_token, - refresh_token, - valid_until_ms, - last_validated, - } = token_res.into_synapse("reading Synapse refresh token")?; + // create a new RNG seeded from the passed RNG so that we can move it into the + // spawned task + let mut rng = rand_chacha::ChaChaRng::from_rng(rng).expect("failed to seed rng"); + let now = clock.now(); + let task = tokio::spawn( + async move { + let mut access_token_write_buffer = + MasWriteBuffer::new(&mas, MasWriter::write_compat_access_tokens); + let mut refresh_token_write_buffer = + MasWriteBuffer::new(&mas, MasWriter::write_compat_refresh_tokens); - let username = synapse_user_id - .extract_localpart(&state.server_name) - .into_extract_localpart(synapse_user_id.clone())? - .to_owned(); - let Some(user_infos) = state.users.get(username.as_str()).copied() else { - return Err(Error::MissingUserFromDependentTable { - table: "refresh_tokens".to_owned(), - user: synapse_user_id, - }); - }; - - let Some(mas_user_id) = user_infos.mas_user_id else { - progress_counter.increment_skipped(); - continue; - }; - - if user_infos.flags.is_deactivated() - || user_infos.flags.is_guest() - || user_infos.flags.is_appservice() - { - progress_counter.increment_skipped(); - continue; - } - - // It's not always accurate, but last_validated is *often* the creation time of - // the device If we don't have one, then use the current time as a - // fallback. - let created_at = last_validated.map_or_else(|| clock.now(), DateTime::from); - - // Use the existing device_id if this is the second token for a device - let session_id = *state - .devices_to_compat_sessions - .entry((mas_user_id, CompactString::new(&device_id))) - .or_insert_with(|| Uuid::from(Ulid::from_datetime_with_source(created_at.into(), rng))); - - let access_token_id = Uuid::from(Ulid::from_datetime_with_source(created_at.into(), rng)); - let refresh_token_id = Uuid::from(Ulid::from_datetime_with_source(created_at.into(), rng)); - - access_token_write_buffer - .write( - &mut mas, - MasNewCompatAccessToken { - token_id: access_token_id, - session_id, + while let Some(token) = rx.recv().await { + let SynapseRefreshableTokenPair { + user_id: synapse_user_id, + device_id, access_token, - created_at, - expires_at: valid_until_ms.map(DateTime::from), - }, - ) - .await - .into_mas("writing compat access tokens")?; - refresh_token_write_buffer - .write( - &mut mas, - MasNewCompatRefreshToken { - refresh_token_id, - session_id, - access_token_id, refresh_token, - created_at, - }, - ) - .await - .into_mas("writing compat refresh tokens")?; + valid_until_ms, + last_validated, + } = token; - progress_counter.increment_migrated(); - } + let username = synapse_user_id + .extract_localpart(&state.server_name) + .into_extract_localpart(synapse_user_id.clone())? + .to_owned(); + let Some(user_infos) = state.users.get(username.as_str()).copied() else { + return Err(Error::MissingUserFromDependentTable { + table: "refresh_tokens".to_owned(), + user: synapse_user_id, + }); + }; - access_token_write_buffer - .finish(&mut mas) - .await - .into_mas("writing compat access tokens")?; + let Some(mas_user_id) = user_infos.mas_user_id else { + progress_counter.increment_skipped(); + continue; + }; - refresh_token_write_buffer - .finish(&mut mas) - .await - .into_mas("writing compat refresh tokens")?; + if user_infos.flags.is_deactivated() + || user_infos.flags.is_guest() + || user_infos.flags.is_appservice() + { + progress_counter.increment_skipped(); + continue; + } + + // It's not always accurate, but last_validated is *often* the creation time of + // the device If we don't have one, then use the current time as a + // fallback. + let created_at = last_validated.map_or_else(|| now, DateTime::from); + + // Use the existing device_id if this is the second token for a device + let session_id = *state + .devices_to_compat_sessions + .entry((mas_user_id, CompactString::new(&device_id))) + .or_insert_with(|| { + Uuid::from(Ulid::from_datetime_with_source(created_at.into(), &mut rng)) + }); + + let access_token_id = + Uuid::from(Ulid::from_datetime_with_source(created_at.into(), &mut rng)); + let refresh_token_id = + Uuid::from(Ulid::from_datetime_with_source(created_at.into(), &mut rng)); + + access_token_write_buffer + .write( + &mut mas, + MasNewCompatAccessToken { + token_id: access_token_id, + session_id, + access_token, + created_at, + expires_at: valid_until_ms.map(DateTime::from), + }, + ) + .await + .into_mas("writing compat access tokens")?; + refresh_token_write_buffer + .write( + &mut mas, + MasNewCompatRefreshToken { + refresh_token_id, + session_id, + access_token_id, + refresh_token, + created_at, + }, + ) + .await + .into_mas("writing compat refresh tokens")?; + + progress_counter.increment_migrated(); + } + + access_token_write_buffer + .finish(&mut mas) + .await + .into_mas("writing compat access tokens")?; + + refresh_token_write_buffer + .finish(&mut mas) + .await + .into_mas("writing compat refresh tokens")?; + Ok((mas, state)) + } + .instrument(tracing::info_span!("ingest_task")), + ); + + // In case this has an error, we still want to join the task, so we look at the + // error later + let res = synapse + .read_refreshable_token_pairs() + .map_err(|e| e.into_synapse("reading refresh token pairs")) + .forward(PollSender::new(tx).sink_map_err(|_| Error::ChannelClosed)) + .inspect_err(|e| tracing::error!(error = e as &dyn std::error::Error)) + .await; + + let (mas, state) = task.await.into_join("refresh token write task")??; + + res?; info!( "refreshable token pairs migrated in {:.1}s", From b3538219e6343252cdc57d8c3728d43910ed7b72 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 22 Apr 2025 14:14:20 +0200 Subject: [PATCH 10/35] syn2mas: introduce a WriteBatch trait to refactor how we write to MAS --- crates/syn2mas/src/mas_writer/mod.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/syn2mas/src/mas_writer/mod.rs b/crates/syn2mas/src/mas_writer/mod.rs index fe52e0eed..33fc856e5 100644 --- a/crates/syn2mas/src/mas_writer/mod.rs +++ b/crates/syn2mas/src/mas_writer/mod.rs @@ -250,6 +250,13 @@ pub struct MasWriter { write_buffer_finish_checker: FinishChecker, } +trait WriteBatch: Sized { + fn write_batch( + conn: &mut PgConnection, + batch: Vec, + ) -> impl Future>; +} + pub struct MasNewUser { pub user_id: NonNilUuid, pub username: String, From 848847636866c081f236f9739081253644b5c198 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 22 Apr 2025 14:14:44 +0200 Subject: [PATCH 11/35] syn2mas: implement WriteBatch for MasNewUser --- ...85277958b66e4534561686c073e282fafaf2a.json | 20 +++ ...817cb6fe71203b2d3471e838f841b53e688d1.json | 20 --- crates/syn2mas/src/mas_writer/mod.rs | 124 +++++++++--------- 3 files changed, 85 insertions(+), 79 deletions(-) create mode 100644 crates/syn2mas/.sqlx/query-207b880ec2dd484ad05a7138ba485277958b66e4534561686c073e282fafaf2a.json delete mode 100644 crates/syn2mas/.sqlx/query-f2820b3752cf66669551ef90a10817cb6fe71203b2d3471e838f841b53e688d1.json diff --git a/crates/syn2mas/.sqlx/query-207b880ec2dd484ad05a7138ba485277958b66e4534561686c073e282fafaf2a.json b/crates/syn2mas/.sqlx/query-207b880ec2dd484ad05a7138ba485277958b66e4534561686c073e282fafaf2a.json new file mode 100644 index 000000000..79688d807 --- /dev/null +++ b/crates/syn2mas/.sqlx/query-207b880ec2dd484ad05a7138ba485277958b66e4534561686c073e282fafaf2a.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO syn2mas__users (\n user_id, username,\n created_at, locked_at,\n deactivated_at,\n can_request_admin, is_guest)\n SELECT * FROM UNNEST(\n $1::UUID[], $2::TEXT[],\n $3::TIMESTAMP WITH TIME ZONE[], $4::TIMESTAMP WITH TIME ZONE[],\n $5::TIMESTAMP WITH TIME ZONE[],\n $6::BOOL[], $7::BOOL[])\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray", + "TextArray", + "TimestamptzArray", + "TimestamptzArray", + "TimestamptzArray", + "BoolArray", + "BoolArray" + ] + }, + "nullable": [] + }, + "hash": "207b880ec2dd484ad05a7138ba485277958b66e4534561686c073e282fafaf2a" +} diff --git a/crates/syn2mas/.sqlx/query-f2820b3752cf66669551ef90a10817cb6fe71203b2d3471e838f841b53e688d1.json b/crates/syn2mas/.sqlx/query-f2820b3752cf66669551ef90a10817cb6fe71203b2d3471e838f841b53e688d1.json deleted file mode 100644 index 66979a67e..000000000 --- a/crates/syn2mas/.sqlx/query-f2820b3752cf66669551ef90a10817cb6fe71203b2d3471e838f841b53e688d1.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO syn2mas__users (\n user_id, username,\n created_at, locked_at,\n deactivated_at,\n can_request_admin, is_guest)\n SELECT * FROM UNNEST(\n $1::UUID[], $2::TEXT[],\n $3::TIMESTAMP WITH TIME ZONE[], $4::TIMESTAMP WITH TIME ZONE[],\n $5::TIMESTAMP WITH TIME ZONE[],\n $6::BOOL[], $7::BOOL[])\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "UuidArray", - "TextArray", - "TimestamptzArray", - "TimestamptzArray", - "TimestamptzArray", - "BoolArray", - "BoolArray" - ] - }, - "nullable": [] - }, - "hash": "f2820b3752cf66669551ef90a10817cb6fe71203b2d3471e838f841b53e688d1" -} diff --git a/crates/syn2mas/src/mas_writer/mod.rs b/crates/syn2mas/src/mas_writer/mod.rs index 33fc856e5..cd64650ac 100644 --- a/crates/syn2mas/src/mas_writer/mod.rs +++ b/crates/syn2mas/src/mas_writer/mod.rs @@ -270,6 +270,70 @@ pub struct MasNewUser { pub is_guest: bool, } +impl WriteBatch for MasNewUser { + async fn write_batch(conn: &mut PgConnection, batch: Vec) -> Result<(), Error> { + // `UNNEST` is a fast way to do bulk inserts, as it lets us send multiple rows + // in one statement without having to change the statement + // SQL thus altering the query plan. See . + // In the future we could consider using sqlx's support for `PgCopyIn` / the + // `COPY FROM STDIN` statement, which is allegedly the best + // for insert performance, but is less simple to encode. + let mut user_ids: Vec = Vec::with_capacity(batch.len()); + let mut usernames: Vec = Vec::with_capacity(batch.len()); + let mut created_ats: Vec> = Vec::with_capacity(batch.len()); + let mut locked_ats: Vec>> = Vec::with_capacity(batch.len()); + let mut deactivated_ats: Vec>> = Vec::with_capacity(batch.len()); + let mut can_request_admins: Vec = Vec::with_capacity(batch.len()); + let mut is_guests: Vec = Vec::with_capacity(batch.len()); + for MasNewUser { + user_id, + username, + created_at, + locked_at, + deactivated_at, + can_request_admin, + is_guest, + } in batch + { + user_ids.push(user_id.get()); + usernames.push(username); + created_ats.push(created_at); + locked_ats.push(locked_at); + deactivated_ats.push(deactivated_at); + can_request_admins.push(can_request_admin); + is_guests.push(is_guest); + } + + sqlx::query!( + r#" + INSERT INTO syn2mas__users ( + user_id, username, + created_at, locked_at, + deactivated_at, + can_request_admin, is_guest) + SELECT * FROM UNNEST( + $1::UUID[], $2::TEXT[], + $3::TIMESTAMP WITH TIME ZONE[], $4::TIMESTAMP WITH TIME ZONE[], + $5::TIMESTAMP WITH TIME ZONE[], + $6::BOOL[], $7::BOOL[]) + "#, + &user_ids[..], + &usernames[..], + &created_ats[..], + // We need to override the typing for arrays of optionals (sqlx limitation) + &locked_ats[..] as &[Option>], + &deactivated_ats[..] as &[Option>], + &can_request_admins[..], + &is_guests[..], + ) + .execute(&mut *conn) + .await + .into_database("writing users to MAS")?; + + Ok(()) + } +} + pub struct MasNewUserPassword { pub user_password_id: Uuid, pub user_id: NonNilUuid, @@ -643,65 +707,7 @@ impl MasWriter { self.writer_pool .spawn_with_connection(move |conn| { Box::pin(async move { - // `UNNEST` is a fast way to do bulk inserts, as it lets us send multiple rows - // in one statement without having to change the statement - // SQL thus altering the query plan. See . - // In the future we could consider using sqlx's support for `PgCopyIn` / the - // `COPY FROM STDIN` statement, which is allegedly the best - // for insert performance, but is less simple to encode. - let mut user_ids: Vec = Vec::with_capacity(users.len()); - let mut usernames: Vec = Vec::with_capacity(users.len()); - let mut created_ats: Vec> = Vec::with_capacity(users.len()); - let mut locked_ats: Vec>> = - Vec::with_capacity(users.len()); - let mut deactivated_ats: Vec>> = - Vec::with_capacity(users.len()); - let mut can_request_admins: Vec = Vec::with_capacity(users.len()); - let mut is_guests: Vec = Vec::with_capacity(users.len()); - for MasNewUser { - user_id, - username, - created_at, - locked_at, - deactivated_at, - can_request_admin, - is_guest, - } in users - { - user_ids.push(user_id.get()); - usernames.push(username); - created_ats.push(created_at); - locked_ats.push(locked_at); - deactivated_ats.push(deactivated_at); - can_request_admins.push(can_request_admin); - is_guests.push(is_guest); - } - - sqlx::query!( - r#" - INSERT INTO syn2mas__users ( - user_id, username, - created_at, locked_at, - deactivated_at, - can_request_admin, is_guest) - SELECT * FROM UNNEST( - $1::UUID[], $2::TEXT[], - $3::TIMESTAMP WITH TIME ZONE[], $4::TIMESTAMP WITH TIME ZONE[], - $5::TIMESTAMP WITH TIME ZONE[], - $6::BOOL[], $7::BOOL[]) - "#, - &user_ids[..], - &usernames[..], - &created_ats[..], - // We need to override the typing for arrays of optionals (sqlx limitation) - &locked_ats[..] as &[Option>], - &deactivated_ats[..] as &[Option>], - &can_request_admins[..], - &is_guests[..], - ) - .execute(&mut *conn) - .await - .into_database("writing users to MAS")?; + MasNewUser::write_batch(conn, users).await?; Ok(()) }) From 4c081152a910a13e01038692e8b5a9ebaecf60e9 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 22 Apr 2025 14:17:04 +0200 Subject: [PATCH 12/35] syn2mas: implement WriteBatch for MasNewUserPassword --- ...e3db8ff7a686180d71052911879f186ed1c8e.json | 18 +++++ ...7cafe3f85d639452fd0593b2773997dfc7425.json | 18 ----- crates/syn2mas/src/mas_writer/mod.rs | 80 +++++++++++-------- 3 files changed, 64 insertions(+), 52 deletions(-) create mode 100644 crates/syn2mas/.sqlx/query-24f6ce6280dc6675ab1ebdde0c5e3db8ff7a686180d71052911879f186ed1c8e.json delete mode 100644 crates/syn2mas/.sqlx/query-c6c7db1d578efc45b9e8c8bfea47cafe3f85d639452fd0593b2773997dfc7425.json diff --git a/crates/syn2mas/.sqlx/query-24f6ce6280dc6675ab1ebdde0c5e3db8ff7a686180d71052911879f186ed1c8e.json b/crates/syn2mas/.sqlx/query-24f6ce6280dc6675ab1ebdde0c5e3db8ff7a686180d71052911879f186ed1c8e.json new file mode 100644 index 000000000..d736336f2 --- /dev/null +++ b/crates/syn2mas/.sqlx/query-24f6ce6280dc6675ab1ebdde0c5e3db8ff7a686180d71052911879f186ed1c8e.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO syn2mas__user_passwords\n (user_password_id, user_id, hashed_password, created_at, version)\n SELECT * FROM UNNEST($1::UUID[], $2::UUID[], $3::TEXT[], $4::TIMESTAMP WITH TIME ZONE[], $5::INTEGER[])\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray", + "UuidArray", + "TextArray", + "TimestamptzArray", + "Int4Array" + ] + }, + "nullable": [] + }, + "hash": "24f6ce6280dc6675ab1ebdde0c5e3db8ff7a686180d71052911879f186ed1c8e" +} diff --git a/crates/syn2mas/.sqlx/query-c6c7db1d578efc45b9e8c8bfea47cafe3f85d639452fd0593b2773997dfc7425.json b/crates/syn2mas/.sqlx/query-c6c7db1d578efc45b9e8c8bfea47cafe3f85d639452fd0593b2773997dfc7425.json deleted file mode 100644 index efa2c4d24..000000000 --- a/crates/syn2mas/.sqlx/query-c6c7db1d578efc45b9e8c8bfea47cafe3f85d639452fd0593b2773997dfc7425.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO syn2mas__user_passwords\n (user_password_id, user_id, hashed_password, created_at, version)\n SELECT * FROM UNNEST($1::UUID[], $2::UUID[], $3::TEXT[], $4::TIMESTAMP WITH TIME ZONE[], $5::INTEGER[])\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "UuidArray", - "UuidArray", - "TextArray", - "TimestamptzArray", - "Int4Array" - ] - }, - "nullable": [] - }, - "hash": "c6c7db1d578efc45b9e8c8bfea47cafe3f85d639452fd0593b2773997dfc7425" -} diff --git a/crates/syn2mas/src/mas_writer/mod.rs b/crates/syn2mas/src/mas_writer/mod.rs index cd64650ac..866b615e9 100644 --- a/crates/syn2mas/src/mas_writer/mod.rs +++ b/crates/syn2mas/src/mas_writer/mod.rs @@ -341,6 +341,44 @@ pub struct MasNewUserPassword { pub created_at: DateTime, } +impl WriteBatch for MasNewUserPassword { + async fn write_batch(conn: &mut PgConnection, batch: Vec) -> Result<(), Error> { + let mut user_password_ids: Vec = Vec::with_capacity(batch.len()); + let mut user_ids: Vec = Vec::with_capacity(batch.len()); + let mut hashed_passwords: Vec = Vec::with_capacity(batch.len()); + let mut created_ats: Vec> = Vec::with_capacity(batch.len()); + let mut versions: Vec = Vec::with_capacity(batch.len()); + for MasNewUserPassword { + user_password_id, + user_id, + hashed_password, + created_at, + } in batch + { + user_password_ids.push(user_password_id); + user_ids.push(user_id.get()); + hashed_passwords.push(hashed_password); + created_ats.push(created_at); + versions.push(MIGRATED_PASSWORD_VERSION.into()); + } + + sqlx::query!( + r#" + INSERT INTO syn2mas__user_passwords + (user_password_id, user_id, hashed_password, created_at, version) + SELECT * FROM UNNEST($1::UUID[], $2::UUID[], $3::TEXT[], $4::TIMESTAMP WITH TIME ZONE[], $5::INTEGER[]) + "#, + &user_password_ids[..], + &user_ids[..], + &hashed_passwords[..], + &created_ats[..], + &versions[..], + ).execute(&mut *conn).await.into_database("writing users to MAS")?; + + Ok(()) + } +} + pub struct MasNewEmailThreepid { pub user_email_id: Uuid, pub user_id: NonNilUuid, @@ -727,41 +765,15 @@ impl MasWriter { &mut self, passwords: Vec, ) -> BoxFuture<'_, Result<(), Error>> { - self.writer_pool.spawn_with_connection(move |conn| Box::pin(async move { - let mut user_password_ids: Vec = Vec::with_capacity(passwords.len()); - let mut user_ids: Vec = Vec::with_capacity(passwords.len()); - let mut hashed_passwords: Vec = Vec::with_capacity(passwords.len()); - let mut created_ats: Vec> = Vec::with_capacity(passwords.len()); - let mut versions: Vec = Vec::with_capacity(passwords.len()); - for MasNewUserPassword { - user_password_id, - user_id, - hashed_password, - created_at, - } in passwords - { - user_password_ids.push(user_password_id); - user_ids.push(user_id.get()); - hashed_passwords.push(hashed_password); - created_ats.push(created_at); - versions.push(MIGRATED_PASSWORD_VERSION.into()); - } + self.writer_pool + .spawn_with_connection(move |conn| { + Box::pin(async move { + MasNewUserPassword::write_batch(conn, passwords).await?; - sqlx::query!( - r#" - INSERT INTO syn2mas__user_passwords - (user_password_id, user_id, hashed_password, created_at, version) - SELECT * FROM UNNEST($1::UUID[], $2::UUID[], $3::TEXT[], $4::TIMESTAMP WITH TIME ZONE[], $5::INTEGER[]) - "#, - &user_password_ids[..], - &user_ids[..], - &hashed_passwords[..], - &created_ats[..], - &versions[..], - ).execute(&mut *conn).await.into_database("writing users to MAS")?; - - Ok(()) - })).boxed() + Ok(()) + }) + }) + .boxed() } #[tracing::instrument(skip_all, level = Level::DEBUG)] From 86ff994fb39eff60e03b2b8f237f9ffc31a7ff10 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Tue, 22 Apr 2025 13:21:07 +0100 Subject: [PATCH 13/35] Remove reference to unsupported aws_ses email transport --- docs/reference/configuration.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 30dbbfca9..81a62f6b1 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -583,10 +583,6 @@ email: # Send emails by calling a local sendmail binary #transport: sendmail #command: /usr/sbin/sendmail - - # Send emails through the AWS SESv2 API - # This uses the AWS SDK, so the usual AWS environment variables are supported - #transport: aws_ses ``` ### `upstream_oauth2` From 2450e2e480f6ba49b9c060aab2640f1d604f6b02 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 22 Apr 2025 14:25:13 +0200 Subject: [PATCH 14/35] syn2mas: implement WriteBatch for MasNewEmailThreepid --- ...035e9a087ff27b06e804464a432d93e5a25f1.json | 17 ++++ ...a853a8a7efccdc20b968d99d8c18deda8dd00.json | 17 ---- crates/syn2mas/src/mas_writer/mod.rs | 80 +++++++++++-------- 3 files changed, 62 insertions(+), 52 deletions(-) create mode 100644 crates/syn2mas/.sqlx/query-08ad2855f0baaaed9d6af23c8bf035e9a087ff27b06e804464a432d93e5a25f1.json delete mode 100644 crates/syn2mas/.sqlx/query-dfbd462f7874d3dae551f2a0328a853a8a7efccdc20b968d99d8c18deda8dd00.json diff --git a/crates/syn2mas/.sqlx/query-08ad2855f0baaaed9d6af23c8bf035e9a087ff27b06e804464a432d93e5a25f1.json b/crates/syn2mas/.sqlx/query-08ad2855f0baaaed9d6af23c8bf035e9a087ff27b06e804464a432d93e5a25f1.json new file mode 100644 index 000000000..545389cb6 --- /dev/null +++ b/crates/syn2mas/.sqlx/query-08ad2855f0baaaed9d6af23c8bf035e9a087ff27b06e804464a432d93e5a25f1.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO syn2mas__user_emails\n (user_email_id, user_id, email, created_at, confirmed_at)\n SELECT * FROM UNNEST($1::UUID[], $2::UUID[], $3::TEXT[], $4::TIMESTAMP WITH TIME ZONE[], $4::TIMESTAMP WITH TIME ZONE[])\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray", + "UuidArray", + "TextArray", + "TimestamptzArray" + ] + }, + "nullable": [] + }, + "hash": "08ad2855f0baaaed9d6af23c8bf035e9a087ff27b06e804464a432d93e5a25f1" +} diff --git a/crates/syn2mas/.sqlx/query-dfbd462f7874d3dae551f2a0328a853a8a7efccdc20b968d99d8c18deda8dd00.json b/crates/syn2mas/.sqlx/query-dfbd462f7874d3dae551f2a0328a853a8a7efccdc20b968d99d8c18deda8dd00.json deleted file mode 100644 index cf89130f9..000000000 --- a/crates/syn2mas/.sqlx/query-dfbd462f7874d3dae551f2a0328a853a8a7efccdc20b968d99d8c18deda8dd00.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO syn2mas__user_emails\n (user_email_id, user_id, email, created_at, confirmed_at)\n SELECT * FROM UNNEST($1::UUID[], $2::UUID[], $3::TEXT[], $4::TIMESTAMP WITH TIME ZONE[], $4::TIMESTAMP WITH TIME ZONE[])\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "UuidArray", - "UuidArray", - "TextArray", - "TimestamptzArray" - ] - }, - "nullable": [] - }, - "hash": "dfbd462f7874d3dae551f2a0328a853a8a7efccdc20b968d99d8c18deda8dd00" -} diff --git a/crates/syn2mas/src/mas_writer/mod.rs b/crates/syn2mas/src/mas_writer/mod.rs index 866b615e9..fe2391ae6 100644 --- a/crates/syn2mas/src/mas_writer/mod.rs +++ b/crates/syn2mas/src/mas_writer/mod.rs @@ -386,6 +386,44 @@ pub struct MasNewEmailThreepid { pub created_at: DateTime, } +impl WriteBatch for MasNewEmailThreepid { + async fn write_batch(conn: &mut PgConnection, batch: Vec) -> Result<(), Error> { + let mut user_email_ids: Vec = Vec::with_capacity(batch.len()); + let mut user_ids: Vec = Vec::with_capacity(batch.len()); + let mut emails: Vec = Vec::with_capacity(batch.len()); + let mut created_ats: Vec> = Vec::with_capacity(batch.len()); + + for MasNewEmailThreepid { + user_email_id, + user_id, + email, + created_at, + } in batch + { + user_email_ids.push(user_email_id); + user_ids.push(user_id.get()); + emails.push(email); + created_ats.push(created_at); + } + + // `confirmed_at` is going to get removed in a future MAS release, + // so just populate with `created_at` + sqlx::query!( + r#" + INSERT INTO syn2mas__user_emails + (user_email_id, user_id, email, created_at, confirmed_at) + SELECT * FROM UNNEST($1::UUID[], $2::UUID[], $3::TEXT[], $4::TIMESTAMP WITH TIME ZONE[], $4::TIMESTAMP WITH TIME ZONE[]) + "#, + &user_email_ids[..], + &user_ids[..], + &emails[..], + &created_ats[..], + ).execute(&mut *conn).await.into_database("writing emails to MAS")?; + + Ok(()) + } +} + pub struct MasNewUnsupportedThreepid { pub user_id: NonNilUuid, pub medium: String, @@ -781,43 +819,15 @@ impl MasWriter { &mut self, threepids: Vec, ) -> BoxFuture<'_, Result<(), Error>> { - self.writer_pool.spawn_with_connection(move |conn| { - Box::pin(async move { - let mut user_email_ids: Vec = Vec::with_capacity(threepids.len()); - let mut user_ids: Vec = Vec::with_capacity(threepids.len()); - let mut emails: Vec = Vec::with_capacity(threepids.len()); - let mut created_ats: Vec> = Vec::with_capacity(threepids.len()); + self.writer_pool + .spawn_with_connection(move |conn| { + Box::pin(async move { + MasNewEmailThreepid::write_batch(conn, threepids).await?; - for MasNewEmailThreepid { - user_email_id, - user_id, - email, - created_at, - } in threepids - { - user_email_ids.push(user_email_id); - user_ids.push(user_id.get()); - emails.push(email); - created_ats.push(created_at); - } - - // `confirmed_at` is going to get removed in a future MAS release, - // so just populate with `created_at` - sqlx::query!( - r#" - INSERT INTO syn2mas__user_emails - (user_email_id, user_id, email, created_at, confirmed_at) - SELECT * FROM UNNEST($1::UUID[], $2::UUID[], $3::TEXT[], $4::TIMESTAMP WITH TIME ZONE[], $4::TIMESTAMP WITH TIME ZONE[]) - "#, - &user_email_ids[..], - &user_ids[..], - &emails[..], - &created_ats[..], - ).execute(&mut *conn).await.into_database("writing emails to MAS")?; - - Ok(()) + Ok(()) + }) }) - }).boxed() + .boxed() } #[tracing::instrument(skip_all, level = Level::DEBUG)] From 028a993dc6b309c1867e38942e6d30263b265371 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 22 Apr 2025 14:30:51 +0200 Subject: [PATCH 15/35] syn2mas: implement WriteBatch for MasNewUnsupportedThreepid --- ...a9cd62ac3c9e58155882858c6056e2ef6c30d.json | 17 ++++ ...b5b37ab50db3505712c35610b822cda322b5b.json | 17 ---- crates/syn2mas/src/mas_writer/mod.rs | 79 +++++++++++-------- 3 files changed, 63 insertions(+), 50 deletions(-) create mode 100644 crates/syn2mas/.sqlx/query-204cf4811150a7fdeafa9373647a9cd62ac3c9e58155882858c6056e2ef6c30d.json delete mode 100644 crates/syn2mas/.sqlx/query-b11590549fdd4cdcd36c937a353b5b37ab50db3505712c35610b822cda322b5b.json diff --git a/crates/syn2mas/.sqlx/query-204cf4811150a7fdeafa9373647a9cd62ac3c9e58155882858c6056e2ef6c30d.json b/crates/syn2mas/.sqlx/query-204cf4811150a7fdeafa9373647a9cd62ac3c9e58155882858c6056e2ef6c30d.json new file mode 100644 index 000000000..464dd9007 --- /dev/null +++ b/crates/syn2mas/.sqlx/query-204cf4811150a7fdeafa9373647a9cd62ac3c9e58155882858c6056e2ef6c30d.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO syn2mas__user_unsupported_third_party_ids\n (user_id, medium, address, created_at)\n SELECT * FROM UNNEST($1::UUID[], $2::TEXT[], $3::TEXT[], $4::TIMESTAMP WITH TIME ZONE[])\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray", + "TextArray", + "TextArray", + "TimestamptzArray" + ] + }, + "nullable": [] + }, + "hash": "204cf4811150a7fdeafa9373647a9cd62ac3c9e58155882858c6056e2ef6c30d" +} diff --git a/crates/syn2mas/.sqlx/query-b11590549fdd4cdcd36c937a353b5b37ab50db3505712c35610b822cda322b5b.json b/crates/syn2mas/.sqlx/query-b11590549fdd4cdcd36c937a353b5b37ab50db3505712c35610b822cda322b5b.json deleted file mode 100644 index b44dfc605..000000000 --- a/crates/syn2mas/.sqlx/query-b11590549fdd4cdcd36c937a353b5b37ab50db3505712c35610b822cda322b5b.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO syn2mas__user_unsupported_third_party_ids\n (user_id, medium, address, created_at)\n SELECT * FROM UNNEST($1::UUID[], $2::TEXT[], $3::TEXT[], $4::TIMESTAMP WITH TIME ZONE[])\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "UuidArray", - "TextArray", - "TextArray", - "TimestamptzArray" - ] - }, - "nullable": [] - }, - "hash": "b11590549fdd4cdcd36c937a353b5b37ab50db3505712c35610b822cda322b5b" -} diff --git a/crates/syn2mas/src/mas_writer/mod.rs b/crates/syn2mas/src/mas_writer/mod.rs index fe2391ae6..cbb4ad3b0 100644 --- a/crates/syn2mas/src/mas_writer/mod.rs +++ b/crates/syn2mas/src/mas_writer/mod.rs @@ -431,6 +431,45 @@ pub struct MasNewUnsupportedThreepid { pub created_at: DateTime, } +impl WriteBatch for MasNewUnsupportedThreepid { + async fn write_batch(conn: &mut PgConnection, batch: Vec) -> Result<(), Error> { + let mut user_ids: Vec = Vec::with_capacity(batch.len()); + let mut mediums: Vec = Vec::with_capacity(batch.len()); + let mut addresses: Vec = Vec::with_capacity(batch.len()); + let mut created_ats: Vec> = Vec::with_capacity(batch.len()); + + for MasNewUnsupportedThreepid { + user_id, + medium, + address, + created_at, + } in batch + { + user_ids.push(user_id.get()); + mediums.push(medium); + addresses.push(address); + created_ats.push(created_at); + } + + sqlx::query!( + r#" + INSERT INTO syn2mas__user_unsupported_third_party_ids + (user_id, medium, address, created_at) + SELECT * FROM UNNEST($1::UUID[], $2::TEXT[], $3::TEXT[], $4::TIMESTAMP WITH TIME ZONE[]) + "#, + &user_ids[..], + &mediums[..], + &addresses[..], + &created_ats[..], + ) + .execute(&mut *conn) + .await + .into_database("writing unsupported threepids to MAS")?; + + Ok(()) + } +} + pub struct MasNewUpstreamOauthLink { pub link_id: Uuid, pub user_id: NonNilUuid, @@ -835,41 +874,15 @@ impl MasWriter { &mut self, threepids: Vec, ) -> BoxFuture<'_, Result<(), Error>> { - self.writer_pool.spawn_with_connection(move |conn| { - Box::pin(async move { - let mut user_ids: Vec = Vec::with_capacity(threepids.len()); - let mut mediums: Vec = Vec::with_capacity(threepids.len()); - let mut addresses: Vec = Vec::with_capacity(threepids.len()); - let mut created_ats: Vec> = Vec::with_capacity(threepids.len()); + self.writer_pool + .spawn_with_connection(move |conn| { + Box::pin(async move { + MasNewUnsupportedThreepid::write_batch(conn, threepids).await?; - for MasNewUnsupportedThreepid { - user_id, - medium, - address, - created_at, - } in threepids - { - user_ids.push(user_id.get()); - mediums.push(medium); - addresses.push(address); - created_ats.push(created_at); - } - - sqlx::query!( - r#" - INSERT INTO syn2mas__user_unsupported_third_party_ids - (user_id, medium, address, created_at) - SELECT * FROM UNNEST($1::UUID[], $2::TEXT[], $3::TEXT[], $4::TIMESTAMP WITH TIME ZONE[]) - "#, - &user_ids[..], - &mediums[..], - &addresses[..], - &created_ats[..], - ).execute(&mut *conn).await.into_database("writing unsupported threepids to MAS")?; - - Ok(()) + Ok(()) + }) }) - }).boxed() + .boxed() } #[tracing::instrument(skip_all, level = Level::DEBUG)] From 07536cbd3b5cb8b9d9be4b43216b61773712edfa Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 22 Apr 2025 14:33:16 +0200 Subject: [PATCH 16/35] syn2mas: implement WriteBatch for MasNewUpstreamOauthLink --- ...110039b9a4a0425fd566e401f56ea215de0dd.json | 18 ++++ ...096c848ae87c43b6430246ef3b6a1dc6a7a32.json | 18 ---- crates/syn2mas/src/mas_writer/mod.rs | 84 +++++++++++-------- 3 files changed, 65 insertions(+), 55 deletions(-) create mode 100644 crates/syn2mas/.sqlx/query-026adeffc646b41ebc096bb874d110039b9a4a0425fd566e401f56ea215de0dd.json delete mode 100644 crates/syn2mas/.sqlx/query-d79fd99ebed9033711f96113005096c848ae87c43b6430246ef3b6a1dc6a7a32.json diff --git a/crates/syn2mas/.sqlx/query-026adeffc646b41ebc096bb874d110039b9a4a0425fd566e401f56ea215de0dd.json b/crates/syn2mas/.sqlx/query-026adeffc646b41ebc096bb874d110039b9a4a0425fd566e401f56ea215de0dd.json new file mode 100644 index 000000000..fa5f442ed --- /dev/null +++ b/crates/syn2mas/.sqlx/query-026adeffc646b41ebc096bb874d110039b9a4a0425fd566e401f56ea215de0dd.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO syn2mas__upstream_oauth_links\n (upstream_oauth_link_id, user_id, upstream_oauth_provider_id, subject, created_at)\n SELECT * FROM UNNEST($1::UUID[], $2::UUID[], $3::UUID[], $4::TEXT[], $5::TIMESTAMP WITH TIME ZONE[])\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray", + "UuidArray", + "UuidArray", + "TextArray", + "TimestamptzArray" + ] + }, + "nullable": [] + }, + "hash": "026adeffc646b41ebc096bb874d110039b9a4a0425fd566e401f56ea215de0dd" +} diff --git a/crates/syn2mas/.sqlx/query-d79fd99ebed9033711f96113005096c848ae87c43b6430246ef3b6a1dc6a7a32.json b/crates/syn2mas/.sqlx/query-d79fd99ebed9033711f96113005096c848ae87c43b6430246ef3b6a1dc6a7a32.json deleted file mode 100644 index f6ac32781..000000000 --- a/crates/syn2mas/.sqlx/query-d79fd99ebed9033711f96113005096c848ae87c43b6430246ef3b6a1dc6a7a32.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO syn2mas__upstream_oauth_links\n (upstream_oauth_link_id, user_id, upstream_oauth_provider_id, subject, created_at)\n SELECT * FROM UNNEST($1::UUID[], $2::UUID[], $3::UUID[], $4::TEXT[], $5::TIMESTAMP WITH TIME ZONE[])\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "UuidArray", - "UuidArray", - "UuidArray", - "TextArray", - "TimestamptzArray" - ] - }, - "nullable": [] - }, - "hash": "d79fd99ebed9033711f96113005096c848ae87c43b6430246ef3b6a1dc6a7a32" -} diff --git a/crates/syn2mas/src/mas_writer/mod.rs b/crates/syn2mas/src/mas_writer/mod.rs index cbb4ad3b0..0f7ccf715 100644 --- a/crates/syn2mas/src/mas_writer/mod.rs +++ b/crates/syn2mas/src/mas_writer/mod.rs @@ -478,6 +478,46 @@ pub struct MasNewUpstreamOauthLink { pub created_at: DateTime, } +impl WriteBatch for MasNewUpstreamOauthLink { + async fn write_batch(conn: &mut PgConnection, batch: Vec) -> Result<(), Error> { + let mut link_ids: Vec = Vec::with_capacity(batch.len()); + let mut user_ids: Vec = Vec::with_capacity(batch.len()); + let mut upstream_provider_ids: Vec = Vec::with_capacity(batch.len()); + let mut subjects: Vec = Vec::with_capacity(batch.len()); + let mut created_ats: Vec> = Vec::with_capacity(batch.len()); + + for MasNewUpstreamOauthLink { + link_id, + user_id, + upstream_provider_id, + subject, + created_at, + } in batch + { + link_ids.push(link_id); + user_ids.push(user_id.get()); + upstream_provider_ids.push(upstream_provider_id); + subjects.push(subject); + created_ats.push(created_at); + } + + sqlx::query!( + r#" + INSERT INTO syn2mas__upstream_oauth_links + (upstream_oauth_link_id, user_id, upstream_oauth_provider_id, subject, created_at) + SELECT * FROM UNNEST($1::UUID[], $2::UUID[], $3::UUID[], $4::TEXT[], $5::TIMESTAMP WITH TIME ZONE[]) + "#, + &link_ids[..], + &user_ids[..], + &upstream_provider_ids[..], + &subjects[..], + &created_ats[..], + ).execute(&mut *conn).await.into_database("writing unsupported threepids to MAS")?; + + Ok(()) + } +} + pub struct MasNewCompatSession { pub session_id: Uuid, pub user_id: NonNilUuid, @@ -890,45 +930,15 @@ impl MasWriter { &mut self, links: Vec, ) -> BoxFuture<'_, Result<(), Error>> { - self.writer_pool.spawn_with_connection(move |conn| { - Box::pin(async move { - let mut link_ids: Vec = Vec::with_capacity(links.len()); - let mut user_ids: Vec = Vec::with_capacity(links.len()); - let mut upstream_provider_ids: Vec = Vec::with_capacity(links.len()); - let mut subjects: Vec = Vec::with_capacity(links.len()); - let mut created_ats: Vec> = Vec::with_capacity(links.len()); + self.writer_pool + .spawn_with_connection(move |conn| { + Box::pin(async move { + MasNewUpstreamOauthLink::write_batch(conn, links).await?; - for MasNewUpstreamOauthLink { - link_id, - user_id, - upstream_provider_id, - subject, - created_at, - } in links - { - link_ids.push(link_id); - user_ids.push(user_id.get()); - upstream_provider_ids.push(upstream_provider_id); - subjects.push(subject); - created_ats.push(created_at); - } - - sqlx::query!( - r#" - INSERT INTO syn2mas__upstream_oauth_links - (upstream_oauth_link_id, user_id, upstream_oauth_provider_id, subject, created_at) - SELECT * FROM UNNEST($1::UUID[], $2::UUID[], $3::UUID[], $4::TEXT[], $5::TIMESTAMP WITH TIME ZONE[]) - "#, - &link_ids[..], - &user_ids[..], - &upstream_provider_ids[..], - &subjects[..], - &created_ats[..], - ).execute(&mut *conn).await.into_database("writing unsupported threepids to MAS")?; - - Ok(()) + Ok(()) + }) }) - }).boxed() + .boxed() } #[tracing::instrument(skip_all, level = Level::DEBUG)] From 86000613ac2f0a578cdf1dd99b6ee51104f133a5 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 22 Apr 2025 14:37:32 +0200 Subject: [PATCH 17/35] syn2mas: implement WriteBatch for MasNewCompatSession --- ...233e5c9aabfdae1f0ee9b77c909b2bb2f3e25.json | 22 +++ ...e36f4ef03e1224a0a89a921e5a3d398a5d35c.json | 22 --- crates/syn2mas/src/mas_writer/mod.rs | 134 +++++++++--------- 3 files changed, 92 insertions(+), 86 deletions(-) create mode 100644 crates/syn2mas/.sqlx/query-09db58b250c20ab9d1701653165233e5c9aabfdae1f0ee9b77c909b2bb2f3e25.json delete mode 100644 crates/syn2mas/.sqlx/query-396c97dbfbc932c73301daa7376e36f4ef03e1224a0a89a921e5a3d398a5d35c.json diff --git a/crates/syn2mas/.sqlx/query-09db58b250c20ab9d1701653165233e5c9aabfdae1f0ee9b77c909b2bb2f3e25.json b/crates/syn2mas/.sqlx/query-09db58b250c20ab9d1701653165233e5c9aabfdae1f0ee9b77c909b2bb2f3e25.json new file mode 100644 index 000000000..97e8a07a0 --- /dev/null +++ b/crates/syn2mas/.sqlx/query-09db58b250c20ab9d1701653165233e5c9aabfdae1f0ee9b77c909b2bb2f3e25.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO syn2mas__compat_sessions (\n compat_session_id, user_id,\n device_id, human_name,\n created_at, is_synapse_admin,\n last_active_at, last_active_ip,\n user_agent)\n SELECT * FROM UNNEST(\n $1::UUID[], $2::UUID[],\n $3::TEXT[], $4::TEXT[],\n $5::TIMESTAMP WITH TIME ZONE[], $6::BOOLEAN[],\n $7::TIMESTAMP WITH TIME ZONE[], $8::INET[],\n $9::TEXT[])\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray", + "UuidArray", + "TextArray", + "TextArray", + "TimestamptzArray", + "BoolArray", + "TimestamptzArray", + "InetArray", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "09db58b250c20ab9d1701653165233e5c9aabfdae1f0ee9b77c909b2bb2f3e25" +} diff --git a/crates/syn2mas/.sqlx/query-396c97dbfbc932c73301daa7376e36f4ef03e1224a0a89a921e5a3d398a5d35c.json b/crates/syn2mas/.sqlx/query-396c97dbfbc932c73301daa7376e36f4ef03e1224a0a89a921e5a3d398a5d35c.json deleted file mode 100644 index 521e4facd..000000000 --- a/crates/syn2mas/.sqlx/query-396c97dbfbc932c73301daa7376e36f4ef03e1224a0a89a921e5a3d398a5d35c.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO syn2mas__compat_sessions (\n compat_session_id, user_id,\n device_id, human_name,\n created_at, is_synapse_admin,\n last_active_at, last_active_ip,\n user_agent)\n SELECT * FROM UNNEST(\n $1::UUID[], $2::UUID[],\n $3::TEXT[], $4::TEXT[],\n $5::TIMESTAMP WITH TIME ZONE[], $6::BOOLEAN[],\n $7::TIMESTAMP WITH TIME ZONE[], $8::INET[],\n $9::TEXT[])\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "UuidArray", - "UuidArray", - "TextArray", - "TextArray", - "TimestamptzArray", - "BoolArray", - "TimestamptzArray", - "InetArray", - "TextArray" - ] - }, - "nullable": [] - }, - "hash": "396c97dbfbc932c73301daa7376e36f4ef03e1224a0a89a921e5a3d398a5d35c" -} diff --git a/crates/syn2mas/src/mas_writer/mod.rs b/crates/syn2mas/src/mas_writer/mod.rs index 0f7ccf715..b0b8250c1 100644 --- a/crates/syn2mas/src/mas_writer/mod.rs +++ b/crates/syn2mas/src/mas_writer/mod.rs @@ -530,6 +530,75 @@ pub struct MasNewCompatSession { pub user_agent: Option, } +impl WriteBatch for MasNewCompatSession { + async fn write_batch(conn: &mut PgConnection, batch: Vec) -> Result<(), Error> { + let mut session_ids: Vec = Vec::with_capacity(batch.len()); + let mut user_ids: Vec = Vec::with_capacity(batch.len()); + let mut device_ids: Vec> = Vec::with_capacity(batch.len()); + let mut human_names: Vec> = Vec::with_capacity(batch.len()); + let mut created_ats: Vec> = Vec::with_capacity(batch.len()); + let mut is_synapse_admins: Vec = Vec::with_capacity(batch.len()); + let mut last_active_ats: Vec>> = Vec::with_capacity(batch.len()); + let mut last_active_ips: Vec> = Vec::with_capacity(batch.len()); + let mut user_agents: Vec> = Vec::with_capacity(batch.len()); + + for MasNewCompatSession { + session_id, + user_id, + device_id, + human_name, + created_at, + is_synapse_admin, + last_active_at, + last_active_ip, + user_agent, + } in batch + { + session_ids.push(session_id); + user_ids.push(user_id.get()); + device_ids.push(device_id); + human_names.push(human_name); + created_ats.push(created_at); + is_synapse_admins.push(is_synapse_admin); + last_active_ats.push(last_active_at); + last_active_ips.push(last_active_ip); + user_agents.push(user_agent); + } + + sqlx::query!( + r#" + INSERT INTO syn2mas__compat_sessions ( + compat_session_id, user_id, + device_id, human_name, + created_at, is_synapse_admin, + last_active_at, last_active_ip, + user_agent) + SELECT * FROM UNNEST( + $1::UUID[], $2::UUID[], + $3::TEXT[], $4::TEXT[], + $5::TIMESTAMP WITH TIME ZONE[], $6::BOOLEAN[], + $7::TIMESTAMP WITH TIME ZONE[], $8::INET[], + $9::TEXT[]) + "#, + &session_ids[..], + &user_ids[..], + &device_ids[..] as &[Option], + &human_names[..] as &[Option], + &created_ats[..], + &is_synapse_admins[..], + // We need to override the typing for arrays of optionals (sqlx limitation) + &last_active_ats[..] as &[Option>], + &last_active_ips[..] as &[Option], + &user_agents[..] as &[Option], + ) + .execute(&mut *conn) + .await + .into_database("writing compat sessions to MAS")?; + + Ok(()) + } +} + pub struct MasNewCompatAccessToken { pub token_id: Uuid, pub session_id: Uuid, @@ -949,70 +1018,7 @@ impl MasWriter { self.writer_pool .spawn_with_connection(move |conn| { Box::pin(async move { - let mut session_ids: Vec = Vec::with_capacity(sessions.len()); - let mut user_ids: Vec = Vec::with_capacity(sessions.len()); - let mut device_ids: Vec> = Vec::with_capacity(sessions.len()); - let mut human_names: Vec> = Vec::with_capacity(sessions.len()); - let mut created_ats: Vec> = Vec::with_capacity(sessions.len()); - let mut is_synapse_admins: Vec = Vec::with_capacity(sessions.len()); - let mut last_active_ats: Vec>> = - Vec::with_capacity(sessions.len()); - let mut last_active_ips: Vec> = - Vec::with_capacity(sessions.len()); - let mut user_agents: Vec> = Vec::with_capacity(sessions.len()); - - for MasNewCompatSession { - session_id, - user_id, - device_id, - human_name, - created_at, - is_synapse_admin, - last_active_at, - last_active_ip, - user_agent, - } in sessions - { - session_ids.push(session_id); - user_ids.push(user_id.get()); - device_ids.push(device_id); - human_names.push(human_name); - created_ats.push(created_at); - is_synapse_admins.push(is_synapse_admin); - last_active_ats.push(last_active_at); - last_active_ips.push(last_active_ip); - user_agents.push(user_agent); - } - - sqlx::query!( - r#" - INSERT INTO syn2mas__compat_sessions ( - compat_session_id, user_id, - device_id, human_name, - created_at, is_synapse_admin, - last_active_at, last_active_ip, - user_agent) - SELECT * FROM UNNEST( - $1::UUID[], $2::UUID[], - $3::TEXT[], $4::TEXT[], - $5::TIMESTAMP WITH TIME ZONE[], $6::BOOLEAN[], - $7::TIMESTAMP WITH TIME ZONE[], $8::INET[], - $9::TEXT[]) - "#, - &session_ids[..], - &user_ids[..], - &device_ids[..] as &[Option], - &human_names[..] as &[Option], - &created_ats[..], - &is_synapse_admins[..], - // We need to override the typing for arrays of optionals (sqlx limitation) - &last_active_ats[..] as &[Option>], - &last_active_ips[..] as &[Option], - &user_agents[..] as &[Option], - ) - .execute(&mut *conn) - .await - .into_database("writing compat sessions to MAS")?; + MasNewCompatSession::write_batch(conn, sessions).await?; Ok(()) }) From aa2e2825fe87649a89cbb6edb245d334eac44662 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 22 Apr 2025 14:39:59 +0200 Subject: [PATCH 18/35] syn2mas: implement WriteBatch for MasNewCompatAccessToken --- ...3c67bf00fd3e411f769b9f25dec27428489ed.json | 18 ++++ ...ad976c3a0ff238046872b17d3f412beda62c7.json | 18 ---- crates/syn2mas/src/mas_writer/mod.rs | 101 ++++++++++-------- 3 files changed, 72 insertions(+), 65 deletions(-) create mode 100644 crates/syn2mas/.sqlx/query-86b2b02fbb6350100d794e4d0fa3c67bf00fd3e411f769b9f25dec27428489ed.json delete mode 100644 crates/syn2mas/.sqlx/query-d55adc78a0c222e19688e6ac810ad976c3a0ff238046872b17d3f412beda62c7.json diff --git a/crates/syn2mas/.sqlx/query-86b2b02fbb6350100d794e4d0fa3c67bf00fd3e411f769b9f25dec27428489ed.json b/crates/syn2mas/.sqlx/query-86b2b02fbb6350100d794e4d0fa3c67bf00fd3e411f769b9f25dec27428489ed.json new file mode 100644 index 000000000..dd8a8e306 --- /dev/null +++ b/crates/syn2mas/.sqlx/query-86b2b02fbb6350100d794e4d0fa3c67bf00fd3e411f769b9f25dec27428489ed.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO syn2mas__compat_access_tokens (\n compat_access_token_id,\n compat_session_id,\n access_token,\n created_at,\n expires_at)\n SELECT * FROM UNNEST(\n $1::UUID[],\n $2::UUID[],\n $3::TEXT[],\n $4::TIMESTAMP WITH TIME ZONE[],\n $5::TIMESTAMP WITH TIME ZONE[])\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray", + "UuidArray", + "TextArray", + "TimestamptzArray", + "TimestamptzArray" + ] + }, + "nullable": [] + }, + "hash": "86b2b02fbb6350100d794e4d0fa3c67bf00fd3e411f769b9f25dec27428489ed" +} diff --git a/crates/syn2mas/.sqlx/query-d55adc78a0c222e19688e6ac810ad976c3a0ff238046872b17d3f412beda62c7.json b/crates/syn2mas/.sqlx/query-d55adc78a0c222e19688e6ac810ad976c3a0ff238046872b17d3f412beda62c7.json deleted file mode 100644 index eb406d23b..000000000 --- a/crates/syn2mas/.sqlx/query-d55adc78a0c222e19688e6ac810ad976c3a0ff238046872b17d3f412beda62c7.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO syn2mas__compat_access_tokens (\n compat_access_token_id,\n compat_session_id,\n access_token,\n created_at,\n expires_at)\n SELECT * FROM UNNEST(\n $1::UUID[],\n $2::UUID[],\n $3::TEXT[],\n $4::TIMESTAMP WITH TIME ZONE[],\n $5::TIMESTAMP WITH TIME ZONE[])\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "UuidArray", - "UuidArray", - "TextArray", - "TimestamptzArray", - "TimestamptzArray" - ] - }, - "nullable": [] - }, - "hash": "d55adc78a0c222e19688e6ac810ad976c3a0ff238046872b17d3f412beda62c7" -} diff --git a/crates/syn2mas/src/mas_writer/mod.rs b/crates/syn2mas/src/mas_writer/mod.rs index b0b8250c1..182a4795b 100644 --- a/crates/syn2mas/src/mas_writer/mod.rs +++ b/crates/syn2mas/src/mas_writer/mod.rs @@ -607,6 +607,59 @@ pub struct MasNewCompatAccessToken { pub expires_at: Option>, } +impl WriteBatch for MasNewCompatAccessToken { + async fn write_batch(conn: &mut PgConnection, batch: Vec) -> Result<(), Error> { + let mut token_ids: Vec = Vec::with_capacity(batch.len()); + let mut session_ids: Vec = Vec::with_capacity(batch.len()); + let mut access_tokens: Vec = Vec::with_capacity(batch.len()); + let mut created_ats: Vec> = Vec::with_capacity(batch.len()); + let mut expires_ats: Vec>> = Vec::with_capacity(batch.len()); + + for MasNewCompatAccessToken { + token_id, + session_id, + access_token, + created_at, + expires_at, + } in batch + { + token_ids.push(token_id); + session_ids.push(session_id); + access_tokens.push(access_token); + created_ats.push(created_at); + expires_ats.push(expires_at); + } + + sqlx::query!( + r#" + INSERT INTO syn2mas__compat_access_tokens ( + compat_access_token_id, + compat_session_id, + access_token, + created_at, + expires_at) + SELECT * FROM UNNEST( + $1::UUID[], + $2::UUID[], + $3::TEXT[], + $4::TIMESTAMP WITH TIME ZONE[], + $5::TIMESTAMP WITH TIME ZONE[]) + "#, + &token_ids[..], + &session_ids[..], + &access_tokens[..], + &created_ats[..], + // We need to override the typing for arrays of optionals (sqlx limitation) + &expires_ats[..] as &[Option>], + ) + .execute(&mut *conn) + .await + .into_database("writing compat access tokens to MAS")?; + + Ok(()) + } +} + pub struct MasNewCompatRefreshToken { pub refresh_token_id: Uuid, pub session_id: Uuid, @@ -1034,53 +1087,7 @@ impl MasWriter { self.writer_pool .spawn_with_connection(move |conn| { Box::pin(async move { - let mut token_ids: Vec = Vec::with_capacity(tokens.len()); - let mut session_ids: Vec = Vec::with_capacity(tokens.len()); - let mut access_tokens: Vec = Vec::with_capacity(tokens.len()); - let mut created_ats: Vec> = Vec::with_capacity(tokens.len()); - let mut expires_ats: Vec>> = - Vec::with_capacity(tokens.len()); - - for MasNewCompatAccessToken { - token_id, - session_id, - access_token, - created_at, - expires_at, - } in tokens - { - token_ids.push(token_id); - session_ids.push(session_id); - access_tokens.push(access_token); - created_ats.push(created_at); - expires_ats.push(expires_at); - } - - sqlx::query!( - r#" - INSERT INTO syn2mas__compat_access_tokens ( - compat_access_token_id, - compat_session_id, - access_token, - created_at, - expires_at) - SELECT * FROM UNNEST( - $1::UUID[], - $2::UUID[], - $3::TEXT[], - $4::TIMESTAMP WITH TIME ZONE[], - $5::TIMESTAMP WITH TIME ZONE[]) - "#, - &token_ids[..], - &session_ids[..], - &access_tokens[..], - &created_ats[..], - // We need to override the typing for arrays of optionals (sqlx limitation) - &expires_ats[..] as &[Option>], - ) - .execute(&mut *conn) - .await - .into_database("writing compat access tokens to MAS")?; + MasNewCompatAccessToken::write_batch(conn, tokens).await?; Ok(()) }) From d2f1ab08fb779f1214ac288a8a8bea81653bb1bb Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 22 Apr 2025 14:42:52 +0200 Subject: [PATCH 19/35] syn2mas: implement WriteBatch for MasNewCompatRefreshToken --- ...b986b1b4864a778525d0b8b0ad6678aef3e9f.json | 18 ++++ ...e5d8cac3836701fc24922f4f0e8b98d330796.json | 18 ---- crates/syn2mas/src/mas_writer/mod.rs | 99 ++++++++++--------- 3 files changed, 71 insertions(+), 64 deletions(-) create mode 100644 crates/syn2mas/.sqlx/query-1d1004d0fb5939fbf30c1986b80b986b1b4864a778525d0b8b0ad6678aef3e9f.json delete mode 100644 crates/syn2mas/.sqlx/query-88975196c4c174d464b33aa015ce5d8cac3836701fc24922f4f0e8b98d330796.json diff --git a/crates/syn2mas/.sqlx/query-1d1004d0fb5939fbf30c1986b80b986b1b4864a778525d0b8b0ad6678aef3e9f.json b/crates/syn2mas/.sqlx/query-1d1004d0fb5939fbf30c1986b80b986b1b4864a778525d0b8b0ad6678aef3e9f.json new file mode 100644 index 000000000..c65dfb7a4 --- /dev/null +++ b/crates/syn2mas/.sqlx/query-1d1004d0fb5939fbf30c1986b80b986b1b4864a778525d0b8b0ad6678aef3e9f.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO syn2mas__compat_refresh_tokens (\n compat_refresh_token_id,\n compat_session_id,\n compat_access_token_id,\n refresh_token,\n created_at)\n SELECT * FROM UNNEST(\n $1::UUID[],\n $2::UUID[],\n $3::UUID[],\n $4::TEXT[],\n $5::TIMESTAMP WITH TIME ZONE[])\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray", + "UuidArray", + "UuidArray", + "TextArray", + "TimestamptzArray" + ] + }, + "nullable": [] + }, + "hash": "1d1004d0fb5939fbf30c1986b80b986b1b4864a778525d0b8b0ad6678aef3e9f" +} diff --git a/crates/syn2mas/.sqlx/query-88975196c4c174d464b33aa015ce5d8cac3836701fc24922f4f0e8b98d330796.json b/crates/syn2mas/.sqlx/query-88975196c4c174d464b33aa015ce5d8cac3836701fc24922f4f0e8b98d330796.json deleted file mode 100644 index cb251624d..000000000 --- a/crates/syn2mas/.sqlx/query-88975196c4c174d464b33aa015ce5d8cac3836701fc24922f4f0e8b98d330796.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO syn2mas__compat_refresh_tokens (\n compat_refresh_token_id,\n compat_session_id,\n compat_access_token_id,\n refresh_token,\n created_at)\n SELECT * FROM UNNEST(\n $1::UUID[],\n $2::UUID[],\n $3::UUID[],\n $4::TEXT[],\n $5::TIMESTAMP WITH TIME ZONE[])\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "UuidArray", - "UuidArray", - "UuidArray", - "TextArray", - "TimestamptzArray" - ] - }, - "nullable": [] - }, - "hash": "88975196c4c174d464b33aa015ce5d8cac3836701fc24922f4f0e8b98d330796" -} diff --git a/crates/syn2mas/src/mas_writer/mod.rs b/crates/syn2mas/src/mas_writer/mod.rs index 182a4795b..0012ef04a 100644 --- a/crates/syn2mas/src/mas_writer/mod.rs +++ b/crates/syn2mas/src/mas_writer/mod.rs @@ -668,6 +668,58 @@ pub struct MasNewCompatRefreshToken { pub created_at: DateTime, } +impl WriteBatch for MasNewCompatRefreshToken { + async fn write_batch(conn: &mut PgConnection, batch: Vec) -> Result<(), Error> { + let mut refresh_token_ids: Vec = Vec::with_capacity(batch.len()); + let mut session_ids: Vec = Vec::with_capacity(batch.len()); + let mut access_token_ids: Vec = Vec::with_capacity(batch.len()); + let mut refresh_tokens: Vec = Vec::with_capacity(batch.len()); + let mut created_ats: Vec> = Vec::with_capacity(batch.len()); + + for MasNewCompatRefreshToken { + refresh_token_id, + session_id, + access_token_id, + refresh_token, + created_at, + } in batch + { + refresh_token_ids.push(refresh_token_id); + session_ids.push(session_id); + access_token_ids.push(access_token_id); + refresh_tokens.push(refresh_token); + created_ats.push(created_at); + } + + sqlx::query!( + r#" + INSERT INTO syn2mas__compat_refresh_tokens ( + compat_refresh_token_id, + compat_session_id, + compat_access_token_id, + refresh_token, + created_at) + SELECT * FROM UNNEST( + $1::UUID[], + $2::UUID[], + $3::UUID[], + $4::TEXT[], + $5::TIMESTAMP WITH TIME ZONE[]) + "#, + &refresh_token_ids[..], + &session_ids[..], + &access_token_ids[..], + &refresh_tokens[..], + &created_ats[..], + ) + .execute(&mut *conn) + .await + .into_database("writing compat refresh tokens to MAS")?; + + Ok(()) + } +} + /// The 'version' of the password hashing scheme used for passwords when they /// are migrated from Synapse to MAS. /// This is version 1, as in the previous syn2mas script. @@ -1103,52 +1155,7 @@ impl MasWriter { self.writer_pool .spawn_with_connection(move |conn| { Box::pin(async move { - let mut refresh_token_ids: Vec = Vec::with_capacity(tokens.len()); - let mut session_ids: Vec = Vec::with_capacity(tokens.len()); - let mut access_token_ids: Vec = Vec::with_capacity(tokens.len()); - let mut refresh_tokens: Vec = Vec::with_capacity(tokens.len()); - let mut created_ats: Vec> = Vec::with_capacity(tokens.len()); - - for MasNewCompatRefreshToken { - refresh_token_id, - session_id, - access_token_id, - refresh_token, - created_at, - } in tokens - { - refresh_token_ids.push(refresh_token_id); - session_ids.push(session_id); - access_token_ids.push(access_token_id); - refresh_tokens.push(refresh_token); - created_ats.push(created_at); - } - - sqlx::query!( - r#" - INSERT INTO syn2mas__compat_refresh_tokens ( - compat_refresh_token_id, - compat_session_id, - compat_access_token_id, - refresh_token, - created_at) - SELECT * FROM UNNEST( - $1::UUID[], - $2::UUID[], - $3::UUID[], - $4::TEXT[], - $5::TIMESTAMP WITH TIME ZONE[]) - "#, - &refresh_token_ids[..], - &session_ids[..], - &access_token_ids[..], - &refresh_tokens[..], - &created_ats[..], - ) - .execute(&mut *conn) - .await - .into_database("writing compat refresh tokens to MAS")?; - + MasNewCompatRefreshToken::write_batch(conn, tokens).await?; Ok(()) }) }) From 47009a8800b85fe9403a0908900cadddad54b5ef Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 22 Apr 2025 15:49:17 +0200 Subject: [PATCH 20/35] syn2mas: make the MasWriteBuffer use the WriteBatch trait --- crates/syn2mas/src/mas_writer/mod.rs | 25 ++++++++++++------------- crates/syn2mas/src/migration.rs | 24 ++++++++++-------------- 2 files changed, 22 insertions(+), 27 deletions(-) diff --git a/crates/syn2mas/src/mas_writer/mod.rs b/crates/syn2mas/src/mas_writer/mod.rs index 0012ef04a..257e07a11 100644 --- a/crates/syn2mas/src/mas_writer/mod.rs +++ b/crates/syn2mas/src/mas_writer/mod.rs @@ -114,7 +114,6 @@ impl WriterConnectionPool { where F: for<'conn> FnOnce(&'conn mut PgConnection) -> BoxFuture<'conn, Result<(), Error>> + Send - + Sync + 'static, { match self.connection_rx.recv().await { @@ -250,11 +249,11 @@ pub struct MasWriter { write_buffer_finish_checker: FinishChecker, } -trait WriteBatch: Sized { +pub trait WriteBatch: Send + Sync + Sized + 'static { fn write_batch( conn: &mut PgConnection, batch: Vec, - ) -> impl Future>; + ) -> impl Future> + Send; } pub struct MasNewUser { @@ -1167,24 +1166,20 @@ impl MasWriter { // database. const WRITE_BUFFER_BATCH_SIZE: usize = 4096; -/// A function that can accept and flush buffers from a `MasWriteBuffer`. -/// Intended uses are the methods on `MasWriter` such as `write_users`. -type WriteBufferFlusher = - for<'a> fn(&'a mut MasWriter, Vec) -> BoxFuture<'a, Result<(), Error>>; - /// A buffer for writing rows to the MAS database. /// Generic over the type of rows. pub struct MasWriteBuffer { rows: Vec, - flusher: WriteBufferFlusher, finish_checker_handle: FinishCheckerHandle, } -impl MasWriteBuffer { - pub fn new(writer: &MasWriter, flusher: WriteBufferFlusher) -> Self { +impl MasWriteBuffer +where + T: WriteBatch, +{ + pub fn new(writer: &MasWriter) -> Self { MasWriteBuffer { rows: Vec::with_capacity(WRITE_BUFFER_BATCH_SIZE), - flusher, finish_checker_handle: writer.write_buffer_finish_checker.handle(), } } @@ -1201,7 +1196,11 @@ impl MasWriteBuffer { } let rows = std::mem::take(&mut self.rows); self.rows.reserve_exact(WRITE_BUFFER_BATCH_SIZE); - (self.flusher)(writer, rows).await?; + writer + .writer_pool + .spawn_with_connection(move |conn| T::write_batch(conn, rows).boxed()) + .boxed() + .await?; Ok(()) } diff --git a/crates/syn2mas/src/migration.rs b/crates/syn2mas/src/migration.rs index 3388ff387..7b34baf62 100644 --- a/crates/syn2mas/src/migration.rs +++ b/crates/syn2mas/src/migration.rs @@ -220,8 +220,8 @@ async fn migrate_users( let mut rng = rand_chacha::ChaChaRng::from_rng(rng).expect("failed to seed rng"); let task = tokio::spawn( async move { - let mut user_buffer = MasWriteBuffer::new(&mas, MasWriter::write_users); - let mut password_buffer = MasWriteBuffer::new(&mas, MasWriter::write_passwords); + let mut user_buffer = MasWriteBuffer::new(&mas); + let mut password_buffer = MasWriteBuffer::new(&mas); while let Some(user) = rx.recv().await { // Handling an edge case: some AS users may have invalid localparts containing @@ -342,9 +342,8 @@ async fn migrate_threepids( let mut rng = rand_chacha::ChaChaRng::from_rng(rng).expect("failed to seed rng"); let task = tokio::spawn( async move { - let mut email_buffer = MasWriteBuffer::new(&mas, MasWriter::write_email_threepids); - let mut unsupported_buffer = - MasWriteBuffer::new(&mas, MasWriter::write_unsupported_threepids); + let mut email_buffer = MasWriteBuffer::new(&mas); + let mut unsupported_buffer = MasWriteBuffer::new(&mas); while let Some(threepid) = rx.recv().await { let SynapseThreepid { @@ -457,7 +456,7 @@ async fn migrate_external_ids( let mut rng = rand_chacha::ChaChaRng::from_rng(rng).expect("failed to seed rng"); let task = tokio::spawn( async move { - let mut write_buffer = MasWriteBuffer::new(&mas, MasWriter::write_upstream_oauth_links); + let mut write_buffer = MasWriteBuffer::new(&mas); while let Some(extid) = rx.recv().await { let SynapseExternalId { @@ -569,7 +568,7 @@ async fn migrate_devices( let mut rng = rand_chacha::ChaChaRng::from_rng(rng).expect("failed to seed rng"); let task = tokio::spawn( async move { - let mut write_buffer = MasWriteBuffer::new(&mas, MasWriter::write_compat_sessions); + let mut write_buffer = MasWriteBuffer::new(&mas); while let Some(device) = rx.recv().await { let SynapseDevice { @@ -704,9 +703,8 @@ async fn migrate_unrefreshable_access_tokens( let mut rng = rand_chacha::ChaChaRng::from_rng(rng).expect("failed to seed rng"); let task = tokio::spawn( async move { - let mut write_buffer = MasWriteBuffer::new(&mas, MasWriter::write_compat_access_tokens); - let mut deviceless_session_write_buffer = - MasWriteBuffer::new(&mas, MasWriter::write_compat_sessions); + let mut write_buffer = MasWriteBuffer::new(&mas); + let mut deviceless_session_write_buffer = MasWriteBuffer::new(&mas); while let Some(token) = rx.recv().await { let SynapseAccessToken { @@ -855,10 +853,8 @@ async fn migrate_refreshable_token_pairs( let now = clock.now(); let task = tokio::spawn( async move { - let mut access_token_write_buffer = - MasWriteBuffer::new(&mas, MasWriter::write_compat_access_tokens); - let mut refresh_token_write_buffer = - MasWriteBuffer::new(&mas, MasWriter::write_compat_refresh_tokens); + let mut access_token_write_buffer = MasWriteBuffer::new(&mas); + let mut refresh_token_write_buffer = MasWriteBuffer::new(&mas); while let Some(token) = rx.recv().await { let SynapseRefreshableTokenPair { From 1056949149484b4f3b29436670dfb458ced15ae4 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 22 Apr 2025 16:30:40 +0200 Subject: [PATCH 21/35] syn2mas: remove the `MasWriter::write_` methods and replaced them in tests --- crates/syn2mas/src/mas_writer/mod.rs | 640 ++++++++++++++------------- 1 file changed, 331 insertions(+), 309 deletions(-) diff --git a/crates/syn2mas/src/mas_writer/mod.rs b/crates/syn2mas/src/mas_writer/mod.rs index 257e07a11..5e3a0f0ef 100644 --- a/crates/syn2mas/src/mas_writer/mod.rs +++ b/crates/syn2mas/src/mas_writer/mod.rs @@ -22,7 +22,7 @@ use sqlx::{Executor, PgConnection, query, query_as}; use thiserror::Error; use thiserror_ext::{Construct, ContextInto}; use tokio::sync::mpsc::{self, Receiver, Sender}; -use tracing::{Instrument, Level, error, info, warn}; +use tracing::{Instrument, error, info, warn}; use uuid::{NonNilUuid, Uuid}; use self::{ @@ -1022,144 +1022,6 @@ impl MasWriter { Ok(conn) } - - /// Write a batch of users to the database. - /// - /// # Errors - /// - /// Errors are returned in the following conditions: - /// - /// - If the database writer connection pool had an error. - #[tracing::instrument(skip_all, level = Level::DEBUG)] - pub fn write_users(&mut self, users: Vec) -> BoxFuture<'_, Result<(), Error>> { - self.writer_pool - .spawn_with_connection(move |conn| { - Box::pin(async move { - MasNewUser::write_batch(conn, users).await?; - - Ok(()) - }) - }) - .boxed() - } - - /// Write a batch of user passwords to the database. - /// - /// # Errors - /// - /// Errors are returned in the following conditions: - /// - /// - If the database writer connection pool had an error. - #[tracing::instrument(skip_all, level = Level::DEBUG)] - pub fn write_passwords( - &mut self, - passwords: Vec, - ) -> BoxFuture<'_, Result<(), Error>> { - self.writer_pool - .spawn_with_connection(move |conn| { - Box::pin(async move { - MasNewUserPassword::write_batch(conn, passwords).await?; - - Ok(()) - }) - }) - .boxed() - } - - #[tracing::instrument(skip_all, level = Level::DEBUG)] - pub fn write_email_threepids( - &mut self, - threepids: Vec, - ) -> BoxFuture<'_, Result<(), Error>> { - self.writer_pool - .spawn_with_connection(move |conn| { - Box::pin(async move { - MasNewEmailThreepid::write_batch(conn, threepids).await?; - - Ok(()) - }) - }) - .boxed() - } - - #[tracing::instrument(skip_all, level = Level::DEBUG)] - pub fn write_unsupported_threepids( - &mut self, - threepids: Vec, - ) -> BoxFuture<'_, Result<(), Error>> { - self.writer_pool - .spawn_with_connection(move |conn| { - Box::pin(async move { - MasNewUnsupportedThreepid::write_batch(conn, threepids).await?; - - Ok(()) - }) - }) - .boxed() - } - - #[tracing::instrument(skip_all, level = Level::DEBUG)] - pub fn write_upstream_oauth_links( - &mut self, - links: Vec, - ) -> BoxFuture<'_, Result<(), Error>> { - self.writer_pool - .spawn_with_connection(move |conn| { - Box::pin(async move { - MasNewUpstreamOauthLink::write_batch(conn, links).await?; - - Ok(()) - }) - }) - .boxed() - } - - #[tracing::instrument(skip_all, level = Level::DEBUG)] - pub fn write_compat_sessions( - &mut self, - sessions: Vec, - ) -> BoxFuture<'_, Result<(), Error>> { - self.writer_pool - .spawn_with_connection(move |conn| { - Box::pin(async move { - MasNewCompatSession::write_batch(conn, sessions).await?; - - Ok(()) - }) - }) - .boxed() - } - - #[tracing::instrument(skip_all, level = Level::DEBUG)] - pub fn write_compat_access_tokens( - &mut self, - tokens: Vec, - ) -> BoxFuture<'_, Result<(), Error>> { - self.writer_pool - .spawn_with_connection(move |conn| { - Box::pin(async move { - MasNewCompatAccessToken::write_batch(conn, tokens).await?; - - Ok(()) - }) - }) - .boxed() - } - - #[tracing::instrument(skip_all, level = Level::DEBUG)] - pub fn write_compat_refresh_tokens( - &mut self, - tokens: Vec, - ) -> BoxFuture<'_, Result<(), Error>> { - self.writer_pool - .spawn_with_connection(move |conn| { - Box::pin(async move { - MasNewCompatRefreshToken::write_batch(conn, tokens).await?; - Ok(()) - }) - }) - .boxed() - } } // How many entries to buffer at once, before writing a batch of rows to the @@ -1228,7 +1090,7 @@ mod test { mas_writer::{ MasNewCompatAccessToken, MasNewCompatRefreshToken, MasNewCompatSession, MasNewEmailThreepid, MasNewUnsupportedThreepid, MasNewUpstreamOauthLink, MasNewUser, - MasNewUserPassword, + MasNewUserPassword, MasWriteBuffer, }, }; @@ -1340,20 +1202,29 @@ mod test { #[sqlx::test(migrator = "mas_storage_pg::MIGRATOR")] async fn test_write_user(pool: PgPool) { let mut writer = make_mas_writer(&pool).await; + let mut buffer = MasWriteBuffer::new(&writer); - writer - .write_users(vec![MasNewUser { - user_id: NonNilUuid::new(Uuid::from_u128(1u128)).unwrap(), - username: "alice".to_owned(), - created_at: DateTime::default(), - locked_at: None, - deactivated_at: None, - can_request_admin: false, - is_guest: false, - }]) + buffer + .write( + &mut writer, + MasNewUser { + user_id: NonNilUuid::new(Uuid::from_u128(1u128)).unwrap(), + username: "alice".to_owned(), + created_at: DateTime::default(), + locked_at: None, + deactivated_at: None, + can_request_admin: false, + is_guest: false, + }, + ) .await .expect("failed to write user"); + buffer + .finish(&mut writer) + .await + .expect("failed to finish MasWriter"); + let mut conn = writer .finish(&Progress::default()) .await @@ -1369,28 +1240,47 @@ mod test { let mut writer = make_mas_writer(&pool).await; - writer - .write_users(vec![MasNewUser { - user_id: USER_ID, - username: "alice".to_owned(), - created_at: DateTime::default(), - locked_at: None, - deactivated_at: None, - can_request_admin: false, - is_guest: false, - }]) + let mut user_buffer = MasWriteBuffer::new(&writer); + let mut password_buffer = MasWriteBuffer::new(&writer); + + user_buffer + .write( + &mut writer, + MasNewUser { + user_id: USER_ID, + username: "alice".to_owned(), + created_at: DateTime::default(), + locked_at: None, + deactivated_at: None, + can_request_admin: false, + is_guest: false, + }, + ) .await .expect("failed to write user"); - writer - .write_passwords(vec![MasNewUserPassword { - user_password_id: Uuid::from_u128(42u128), - user_id: USER_ID, - hashed_password: "$bcrypt$aaaaaaaaaaa".to_owned(), - created_at: DateTime::default(), - }]) + + password_buffer + .write( + &mut writer, + MasNewUserPassword { + user_password_id: Uuid::from_u128(42u128), + user_id: USER_ID, + hashed_password: "$bcrypt$aaaaaaaaaaa".to_owned(), + created_at: DateTime::default(), + }, + ) .await .expect("failed to write password"); + user_buffer + .finish(&mut writer) + .await + .expect("failed to finish MasWriteBuffer"); + password_buffer + .finish(&mut writer) + .await + .expect("failed to finish MasWriteBuffer"); + let mut conn = writer .finish(&Progress::default()) .await @@ -1404,29 +1294,47 @@ mod test { async fn test_write_user_with_email(pool: PgPool) { let mut writer = make_mas_writer(&pool).await; - writer - .write_users(vec![MasNewUser { - user_id: NonNilUuid::new(Uuid::from_u128(1u128)).unwrap(), - username: "alice".to_owned(), - created_at: DateTime::default(), - locked_at: None, - deactivated_at: None, - can_request_admin: false, - is_guest: false, - }]) + let mut user_buffer = MasWriteBuffer::new(&writer); + let mut email_buffer = MasWriteBuffer::new(&writer); + + user_buffer + .write( + &mut writer, + MasNewUser { + user_id: NonNilUuid::new(Uuid::from_u128(1u128)).unwrap(), + username: "alice".to_owned(), + created_at: DateTime::default(), + locked_at: None, + deactivated_at: None, + can_request_admin: false, + is_guest: false, + }, + ) .await .expect("failed to write user"); - writer - .write_email_threepids(vec![MasNewEmailThreepid { - user_email_id: Uuid::from_u128(2u128), - user_id: NonNilUuid::new(Uuid::from_u128(1u128)).unwrap(), - email: "alice@example.org".to_owned(), - created_at: DateTime::default(), - }]) + email_buffer + .write( + &mut writer, + MasNewEmailThreepid { + user_email_id: Uuid::from_u128(2u128), + user_id: NonNilUuid::new(Uuid::from_u128(1u128)).unwrap(), + email: "alice@example.org".to_owned(), + created_at: DateTime::default(), + }, + ) .await .expect("failed to write e-mail"); + user_buffer + .finish(&mut writer) + .await + .expect("failed to finish user buffer"); + email_buffer + .finish(&mut writer) + .await + .expect("failed to finish email buffer"); + let mut conn = writer .finish(&Progress::default()) .await @@ -1441,29 +1349,47 @@ mod test { async fn test_write_user_with_unsupported_threepid(pool: PgPool) { let mut writer = make_mas_writer(&pool).await; - writer - .write_users(vec![MasNewUser { - user_id: NonNilUuid::new(Uuid::from_u128(1u128)).unwrap(), - username: "alice".to_owned(), - created_at: DateTime::default(), - locked_at: None, - deactivated_at: None, - can_request_admin: false, - is_guest: false, - }]) + let mut user_buffer = MasWriteBuffer::new(&writer); + let mut threepid_buffer = MasWriteBuffer::new(&writer); + + user_buffer + .write( + &mut writer, + MasNewUser { + user_id: NonNilUuid::new(Uuid::from_u128(1u128)).unwrap(), + username: "alice".to_owned(), + created_at: DateTime::default(), + locked_at: None, + deactivated_at: None, + can_request_admin: false, + is_guest: false, + }, + ) .await .expect("failed to write user"); - writer - .write_unsupported_threepids(vec![MasNewUnsupportedThreepid { - user_id: NonNilUuid::new(Uuid::from_u128(1u128)).unwrap(), - medium: "msisdn".to_owned(), - address: "441189998819991197253".to_owned(), - created_at: DateTime::default(), - }]) + threepid_buffer + .write( + &mut writer, + MasNewUnsupportedThreepid { + user_id: NonNilUuid::new(Uuid::from_u128(1u128)).unwrap(), + medium: "msisdn".to_owned(), + address: "441189998819991197253".to_owned(), + created_at: DateTime::default(), + }, + ) .await .expect("failed to write phone number (unsupported threepid)"); + user_buffer + .finish(&mut writer) + .await + .expect("failed to finish user buffer"); + threepid_buffer + .finish(&mut writer) + .await + .expect("failed to finish threepid buffer"); + let mut conn = writer .finish(&Progress::default()) .await @@ -1479,30 +1405,48 @@ mod test { async fn test_write_user_with_upstream_provider_link(pool: PgPool) { let mut writer = make_mas_writer(&pool).await; - writer - .write_users(vec![MasNewUser { - user_id: NonNilUuid::new(Uuid::from_u128(1u128)).unwrap(), - username: "alice".to_owned(), - created_at: DateTime::default(), - locked_at: None, - deactivated_at: None, - can_request_admin: false, - is_guest: false, - }]) + let mut user_buffer = MasWriteBuffer::new(&writer); + let mut link_buffer = MasWriteBuffer::new(&writer); + + user_buffer + .write( + &mut writer, + MasNewUser { + user_id: NonNilUuid::new(Uuid::from_u128(1u128)).unwrap(), + username: "alice".to_owned(), + created_at: DateTime::default(), + locked_at: None, + deactivated_at: None, + can_request_admin: false, + is_guest: false, + }, + ) .await .expect("failed to write user"); - writer - .write_upstream_oauth_links(vec![MasNewUpstreamOauthLink { - user_id: NonNilUuid::new(Uuid::from_u128(1u128)).unwrap(), - link_id: Uuid::from_u128(3u128), - upstream_provider_id: Uuid::from_u128(4u128), - subject: "12345.67890".to_owned(), - created_at: DateTime::default(), - }]) + link_buffer + .write( + &mut writer, + MasNewUpstreamOauthLink { + user_id: NonNilUuid::new(Uuid::from_u128(1u128)).unwrap(), + link_id: Uuid::from_u128(3u128), + upstream_provider_id: Uuid::from_u128(4u128), + subject: "12345.67890".to_owned(), + created_at: DateTime::default(), + }, + ) .await .expect("failed to write link"); + user_buffer + .finish(&mut writer) + .await + .expect("failed to finish user buffer"); + link_buffer + .finish(&mut writer) + .await + .expect("failed to finish link buffer"); + let mut conn = writer .finish(&Progress::default()) .await @@ -1516,34 +1460,52 @@ mod test { async fn test_write_user_with_device(pool: PgPool) { let mut writer = make_mas_writer(&pool).await; - writer - .write_users(vec![MasNewUser { - user_id: NonNilUuid::new(Uuid::from_u128(1u128)).unwrap(), - username: "alice".to_owned(), - created_at: DateTime::default(), - locked_at: None, - deactivated_at: None, - can_request_admin: false, - is_guest: false, - }]) + let mut user_buffer = MasWriteBuffer::new(&writer); + let mut session_buffer = MasWriteBuffer::new(&writer); + + user_buffer + .write( + &mut writer, + MasNewUser { + user_id: NonNilUuid::new(Uuid::from_u128(1u128)).unwrap(), + username: "alice".to_owned(), + created_at: DateTime::default(), + locked_at: None, + deactivated_at: None, + can_request_admin: false, + is_guest: false, + }, + ) .await .expect("failed to write user"); - writer - .write_compat_sessions(vec![MasNewCompatSession { - user_id: NonNilUuid::new(Uuid::from_u128(1u128)).unwrap(), - session_id: Uuid::from_u128(5u128), - created_at: DateTime::default(), - device_id: Some("ADEVICE".to_owned()), - human_name: Some("alice's pinephone".to_owned()), - is_synapse_admin: true, - last_active_at: Some(DateTime::default()), - last_active_ip: Some("203.0.113.1".parse().unwrap()), - user_agent: Some("Browser/5.0".to_owned()), - }]) + session_buffer + .write( + &mut writer, + MasNewCompatSession { + user_id: NonNilUuid::new(Uuid::from_u128(1u128)).unwrap(), + session_id: Uuid::from_u128(5u128), + created_at: DateTime::default(), + device_id: Some("ADEVICE".to_owned()), + human_name: Some("alice's pinephone".to_owned()), + is_synapse_admin: true, + last_active_at: Some(DateTime::default()), + last_active_ip: Some("203.0.113.1".parse().unwrap()), + user_agent: Some("Browser/5.0".to_owned()), + }, + ) .await .expect("failed to write compat session"); + user_buffer + .finish(&mut writer) + .await + .expect("failed to finish user buffer"); + session_buffer + .finish(&mut writer) + .await + .expect("failed to finish session buffer"); + let mut conn = writer .finish(&Progress::default()) .await @@ -1557,45 +1519,71 @@ mod test { async fn test_write_user_with_access_token(pool: PgPool) { let mut writer = make_mas_writer(&pool).await; - writer - .write_users(vec![MasNewUser { - user_id: NonNilUuid::new(Uuid::from_u128(1u128)).unwrap(), - username: "alice".to_owned(), - created_at: DateTime::default(), - locked_at: None, - deactivated_at: None, - can_request_admin: false, - is_guest: false, - }]) + let mut user_buffer = MasWriteBuffer::new(&writer); + let mut session_buffer = MasWriteBuffer::new(&writer); + let mut token_buffer = MasWriteBuffer::new(&writer); + + user_buffer + .write( + &mut writer, + MasNewUser { + user_id: NonNilUuid::new(Uuid::from_u128(1u128)).unwrap(), + username: "alice".to_owned(), + created_at: DateTime::default(), + locked_at: None, + deactivated_at: None, + can_request_admin: false, + is_guest: false, + }, + ) .await .expect("failed to write user"); - writer - .write_compat_sessions(vec![MasNewCompatSession { - user_id: NonNilUuid::new(Uuid::from_u128(1u128)).unwrap(), - session_id: Uuid::from_u128(5u128), - created_at: DateTime::default(), - device_id: Some("ADEVICE".to_owned()), - human_name: None, - is_synapse_admin: false, - last_active_at: None, - last_active_ip: None, - user_agent: None, - }]) + session_buffer + .write( + &mut writer, + MasNewCompatSession { + user_id: NonNilUuid::new(Uuid::from_u128(1u128)).unwrap(), + session_id: Uuid::from_u128(5u128), + created_at: DateTime::default(), + device_id: Some("ADEVICE".to_owned()), + human_name: None, + is_synapse_admin: false, + last_active_at: None, + last_active_ip: None, + user_agent: None, + }, + ) .await .expect("failed to write compat session"); - writer - .write_compat_access_tokens(vec![MasNewCompatAccessToken { - token_id: Uuid::from_u128(6u128), - session_id: Uuid::from_u128(5u128), - access_token: "syt_zxcvzxcvzxcvzxcv_zxcv".to_owned(), - created_at: DateTime::default(), - expires_at: None, - }]) + token_buffer + .write( + &mut writer, + MasNewCompatAccessToken { + token_id: Uuid::from_u128(6u128), + session_id: Uuid::from_u128(5u128), + access_token: "syt_zxcvzxcvzxcvzxcv_zxcv".to_owned(), + created_at: DateTime::default(), + expires_at: None, + }, + ) .await .expect("failed to write access token"); + user_buffer + .finish(&mut writer) + .await + .expect("failed to finish user buffer"); + session_buffer + .finish(&mut writer) + .await + .expect("failed to finish session buffer"); + token_buffer + .finish(&mut writer) + .await + .expect("failed to finish token buffer"); + let mut conn = writer .finish(&Progress::default()) .await @@ -1610,56 +1598,90 @@ mod test { async fn test_write_user_with_refresh_token(pool: PgPool) { let mut writer = make_mas_writer(&pool).await; - writer - .write_users(vec![MasNewUser { - user_id: NonNilUuid::new(Uuid::from_u128(1u128)).unwrap(), - username: "alice".to_owned(), - created_at: DateTime::default(), - locked_at: None, - deactivated_at: None, - can_request_admin: false, - is_guest: false, - }]) + let mut user_buffer = MasWriteBuffer::new(&writer); + let mut session_buffer = MasWriteBuffer::new(&writer); + let mut token_buffer = MasWriteBuffer::new(&writer); + let mut refresh_token_buffer = MasWriteBuffer::new(&writer); + + user_buffer + .write( + &mut writer, + MasNewUser { + user_id: NonNilUuid::new(Uuid::from_u128(1u128)).unwrap(), + username: "alice".to_owned(), + created_at: DateTime::default(), + locked_at: None, + deactivated_at: None, + can_request_admin: false, + is_guest: false, + }, + ) .await .expect("failed to write user"); - writer - .write_compat_sessions(vec![MasNewCompatSession { - user_id: NonNilUuid::new(Uuid::from_u128(1u128)).unwrap(), - session_id: Uuid::from_u128(5u128), - created_at: DateTime::default(), - device_id: Some("ADEVICE".to_owned()), - human_name: None, - is_synapse_admin: false, - last_active_at: None, - last_active_ip: None, - user_agent: None, - }]) + session_buffer + .write( + &mut writer, + MasNewCompatSession { + user_id: NonNilUuid::new(Uuid::from_u128(1u128)).unwrap(), + session_id: Uuid::from_u128(5u128), + created_at: DateTime::default(), + device_id: Some("ADEVICE".to_owned()), + human_name: None, + is_synapse_admin: false, + last_active_at: None, + last_active_ip: None, + user_agent: None, + }, + ) .await .expect("failed to write compat session"); - writer - .write_compat_access_tokens(vec![MasNewCompatAccessToken { - token_id: Uuid::from_u128(6u128), - session_id: Uuid::from_u128(5u128), - access_token: "syt_zxcvzxcvzxcvzxcv_zxcv".to_owned(), - created_at: DateTime::default(), - expires_at: None, - }]) + token_buffer + .write( + &mut writer, + MasNewCompatAccessToken { + token_id: Uuid::from_u128(6u128), + session_id: Uuid::from_u128(5u128), + access_token: "syt_zxcvzxcvzxcvzxcv_zxcv".to_owned(), + created_at: DateTime::default(), + expires_at: None, + }, + ) .await .expect("failed to write access token"); - writer - .write_compat_refresh_tokens(vec![MasNewCompatRefreshToken { - refresh_token_id: Uuid::from_u128(7u128), - session_id: Uuid::from_u128(5u128), - access_token_id: Uuid::from_u128(6u128), - refresh_token: "syr_zxcvzxcvzxcvzxcv_zxcv".to_owned(), - created_at: DateTime::default(), - }]) + refresh_token_buffer + .write( + &mut writer, + MasNewCompatRefreshToken { + refresh_token_id: Uuid::from_u128(7u128), + session_id: Uuid::from_u128(5u128), + access_token_id: Uuid::from_u128(6u128), + refresh_token: "syr_zxcvzxcvzxcvzxcv_zxcv".to_owned(), + created_at: DateTime::default(), + }, + ) .await .expect("failed to write refresh token"); + user_buffer + .finish(&mut writer) + .await + .expect("failed to finish user buffer"); + session_buffer + .finish(&mut writer) + .await + .expect("failed to finish session buffer"); + token_buffer + .finish(&mut writer) + .await + .expect("failed to finish token buffer"); + refresh_token_buffer + .finish(&mut writer) + .await + .expect("failed to finish refresh token buffer"); + let mut conn = writer .finish(&Progress::default()) .await From ad2c183c713f381eb7a47f3eea9d09e2c826c11b Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 22 Apr 2025 17:35:15 +0200 Subject: [PATCH 22/35] syn2mas: reduce the channel buffer size --- crates/syn2mas/src/migration.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/syn2mas/src/migration.rs b/crates/syn2mas/src/migration.rs index 7b34baf62..b79718056 100644 --- a/crates/syn2mas/src/migration.rs +++ b/crates/syn2mas/src/migration.rs @@ -213,7 +213,7 @@ async fn migrate_users( ) -> Result<(MasWriter, MigrationState), Error> { let start = Instant::now(); - let (tx, mut rx) = tokio::sync::mpsc::channel::(10 * 1024 * 1024); + let (tx, mut rx) = tokio::sync::mpsc::channel::(100 * 1024); // create a new RNG seeded from the passed RNG so that we can move it into the // spawned task @@ -335,7 +335,7 @@ async fn migrate_threepids( ) -> Result<(MasWriter, MigrationState), Error> { let start = Instant::now(); - let (tx, mut rx) = tokio::sync::mpsc::channel::(10 * 1024 * 1024); + let (tx, mut rx) = tokio::sync::mpsc::channel::(100 * 1024); // create a new RNG seeded from the passed RNG so that we can move it into the // spawned task @@ -449,7 +449,7 @@ async fn migrate_external_ids( ) -> Result<(MasWriter, MigrationState), Error> { let start = Instant::now(); - let (tx, mut rx) = tokio::sync::mpsc::channel::(10 * 1024 * 1024); + let (tx, mut rx) = tokio::sync::mpsc::channel::(100 * 1024); // create a new RNG seeded from the passed RNG so that we can move it into the // spawned task @@ -561,7 +561,7 @@ async fn migrate_devices( ) -> Result<(MasWriter, MigrationState), Error> { let start = Instant::now(); - let (tx, mut rx) = tokio::sync::mpsc::channel(10 * 1024 * 1024); + let (tx, mut rx) = tokio::sync::mpsc::channel(100 * 1024); // create a new RNG seeded from the passed RNG so that we can move it into the // spawned task @@ -695,7 +695,7 @@ async fn migrate_unrefreshable_access_tokens( ) -> Result<(MasWriter, MigrationState), Error> { let start = Instant::now(); - let (tx, mut rx) = tokio::sync::mpsc::channel(10 * 1024 * 1024); + let (tx, mut rx) = tokio::sync::mpsc::channel(100 * 1024); let now = clock.now(); // create a new RNG seeded from the passed RNG so that we can move it into the @@ -845,7 +845,7 @@ async fn migrate_refreshable_token_pairs( ) -> Result<(MasWriter, MigrationState), Error> { let start = Instant::now(); - let (tx, mut rx) = tokio::sync::mpsc::channel::(10 * 1024 * 1024); + let (tx, mut rx) = tokio::sync::mpsc::channel::(100 * 1024); // create a new RNG seeded from the passed RNG so that we can move it into the // spawned task From 9aafc576a193763eed0d6612cac285385e99530e Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 22 Apr 2025 17:38:58 +0200 Subject: [PATCH 23/35] syn2mas: log the number of entities migrated at each step --- crates/syn2mas/src/migration.rs | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/crates/syn2mas/src/migration.rs b/crates/syn2mas/src/migration.rs index b79718056..2a906d933 100644 --- a/crates/syn2mas/src/migration.rs +++ b/crates/syn2mas/src/migration.rs @@ -212,6 +212,7 @@ async fn migrate_users( progress_counter: ProgressCounter, ) -> Result<(MasWriter, MigrationState), Error> { let start = Instant::now(); + let progress_counter_ = progress_counter.clone(); let (tx, mut rx) = tokio::sync::mpsc::channel::(100 * 1024); @@ -318,7 +319,9 @@ async fn migrate_users( res?; info!( - "users migrated in {:.1}s", + "{} users migrated ({} skipped) in {:.1}s", + progress_counter_.migrated(), + progress_counter_.skipped(), Instant::now().duration_since(start).as_secs_f64() ); @@ -334,6 +337,7 @@ async fn migrate_threepids( progress_counter: ProgressCounter, ) -> Result<(MasWriter, MigrationState), Error> { let start = Instant::now(); + let progress_counter_ = progress_counter.clone(); let (tx, mut rx) = tokio::sync::mpsc::channel::(100 * 1024); @@ -432,7 +436,9 @@ async fn migrate_threepids( res?; info!( - "third-party IDs migrated in {:.1}s", + "{} third-party IDs migrated ({} skipped) in {:.1}s", + progress_counter_.migrated(), + progress_counter_.skipped(), Instant::now().duration_since(start).as_secs_f64() ); @@ -448,6 +454,7 @@ async fn migrate_external_ids( progress_counter: ProgressCounter, ) -> Result<(MasWriter, MigrationState), Error> { let start = Instant::now(); + let progress_counter_ = progress_counter.clone(); let (tx, mut rx) = tokio::sync::mpsc::channel::(100 * 1024); @@ -536,7 +543,9 @@ async fn migrate_external_ids( res?; info!( - "upstream links (external IDs) migrated in {:.1}s", + "{} upstream links (external IDs) migrated ({} skipped) in {:.1}s", + progress_counter_.migrated(), + progress_counter_.skipped(), Instant::now().duration_since(start).as_secs_f64() ); @@ -560,6 +569,7 @@ async fn migrate_devices( progress_counter: ProgressCounter, ) -> Result<(MasWriter, MigrationState), Error> { let start = Instant::now(); + let progress_counter_ = progress_counter.clone(); let (tx, mut rx) = tokio::sync::mpsc::channel(100 * 1024); @@ -675,7 +685,9 @@ async fn migrate_devices( res?; info!( - "devices migrated in {:.1}s", + "{} devices migrated ({} skipped) in {:.1}s", + progress_counter_.migrated(), + progress_counter_.skipped(), Instant::now().duration_since(start).as_secs_f64() ); @@ -694,6 +706,7 @@ async fn migrate_unrefreshable_access_tokens( progress_counter: ProgressCounter, ) -> Result<(MasWriter, MigrationState), Error> { let start = Instant::now(); + let progress_counter_ = progress_counter.clone(); let (tx, mut rx) = tokio::sync::mpsc::channel(100 * 1024); @@ -825,7 +838,9 @@ async fn migrate_unrefreshable_access_tokens( res?; info!( - "non-refreshable access tokens migrated in {:.1}s", + "{} non-refreshable access tokens migrated ({} skipped) in {:.1}s", + progress_counter_.migrated(), + progress_counter_.skipped(), Instant::now().duration_since(start).as_secs_f64() ); @@ -844,6 +859,7 @@ async fn migrate_refreshable_token_pairs( progress_counter: ProgressCounter, ) -> Result<(MasWriter, MigrationState), Error> { let start = Instant::now(); + let progress_counter_ = progress_counter.clone(); let (tx, mut rx) = tokio::sync::mpsc::channel::(100 * 1024); @@ -966,7 +982,9 @@ async fn migrate_refreshable_token_pairs( res?; info!( - "refreshable token pairs migrated in {:.1}s", + "{} refreshable token pairs migrated ({} skipped) in {:.1}s", + progress_counter_.migrated(), + progress_counter_.skipped(), Instant::now().duration_since(start).as_secs_f64() ); From 853b987a4acc5d81297d21d7f42562c736646b21 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 22 Apr 2025 17:46:12 +0200 Subject: [PATCH 24/35] syn2mas: only log once when rebuilding constraints --- crates/syn2mas/src/mas_writer/constraint_pausing.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/syn2mas/src/mas_writer/constraint_pausing.rs b/crates/syn2mas/src/mas_writer/constraint_pausing.rs index 36783215f..d350bd89f 100644 --- a/crates/syn2mas/src/mas_writer/constraint_pausing.rs +++ b/crates/syn2mas/src/mas_writer/constraint_pausing.rs @@ -123,7 +123,6 @@ pub async fn restore_constraint( table_name, definition, } = &constraint; - info!("rebuilding constraint {name}"); sqlx::query(&format!( "ALTER TABLE {table_name} ADD CONSTRAINT {name} {definition};" From 31c8a08dd1309069bb1144fe4716ad1f8eb657ea Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Tue, 22 Apr 2025 17:57:30 +0200 Subject: [PATCH 25/35] syn2mas: spawn the writer connections in parallel Also make sure we have a single span for them --- crates/cli/src/commands/syn2mas.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/crates/cli/src/commands/syn2mas.rs b/crates/cli/src/commands/syn2mas.rs index 473afed54..bc1ae72f1 100644 --- a/crates/cli/src/commands/syn2mas.rs +++ b/crates/cli/src/commands/syn2mas.rs @@ -231,18 +231,17 @@ impl Options { // TODO this dry-run flag should be set to false in real circumstances !!! let reader = SynapseReader::new(&mut syn_conn, true).await?; - let mut writer_mas_connections = Vec::with_capacity(NUM_WRITER_CONNECTIONS); - for _ in 0..NUM_WRITER_CONNECTIONS { - writer_mas_connections.push( + let writer_mas_connections = + futures_util::future::try_join_all((0..NUM_WRITER_CONNECTIONS).map(|_| { database_connection_from_config_with_options( &config, &DatabaseConnectOptions { log_slow_statements: false, }, ) - .await?, - ); - } + })) + .instrument(tracing::info_span!("syn2mas.mas_writer_connections")) + .await?; let writer = MasWriter::new(mas_connection, writer_mas_connections).await?; let clock = SystemClock::default(); @@ -256,7 +255,6 @@ impl Options { tokio::spawn(occasional_progress_logger(progress.clone())); let mas_matrix = MatrixConfig::extract(figment)?; - eprintln!("\n\n"); syn2mas::migrate( reader, writer, From f84f6142b9811eec1f029fba9748a2332706334b Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Wed, 23 Apr 2025 10:47:57 +0200 Subject: [PATCH 26/35] Fix starting up when no telemetry config is set --- crates/cli/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 85c5a89f1..3235cf443 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -10,7 +10,7 @@ use std::{io::IsTerminal, process::ExitCode, sync::Arc}; use anyhow::Context; use clap::Parser; -use mas_config::{ConfigurationSection, TelemetryConfig}; +use mas_config::{ConfigurationSectionExt, TelemetryConfig}; use sentry_tracing::EventFilter; use tracing_subscriber::{ EnvFilter, Layer, Registry, filter::LevelFilter, layer::SubscriberExt, util::SubscriberInitExt, @@ -110,7 +110,7 @@ async fn try_main() -> anyhow::Result { let figment = opts.figment(); let telemetry_config = - TelemetryConfig::extract(&figment).context("Failed to load telemetry config")?; + TelemetryConfig::extract_or_default(&figment).context("Failed to load telemetry config")?; // Setup Sentry let sentry = sentry::init(( From ed319dc43cff91e68ebcbeae092446e4fa6b91c6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 23 Apr 2025 09:22:20 +0000 Subject: [PATCH 27/35] Translations updates --- frontend/.storybook/locales.ts | 76 ++++--- frontend/locales/hu.json | 395 +++++++++++++++++++++++++++++++++ frontend/locales/nb-NO.json | 395 +++++++++++++++++++++++++++++++++ translations/hu.json | 265 ++++++++++++++++++++++ translations/nb-NO.json | 265 ++++++++++++++++++++++ 5 files changed, 1369 insertions(+), 27 deletions(-) create mode 100644 frontend/locales/hu.json create mode 100644 frontend/locales/nb-NO.json create mode 100644 translations/hu.json create mode 100644 translations/nb-NO.json diff --git a/frontend/.storybook/locales.ts b/frontend/.storybook/locales.ts index 61129bca2..41eba3c10 100644 --- a/frontend/.storybook/locales.ts +++ b/frontend/.storybook/locales.ts @@ -27,7 +27,7 @@ export type LocalazyMetadata = { }; const localazyMetadata: LocalazyMetadata = { - projectUrl: "https://localazy.com/p/matrix-authentication-service", + projectUrl: "https://localazy.com/p/matrix-authentication-service!v0.15", baseLocale: "en", languages: [ { @@ -93,6 +93,24 @@ const localazyMetadata: LocalazyMetadata = { localizedName: "Français", pluralType: (n) => { return (n===0 || n===1) ? "one" : "other"; } }, + { + language: "hu", + region: "", + script: "", + isRtl: false, + name: "Hungarian", + localizedName: "Magyar", + pluralType: (n) => { return (n===1) ? "one" : "other"; } + }, + { + language: "nb", + region: "NO", + script: "", + isRtl: false, + name: "Norwegian Bokmål (Norway)", + localizedName: "Norsk bokmål (Norge)", + pluralType: (n) => { return (n===1) ? "one" : "other"; } + }, { language: "nl", region: "", @@ -154,19 +172,21 @@ const localazyMetadata: LocalazyMetadata = { file: "frontend.json", path: "", cdnFiles: { - "cs": "https://delivery.localazy.com/_a7686032324574572744739e0707/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/cs/frontend.json", - "da": "https://delivery.localazy.com/_a7686032324574572744739e0707/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/da/frontend.json", - "de": "https://delivery.localazy.com/_a7686032324574572744739e0707/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/de/frontend.json", - "en": "https://delivery.localazy.com/_a7686032324574572744739e0707/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/en/frontend.json", - "et": "https://delivery.localazy.com/_a7686032324574572744739e0707/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/et/frontend.json", - "fi": "https://delivery.localazy.com/_a7686032324574572744739e0707/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/fi/frontend.json", - "fr": "https://delivery.localazy.com/_a7686032324574572744739e0707/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/fr/frontend.json", - "nl": "https://delivery.localazy.com/_a7686032324574572744739e0707/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/nl/frontend.json", - "pt": "https://delivery.localazy.com/_a7686032324574572744739e0707/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/pt/frontend.json", - "ru": "https://delivery.localazy.com/_a7686032324574572744739e0707/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/ru/frontend.json", - "sv": "https://delivery.localazy.com/_a7686032324574572744739e0707/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/sv/frontend.json", - "uk": "https://delivery.localazy.com/_a7686032324574572744739e0707/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/uk/frontend.json", - "zh#Hans": "https://delivery.localazy.com/_a7686032324574572744739e0707/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/zh-Hans/frontend.json" + "cs": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/cs/frontend.json", + "da": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/da/frontend.json", + "de": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/de/frontend.json", + "en": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/en/frontend.json", + "et": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/et/frontend.json", + "fi": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/fi/frontend.json", + "fr": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/fr/frontend.json", + "hu": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/hu/frontend.json", + "nb_NO": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/nb-NO/frontend.json", + "nl": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/nl/frontend.json", + "pt": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/pt/frontend.json", + "ru": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/ru/frontend.json", + "sv": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/sv/frontend.json", + "uk": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/uk/frontend.json", + "zh#Hans": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/7c203a8ac8bd48c3c4609a8effcd0fbac430f9b2/zh-Hans/frontend.json" } }, { @@ -174,19 +194,21 @@ const localazyMetadata: LocalazyMetadata = { file: "file.json", path: "", cdnFiles: { - "cs": "https://delivery.localazy.com/_a7686032324574572744739e0707/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/cs/file.json", - "da": "https://delivery.localazy.com/_a7686032324574572744739e0707/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/da/file.json", - "de": "https://delivery.localazy.com/_a7686032324574572744739e0707/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/de/file.json", - "en": "https://delivery.localazy.com/_a7686032324574572744739e0707/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/en/file.json", - "et": "https://delivery.localazy.com/_a7686032324574572744739e0707/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/et/file.json", - "fi": "https://delivery.localazy.com/_a7686032324574572744739e0707/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/fi/file.json", - "fr": "https://delivery.localazy.com/_a7686032324574572744739e0707/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/fr/file.json", - "nl": "https://delivery.localazy.com/_a7686032324574572744739e0707/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/nl/file.json", - "pt": "https://delivery.localazy.com/_a7686032324574572744739e0707/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/pt/file.json", - "ru": "https://delivery.localazy.com/_a7686032324574572744739e0707/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/ru/file.json", - "sv": "https://delivery.localazy.com/_a7686032324574572744739e0707/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/sv/file.json", - "uk": "https://delivery.localazy.com/_a7686032324574572744739e0707/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/uk/file.json", - "zh#Hans": "https://delivery.localazy.com/_a7686032324574572744739e0707/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/zh-Hans/file.json" + "cs": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/cs/file.json", + "da": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/da/file.json", + "de": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/de/file.json", + "en": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/en/file.json", + "et": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/et/file.json", + "fi": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/fi/file.json", + "fr": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/fr/file.json", + "hu": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/hu/file.json", + "nb_NO": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/nb-NO/file.json", + "nl": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/nl/file.json", + "pt": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/pt/file.json", + "ru": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/ru/file.json", + "sv": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/sv/file.json", + "uk": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/uk/file.json", + "zh#Hans": "https://delivery.localazy.com/_a69844934179046823895c13fb73/_e0/5b69b0350dccfd47c245a5d41c1b9fdf6912cc6e/zh-Hans/file.json" } } ] diff --git a/frontend/locales/hu.json b/frontend/locales/hu.json new file mode 100644 index 000000000..fc27e6b2d --- /dev/null +++ b/frontend/locales/hu.json @@ -0,0 +1,395 @@ +{ + "action": { + "back": "Vissza", + "cancel": "Mégse", + "clear": "Törlés", + "close": "Bezárás", + "collapse": "Összecsukás", + "confirm": "Megerősítés", + "continue": "Folytatás", + "edit": "Szerkesztés", + "expand": "Kibontás", + "save": "Mentés", + "save_and_continue": "Mentés és folytatás", + "sign_out": "Kijelentkezés", + "start_over": "Újrakezdés" + }, + "branding": { + "privacy_policy": { + "alt": "Hivatkozás a szolgáltatás adatvédelmi irányelveire", + "link": "Adatvédelmi irányelvek" + }, + "terms_and_conditions": { + "alt": "Hivatkozás a szolgáltatási feltételekre", + "link": "Szolgáltatási feltételek" + } + }, + "common": { + "add": "Hozzáadás", + "e2ee": "Végpontok közti titkosítás", + "error": "Hiba", + "loading": "Betöltés…", + "next": "Következő", + "password": "Jelszó", + "previous": "Előző", + "saved": "Mentve", + "saving": "Mentés…" + }, + "frontend": { + "account": { + "account_password": "Fiók jelszava", + "contact_info": "Kapcsolati információ", + "delete_account": { + "alert_description": "Ez a fiók véglegesen törölve lesz, és többé nem fog hozzáférni az üzeneteihez.", + "alert_title": "Hamarosan elveszíti az összes adatát", + "button": "Fiók törlése", + "dialog_description": "Erősítse meg, hogy törölné a fiókját:\n\n\nNem fogja tudni újraaktiválni a fiókját\nTöbbé nem fog tudni bejelentkezni\nSenki sem fogja tudni használni a felhasználónevét (MXID), Önt is beleértve\nElhagyja az összes szobáját és a közvetlen üzeneteit\nEl lesz távolítva az azonosítási kiszolgálóról, és senki sem fogja tudni megtalálni az e-mail-címe vagy telefonszáma alapján\n\nA régi üzenetei továbbra is láthatóak lesznek azok számára, akik megkapták azokat. Elrejti az elküldött üzeneteit azok elől, akik a jövőben csatlakoznak a szobákhoz?", + "dialog_title": "Törli ezt a fiókot?", + "erase_checkbox_label": "Igen, az összes üzenet elrejtése az új érkezők elől", + "incorrect_password": "Helytelen jelszó, próbálja újra", + "mxid_label": "Erősítse meg a Matrix-azonosítóját ({{ mxid }})", + "mxid_mismatch": "Ez az érték nem egyezik a Matrix-azonosítójával", + "password_label": "A folytatáshoz adja meg a jelszavát" + }, + "edit_profile": { + "display_name_help": "Ez az, amit mások látni fognak, ha be van jelentkezve.", + "display_name_label": "Megjelenítési név", + "title": "Profil szerkesztése", + "username_label": "Felhasználói név" + }, + "password": { + "change": "Jelszó módosítása", + "change_disabled": "A jelszóváltoztatást letiltotta a rendszergazda.", + "label": "Jelszó" + }, + "sign_out": { + "button": "Kijelentkezés a fiókból", + "dialog": "Kijelentkezik ebből a fiókból?" + }, + "title": "Saját fiók" + }, + "add_email_form": { + "email_denied_alert": { + "text": "A megadott e-mail-címet nem engedélyezi a kiszolgáló házirendje.", + "title": "E-mail-cím házirend alapján elutasítva" + }, + "email_denied_error": "A megadott e-mail-címet nem engedélyezi a kiszolgáló házirendje", + "email_exists_alert": { + "text": "A megadott e-mail-cím már hozzá lett adva ehhez a fiókhoz", + "title": "Az e-mail-cím már létezik" + }, + "email_exists_error": "A megadott e-mail-cím már hozzá lett adva ehhez a fiókhoz", + "email_field_help": "Alternatív e-mail-cím hozzáadása, mellyel hozzáférhet ehhez a fiókhoz.", + "email_field_label": "E-mail-cím hozzáadása", + "email_in_use_error": "A megadott e-mail-cím már használatban van", + "email_invalid_alert": { + "text": "A megadott e-mail-cím érvénytelen", + "title": "Érvénytelen e-mail-cím" + }, + "email_invalid_error": "A megadott e-mail-cím érvénytelen", + "incorrect_password_error": "Helytelen jelszó, próbálja újra", + "password_confirmation": "Erősítse meg a fiókja jelszavát az e-mail-cím hozzáadásához" + }, + "app_sessions_list": { + "error": "Az alkalmazás munkameneteinek betöltése sikertelen", + "heading": "Alkalmazások" + }, + "browser_session_details": { + "current_badge": "Jelenlegi", + "session_details_title": "Munkamenet" + }, + "browser_sessions_overview": { + "body:one": "{{count}} aktív munkamenet", + "body:other": "{{count}} aktív munkamenet", + "heading": "Böngészők", + "no_active_sessions": { + "default": "Még egyetlen webböngészőben sem jelentkezett be.", + "inactive_90_days": "Az összes munkamenete aktív volt az elmúlt 90 napban." + }, + "view_all_button": "Összes megtekintése" + }, + "compat_session_detail": { + "client_details_title": "Kliensinformációk", + "name": "Név", + "session_details_title": "Munkamenet" + }, + "device_type_icon_label": { + "desktop": "Asztali számítógép", + "mobile": "Mobil", + "pc": "Számítógép", + "tablet": "Táblagép", + "unknown": "Ismeretlen eszköztípus", + "web": "Web" + }, + "email_in_use": { + "heading": "A(z) {{email}} e-mail-cím már használatban van." + }, + "end_session_button": { + "confirmation_modal_title": "Biztos, hogy befejezi a munkamenetet?", + "text": "Eszköz eltávolítása" + }, + "error": { + "hideDetails": "Részletek elrejtése", + "showDetails": "Részletek megjelenítése", + "subtitle": "Váratlan hiba történt. Próbálja újra.", + "title": "Valamilyen hiba történt" + }, + "error_boundary_title": "Valamilyen hiba történt", + "errors": { + "field_required": "Ez a mező kötelező", + "rate_limit_exceeded": "Túl sok kérést adott fel egy rövid időszak alatt. Várjon néhány percet, és próbálja újra." + }, + "last_active": { + "active_date": "{{relativeDate}} aktív", + "active_now": "Jelenleg aktív", + "inactive_90_days": "90+ napja inaktív" + }, + "nav": { + "devices": "Eszközök", + "profile": "Profil", + "sessions": "Munkamenetek", + "settings": "Beállítások" + }, + "not_found_alert_title": "Nem található.", + "not_logged_in_alert": "Nincs bejelentkezve.", + "oauth2_client_detail": { + "details_title": "Kliensinformációk", + "id": "Kliensazonosító", + "name": "Név", + "policy": "Házirend", + "terms": "Szolgáltatás feltételei" + }, + "oauth2_session_detail": { + "client_details_name": "Név", + "client_title": "Kliensinformációk", + "session_details_title": "Munkamenet" + }, + "pagination_controls": { + "total": "Összesen: {{totalCount}}" + }, + "password_change": { + "current_password_label": "Jelenlegi jelszó", + "failure": { + "description": { + "account_locked": "A fiókja zárolva van, és jelenleg nem állítható helyre. Ha erre nem számított, akkor lépjen kapcsolatba a kiszolgáló rendszergazdájával.", + "expired_recovery_ticket": "A helyreállítási hivatkozás lejárt. Kezdje az elejéről a fiókja helyreállítási folyamatát.", + "invalid_new_password": "A választott új jelszó érvénytelen; lehet, hogy nem felel meg a beállított biztonsági házirendnek.", + "no_current_password": "Nincs jelenlegi jelszava.", + "no_such_recovery_ticket": "A helyreállítási hivatkozás érvénytelen. Ha a helyreállítási üzenetből másolta ki a hivatkozást, akkor ellenőrizze, hogy a teljes hivatkozást átmásolta-e.", + "password_changes_disabled": "A jelszómódosítás le van tiltva.", + "recovery_ticket_already_used": "A helyreállítási hivatkozás már fel lett használva. Többé nem használható.", + "unspecified": "Ez ideiglenes probléma is lehet, így próbálja újra később. Ha a probléma továbbra is fennáll, lépjen kapcsolatba a kiszolgáló rendszergazdájával.", + "wrong_password": "A jelenlegi jelszóként megadott jelszó helytelen. Próbálja újra." + }, + "title": "A jelszó frissítése sikertelen" + }, + "new_password_again_label": "Adja meg a jelszót újból", + "new_password_label": "Új jelszó", + "passwords_match": "A jelszavak megegyeznek!", + "passwords_no_match": "A jelszavak nem egyeznek", + "subtitle": "Válasszon egy új jelszót a fiókjához.", + "success": { + "description": "A jelszava sikeresen frissült.", + "title": "Jelszó frissítve" + }, + "title": "Jelszó módosítása" + }, + "password_reset": { + "consumed": { + "subtitle": "Új jelszó létrehozásához kezdje elölről, és válassza a „Elfelejtett jelszó” lehetőséget.", + "title": "A jelszóhelyreállítási hivatkozás már fel lett használva" + }, + "expired": { + "resend_email": "Levél újraküldése", + "subtitle": "Új levél kérése, amely ide lesz elküldve: {{email}}", + "title": "A jelszóvisszaállítási hivatkozás lejárt" + }, + "subtitle": "Válasszon egy új jelszót a fiókjához.", + "title": "Jelszó visszaállítása" + }, + "password_strength": { + "placeholder": "Jelszó erőssége", + "score": { + "0": "Rendkívül gyenge jelszó", + "1": "Nagyon gyenge jelszó", + "2": "Gyenge jelszó", + "3": "Erős jelszó", + "4": "Nagyon erős jelszó" + }, + "suggestion": { + "all_uppercase": "Használjon nagybetűket, de nem mindnél.", + "another_word": "Adjon hozzá néhány kevésbé gyakori szót.", + "associated_years": "Kerülje az Önhöz köthető éveket.", + "capitalization": "Ne csak az első betű legyen nagybetűs.", + "dates": "Kerülje az Önhöz köthető dátumokat és éveket.", + "l33t": "Kerülje a kiszámítható betűhelyettesítéseket, mint az „a” helyetti „@”.", + "longer_keyboard_pattern": "Használjon hosszabb billentyűzetmintát, és többször módosítsa a gépelési irányt.", + "no_need": "Anélkül is hozhat létre erős jelszavakat, hogy szimbólumokat, számokat vagy nagybetűket használna.", + "pwned": "Ha máshol is használja ezt a jelszót, akkor változtassa meg.", + "recent_years": "Kerülje a közelmúltbeli éveket.", + "repeated": "Kerülje az ismétlődő szavakat és karaktereket.", + "reverse_words": "Kerülje a gyakori szavak fordított betűzését.", + "sequences": "Kerülje a gyakori karaktersorozatokat.", + "use_words": "Használjon több szót, de kerülje a gyakori kifejezéseket." + }, + "too_weak": "Ez a jelszó túl gyenge", + "warning": { + "common": "Ez egy gyakran használt jelszó.", + "common_names": "A gyakori nevek és vezetéknevek könnyen kitalálhatóak.", + "dates": "A dátumok könnyen kitalálhatóak.", + "extended_repeat": "Az ismétlődő karaktersorozatok, mint az „abcabcabc” könnyen kitalálhatóak.", + "key_pattern": "A rövid billentyűzetminták könnyen kitalálhatóak.", + "names_by_themselves": "Az egy nevet vagy vezetéknevet tartalmazó jelszók könnyen kitalálhatók.", + "pwned": "A jelszava egy adatvédelmi incidensben kikerült az internetre.", + "recent_years": "A közelmúltbeli évek könnyen kitalálhatóak.", + "sequences": "A gyakori karaktersorozatok, mint az „abc”, könnyen kitalálhatóak.", + "similar_to_common": "Ez hasonlít egy gyakran használt jelszóhoz.", + "simple_repeat": "Az ismétlődő karakterek, mint az „aaa” könnyen kitalálhatóak.", + "straight_row": "A billentyűzeten szereplő karaktersorok könnyen kitalálhatóak.", + "top_hundred": "Ez egy gyakran használt jelszó.", + "top_ten": "Ez egy nagyon gyakran használt jelszó.", + "user_inputs": "Ne legyen benne személyes, vagy az oldallal kapcsolatos adat.", + "word_by_itself": "Az egy szavas jelszavak könnyen kitalálhatóak." + } + }, + "reset_cross_signing": { + "button": "Személyazonosság visszaállítása", + "cancelled": { + "description_1": "A folytatáshoz bezárhatja ezt az ablakot, és visszatérhet az alkalmazáshoz.", + "description_2": "Ha mindenhonnan kijelentkezett, és nem emlékszik a helyreállítási kódjára, akkor vissza kell állítania a személyazonosságát.", + "heading": "Személyazonosság visszaállítása megszakítva." + }, + "description": "Ha nincs bejelentkezve egyetlen más eszközön sem, és elvesztette a helyreállítási kulcsát, akkor az alkalmazás használatának folytatásához alaphelyzetbe kell állítania a személyazonosságát.", + "effect_list": { + "negative_1": "El fogja veszíteni a meglévő üzenetelőzményeit", + "negative_2": "Újból ellenőriznie kell az összes meglévő eszközét és kapcsolatát", + "neutral_1": "El fogja veszíteni a csak a kiszolgálón tárolt üzenetelőzményeit", + "neutral_2": "Újból ellenőriznie kell az összes meglévő eszközét és kapcsolatát", + "positive_1": "A fiókja részletei, a névjegyei, a beállításai és a csevegéslistája meg lesz tartva" + }, + "failure": { + "description": "Ez ideiglenes probléma is lehet, így próbálja újra később. Ha a probléma továbbra is fennáll, lépjen kapcsolatba a kiszolgáló rendszergazdájával.", + "heading": "A kriptográfiai személyazonossága alaphelyzetbe állításának engedélyezése sikertelen", + "title": "A kriptográfiai személyazonosságának engedélyezése sikertelen" + }, + "finish_reset": "Alaphelyzetbe állítás befejezése", + "heading": "Állítsa alaphelyzetbe a személyazonosságát, ha semmilyen más módon nem tudja megerősíteni", + "start_reset": "Alaphelyzetbe állítás elkezdése", + "success": { + "description": "A személyazonosság alaphelyzetbe állítása engedélyezve a következő {{minutes}} percre. A folytatáshoz bezárhatja azt az ablakot, és visszatérhet az alkalmazáshoz.", + "heading": "Személyazonosság sikeresen alaphelyzetbe állítva. A folyamat befejezéséhez térjen vissza az alkalmazáshoz.", + "title": "A kriptográfiai személyazonossága alaphelyzetbe állítása ideiglenesen engedélyezve" + }, + "warning": "Csak akkor állítsa alaphelyzetbe a személyazonosságát, ha nem fér hozzá más bejelentkezett eszközhöz, és elveszítette a helyreállítási kulcsát." + }, + "selectable_session": { + "label": "Munkamenet kiválasztása" + }, + "session": { + "client_id_label": "Kliensazonosító", + "current": "Jelenlegi", + "current_badge": "Jelenlegi", + "device_id_label": "Eszközazonosító", + "finished_date": "Befejezve: ", + "finished_label": "Befejezve", + "generic_browser_session": "Böngésző-munkamenet", + "id_label": "Azonosító", + "ip_label": "IP-cím", + "last_active_label": "Legutóbb aktív", + "last_auth_label": "Legutóbbi hitelesítés", + "name_for_platform": "{{name}} erre: {{platform}}", + "scopes_label": "Hatókörök", + "signed_in_date": "Bejelentkezve: ", + "signed_in_label": "Bejelentkezve", + "title": "Eszköz részletei", + "unknown_browser": "Ismeretlen böngésző", + "unknown_device": "Ismeretlen eszköz", + "uri_label": "URI", + "user_id_label": "Felhasználóazonosító", + "username_label": "Felhasználónév" + }, + "session_detail": { + "alert": { + "button": "Vissza", + "text": "Ez a munkamenet nem létezik, vagy már nem aktív.", + "title": "A munkamenet nem található: {{deviceId}}" + } + }, + "unknown_route": "Ismeretlen útvonal: {{route}}", + "unverified_email_alert": { + "button": "Áttekintés és ellenőrzés", + "text:one": "{{count}} nem ellenőrzött e-mail-címe van.", + "text:other": "{{count}} nem ellenőrzött e-mail-címe van.", + "title": "Nem megerősített e-mail-cím" + }, + "user_email": { + "cant_delete_primary": "Válasszon egy másik elsődleges e-mail-címet, hogy törölhesse ezt.", + "delete_button_confirmation_modal": { + "action": "E-mail-cím törlése", + "body": "Törli ezt az e-mail-címet?", + "incorrect_password": "Helytelen jelszó, próbálja újra", + "password_confirmation": "Erősítse meg a fiókja jelszavát az e-mail-cím törléséhez" + }, + "delete_button_title": "E-mail-cím eltávolítása", + "email": "E-mail", + "make_primary_button": "Elsődlegessé tétel", + "not_verified": "Nincs ellenőrizve", + "primary_email": "Elsődleges e-mail-cím", + "retry_button": "Kód újraküldése", + "unverified": "Ellenőrizetlen" + }, + "user_email_list": { + "heading": "Levelek", + "no_primary_email_alert": "Nincs elsődleges e-mail-cím" + }, + "user_greeting": { + "error": "A felhasználó betöltése sikertelen" + }, + "user_name": { + "display_name_field_label": "Megjelenítési név" + }, + "user_sessions_overview": { + "active_sessions:one": "{{count}} aktív munkamenet", + "active_sessions:other": "{{count}} aktív munkamenet", + "heading": "Hol jelentkezett be", + "no_active_sessions": { + "default": "Egyetlen alkalmazásba sincs bejelentkezve.", + "inactive_90_days": "Az összes munkamenete aktív volt az elmúlt 90 napban." + } + }, + "verify_email": { + "code_expired_alert": { + "description": "A kód lejárt. Kérjen egy újat.", + "title": "A kód lejárt" + }, + "code_field_error": "A kód nem ismerhető fel", + "code_field_label": "6 számjegyű kód", + "code_field_wrong_shape": "A kódnak 6 számjegyűnek kell lennie", + "email_sent_alert": { + "description": "Adja meg alább az új kódot.", + "title": "Új kód küldve" + }, + "enter_code_prompt": "Adja meg az ide küldött 6 számjegyű kódot: {{email}}", + "heading": "E-mail-cím ellenőrzése", + "invalid_code_alert": { + "description": "A folytatáshoz ellenőrizze az e-mail-címére küldött kódot, és frissítse a lenti mezőket.", + "title": "Hibás kódot adott meg" + }, + "resend_code": "Kód újraküldése", + "resend_email": "Levél újraküldése", + "sent": "Elküldve!", + "unknown_email": "Ismeretlen e-mail-cím" + } + }, + "mas": { + "scope": { + "edit_profile": "Profil és elérhetőségek szerkesztése", + "manage_sessions": "Eszközök és munkamenetek kezelése", + "mas_admin": "Bármely felhasználó kezelése a matrix-authentication-service szolgáltatásban", + "send_messages": "Új üzenetek küldése az Ön nevében", + "synapse_admin": "A Synapse Matrix-kiszolgáló kezelése", + "view_messages": "Meglévő üzenetek és adatok megtekintése", + "view_profile": "Saját profilinformációk és kapcsolati részletek megtekintése" + } + } +} \ No newline at end of file diff --git a/frontend/locales/nb-NO.json b/frontend/locales/nb-NO.json new file mode 100644 index 000000000..ad2e7b8f6 --- /dev/null +++ b/frontend/locales/nb-NO.json @@ -0,0 +1,395 @@ +{ + "action": { + "back": "Tilbake", + "cancel": "Avbryt", + "clear": "Tøm", + "close": "Lukk", + "collapse": "Skjul", + "confirm": "Bekreft", + "continue": "Fortsett", + "edit": "Rediger", + "expand": "Utvid", + "save": "Lagre", + "save_and_continue": "Lagre og fortsett", + "sign_out": "Logg ut", + "start_over": "Begynn på nytt" + }, + "branding": { + "privacy_policy": { + "alt": "Lenke til tjenestens personvernerklæring", + "link": "Personvernerklæring" + }, + "terms_and_conditions": { + "alt": "Lenke til tjenestens vilkår og betingelser", + "link": "Vilkår og betingelser" + } + }, + "common": { + "add": "Legg til", + "e2ee": "Ende-til-ende-kryptering", + "error": "Feil", + "loading": "Laster inn...", + "next": "Neste", + "password": "Passord", + "previous": "Forrige", + "saved": "Lagret", + "saving": "Lagrer…" + }, + "frontend": { + "account": { + "account_password": "Passord for konto", + "contact_info": "Kontaktopplysninger", + "delete_account": { + "alert_description": "Denne kontoen vil bli slettet permanent, og du vil ikke lenger ha tilgang til noen av meldingene dine.", + "alert_title": "Du er i ferd med å miste alle dataene dine", + "button": "Slett konto", + "dialog_description": "Bekreft at du ønsker å slette kontoen din:\n\n\nDu vil ikke kunne aktivere kontoen din på nytt\nDu vil ikke lenger kunne logge på\nIngen vil kunne gjenbruke brukernavnet ditt (MXID), heller ikke du\nDu vil forlate alle rom og direktemeldinger du er en del av\nDu vil bli fjernet fra identitetsserveren, og ingen vil kunne finne deg med e-postadressen eller telefonnummeret ditt\n\nDine gamle meldinger vil fortsatt være synlige for personer som har mottatt dem. Ønsker du å skjule dine sendte meldinger for personer som blir med i rommene i fremtiden?", + "dialog_title": "Slett denne kontoen?", + "erase_checkbox_label": "Ja, skjul alle meldingene mine for nye medlemmer", + "incorrect_password": "Feil passord, prøv igjen", + "mxid_label": "Bekreft din Matrix ID ({{ mxid }})", + "mxid_mismatch": "Denne verdien samsvarer ikke med din Matrix ID", + "password_label": "Skriv inn passordet ditt for å fortsette" + }, + "edit_profile": { + "display_name_help": "Dette er det andre vil se uansett hvor du er logget inn.", + "display_name_label": "Visningsnavn", + "title": "Rediger profil", + "username_label": "Brukernavn" + }, + "password": { + "change": "Endre passord", + "change_disabled": "Endring av passord er deaktivert av administrator.", + "label": "Passord" + }, + "sign_out": { + "button": "Logg av konto", + "dialog": "Logg ut av denne kontoen?" + }, + "title": "Din konto" + }, + "add_email_form": { + "email_denied_alert": { + "text": "Den angitte e-postadressen er ikke tillatt av serverpolicyen.", + "title": "E-post avvist av policy" + }, + "email_denied_error": "Den angitte e-postadressen er ikke tillatt av serverpolicyen", + "email_exists_alert": { + "text": "Den angitte e-postadressen er allerede lagt til denne kontoen", + "title": "E-posten finnes allerede" + }, + "email_exists_error": "Den angitte e-postadressen er allerede lagt til denne kontoen", + "email_field_help": "Legg til en alternativ e-postadresse du kan bruke for å få tilgang til denne kontoen.", + "email_field_label": "Legg til e-post", + "email_in_use_error": "Den angitte e-postadressen er allerede i bruk", + "email_invalid_alert": { + "text": "Den angitte e-postadressen er ugyldig", + "title": "Ugyldig e-post" + }, + "email_invalid_error": "Den angitte e-postadressen er ugyldig", + "incorrect_password_error": "Feil passord, prøv igjen", + "password_confirmation": "Bekreft passordet ditt for å legge til denne e-postadressen" + }, + "app_sessions_list": { + "error": "Kunne ikke laste inn appsesjoner", + "heading": "Applikasjoner" + }, + "browser_session_details": { + "current_badge": "Nåværende", + "session_details_title": "Sesjon" + }, + "browser_sessions_overview": { + "body:one": "{{count}} aktiv sesjon", + "body:other": "{{count}} aktive sesjoner", + "heading": "Nettlesere", + "no_active_sessions": { + "default": "Du er ikke logget inn på noen nettlesere.", + "inactive_90_days": "Alle sesjonene dine har vært aktive de siste 90 dagene." + }, + "view_all_button": "Vis alle" + }, + "compat_session_detail": { + "client_details_title": "Klient informasjon", + "name": "Navn", + "session_details_title": "Sesjon" + }, + "device_type_icon_label": { + "desktop": "Skrivebord", + "mobile": "Mobil", + "pc": "Datamaskin", + "tablet": "Nettbrett", + "unknown": "Ukjent enhetstype", + "web": "Web" + }, + "email_in_use": { + "heading": "E-postadressen {{email}} er allerede i bruk." + }, + "end_session_button": { + "confirmation_modal_title": "Er du sikker på at du vil avslutte denne sesjonen?", + "text": "Fjern enheten" + }, + "error": { + "hideDetails": "Skjul detaljer", + "showDetails": "Vis detaljer", + "subtitle": "Det oppstod en uventet feil. Vennligst prøv igjen.", + "title": "Noe gikk galt" + }, + "error_boundary_title": "Noe gikk galt", + "errors": { + "field_required": "Dette feltet er obligatorisk", + "rate_limit_exceeded": "Du har kommet med for mange forespørsler på kort tid. Vent noen minutter og prøv igjen." + }, + "last_active": { + "active_date": "Aktiv {{relativeDate}}", + "active_now": "Aktiv nå", + "inactive_90_days": "Inaktiv i 90+ dager" + }, + "nav": { + "devices": "Enheter", + "profile": "Profil", + "sessions": "Sesjoner", + "settings": "Innstillinger" + }, + "not_found_alert_title": "Ikke funnet.", + "not_logged_in_alert": "Du er ikke innlogget.", + "oauth2_client_detail": { + "details_title": "Klient informasjon", + "id": "Klient-ID", + "name": "Navn", + "policy": "Retningslinjer", + "terms": "Vilkår for bruk" + }, + "oauth2_session_detail": { + "client_details_name": "Navn", + "client_title": "Klient informasjon", + "session_details_title": "Sesjon" + }, + "pagination_controls": { + "total": "Totalt: {{totalCount}}" + }, + "password_change": { + "current_password_label": "Nåværende passord", + "failure": { + "description": { + "account_locked": "Kontoen din er låst og kan ikke gjenopprettes på dette tidspunktet. Hvis dette ikke er forventet, kan du kontakte serveradministratoren din.", + "expired_recovery_ticket": "Gjenopprettingslenken er utløpt. Start kontogjenopprettingsprosessen på nytt.", + "invalid_new_password": "Det nye passordet du valgte er ugyldig. Det kan hende at den ikke oppfyller den gjeldende sikkerhetspolicyen.", + "no_current_password": "Du har ikke et gjeldende passord.", + "no_such_recovery_ticket": "Gjenopprettingslenken er ugyldig. Hvis du kopierte lenken fra gjenopprettingseposten, vennligst sjekk at hele lenken ble kopiert.", + "password_changes_disabled": "Endring av passord er deaktivert.", + "recovery_ticket_already_used": "Gjenopprettingslenken er allerede brukt. Den kan ikke brukes igjen.", + "unspecified": "Dette kan være et midlertidig problem, så prøv igjen senere. Hvis problemet vedvarer, vennligst kontakt serveradministratoren din.", + "wrong_password": "Passordet du oppga som ditt nåværende passord er feil. Prøv igjen." + }, + "title": "Kunne ikke oppdatere passordet" + }, + "new_password_again_label": "Skriv inn nytt passord igjen", + "new_password_label": "Nytt passord", + "passwords_match": "Passordene stemmer overens!", + "passwords_no_match": "Passord stemmer ikke overens", + "subtitle": "Velg et nytt passord for kontoen din.", + "success": { + "description": "Passordet ditt har blitt oppdatert.", + "title": "Passord oppdatert" + }, + "title": "Bytt passordet ditt" + }, + "password_reset": { + "consumed": { + "subtitle": "For å opprette et nytt passord, start på nytt og velg «Glemt passord».", + "title": "Lenken for å tilbakestille passordet ditt har allerede blitt brukt" + }, + "expired": { + "resend_email": "Send e-post på nytt", + "subtitle": "Be om en ny e-post som vil bli sendt til: {{email}}", + "title": "Lenken for å tilbakestille passordet ditt er utløpt" + }, + "subtitle": "Velg et nytt passord for kontoen din.", + "title": "Tilbakestill passordet ditt" + }, + "password_strength": { + "placeholder": "Passordstyrke", + "score": { + "0": "Ekstremt svakt passord", + "1": "Veldig svakt passord", + "2": "Svakt passord", + "3": "Sterkt passord", + "4": "Veldig sterkt passord" + }, + "suggestion": { + "all_uppercase": "Bruk store bokstaver, men ikke for alle bokstaver.", + "another_word": "Legg til flere ord som er mindre vanlige.", + "associated_years": "Unngå år som er knyttet til deg", + "capitalization": "Bruk stor bokstav på mer enn den første bokstaven.", + "dates": "Unngå datoer og år som er knyttet til deg", + "l33t": "Unngå forutsigbare bokstavbytter som \"@\" i stedet for \"a\".", + "longer_keyboard_pattern": "Bruk lengre tastaturmønstre og endre skriveretning flere ganger.", + "no_need": "Du kan lage sterke passord uten å bruke symboler, tall eller store bokstaver.", + "pwned": "Hvis du bruker dette passordet andre steder, bør du endre det.", + "recent_years": "Unngå nylige år", + "repeated": "Unngå gjentatte ord og tegn.", + "reverse_words": "Unngå omvendt staving av vanlige ord.", + "sequences": "Unngå vanlige tegnsekvenser.", + "use_words": "Bruk flere ord, men unngå vanlige fraser." + }, + "too_weak": "Dette passordet er for svakt", + "warning": { + "common": "Dette er et ofte brukt passord.", + "common_names": "Vanlige navn og etternavn er lette å gjette seg til.", + "dates": "Datoer er enkle å gjette seg til.", + "extended_repeat": "Gjentatte tegnmønstre som \"abcabcabc\" er lette å gjette seg til.", + "key_pattern": "Korte tastaturmønstre er enkle å gjette.", + "names_by_themselves": "Enkeltnavn eller etternavn er lette å gjette.", + "pwned": "Passordet ditt ble eksponert ved et datainnbrudd på Internett.", + "recent_years": "De siste årene er enkle å gjette seg til.", + "sequences": "Vanlige tegnsekvenser som «abc» er enkle å gjette.", + "similar_to_common": "Dette ligner på et ofte brukt passord.", + "simple_repeat": "Gjentatte tegn som \"aaa\" er lette å gjette.", + "straight_row": "Rette tasterader på tastaturet er enkle å gjette seg til.", + "top_hundred": "Dette er et ofte brukt passord.", + "top_ten": "Dette er et mye brukt passord.", + "user_inputs": "Det skal ikke være noen personlige eller siderelaterte data.", + "word_by_itself": "Enkeltord er lette å gjette." + } + }, + "reset_cross_signing": { + "button": "Tilbakestill identitet", + "cancelled": { + "description_1": "Du kan lukke dette vinduet og gå tilbake til appen for å fortsette.", + "description_2": "Hvis du er logget av overalt og ikke husker gjenopprettingskoden, må du fortsatt tilbakestille identiteten din.", + "heading": "Tilbakestilling av identitet kansellert." + }, + "description": "Hvis du ikke er logget på andre enheter, og du har mistet gjenopprettingsnøkkelen, må du tilbakestille identiteten din for å fortsette å bruke appen.", + "effect_list": { + "negative_1": "Du vil miste din eksisterende meldingshistorikk", + "negative_2": "Du må bekrefte alle eksisterende enheter og kontakter på nytt", + "neutral_1": "Du vil miste all meldingshistorikk som bare er lagret på serveren", + "neutral_2": "Du må bekrefte alle eksisterende enheter og kontakter på nytt", + "positive_1": "Dine kontodetaljer, kontakter, preferanser og chatteliste vil bli beholdt" + }, + "failure": { + "description": "Dette kan være et midlertidig problem, så prøv igjen senere. Hvis problemet vedvarer, vennligst kontakt serveradministratoren din.", + "heading": "Kunne ikke tillate tilbakestilling av kryptoidentitet", + "title": "Kunne ikke tillate kryptoidentitet" + }, + "finish_reset": "Fullfør tilbakestillingen", + "heading": "Tilbakestill identiteten din i tilfelle du ikke kan bekrefte på en annen måte", + "start_reset": "Start tilbakestilling", + "success": { + "description": "Tilbakestillingen av identiteten er godkjent for de neste {{minutes}} minuttene. Du kan lukke dette vinduet og gå tilbake til appen for å fortsette.", + "heading": "Identitet tilbakestilt. Gå tilbake til appen for å fullføre prosessen.", + "title": "Tilbakestilling av kryptoidentitet midlertidig tillatt" + }, + "warning": "Tilbakestill identiteten din bare hvis du ikke har tilgang til en annen pålogget enhet og du har mistet gjenopprettingsnøkkelen." + }, + "selectable_session": { + "label": "Velg sesjon" + }, + "session": { + "client_id_label": "Klient-ID", + "current": "Nåværende", + "current_badge": "Nåværende", + "device_id_label": "Enhets-ID", + "finished_date": "Fullført ", + "finished_label": "Fullført", + "generic_browser_session": "Nettlesersesjon", + "id_label": "ID", + "ip_label": "IP-adresse", + "last_active_label": "Sist aktiv", + "last_auth_label": "Siste autentisering", + "name_for_platform": "{{name}} for {{platform}}", + "scopes_label": "Omfang", + "signed_in_date": "Logget på ", + "signed_in_label": "Logget på", + "title": "Detaljer om enheten", + "unknown_browser": "Ukjent nettleser", + "unknown_device": "Ukjent enhet", + "uri_label": "Uri", + "user_id_label": "Bruker ID", + "username_label": "Brukernavn" + }, + "session_detail": { + "alert": { + "button": "Gå tilbake", + "text": "Denne sesjonen finnes ikke, eller er ikke lenger aktiv.", + "title": "Finner ikke sesjonen: {{deviceId}}" + } + }, + "unknown_route": "Ukjent rute {{route}}", + "unverified_email_alert": { + "button": "Gjennomgå og verifiser", + "text:one": "Du har {{count}} ubekreftet e-postadresse.", + "text:other": "Du har {{count}} ubekreftede e-postadresser.", + "title": "Ubekreftet e-post" + }, + "user_email": { + "cant_delete_primary": "Velg en annen primær e-postadresse for å slette denne.", + "delete_button_confirmation_modal": { + "action": "Slett e-post", + "body": "Vil du slette denne e-posten?", + "incorrect_password": "Feil passord, prøv igjen", + "password_confirmation": "Bekreft kontopassordet ditt for å slette denne e-postadressen" + }, + "delete_button_title": "Fjern e-postadresse", + "email": "E-post", + "make_primary_button": "Gjøre til primær", + "not_verified": "Ikke verifisert", + "primary_email": "Primær e-postadresse", + "retry_button": "Send kode på nytt", + "unverified": "Ikke verifisert" + }, + "user_email_list": { + "heading": "E-poster", + "no_primary_email_alert": "Ingen primær e-postadresse" + }, + "user_greeting": { + "error": "Kunne ikke laste inn bruker" + }, + "user_name": { + "display_name_field_label": "Visningsnavn" + }, + "user_sessions_overview": { + "active_sessions:one": "{{count}} aktiv sesjon", + "active_sessions:other": "{{count}} aktive sesjoner", + "heading": "Hvor du er logget inn", + "no_active_sessions": { + "default": "Du er ikke logget på noen applikasjoner.", + "inactive_90_days": "Alle sesjonene dine har vært aktive de siste 90 dagene." + } + }, + "verify_email": { + "code_expired_alert": { + "description": "Koden er utløpt. Be om en ny kode.", + "title": "Koden er utløpt" + }, + "code_field_error": "Kode ikke gjenkjent", + "code_field_label": "6-sifret kode", + "code_field_wrong_shape": "Koden må være 6 sifre", + "email_sent_alert": { + "description": "Skriv inn den nye koden nedenfor.", + "title": "Ny kode sendt" + }, + "enter_code_prompt": "Skriv inn den 6-sifrede koden sendt til: {{email}}", + "heading": "Bekreft e-postadressen din", + "invalid_code_alert": { + "description": "Sjekk koden som er sendt til e-posten din, og oppdater feltene nedenfor for å fortsette.", + "title": "Du skrev inn feil kode" + }, + "resend_code": "Send kode på nytt", + "resend_email": "Send e-post på nytt", + "sent": "Sendt!", + "unknown_email": "Ukjent e-postadresse" + } + }, + "mas": { + "scope": { + "edit_profile": "Rediger din profil og kontaktdetaljer", + "manage_sessions": "Administrer enhetene og sesjonene dine", + "mas_admin": "Administrer alle brukere på matrix-authentication-service", + "send_messages": "Send nye meldinger på dine vegne", + "synapse_admin": "Administrer Synapse-hjemmeserveren", + "view_messages": "Se dine eksisterende meldinger og data", + "view_profile": "Se din profilinformasjon og kontaktdetaljer" + } + } +} \ No newline at end of file diff --git a/translations/hu.json b/translations/hu.json new file mode 100644 index 000000000..6fdff188b --- /dev/null +++ b/translations/hu.json @@ -0,0 +1,265 @@ +{ + "action": { + "back": "Vissza", + "cancel": "Mégse", + "continue": "Folytatás", + "create_account": "Fiók létrehozása", + "sign_in": "Bejelentkezés", + "sign_out": "Kijelentkezés", + "skip": "Kihagyás", + "start_over": "Újrakezdés", + "submit": "Elküldés" + }, + "app": { + "human_name": "Matrix hitelesítési szolgáltatás", + "name": "matrix-authentication-service", + "technical_description": "OpenID Connect felfedezési dokumentum: %(discovery_url)s" + }, + "branding": { + "privacy_policy": { + "alt": "Hivatkozás a szolgáltatás adatvédelmi irányelveire", + "link": "Adatvédelmi irányelvek" + }, + "terms_and_conditions": { + "alt": "Hivatkozás a szolgáltatási feltételekre", + "link": "Szolgáltatási feltételek" + } + }, + "common": { + "display_name": "Megjelenítési név", + "email_address": "E-mail-cím", + "loading": "Betöltés…", + "mxid": "Matrix-azonosító", + "password": "Jelszó", + "password_confirm": "Jelszó megerősítése", + "username": "Felhasználói név" + }, + "error": { + "unexpected": "Váratlan hiba" + }, + "mas": { + "account": { + "deactivated": { + "description": "Ez a fiók (%(mxid)s) törölve lett. Ha erre nem számított, vegye fel a kapcsolatot a kiszolgáló rendszergazdájával.", + "heading": "Fiók törölve" + }, + "locked": { + "description": "Ez a fiók (%(mxid)s) zárolva lett. Ha erre nem számított, vegye fel a kapcsolatot a kiszolgáló rendszergazdájával.", + "heading": "Fiók zárolva" + }, + "logged_out": { + "description": "Ez a munkamenet véget ért. Jelentkezzen ki, hogy újra bejelentkezhessen.", + "heading": "Munkamenet befejezve" + } + }, + "add_email": { + "description": "Adjon meg egy e-mail-címet a fiókja helyreállításához, arra az esetre, ha elveszítené a hozzáférését.", + "heading": "E-mail-cím hozzáadása" + }, + "back_to_homepage": "Vissza a kezdőlaphoz", + "captcha": { + "noscript": "Ezt az űrlapot CAPTCHA védi, és az elküldéséhez engedélyezni kell a JavaScriptet. Engedélyezze a böngészőben a JavaScriptet, és töltse újra az oldalt." + }, + "change_password": { + "change": "Jelszó módosítása", + "confirm": "Jelszó megerősítése", + "current": "Jelenlegi jelszó", + "description": "Ez módosítja a fiókja jelszavát.", + "heading": "Saját jelszó módosítása", + "new": "Új jelszó" + }, + "choose_display_name": { + "description": "Ez az a név, melyet a többi ember látni fog. Ezt bármikor módosíthatja.", + "headline": "Válasszon megjelenítési nevet" + }, + "consent": { + "client_wants_access": "A(z) %(client_name)s (itt: %(redirect_uri)s) hozzá akar férni a fiókjához.", + "heading": "Engedélyezi a hozzáférést a fiókjához?", + "make_sure_you_trust": "Győződjön meg arról, hogy megbízik a(z) %(client_name)s kliensben.", + "this_will_allow": "Ez lehetővé teszi, hogy a(z) %(client_name)s a következőket tegye:", + "you_may_be_sharing": "Bizalmas információkat oszthat meg ezzel az oldallal vagy alkalmazással." + }, + "device_card": { + "access_requested": "Hozzáférés kérve", + "device_code": "Kód", + "generic_device": "Eszköz", + "ip_address": "IP-cím" + }, + "device_code_link": { + "description": "Eszköz összekapcsolása", + "headline": "Adja meg az eszközén megjelenített kódot" + }, + "device_consent": { + "another_device_access": "Egy másik eszköz akarja elérni a fiókját.", + "denied": { + "description": "Elutasította a(z) %(client_name)s hozzáférését. Bezárhatja ezt az ablakot.", + "heading": "Hozzáférés megtagadva" + }, + "granted": { + "description": "Megadta a(z) %(client_name)s hozzáférését. Bezárhatja ezt az ablakot.", + "heading": "Hozzáférés megadva" + } + }, + "email_in_use": { + "description": "Ha elfelejtette a fiókja hitelesítő adatait, akkor helyreállíthatja a fiókját. Újra is kezdheti egy másik e-mail-címmel.", + "title": "A(z) %(email)s e-mail-cím már használatban van" + }, + "emails": { + "greeting": "Kedves %(username)s!", + "recovery": { + "click_button": "Kattintson a lenti gombra az új jelszava létrehozásához:", + "copy_link": "Másolja a következő hivatkozást, és illessze be a böngészőbe egy új jelszó létrehozásához:", + "create_new_password": "Új jelszó létrehozása", + "fallback": "A gomb nem működik?", + "headline": "Jelszóvisszaállítást kért a(z) %(client_name)s fiókjához.", + "subject": "Fiókjelszó visszaállítása (%(mxid)s)", + "you_can_ignore": "Ha nem kért új jelszót, akkor figyelmen kívül hagyhatja ezt a levelet. A jelenlegi jelszava továbbra is működni fog." + }, + "verify": { + "body_html": "Az ellenőrzőkódja az e-mail-címe megerősítéséhez: %(code)s", + "body_text": "Az ellenőrzőkódja az e-mail-címe megerősítéséhez: %(code)s", + "subject": "Az e-mail-elleőrzőkódja: %(code)s" + } + }, + "errors": { + "captcha": "A CAPTCHA ellenőrzés sikertelen, próbálja újra", + "denied_policy": "Házirend alapján elutasítva: %(policy)s", + "email_banned": "Az e-mail-címet a kiszolgáló-házirend tiltja", + "email_domain_banned": "Az e-mail-tartományt a kiszolgáló-házirend tiltja", + "email_domain_not_allowed": "Az e-mail-tartományt nem engedélyezi a kiszolgáló-házirend", + "email_not_allowed": "Az e-mail-címet nem engedélyezi a kiszolgáló-házirend", + "field_required": "Ez a mező kötelező", + "invalid_credentials": "Érvénytelen hitelesítő adatok", + "password_mismatch": "A jelszómezők nem egyeznek", + "rate_limit_exceeded": "Túl sok kérést adott fel egy rövid időszak alatt. Várjon néhány percet, és próbálja újra.", + "username_all_numeric": "A felhasználónév nem állhat pusztán számokból", + "username_banned": "A felhasználónevet a kiszolgáló-házirend tiltja", + "username_invalid_chars": "A felhasználónév érvénytelen karaktereket tartalmaz. Csak kisbetűket, számokat, kötőjeleket és aláhúzásokat használjon.", + "username_not_allowed": "A felhasználónevet nem engedélyezi a kiszolgáló-házirend", + "username_taken": "A felhasználónév már foglalt", + "username_too_long": "A felhasználónév túl hosszú", + "username_too_short": "A felhasználónév túl rövid" + }, + "login": { + "call_to_register": "Még nincs fiókja?", + "continue_with_provider": "Folytatás ezzel a szolgáltatóval: %(provider)s", + "description": "Jelentkezzen be a folytatáshoz:", + "forgot_password": "Elfelejtette a jelszavát?", + "headline": "Bejelentkezés", + "link": { + "description": "A(z) %(provider)s fiókjának összekötése", + "headline": "Bejelentkezés az összekötéshez" + }, + "no_login_methods": "Nem érhetőek el bejelentkezési módok.", + "separator": "vagy", + "username_or_email": "Felhasználónév vagy e-mail-cím" + }, + "navbar": { + "my_account": "Saját fiók", + "register": "Fiók létrehozása", + "signed_in_as": "Bejelentkezve mint %(username)s." + }, + "not_found": { + "description": "A keresett oldal nem létezik vagy áthelyezték", + "heading": "Az oldal nem található" + }, + "not_you": "Nem %(username)s?", + "or_separator": "vagy", + "policy_violation": { + "description": "Ez a kérést feladó kliens, a jelenleg bejelentkezett felhasználó, vagy maga a kérés miatt is lehet.", + "heading": "Az engedélyezési kérés a szolgáltatás által betartatott házirend alapján elutasítva", + "logged_as": "Bejelentkezve mint %(username)s" + }, + "recovery": { + "consumed": { + "description": "Új jelszó létrehozásához kezdje elölről, és válassza a „Elfelejtett jelszó” lehetőséget.", + "heading": "A jelszóhelyreállítási hivatkozás már fel lett használva" + }, + "disabled": { + "description": "Ha elvesztette a hitelesítő adatait, akkor a fiókja visszaszerzéséhez lépjen kapcsolatba a rendszergazdával.", + "heading": "A fiókhelyreállítás le van tiltva" + }, + "expired": { + "description": "Új e-mail kérése, amely ide lesz küldve: %(email)s.", + "heading": "A jelszóvisszaállítási hivatkozás lejárt", + "resend_email": "Levél újraküldése" + }, + "finish": { + "confirm": "Adja meg a jelszót újból", + "description": "Válasszon egy új jelszót a fiókjához.", + "heading": "Jelszó visszaállítása", + "new": "Új jelszó", + "save_and_continue": "Mentés és folytatás" + }, + "progress": { + "change_email": "Próbálkozás másik e-mail-címmel", + "description": "Egy jelszóvisszaállítási hivatkozást tartalmazó levelet küldtünk, ha van fiókja a(z) %(email)s címmel.", + "heading": "Nézze meg a leveleit", + "resend_email": "Levél újraküldése" + }, + "start": { + "description": "Egy jelszóvisszaállítási hivatkozást tartalmazó levél lesz küldve.", + "heading": "A folytatáshoz adja meg az e-mail-címét" + } + }, + "register": { + "call_to_login": "Már van fiókja?", + "continue_with_email": "Folytatás az e-mail-címével", + "create_account": { + "description": "A folytatáshoz válasszon felhasználónevet.", + "heading": "Fiók létrehozása" + }, + "sign_in_instead": "Bejelentkezés", + "terms_of_service": "Elfogadom a Szolgáltatási feltételeket" + }, + "scope": { + "edit_profile": "Profil és elérhetőségek szerkesztése", + "manage_sessions": "Eszközök és munkamenetek kezelése", + "mas_admin": "Bármely felhasználó kezelése a matrix-authentication-service szolgáltatásban", + "send_messages": "Új üzenetek küldése az Ön nevében", + "synapse_admin": "A Synapse Matrix-kiszolgáló kezelése", + "view_messages": "Meglévő üzenetek és adatok megtekintése", + "view_profile": "Saját profilinformációk és kapcsolati részletek megtekintése" + }, + "upstream_oauth2": { + "link_mismatch": { + "heading": "Ez a forrásfiók már egy másik fiókkal van összekötve." + }, + "register": { + "choose_username": { + "description": "Ez később nem változtatható meg.", + "heading": "Válasszon felhasználónevet" + }, + "create_account": "Új fiók létrehozása", + "enforced_by_policy": "Kiszolgáló-házirend által betartatva", + "forced_display_name": "A következő megjelenítési nevet fogja használni", + "forced_email": "A következő e-mail-címet fogja használni", + "forced_localpart": "A következő felhasználónevet fogja használni", + "import_data": { + "description": "Erősítse meg az információkat, melyek a(z) %(server_name)s fiókjához lesznek kapcsolva.", + "heading": "Saját adatok importálása" + }, + "imported_from_upstream": "Importálva a forrásfiókjából", + "imported_from_upstream_with_name": "Importálva a(z) %(human_name)s fiókjából", + "link_existing": "Hivatkozás egy meglévő fiókra", + "provider_name": "%(human_name)s fiók", + "signup_with_upstream": { + "heading": "Folytassa a regisztrációt a(z) %(human_name)s fiókjával" + }, + "suggested_display_name": "Megjelenítési név importálása", + "suggested_email": "E-mail-cím importálása", + "use": "Használat" + }, + "suggest_link": { + "action": "Hivatkozás", + "heading": "Meglévő fiók összekötése" + } + }, + "verify_email": { + "6_digit_code": "6 számjegyű kód", + "code": "Kód", + "description": "Adja meg az ide küldött 6 számjegyű kódot: %(email)s", + "headline": "E-mail-cím ellenőrzése" + } + } +} \ No newline at end of file diff --git a/translations/nb-NO.json b/translations/nb-NO.json new file mode 100644 index 000000000..0ac50e862 --- /dev/null +++ b/translations/nb-NO.json @@ -0,0 +1,265 @@ +{ + "action": { + "back": "Tilbake", + "cancel": "Avbryt", + "continue": "Fortsett", + "create_account": "Opprett konto", + "sign_in": "Logg inn", + "sign_out": "Logg ut", + "skip": "Hopp over", + "start_over": "Begynn på nytt", + "submit": "Send" + }, + "app": { + "human_name": "Matrix Authentication Service", + "name": "matrix-authentication-service", + "technical_description": "OpenID Connect oppdagelsesdokument: %(discovery_url)s" + }, + "branding": { + "privacy_policy": { + "alt": "Lenke til tjenestens personvernerklæring", + "link": "Personvernerklæring" + }, + "terms_and_conditions": { + "alt": "Lenke til tjenestens vilkår og betingelser", + "link": "Vilkår og betingelser" + } + }, + "common": { + "display_name": "Visningsnavn", + "email_address": "E-postadresse", + "loading": "Laster inn...", + "mxid": "Matrix ID", + "password": "Passord", + "password_confirm": "Bekreft passord", + "username": "Brukernavn" + }, + "error": { + "unexpected": "Uventet feil" + }, + "mas": { + "account": { + "deactivated": { + "description": "Denne kontoen (%(mxid)s) har blitt slettet. Hvis dette ikke er forventet, må du kontakte serveradministratoren.", + "heading": "Konto slettet" + }, + "locked": { + "description": "Denne kontoen (%(mxid)s ) har blitt låst. Hvis dette ikke er forventet, kontakt serveradministratoren din.", + "heading": "Konto låst" + }, + "logged_out": { + "description": "Denne økten har blitt avsluttet. Logg ut for å kunne logge inn igjen", + "heading": "Sesjonen er avsluttet" + } + }, + "add_email": { + "description": "Skriv inn en e-postadresse for å gjenopprette kontoen din i tilfelle du mister tilgangen til den.", + "heading": "Legg til en e-postadresse" + }, + "back_to_homepage": "Gå tilbake til hjemmesiden", + "captcha": { + "noscript": "Dette skjemaet er beskyttet av en CAPTCHA og krever at JavaScript er aktivert for å kunne sendes inn. Aktiver JavaScript i nettleseren din og last inn denne siden på nytt." + }, + "change_password": { + "change": "Endre passord", + "confirm": "Bekreft passord", + "current": "Nåværende passord", + "description": "Dette vil endre passordet på kontoen din.", + "heading": "Endre passordet mitt", + "new": "Nytt passord" + }, + "choose_display_name": { + "description": "Dette er navnet andre vil se. Du kan endre dette når som helst.", + "headline": "Velg visningsnavnet ditt" + }, + "consent": { + "client_wants_access": "%(client_name)s på %(redirect_uri)s ønsker å få tilgang til kontoen din.", + "heading": "Vil du gi tilgang til kontoen din?", + "make_sure_you_trust": "Vær sikker på at du stoler på %(client_name)s.", + "this_will_allow": "Dette vil tillate %(client_name)s å:", + "you_may_be_sharing": "Det kan hende du deler sensitiv informasjon med denne siden eller appen." + }, + "device_card": { + "access_requested": "Tilgang forespurt", + "device_code": "Kode", + "generic_device": "Enhet", + "ip_address": "IP adresse" + }, + "device_code_link": { + "description": "Koble til en enhet", + "headline": "Skriv inn koden som vises på enheten din" + }, + "device_consent": { + "another_device_access": "En annen enhet vil ha tilgang til kontoen din.", + "denied": { + "description": "Du nektet tilgang til %(client_name)s. Du kan lukke dette vinduet.", + "heading": "Tilgang nektet" + }, + "granted": { + "description": "Du har gitt tilgang til %(client_name)s. Du kan lukke dette vinduet.", + "heading": "Tilgang gitt" + } + }, + "email_in_use": { + "description": "Hvis du har glemt kontolegitimasjonen din, kan du gjenopprette kontoen din. Du kan også starte på nytt og bruke en annen e-postadresse.", + "title": "E-postadressen %(email)s er allerede i bruk" + }, + "emails": { + "greeting": "Hallo %(username)s,", + "recovery": { + "click_button": "Klikk på knappen nedenfor for å opprette et nytt passord:", + "copy_link": "Kopier følgende lenke og lim den inn i en nettleser for å opprette et nytt passord:", + "create_new_password": "Opprett nytt passord", + "fallback": "Fungerer ikke knappen for deg?", + "headline": "Du har bedt om tilbakestilling av passord for %(server_name)s kontoen din.", + "subject": "Tilbakestill kontopassordet ditt (%(mxid)s)", + "you_can_ignore": "Hvis du ikke ba om et nytt passord, kan du ignorere denne e-posten. Ditt nåværende passord vil fortsette å fungere." + }, + "verify": { + "body_html": "Verifiseringskoden for å bekrefte denne e-postadressen er: %(code)s", + "body_text": "Verifiseringskoden for å bekrefte denne e-postadressen er: %(code)s", + "subject": "Verifiseringskoden for e-posten din er: %(code)s" + } + }, + "errors": { + "captcha": "CAPTCHA-verifisering mislyktes. Prøv igjen", + "denied_policy": "Avvist av policy: %(policy)s", + "email_banned": "E-post er utestengt av serverpolicyen", + "email_domain_banned": "E-postdomenet er utestengt av serverpolicyen", + "email_domain_not_allowed": "E-postdomene er ikke tillatt av serverpolicyen", + "email_not_allowed": "E-post er ikke tillatt av serverpolicyen", + "field_required": "Dette feltet er obligatorisk", + "invalid_credentials": "Ugyldig legitimasjon", + "password_mismatch": "Passordfeltene stemmer ikke overens", + "rate_limit_exceeded": "Du har kommet med for mange forespørsler på kort tid. Vent noen minutter og prøv igjen.", + "username_all_numeric": "Brukernavn kan ikke bare bestå av tall", + "username_banned": "Brukernavn er utestengt av serverpolicyen", + "username_invalid_chars": "Brukernavnet inneholder ugyldige tegn. Bruk bare små bokstaver, tall, bindestrek og understrek.", + "username_not_allowed": "Brukernavnet er ikke tillatt av serverpolicyen", + "username_taken": "Dette brukernavnet er allerede tatt", + "username_too_long": "Brukernavnet er for langt", + "username_too_short": "Brukernavnet er for kort" + }, + "login": { + "call_to_register": "Har du ikke en konto ennå?", + "continue_with_provider": "Fortsett med %(provider)s", + "description": "Logg på for å fortsette:", + "forgot_password": "Glemt passordet?", + "headline": "Logg inn", + "link": { + "description": "Kobler din %(provider)s konto", + "headline": "Logg inn for å koble" + }, + "no_login_methods": "Ingen påloggingsmetoder tilgjengelig.", + "separator": "Eller", + "username_or_email": "Brukernavn eller e-postadresse" + }, + "navbar": { + "my_account": "Min konto", + "register": "Opprett en konto", + "signed_in_as": "Logget på som %(username)s." + }, + "not_found": { + "description": "Siden du lette etter eksisterer ikke eller har blitt flyttet", + "heading": "Side ikke funnet" + }, + "not_you": "Ikke %(username)s?", + "or_separator": "Eller", + "policy_violation": { + "description": "Dette kan skyldes klienten som har opprettet forespørselen, den påloggede brukeren eller selve forespørselen.", + "heading": "Autorisasjonsforespørselen ble avvist av policyen som håndheves av denne tjenesten", + "logged_as": "Logget inn som %(username)s" + }, + "recovery": { + "consumed": { + "description": "For å opprette et nytt passord, start på nytt og velg «Glemt passord».", + "heading": "Lenken for å tilbakestille passordet ditt har allerede blitt brukt" + }, + "disabled": { + "description": "Hvis du har mistet legitimasjonen din, vennligst kontakt administratoren for å gjenopprette kontoen din.", + "heading": "Muligheten for kontogjenoppretting er skrudd av" + }, + "expired": { + "description": "Be om en ny e-post som vil bli sendt til: %(email)s.", + "heading": "Lenken for å tilbakestille passordet ditt er utløpt", + "resend_email": "Send e-post på nytt" + }, + "finish": { + "confirm": "Skriv inn nytt passord igjen", + "description": "Velg et nytt passord for kontoen din.", + "heading": "Tilbakestill passordet ditt", + "new": "Nytt passord", + "save_and_continue": "Lagre og fortsett" + }, + "progress": { + "change_email": "Prøv en annen e-postadresse", + "description": "Vi sendte en e-post med en lenke for å tilbakestille passordet ditt hvis det er en konto som bruker %(email)s.", + "heading": "Sjekk e-posten din", + "resend_email": "Send e-post på nytt" + }, + "start": { + "description": "En e-post vil bli sendt med en lenke for å tilbakestille passordet ditt.", + "heading": "Skriv inn e-postadressen din for å fortsette" + } + }, + "register": { + "call_to_login": "Har du allerede en konto?", + "continue_with_email": "Fortsett med e-postadresse", + "create_account": { + "description": "Velg et brukernavn for å fortsette.", + "heading": "Opprett en konto" + }, + "sign_in_instead": "Logg på i stedet", + "terms_of_service": "Jeg godtar vilkårene og betingelsene" + }, + "scope": { + "edit_profile": "Rediger din profil og kontaktdetaljer", + "manage_sessions": "Administrer enhetene og sesjonene dine", + "mas_admin": "Administrer alle brukere på matrix-authentication-service", + "send_messages": "Send nye meldinger på dine vegne", + "synapse_admin": "Administrer Synapse-hjemmeserveren", + "view_messages": "Se dine eksisterende meldinger og data", + "view_profile": "Se din profilinformasjon og kontaktdetaljer" + }, + "upstream_oauth2": { + "link_mismatch": { + "heading": "Denne oppstrømskontoen er allerede knyttet til en annen konto." + }, + "register": { + "choose_username": { + "description": "Dette kan ikke endres senere.", + "heading": "Velg ditt brukernavn" + }, + "create_account": "Opprett en ny konto", + "enforced_by_policy": "Håndhevet av serverpolicy", + "forced_display_name": "Vil bruke følgende visningsnavn", + "forced_email": "Vil bruke følgende e-postadresse", + "forced_localpart": "Vil bruke følgende brukernavn", + "import_data": { + "description": "Bekreft informasjonen som vil bli knyttet til din nye %(server_name)s konto.", + "heading": "Importer dataene dine" + }, + "imported_from_upstream": "Importert fra oppstrømskontoen din", + "imported_from_upstream_with_name": "Importert fra %(human_name)s kontoen din", + "link_existing": "Koble til en eksisterende konto", + "provider_name": "%(human_name)s konto", + "signup_with_upstream": { + "heading": "Fortsett å registrere deg med din %(human_name)s konto" + }, + "suggested_display_name": "Importer visningsnavn", + "suggested_email": "Importer e-postadresse", + "use": "Bruk" + }, + "suggest_link": { + "action": "Lenke", + "heading": "Koble til din eksisterende konto" + } + }, + "verify_email": { + "6_digit_code": "6-sifret kode", + "code": "Kode", + "description": "Skriv inn den 6-sifrede koden sendt til: %(email)s", + "headline": "Bekreft e-postadressen din" + } + } +} \ No newline at end of file From fa1ecc5216b8a6af1673731735b3e1cfaaeafdf5 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Wed, 23 Apr 2025 13:30:02 +0200 Subject: [PATCH 28/35] syn2mas: warn about existing oauth-delegated user_external_ids --- crates/syn2mas/src/synapse_reader/checks.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/crates/syn2mas/src/synapse_reader/checks.rs b/crates/syn2mas/src/synapse_reader/checks.rs index 4dca03029..0969d3787 100644 --- a/crates/syn2mas/src/synapse_reader/checks.rs +++ b/crates/syn2mas/src/synapse_reader/checks.rs @@ -73,6 +73,11 @@ pub enum CheckError { )] SynapseMissingOAuthProvider { provider: String, num_users: i64 }, + #[error( + "Synapse database has {num_users} mapping entries from a previously-configured MAS instance. If this is from a previous migration attempt, run the following SQL query against the Synapse database: `DELETE FROM user_external_ids WHERE auth_provider = 'oauth-delegated';` and then run the migration again." + )] + ExistingOAuthDelegated { num_users: i64 }, + #[error( "Synapse config contains an OpenID Connect or OAuth2 provider '{provider}' (issuer: {issuer:?}) used by {num_users} users which must also be configured in the MAS configuration as an upstream provider." )] @@ -292,6 +297,14 @@ pub async fn synapse_database_check( let syn_oauth2 = synapse.all_oidc_providers(); let mas_oauth2 = UpstreamOAuth2Config::extract_or_default(mas)?; for row in oauth_provider_user_counts { + // This is a special case of a previous migration attempt to MAS + if row.auth_provider == "oauth-delegated" { + errors.push(CheckError::ExistingOAuthDelegated { + num_users: row.num_users, + }); + continue; + } + let matching_syn = syn_oauth2.get(&row.auth_provider); let Some(matching_syn) = matching_syn else { From d62fe98f5a58d9daf30ca8a1778516d76fc88b90 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Wed, 23 Apr 2025 13:34:34 +0200 Subject: [PATCH 29/35] syn2mas: provide guidance on how to re-do a fresh migration --- crates/syn2mas/src/mas_writer/checks.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/syn2mas/src/mas_writer/checks.rs b/crates/syn2mas/src/mas_writer/checks.rs index d5b51b510..6ddf7619b 100644 --- a/crates/syn2mas/src/mas_writer/checks.rs +++ b/crates/syn2mas/src/mas_writer/checks.rs @@ -16,10 +16,12 @@ use super::{MAS_TABLES_AFFECTED_BY_MIGRATION, is_syn2mas_in_progress, locking::L #[derive(Debug, Error, ContextInto)] pub enum Error { - #[error("the MAS database is not empty: rows found in at least `{table}`")] + #[error( + "The MAS database is not empty: rows found in at least `{table}`. Please drop and recreate the database, then try again." + )] MasDatabaseNotEmpty { table: &'static str }, - #[error("query against {table} failed — is this actually a MAS database?")] + #[error("Query against {table} failed — is this actually a MAS database?")] MaybeNotMas { #[source] source: sqlx::Error, @@ -29,7 +31,7 @@ pub enum Error { #[error(transparent)] Sqlx(#[from] sqlx::Error), - #[error("unable to check if syn2mas is already in progress")] + #[error("Unable to check if syn2mas is already in progress")] UnableToCheckInProgress(#[source] super::Error), } From be09f62bfa7fe80ef340538e23bb9f176c71cd10 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Wed, 23 Apr 2025 13:38:46 +0200 Subject: [PATCH 30/35] syn2mas: drop the experimental flag --- crates/cli/src/commands/syn2mas.rs | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/crates/cli/src/commands/syn2mas.rs b/crates/cli/src/commands/syn2mas.rs index bc1ae72f1..ca6f1422e 100644 --- a/crates/cli/src/commands/syn2mas.rs +++ b/crates/cli/src/commands/syn2mas.rs @@ -15,7 +15,7 @@ use sqlx::{Connection, Either, PgConnection, postgres::PgConnectOptions, types:: use syn2mas::{ LockedMasDatabase, MasWriter, Progress, ProgressStage, SynapseReader, synapse_config, }; -use tracing::{Instrument, error, info, info_span, warn}; +use tracing::{Instrument, error, info, info_span}; use crate::util::{DatabaseConnectOptions, database_connection_from_config_with_options}; @@ -32,15 +32,6 @@ pub(super) struct Options { #[command(subcommand)] subcommand: Subcommand, - /// This version of the syn2mas tool is EXPERIMENTAL and INCOMPLETE. It is - /// only suitable for TESTING. If you want to use this tool anyway, - /// please pass this argument. - /// - /// If you want to migrate from Synapse to MAS today, please use the - /// Node.js-based tool in the MAS repository. - #[clap(long = "i-swear-i-am-just-testing-in-a-staging-environment")] - experimental_accepted: bool, - /// Path to the Synapse configuration (in YAML format). /// May be specified multiple times if multiple Synapse configuration files /// are in use. @@ -85,14 +76,6 @@ impl Options { #[tracing::instrument("cli.syn2mas.run", skip_all)] #[allow(clippy::too_many_lines)] pub async fn run(self, figment: &Figment) -> anyhow::Result { - warn!( - "This version of the syn2mas tool is EXPERIMENTAL and INCOMPLETE. Do not use it, except for TESTING." - ); - if !self.experimental_accepted { - error!("Please agree that you can only use this tool for testing."); - return Ok(ExitCode::FAILURE); - } - if self.synapse_configuration_files.is_empty() { error!("Please specify the path to the Synapse configuration file(s)."); return Ok(ExitCode::FAILURE); From 44727912c07c3c688ac9fc10d4d448d41acc09ac Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Wed, 23 Apr 2025 13:46:44 +0200 Subject: [PATCH 31/35] Add a few missing license headers --- crates/cli/src/commands/syn2mas.rs | 5 +++++ crates/syn2mas/src/mas_writer/checks.rs | 2 +- crates/syn2mas/src/mas_writer/constraint_pausing.rs | 2 +- .../syn2mas/src/mas_writer/fixtures/upstream_provider.sql | 5 +++++ crates/syn2mas/src/mas_writer/locking.rs | 2 +- crates/syn2mas/src/mas_writer/mod.rs | 2 +- .../src/synapse_reader/fixtures/access_token_alice.sql | 5 +++++ .../fixtures/access_token_alice_with_puppet.sql | 5 +++++ .../fixtures/access_token_alice_with_refresh_token.sql | 5 +++++ .../access_token_alice_with_unused_refresh_token.sql | 5 +++++ .../syn2mas/src/synapse_reader/fixtures/devices_alice.sql | 5 +++++ .../src/synapse_reader/fixtures/external_ids_alice.sql | 5 +++++ .../src/synapse_reader/fixtures/threepids_alice.sql | 5 +++++ crates/syn2mas/src/synapse_reader/fixtures/user_alice.sql | 7 +++++-- 14 files changed, 54 insertions(+), 6 deletions(-) diff --git a/crates/cli/src/commands/syn2mas.rs b/crates/cli/src/commands/syn2mas.rs index ca6f1422e..c7851627b 100644 --- a/crates/cli/src/commands/syn2mas.rs +++ b/crates/cli/src/commands/syn2mas.rs @@ -1,3 +1,8 @@ +// Copyright 2024, 2025 New Vector Ltd. +// +// SPDX-License-Identifier: AGPL-3.0-only +// Please see LICENSE in the repository root for full details. + use std::{collections::HashMap, process::ExitCode, time::Duration}; use anyhow::Context; diff --git a/crates/syn2mas/src/mas_writer/checks.rs b/crates/syn2mas/src/mas_writer/checks.rs index 6ddf7619b..288156d8c 100644 --- a/crates/syn2mas/src/mas_writer/checks.rs +++ b/crates/syn2mas/src/mas_writer/checks.rs @@ -1,4 +1,4 @@ -// Copyright 2024 New Vector Ltd. +// Copyright 2024, 2025 New Vector Ltd. // // SPDX-License-Identifier: AGPL-3.0-only // Please see LICENSE in the repository root for full details. diff --git a/crates/syn2mas/src/mas_writer/constraint_pausing.rs b/crates/syn2mas/src/mas_writer/constraint_pausing.rs index d350bd89f..49fd4a8e3 100644 --- a/crates/syn2mas/src/mas_writer/constraint_pausing.rs +++ b/crates/syn2mas/src/mas_writer/constraint_pausing.rs @@ -1,4 +1,4 @@ -// Copyright 2024 New Vector Ltd. +// Copyright 2024, 2025 New Vector Ltd. // // SPDX-License-Identifier: AGPL-3.0-only // Please see LICENSE in the repository root for full details. diff --git a/crates/syn2mas/src/mas_writer/fixtures/upstream_provider.sql b/crates/syn2mas/src/mas_writer/fixtures/upstream_provider.sql index 7d1e98bc8..9da09b174 100644 --- a/crates/syn2mas/src/mas_writer/fixtures/upstream_provider.sql +++ b/crates/syn2mas/src/mas_writer/fixtures/upstream_provider.sql @@ -1,3 +1,8 @@ +-- Copyright 2024, 2025 New Vector Ltd. +-- +-- SPDX-License-Identifier: AGPL-3.0-only +-- Please see LICENSE in the repository root for full details. + INSERT INTO upstream_oauth_providers ( upstream_oauth_provider_id, diff --git a/crates/syn2mas/src/mas_writer/locking.rs b/crates/syn2mas/src/mas_writer/locking.rs index 031ca9ac3..8200924d4 100644 --- a/crates/syn2mas/src/mas_writer/locking.rs +++ b/crates/syn2mas/src/mas_writer/locking.rs @@ -1,4 +1,4 @@ -// Copyright 2024 New Vector Ltd. +// Copyright 2024, 2025 New Vector Ltd. // // SPDX-License-Identifier: AGPL-3.0-only // Please see LICENSE in the repository root for full details. diff --git a/crates/syn2mas/src/mas_writer/mod.rs b/crates/syn2mas/src/mas_writer/mod.rs index 5e3a0f0ef..07fceb85c 100644 --- a/crates/syn2mas/src/mas_writer/mod.rs +++ b/crates/syn2mas/src/mas_writer/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2024 New Vector Ltd. +// Copyright 2024, 2025 New Vector Ltd. // // SPDX-License-Identifier: AGPL-3.0-only // Please see LICENSE in the repository root for full details. diff --git a/crates/syn2mas/src/synapse_reader/fixtures/access_token_alice.sql b/crates/syn2mas/src/synapse_reader/fixtures/access_token_alice.sql index d9f9a4a7b..e92fd21bf 100644 --- a/crates/syn2mas/src/synapse_reader/fixtures/access_token_alice.sql +++ b/crates/syn2mas/src/synapse_reader/fixtures/access_token_alice.sql @@ -1,3 +1,8 @@ +-- Copyright 2024, 2025 New Vector Ltd. +-- +-- SPDX-License-Identifier: AGPL-3.0-only +-- Please see LICENSE in the repository root for full details. + INSERT INTO access_tokens ( id, diff --git a/crates/syn2mas/src/synapse_reader/fixtures/access_token_alice_with_puppet.sql b/crates/syn2mas/src/synapse_reader/fixtures/access_token_alice_with_puppet.sql index 6bdfb0d9c..c8b2850ac 100644 --- a/crates/syn2mas/src/synapse_reader/fixtures/access_token_alice_with_puppet.sql +++ b/crates/syn2mas/src/synapse_reader/fixtures/access_token_alice_with_puppet.sql @@ -1,3 +1,8 @@ +-- Copyright 2024, 2025 New Vector Ltd. +-- +-- SPDX-License-Identifier: AGPL-3.0-only +-- Please see LICENSE in the repository root for full details. + INSERT INTO access_tokens ( id, diff --git a/crates/syn2mas/src/synapse_reader/fixtures/access_token_alice_with_refresh_token.sql b/crates/syn2mas/src/synapse_reader/fixtures/access_token_alice_with_refresh_token.sql index 554ae4458..180a58810 100644 --- a/crates/syn2mas/src/synapse_reader/fixtures/access_token_alice_with_refresh_token.sql +++ b/crates/syn2mas/src/synapse_reader/fixtures/access_token_alice_with_refresh_token.sql @@ -1,3 +1,8 @@ +-- Copyright 2024, 2025 New Vector Ltd. +-- +-- SPDX-License-Identifier: AGPL-3.0-only +-- Please see LICENSE in the repository root for full details. + INSERT INTO access_tokens ( id, diff --git a/crates/syn2mas/src/synapse_reader/fixtures/access_token_alice_with_unused_refresh_token.sql b/crates/syn2mas/src/synapse_reader/fixtures/access_token_alice_with_unused_refresh_token.sql index 42bfddf01..8c7d1c695 100644 --- a/crates/syn2mas/src/synapse_reader/fixtures/access_token_alice_with_unused_refresh_token.sql +++ b/crates/syn2mas/src/synapse_reader/fixtures/access_token_alice_with_unused_refresh_token.sql @@ -1,3 +1,8 @@ +-- Copyright 2024, 2025 New Vector Ltd. +-- +-- SPDX-License-Identifier: AGPL-3.0-only +-- Please see LICENSE in the repository root for full details. + INSERT INTO access_tokens ( id, diff --git a/crates/syn2mas/src/synapse_reader/fixtures/devices_alice.sql b/crates/syn2mas/src/synapse_reader/fixtures/devices_alice.sql index c7f0691d6..8eb50a3ba 100644 --- a/crates/syn2mas/src/synapse_reader/fixtures/devices_alice.sql +++ b/crates/syn2mas/src/synapse_reader/fixtures/devices_alice.sql @@ -1,3 +1,8 @@ +-- Copyright 2024, 2025 New Vector Ltd. +-- +-- SPDX-License-Identifier: AGPL-3.0-only +-- Please see LICENSE in the repository root for full details. + INSERT INTO devices ( user_id, diff --git a/crates/syn2mas/src/synapse_reader/fixtures/external_ids_alice.sql b/crates/syn2mas/src/synapse_reader/fixtures/external_ids_alice.sql index 5a00cebb5..a365faf05 100644 --- a/crates/syn2mas/src/synapse_reader/fixtures/external_ids_alice.sql +++ b/crates/syn2mas/src/synapse_reader/fixtures/external_ids_alice.sql @@ -1,3 +1,8 @@ +-- Copyright 2024, 2025 New Vector Ltd. +-- +-- SPDX-License-Identifier: AGPL-3.0-only +-- Please see LICENSE in the repository root for full details. + INSERT INTO user_external_ids ( user_id, diff --git a/crates/syn2mas/src/synapse_reader/fixtures/threepids_alice.sql b/crates/syn2mas/src/synapse_reader/fixtures/threepids_alice.sql index 526c00c2c..4bf680cce 100644 --- a/crates/syn2mas/src/synapse_reader/fixtures/threepids_alice.sql +++ b/crates/syn2mas/src/synapse_reader/fixtures/threepids_alice.sql @@ -1,3 +1,8 @@ +-- Copyright 2024, 2025 New Vector Ltd. +-- +-- SPDX-License-Identifier: AGPL-3.0-only +-- Please see LICENSE in the repository root for full details. + INSERT INTO user_threepids ( user_id, diff --git a/crates/syn2mas/src/synapse_reader/fixtures/user_alice.sql b/crates/syn2mas/src/synapse_reader/fixtures/user_alice.sql index bf52d6c5c..dc77d5859 100644 --- a/crates/syn2mas/src/synapse_reader/fixtures/user_alice.sql +++ b/crates/syn2mas/src/synapse_reader/fixtures/user_alice.sql @@ -1,4 +1,8 @@ --- +-- Copyright 2024, 2025 New Vector Ltd. +-- +-- SPDX-License-Identifier: AGPL-3.0-only +-- Please see LICENSE in the repository root for full details. + INSERT INTO users ( name, @@ -37,4 +41,3 @@ INSERT INTO users false, false ); - From 30a0e5054af12844efe4b6875862843fb67080f4 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Wed, 23 Apr 2025 14:18:59 +0200 Subject: [PATCH 32/35] Allow syn2mas arguments to be specified after the subcommand --- crates/cli/src/commands/syn2mas.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/cli/src/commands/syn2mas.rs b/crates/cli/src/commands/syn2mas.rs index c7851627b..c32f49bd9 100644 --- a/crates/cli/src/commands/syn2mas.rs +++ b/crates/cli/src/commands/syn2mas.rs @@ -40,7 +40,7 @@ pub(super) struct Options { /// Path to the Synapse configuration (in YAML format). /// May be specified multiple times if multiple Synapse configuration files /// are in use. - #[clap(long = "synapse-config")] + #[clap(long = "synapse-config", global = true)] synapse_configuration_files: Vec, /// Override the Synapse database URI. @@ -60,7 +60,7 @@ pub(super) struct Options { /// environment variables `PGHOST`, `PGPORT`, `PGUSER`, `PGDATABASE`, /// `PGPASSWORD`, etc. It is valid to specify the URL `postgresql:` and /// configure all values through those environment variables. - #[clap(long = "synapse-database-uri")] + #[clap(long = "synapse-database-uri", global = true)] synapse_database_uri: Option, } From ae29ef1f9de6cded5939d2e3b2df86798c48f677 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Wed, 23 Apr 2025 14:21:15 +0200 Subject: [PATCH 33/35] syn2mas: log progress more often --- crates/cli/src/commands/syn2mas.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/cli/src/commands/syn2mas.rs b/crates/cli/src/commands/syn2mas.rs index c32f49bd9..fd84c4eca 100644 --- a/crates/cli/src/commands/syn2mas.rs +++ b/crates/cli/src/commands/syn2mas.rs @@ -262,13 +262,13 @@ impl Options { } } -/// Logs progress every 30 seconds, as a lightweight alternative to a progress -/// bar. For most deployments, the migration will not take 30 seconds so this +/// Logs progress every 5 seconds, as a lightweight alternative to a progress +/// bar. For most deployments, the migration will not take 5 seconds so this /// will not be relevant. In other cases, this will give the operator an idea of /// what's going on. async fn occasional_progress_logger(progress: Progress) { loop { - tokio::time::sleep(Duration::from_secs(30)).await; + tokio::time::sleep(Duration::from_secs(5)).await; match &**progress.get_current_stage() { ProgressStage::SettingUp => { info!(name: "progress", "still setting up"); From 348eb56344db25018135b0cebdb4a166462a77dd Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Wed, 23 Apr 2025 14:42:30 +0200 Subject: [PATCH 34/35] syn2mas: introduce a dry-run mode --- crates/cli/src/commands/syn2mas.rs | 23 +++++++++++++++------- crates/syn2mas/src/mas_writer/mod.rs | 29 +++++++++++++++++++++++++--- 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/crates/cli/src/commands/syn2mas.rs b/crates/cli/src/commands/syn2mas.rs index fd84c4eca..1d1fa06de 100644 --- a/crates/cli/src/commands/syn2mas.rs +++ b/crates/cli/src/commands/syn2mas.rs @@ -70,8 +70,17 @@ enum Subcommand { /// /// It is OK for Synapse to be online during these checks. Check, + /// Perform a migration. Synapse must be offline during this process. - Migrate, + Migrate { + /// Perform a dry-run migration, which is safe to run with Synapse + /// running, and will restore the MAS database to an empty state. + /// + /// This still *does* write to the MAS database, making it more + /// realistic compared to the final migration. + #[clap(long)] + dry_run: bool, + }, } /// The number of parallel writing transactions active against the MAS database. @@ -118,11 +127,10 @@ impl Options { .await .context("could not run migrations")?; - if matches!(&self.subcommand, Subcommand::Migrate) { + if matches!(&self.subcommand, Subcommand::Migrate { .. }) { // First perform a config sync // This is crucial to ensure we register upstream OAuth providers // in the MAS database - // let config = SyncConfig::extract(figment)?; let clock = SystemClock::default(); let encrypter = config.secrets.encrypter(); @@ -201,7 +209,8 @@ impl Options { Ok(ExitCode::SUCCESS) } - Subcommand::Migrate => { + + Subcommand::Migrate { dry_run } => { let provider_id_mappings: HashMap = { let mas_oauth2 = UpstreamOAuth2Config::extract_or_default(figment)?; @@ -217,8 +226,7 @@ impl Options { // TODO how should we handle warnings at this stage? - // TODO this dry-run flag should be set to false in real circumstances !!! - let reader = SynapseReader::new(&mut syn_conn, true).await?; + let reader = SynapseReader::new(&mut syn_conn, dry_run).await?; let writer_mas_connections = futures_util::future::try_join_all((0..NUM_WRITER_CONNECTIONS).map(|_| { database_connection_from_config_with_options( @@ -230,7 +238,8 @@ impl Options { })) .instrument(tracing::info_span!("syn2mas.mas_writer_connections")) .await?; - let writer = MasWriter::new(mas_connection, writer_mas_connections).await?; + let writer = + MasWriter::new(mas_connection, writer_mas_connections, dry_run).await?; let clock = SystemClock::default(); // TODO is this rng ok? diff --git a/crates/syn2mas/src/mas_writer/mod.rs b/crates/syn2mas/src/mas_writer/mod.rs index 07fceb85c..f36851dfd 100644 --- a/crates/syn2mas/src/mas_writer/mod.rs +++ b/crates/syn2mas/src/mas_writer/mod.rs @@ -242,6 +242,7 @@ impl FinishCheckerHandle { pub struct MasWriter { conn: LockedMasDatabase, writer_pool: WriterConnectionPool, + dry_run: bool, indices_to_restore: Vec, constraints_to_restore: Vec, @@ -793,6 +794,7 @@ impl MasWriter { pub async fn new( mut conn: LockedMasDatabase, mut writer_connections: Vec, + dry_run: bool, ) -> Result { // Given that we don't have any concurrent transactions here, // the READ COMMITTED isolation level is sufficient. @@ -902,7 +904,7 @@ impl MasWriter { Ok(Self { conn, - + dry_run, writer_pool: WriterConnectionPool::new(writer_connections), indices_to_restore, constraints_to_restore, @@ -987,7 +989,6 @@ impl MasWriter { // Now all the data has been migrated, finish off by restoring indices and // constraints! - query("BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;") .execute(self.conn.as_mut()) .await @@ -1009,6 +1010,28 @@ impl MasWriter { .await .into_database("could not revert temporary tables")?; + // If we're in dry-run mode, truncate all the tables we've written to + if self.dry_run { + warn!("Migration ran in dry-run mode, deleting all imported data"); + let tables = MAS_TABLES_AFFECTED_BY_MIGRATION + .iter() + .map(|table| format!("\"{table}\"")) + .collect::>() + .join(", "); + + // Note that we do that with CASCADE, because we do that *after* + // restoring the FK constraints. + // + // The alternative would be to list all the tables we have FK to + // those tables, which would be a hassle, or to do that after + // restoring the constraints, which would mean we wouldn't validate + // that we've done valid FKs in dry-run mode. + query(&format!("TRUNCATE TABLE {tables} CASCADE;")) + .execute(self.conn.as_mut()) + .await + .into_database_with(|| "failed to truncate all tables")?; + } + query("COMMIT;") .execute(self.conn.as_mut()) .await @@ -1193,7 +1216,7 @@ mod test { .await .expect("failed to lock MAS database") .expect_left("MAS database is already locked"); - MasWriter::new(locked_main_conn, writer_conns) + MasWriter::new(locked_main_conn, writer_conns, false) .await .expect("failed to construct MasWriter") } From ac41b41d9ca5b3970798e1b9254c82f5f2d56004 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 23 Apr 2025 14:08:02 +0000 Subject: [PATCH 35/35] 0.15.0 --- Cargo.lock | 54 +++++++++++++++--------------- Cargo.toml | 58 ++++++++++++++++----------------- tools/syn2mas/package-lock.json | 4 +-- tools/syn2mas/package.json | 2 +- 4 files changed, 59 insertions(+), 59 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d02df0709..035223777 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3114,7 +3114,7 @@ dependencies = [ [[package]] name = "mas-axum-utils" -version = "0.15.0-rc.0" +version = "0.15.0" dependencies = [ "axum", "axum-extra", @@ -3147,7 +3147,7 @@ dependencies = [ [[package]] name = "mas-cli" -version = "0.15.0-rc.0" +version = "0.15.0" dependencies = [ "anyhow", "axum", @@ -3218,7 +3218,7 @@ dependencies = [ [[package]] name = "mas-config" -version = "0.15.0-rc.0" +version = "0.15.0" dependencies = [ "anyhow", "camino", @@ -3248,7 +3248,7 @@ dependencies = [ [[package]] name = "mas-data-model" -version = "0.15.0-rc.0" +version = "0.15.0" dependencies = [ "base64ct", "chrono", @@ -3269,7 +3269,7 @@ dependencies = [ [[package]] name = "mas-email" -version = "0.15.0-rc.0" +version = "0.15.0" dependencies = [ "async-trait", "lettre", @@ -3280,7 +3280,7 @@ dependencies = [ [[package]] name = "mas-handlers" -version = "0.15.0-rc.0" +version = "0.15.0" dependencies = [ "aide", "anyhow", @@ -3357,7 +3357,7 @@ dependencies = [ [[package]] name = "mas-http" -version = "0.15.0-rc.0" +version = "0.15.0" dependencies = [ "futures-util", "headers", @@ -3378,7 +3378,7 @@ dependencies = [ [[package]] name = "mas-i18n" -version = "0.15.0-rc.0" +version = "0.15.0" dependencies = [ "camino", "icu_calendar", @@ -3400,7 +3400,7 @@ dependencies = [ [[package]] name = "mas-i18n-scan" -version = "0.15.0-rc.0" +version = "0.15.0" dependencies = [ "camino", "clap", @@ -3414,7 +3414,7 @@ dependencies = [ [[package]] name = "mas-iana" -version = "0.15.0-rc.0" +version = "0.15.0" dependencies = [ "schemars", "serde", @@ -3422,7 +3422,7 @@ dependencies = [ [[package]] name = "mas-iana-codegen" -version = "0.15.0-rc.0" +version = "0.15.0" dependencies = [ "anyhow", "async-trait", @@ -3438,7 +3438,7 @@ dependencies = [ [[package]] name = "mas-jose" -version = "0.15.0-rc.0" +version = "0.15.0" dependencies = [ "base64ct", "chrono", @@ -3468,7 +3468,7 @@ dependencies = [ [[package]] name = "mas-keystore" -version = "0.15.0-rc.0" +version = "0.15.0" dependencies = [ "aead", "base64ct", @@ -3496,7 +3496,7 @@ dependencies = [ [[package]] name = "mas-listener" -version = "0.15.0-rc.0" +version = "0.15.0" dependencies = [ "anyhow", "bytes", @@ -3520,7 +3520,7 @@ dependencies = [ [[package]] name = "mas-matrix" -version = "0.15.0-rc.0" +version = "0.15.0" dependencies = [ "anyhow", "async-trait", @@ -3530,7 +3530,7 @@ dependencies = [ [[package]] name = "mas-matrix-synapse" -version = "0.15.0-rc.0" +version = "0.15.0" dependencies = [ "anyhow", "async-trait", @@ -3547,7 +3547,7 @@ dependencies = [ [[package]] name = "mas-oidc-client" -version = "0.15.0-rc.0" +version = "0.15.0" dependencies = [ "assert_matches", "async-trait", @@ -3583,7 +3583,7 @@ dependencies = [ [[package]] name = "mas-policy" -version = "0.15.0-rc.0" +version = "0.15.0" dependencies = [ "anyhow", "arc-swap", @@ -3600,7 +3600,7 @@ dependencies = [ [[package]] name = "mas-router" -version = "0.15.0-rc.0" +version = "0.15.0" dependencies = [ "axum", "serde", @@ -3611,7 +3611,7 @@ dependencies = [ [[package]] name = "mas-spa" -version = "0.15.0-rc.0" +version = "0.15.0" dependencies = [ "camino", "serde", @@ -3620,7 +3620,7 @@ dependencies = [ [[package]] name = "mas-storage" -version = "0.15.0-rc.0" +version = "0.15.0" dependencies = [ "async-trait", "chrono", @@ -3642,7 +3642,7 @@ dependencies = [ [[package]] name = "mas-storage-pg" -version = "0.15.0-rc.0" +version = "0.15.0" dependencies = [ "async-trait", "chrono", @@ -3668,7 +3668,7 @@ dependencies = [ [[package]] name = "mas-tasks" -version = "0.15.0-rc.0" +version = "0.15.0" dependencies = [ "anyhow", "async-trait", @@ -3699,7 +3699,7 @@ dependencies = [ [[package]] name = "mas-templates" -version = "0.15.0-rc.0" +version = "0.15.0" dependencies = [ "anyhow", "arc-swap", @@ -3729,7 +3729,7 @@ dependencies = [ [[package]] name = "mas-tower" -version = "0.15.0-rc.0" +version = "0.15.0" dependencies = [ "http", "opentelemetry", @@ -3999,7 +3999,7 @@ dependencies = [ [[package]] name = "oauth2-types" -version = "0.15.0-rc.0" +version = "0.15.0" dependencies = [ "assert_matches", "base64ct", @@ -6149,7 +6149,7 @@ dependencies = [ [[package]] name = "syn2mas" -version = "0.15.0-rc.0" +version = "0.15.0" dependencies = [ "anyhow", "arc-swap", diff --git a/Cargo.toml b/Cargo.toml index a5f1ee216..a29b83963 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ members = ["crates/*"] resolver = "2" # Updated in the CI with a `sed` command -package.version = "0.15.0-rc.0" +package.version = "0.15.0" package.license = "AGPL-3.0-only" package.authors = ["Element Backend Team"] package.edition = "2024" @@ -27,34 +27,34 @@ broken_intra_doc_links = "deny" [workspace.dependencies] # Workspace crates -mas-axum-utils = { path = "./crates/axum-utils/", version = "=0.15.0-rc.0" } -mas-cli = { path = "./crates/cli/", version = "=0.15.0-rc.0" } -mas-config = { path = "./crates/config/", version = "=0.15.0-rc.0" } -mas-data-model = { path = "./crates/data-model/", version = "=0.15.0-rc.0" } -mas-email = { path = "./crates/email/", version = "=0.15.0-rc.0" } -mas-graphql = { path = "./crates/graphql/", version = "=0.15.0-rc.0" } -mas-handlers = { path = "./crates/handlers/", version = "=0.15.0-rc.0" } -mas-http = { path = "./crates/http/", version = "=0.15.0-rc.0" } -mas-i18n = { path = "./crates/i18n/", version = "=0.15.0-rc.0" } -mas-i18n-scan = { path = "./crates/i18n-scan/", version = "=0.15.0-rc.0" } -mas-iana = { path = "./crates/iana/", version = "=0.15.0-rc.0" } -mas-iana-codegen = { path = "./crates/iana-codegen/", version = "=0.15.0-rc.0" } -mas-jose = { path = "./crates/jose/", version = "=0.15.0-rc.0" } -mas-keystore = { path = "./crates/keystore/", version = "=0.15.0-rc.0" } -mas-listener = { path = "./crates/listener/", version = "=0.15.0-rc.0" } -mas-matrix = { path = "./crates/matrix/", version = "=0.15.0-rc.0" } -mas-matrix-synapse = { path = "./crates/matrix-synapse/", version = "=0.15.0-rc.0" } -mas-oidc-client = { path = "./crates/oidc-client/", version = "=0.15.0-rc.0" } -mas-policy = { path = "./crates/policy/", version = "=0.15.0-rc.0" } -mas-router = { path = "./crates/router/", version = "=0.15.0-rc.0" } -mas-spa = { path = "./crates/spa/", version = "=0.15.0-rc.0" } -mas-storage = { path = "./crates/storage/", version = "=0.15.0-rc.0" } -mas-storage-pg = { path = "./crates/storage-pg/", version = "=0.15.0-rc.0" } -mas-tasks = { path = "./crates/tasks/", version = "=0.15.0-rc.0" } -mas-templates = { path = "./crates/templates/", version = "=0.15.0-rc.0" } -mas-tower = { path = "./crates/tower/", version = "=0.15.0-rc.0" } -oauth2-types = { path = "./crates/oauth2-types/", version = "=0.15.0-rc.0" } -syn2mas = { path = "./crates/syn2mas", version = "=0.15.0-rc.0" } +mas-axum-utils = { path = "./crates/axum-utils/", version = "=0.15.0" } +mas-cli = { path = "./crates/cli/", version = "=0.15.0" } +mas-config = { path = "./crates/config/", version = "=0.15.0" } +mas-data-model = { path = "./crates/data-model/", version = "=0.15.0" } +mas-email = { path = "./crates/email/", version = "=0.15.0" } +mas-graphql = { path = "./crates/graphql/", version = "=0.15.0" } +mas-handlers = { path = "./crates/handlers/", version = "=0.15.0" } +mas-http = { path = "./crates/http/", version = "=0.15.0" } +mas-i18n = { path = "./crates/i18n/", version = "=0.15.0" } +mas-i18n-scan = { path = "./crates/i18n-scan/", version = "=0.15.0" } +mas-iana = { path = "./crates/iana/", version = "=0.15.0" } +mas-iana-codegen = { path = "./crates/iana-codegen/", version = "=0.15.0" } +mas-jose = { path = "./crates/jose/", version = "=0.15.0" } +mas-keystore = { path = "./crates/keystore/", version = "=0.15.0" } +mas-listener = { path = "./crates/listener/", version = "=0.15.0" } +mas-matrix = { path = "./crates/matrix/", version = "=0.15.0" } +mas-matrix-synapse = { path = "./crates/matrix-synapse/", version = "=0.15.0" } +mas-oidc-client = { path = "./crates/oidc-client/", version = "=0.15.0" } +mas-policy = { path = "./crates/policy/", version = "=0.15.0" } +mas-router = { path = "./crates/router/", version = "=0.15.0" } +mas-spa = { path = "./crates/spa/", version = "=0.15.0" } +mas-storage = { path = "./crates/storage/", version = "=0.15.0" } +mas-storage-pg = { path = "./crates/storage-pg/", version = "=0.15.0" } +mas-tasks = { path = "./crates/tasks/", version = "=0.15.0" } +mas-templates = { path = "./crates/templates/", version = "=0.15.0" } +mas-tower = { path = "./crates/tower/", version = "=0.15.0" } +oauth2-types = { path = "./crates/oauth2-types/", version = "=0.15.0" } +syn2mas = { path = "./crates/syn2mas", version = "=0.15.0" } # OpenAPI schema generation and validation [workspace.dependencies.aide] diff --git a/tools/syn2mas/package-lock.json b/tools/syn2mas/package-lock.json index 1e9e16bf7..d85617479 100644 --- a/tools/syn2mas/package-lock.json +++ b/tools/syn2mas/package-lock.json @@ -1,12 +1,12 @@ { "name": "@vector-im/syn2mas", - "version": "0.15.0-rc.0", + "version": "0.15.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@vector-im/syn2mas", - "version": "0.15.0-rc.0", + "version": "0.15.0", "license": "AGPL-3.0-only", "dependencies": { "command-line-args": "^6.0.0", diff --git a/tools/syn2mas/package.json b/tools/syn2mas/package.json index ed777771c..6e0118152 100644 --- a/tools/syn2mas/package.json +++ b/tools/syn2mas/package.json @@ -1,6 +1,6 @@ { "name": "@vector-im/syn2mas", - "version": "0.15.0-rc.0", + "version": "0.15.0", "description": "A tool to migrate Synapse users and sessions to the Matrix Authentication Service", "license": "AGPL-3.0-only", "author": "Matrix.org",