diff --git a/CALCKEY.md b/CALCKEY.md index e561c3a5a..418ab27e9 100644 --- a/CALCKEY.md +++ b/CALCKEY.md @@ -118,6 +118,7 @@ - Skin tone selection support - [DragonflyDB](https://dragonflydb.io/) support as a Redis alternative - Link verification +- Importing posts from other Calckey/Misskey/Mastodon/Akkoma/Pleroma instances ## Implemented (remote) diff --git a/package.json b/package.json index d4460c878..eccc170e0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "calckey", - "version": "14.0.0-dev79", + "version": "14.0.0-dev83", "codename": "aqua", "repository": { "type": "git", diff --git a/packages/backend/native-utils/Cargo.lock b/packages/backend/native-utils/Cargo.lock index 1a20b792c..e5f8af37a 100644 --- a/packages/backend/native-utils/Cargo.lock +++ b/packages/backend/native-utils/Cargo.lock @@ -458,6 +458,20 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "combine" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" +dependencies = [ + "bytes", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util", +] + [[package]] name = "console" version = "0.15.7" @@ -486,6 +500,16 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.4" @@ -1278,11 +1302,14 @@ dependencies = [ "futures", "indicatif", "native-utils", + "redis", + "sea-orm", "sea-orm-migration", "serde", "serde_json", "serde_yaml", "tokio", + "url", "urlencoding", ] @@ -1509,6 +1536,12 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + [[package]] name = "os_str_bytes" version = "6.5.0" @@ -1843,6 +1876,29 @@ dependencies = [ "rand_core", ] +[[package]] +name = "redis" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ea8c51b5dc1d8e5fd3350ec8167f464ec0995e79f2e90a075b63371500d557f" +dependencies = [ + "async-trait", + "bytes", + "combine", + "futures-util", + "itoa", + "percent-encoding", + "pin-project-lite", + "rustls 0.21.3", + "rustls-native-certs", + "ryu", + "sha1_smol", + "tokio", + "tokio-rustls 0.24.1", + "tokio-util", + "url", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -2043,6 +2099,30 @@ dependencies = [ "webpki", ] +[[package]] +name = "rustls" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b19faa85ecb5197342b54f987b142fb3e30d0c90da40f80ef4fa9a726e6676ed" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "1.0.2" @@ -2052,6 +2132,16 @@ dependencies = [ "base64 0.21.2", ] +[[package]] +name = "rustls-webpki" +version = "0.101.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f36a6828982f422756984e47912a7a51dcbc2a197aa791158f8ca61cd8204e" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.12" @@ -2076,6 +2166,15 @@ version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +[[package]] +name = "schannel" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +dependencies = [ + "windows-sys 0.48.0", +] + [[package]] name = "schemars" version = "0.8.12" @@ -2286,6 +2385,29 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "security-framework" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.17" @@ -2370,6 +2492,12 @@ dependencies = [ "digest", ] +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + [[package]] name = "sha2" version = "0.10.6" @@ -2518,7 +2646,7 @@ dependencies = [ "percent-encoding", "rand", "rust_decimal", - "rustls", + "rustls 0.20.8", "rustls-pemfile", "serde", "serde_json", @@ -2564,7 +2692,7 @@ checksum = "804d3f245f894e61b1e6263c84b23ca675d96753b5abfd5cc8597d86806e8024" dependencies = [ "once_cell", "tokio", - "tokio-rustls", + "tokio-rustls 0.23.4", ] [[package]] @@ -2778,11 +2906,21 @@ version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ - "rustls", + "rustls 0.20.8", "tokio", "webpki", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.3", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.14" @@ -2949,6 +3087,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] diff --git a/packages/backend/native-utils/Cargo.toml b/packages/backend/native-utils/Cargo.toml index f93180fe4..6f4dd9175 100644 --- a/packages/backend/native-utils/Cargo.toml +++ b/packages/backend/native-utils/Cargo.toml @@ -9,7 +9,7 @@ members = ["migration"] [features] default = [] noarray = [] -napi = ["dep:napi", "dep:napi-derive", "dep:radix_fmt"] +napi = ["dep:napi", "dep:napi-derive"] [lib] crate-type = ["cdylib", "lib"] @@ -31,11 +31,11 @@ serde_json = "1.0.96" thiserror = "1.0.40" tokio = { version = "1.28.1", features = ["full"] } utoipa = "3.3.0" +radix_fmt = "1.0.0" # Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix napi = { version = "2.13.1", default-features = false, features = ["napi6", "tokio_rt"], optional = true } napi-derive = { version = "2.12.0", optional = true } -radix_fmt = { version = "1.0.0", optional = true } [dev-dependencies] pretty_assertions = "1.3.0" diff --git a/packages/backend/native-utils/__test__/index.spec.mjs b/packages/backend/native-utils/__test__/index.spec.mjs index 6e6a91858..9ff8ead6c 100644 --- a/packages/backend/native-utils/__test__/index.spec.mjs +++ b/packages/backend/native-utils/__test__/index.spec.mjs @@ -12,18 +12,18 @@ test("convert to mastodon id", (t) => { t.is(convertId("9gf61ehcxv", IdConvertType.MastodonId), "960365976481219"); t.is( convertId("9fbr9z0wbrjqyd3u", IdConvertType.MastodonId), - "3954607381600562394", + "2083785058661759970208986", ); t.is( convertId("9fbs680oyviiqrol9md73p8g", IdConvertType.MastodonId), - "3494513243013053824", + "5878598648988104013828532260828151168", ); }); test("create cuid2 with timestamp prefix", (t) => { nativeInitIdGenerator(16, ""); - t.not(nativeCreateId(BigInt(Date.now())), nativeCreateId(BigInt(Date.now()))); - t.is(nativeCreateId(BigInt(Date.now())).length, 16); + t.not(nativeCreateId(Date.now()), nativeCreateId(Date.now())); + t.is(nativeCreateId(Date.now()).length, 16); }); test("create random string", (t) => { diff --git a/packages/backend/native-utils/migration/Cargo.toml b/packages/backend/native-utils/migration/Cargo.toml index 7ed9fd5f0..9bf793e04 100644 --- a/packages/backend/native-utils/migration/Cargo.toml +++ b/packages/backend/native-utils/migration/Cargo.toml @@ -21,14 +21,17 @@ futures = { version = "0.3.28", optional = true } serde_yaml = "0.9.21" serde = { version = "1.0.163", features = ["derive"] } urlencoding = "2.1.2" +redis = { version = "0.23.0", features = ["tokio-rustls-comp"] } +sea-orm = "0.11.3" +url = { version = "2.4.0", features = ["serde"] } [dependencies.sea-orm-migration] version = "0.11.0" features = [ - # Enable at least one `ASYNC_RUNTIME` and `DATABASE_DRIVER` feature if you want to run migration via CLI. - # View the list of supported features at https://www.sea-ql.org/SeaORM/docs/install-and-config/database-and-async-runtime. - # e.g. - "runtime-tokio-rustls", # `ASYNC_RUNTIME` feature - "sqlx-postgres", # `DATABASE_DRIVER` feature + # Enable at least one `ASYNC_RUNTIME` and `DATABASE_DRIVER` feature if you want to run migration via CLI. + # View the list of supported features at https://www.sea-ql.org/SeaORM/docs/install-and-config/database-and-async-runtime. + # e.g. + "runtime-tokio-rustls", # `ASYNC_RUNTIME` feature + "sqlx-postgres", # `DATABASE_DRIVER` feature "sqlx-sqlite", ] diff --git a/packages/backend/native-utils/migration/src/lib.rs b/packages/backend/native-utils/migration/src/lib.rs index 94e2b08cc..5ad23f162 100644 --- a/packages/backend/native-utils/migration/src/lib.rs +++ b/packages/backend/native-utils/migration/src/lib.rs @@ -2,6 +2,7 @@ pub use sea_orm_migration::prelude::*; mod m20230531_180824_drop_reversi; mod m20230627_185451_index_note_url; +mod m20230709_000510_move_antenna_to_cache; pub struct Migrator; @@ -11,6 +12,7 @@ impl MigratorTrait for Migrator { vec![ Box::new(m20230531_180824_drop_reversi::Migration), Box::new(m20230627_185451_index_note_url::Migration), + Box::new(m20230709_000510_move_antenna_to_cache::Migration), ] } } diff --git a/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs b/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs new file mode 100644 index 000000000..83bd0448a --- /dev/null +++ b/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs @@ -0,0 +1,248 @@ +use redis::streams::StreamMaxlen; +use sea_orm::Statement; +use sea_orm_migration::prelude::*; +use std::env; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + let cache_url = env::var("CACHE_URL").unwrap(); + let skip_copy = env::var("ANTENNA_MIGRATION_SKIP").unwrap_or_default(); + let copy_limit = env::var("ANTENNA_MIGRATION_COPY_LIMIT").unwrap_or_default(); + let read_limit: u64 = env::var("ANTENNA_MIGRATION_READ_LIMIT") + .unwrap_or("10000".to_string()) + .parse() + .unwrap(); + let copy_limit: i64 = match copy_limit.parse() { + Ok(limit) => limit, + Err(_) => 0, + }; + + if skip_copy == "true" { + println!("Skipped antenna migration"); + } else { + let prefix = env::var("CACHE_PREFIX").unwrap(); + + let db = manager.get_connection(); + let bk = manager.get_database_backend(); + + let count_stmt = + Statement::from_string(bk, "SELECT COUNT(1) FROM antenna_note".to_owned()); + let total_num = db + .query_one(count_stmt) + .await? + .unwrap() + .try_get_by_index::(0)?; + let copy_limit = if copy_limit > 0 { + copy_limit + } else { + total_num + }; + println!( + "Copying {} out of {} entries in antenna_note.", + copy_limit, total_num + ); + + let stmt_base = Query::select() + .column((AntennaNote::Table, AntennaNote::Id)) + .column(AntennaNote::AntennaId) + .column(AntennaNote::NoteId) + .from(AntennaNote::Table) + .order_by((AntennaNote::Table, AntennaNote::Id), Order::Asc) + .limit(read_limit) + .to_owned(); + + let mut stmt = stmt_base.clone(); + + let client = redis::Client::open(cache_url).unwrap(); + let mut redis_conn = client.get_connection().unwrap(); + + let mut remaining = total_num; + let mut pagination: i64 = 0; + + loop { + let res = db.query_all(bk.build(&stmt)).await?; + if res.len() == 0 { + break; + } + let val: Vec<(String, String, String)> = res + .iter() + .filter_map(|q| q.try_get_many_by_index().ok()) + .collect(); + + remaining -= val.len() as i64; + if remaining <= copy_limit { + let mut pipe = redis::pipe(); + for v in &val { + pipe.xadd_maxlen( + format!("{}:antennaTimeline:{}", prefix, v.1), + StreamMaxlen::Approx(200), + "*", + &[("note", v.2.to_owned())], + ) + .ignore(); + } + pipe.query::<()>(&mut redis_conn).unwrap(); + } + + let copied = total_num - remaining; + let copied = std::cmp::min(copied, total_num); + pagination += 1; + if pagination % 10 == 0 { + println!( + "Migrating antenna [{:.2}%]", + (copied as f64 / total_num as f64) * 100_f64, + ); + } + + if let Some((last_id, _, _)) = val.last() { + stmt = stmt_base + .clone() + .and_where( + Expr::col((AntennaNote::Table, AntennaNote::Id)).gt(last_id.to_owned()), + ) + .to_owned(); + } else { + break; + } + } + + println!("Migrating antenna [100.00%]"); + } + + manager + .drop_table( + Table::drop() + .table(AntennaNote::Table) + .if_exists() + .to_owned(), + ) + .await?; + + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(AntennaNote::Table) + .if_not_exists() + .col( + ColumnDef::new(AntennaNote::Id) + .string_len(32) + .not_null() + .primary_key(), + ) + .col( + ColumnDef::new(AntennaNote::NoteId) + .string_len(32) + .not_null(), + ) + .col( + ColumnDef::new(AntennaNote::AntennaId) + .string_len(32) + .not_null(), + ) + .col( + ColumnDef::new(AntennaNote::Read) + .boolean() + .default(false) + .not_null(), + ) + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .name("IDX_0d775946662d2575dfd2068a5f") + .table(AntennaNote::Table) + .col(AntennaNote::AntennaId) + .if_not_exists() + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .name("IDX_bd0397be22147e17210940e125") + .table(AntennaNote::Table) + .col(AntennaNote::NoteId) + .if_not_exists() + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .name("IDX_335a0bf3f904406f9ef3dd51c2") + .table(AntennaNote::Table) + .col(AntennaNote::NoteId) + .col(AntennaNote::AntennaId) + .unique() + .if_not_exists() + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .name("IDX_9937ea48d7ae97ffb4f3f063a4") + .table(AntennaNote::Table) + .col(AntennaNote::Read) + .if_not_exists() + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_0d775946662d2575dfd2068a5f5") + .from(AntennaNote::Table, AntennaNote::AntennaId) + .to(Antenna::Table, Antenna::Id) + .on_delete(ForeignKeyAction::Cascade) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_bd0397be22147e17210940e125b") + .from(AntennaNote::Table, AntennaNote::NoteId) + .to(Note::Table, Note::Id) + .on_delete(ForeignKeyAction::Cascade) + .to_owned(), + ) + .await?; + + Ok(()) + } +} + +/// Learn more at https://docs.rs/sea-query#iden +#[derive(Iden)] +enum AntennaNote { + Table, + Id, + #[iden = "noteId"] + NoteId, + #[iden = "antennaId"] + AntennaId, + Read, +} + +#[derive(Iden)] +enum Antenna { + Table, + Id, +} + +#[derive(Iden)] +enum Note { + Table, + Id, +} diff --git a/packages/backend/native-utils/migration/src/main.rs b/packages/backend/native-utils/migration/src/main.rs index f0f761f65..896f1ed59 100644 --- a/packages/backend/native-utils/migration/src/main.rs +++ b/packages/backend/native-utils/migration/src/main.rs @@ -5,6 +5,10 @@ use urlencoding::encode; use sea_orm_migration::prelude::*; +const DB_URL_ENV: &str = "DATABASE_URL"; +const CACHE_URL_ENV: &str = "CACHE_URL"; +const CACHE_PREFIX_ENV: &str = "CACHE_PREFIX"; + #[cfg(feature = "convert")] mod vec_to_json; @@ -15,17 +19,48 @@ async fn main() { .expect("Failed to open '.config/default.yml'"); let config: Config = serde_yaml::from_reader(yml).expect("Failed to parse yaml"); - env::set_var( - "DATABASE_URL", - format!( - "postgres://{}:{}@{}:{}/{}", - config.db.user, - encode(&config.db.pass), - config.db.host, - config.db.port, - config.db.db, - ), - ); + if env::var_os(DB_URL_ENV).is_none() { + env::set_var( + DB_URL_ENV, + format!( + "postgres://{}:{}@{}:{}/{}", + config.db.user, + encode(&config.db.pass), + config.db.host, + config.db.port, + config.db.db, + ), + ); + }; + + if env::var_os(CACHE_URL_ENV).is_none() { + let redis_conf = match config.cache_server { + None => config.redis, + Some(conf) => conf, + }; + let redis_proto = match redis_conf.tls { + None => "redis", + Some(_) => "rediss", + }; + let redis_uri_userpass = match redis_conf.user { + None => "".to_string(), + Some(user) => format!("{}:{}@", user, encode(&redis_conf.pass.unwrap_or_default())), + }; + let redis_uri_hostport = format!("{}:{}", redis_conf.host, redis_conf.port); + let redis_uri = format!( + "{}://{}{}/{}", + redis_proto, redis_uri_userpass, redis_uri_hostport, redis_conf.db + ); + env::set_var(CACHE_URL_ENV, redis_uri); + env::set_var( + CACHE_PREFIX_ENV, + if redis_conf.prefix.is_empty() { + config.url.host_str().unwrap() + } else { + &redis_conf.prefix + }, + ); + } cli::run_cli(migration::Migrator).await; @@ -34,13 +69,15 @@ async fn main() { } #[derive(Debug, PartialEq, Deserialize)] -#[serde(rename = "camelCase")] +#[serde(rename_all = "camelCase")] pub struct Config { + pub url: url::Url, pub db: DbConfig, + pub redis: RedisConfig, + pub cache_server: Option, } #[derive(Debug, PartialEq, Deserialize)] -#[serde(rename = "camelCase")] pub struct DbConfig { pub host: String, pub port: u32, @@ -48,3 +85,23 @@ pub struct DbConfig { pub user: String, pub pass: String, } + +#[derive(Debug, PartialEq, Deserialize)] +pub struct RedisConfig { + pub host: String, + pub port: u32, + pub user: Option, + pub pass: Option, + pub tls: Option, + #[serde(default)] + pub db: u32, + #[serde(default)] + pub prefix: String, +} + +#[derive(Debug, PartialEq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TlsConfig { + pub host: String, + pub reject_unauthorized: bool, +} diff --git a/packages/backend/native-utils/src/model/entity.rs b/packages/backend/native-utils/src/model/entity.rs index d71057fde..ccf3f0060 100644 --- a/packages/backend/native-utils/src/model/entity.rs +++ b/packages/backend/native-utils/src/model/entity.rs @@ -8,7 +8,6 @@ pub mod ad; pub mod announcement; pub mod announcement_read; pub mod antenna; -pub mod antenna_note; pub mod app; pub mod attestation_challenge; pub mod auth_session; diff --git a/packages/backend/native-utils/src/model/entity/announcement.rs b/packages/backend/native-utils/src/model/entity/announcement.rs index e8a2a28aa..5cdb690d2 100644 --- a/packages/backend/native-utils/src/model/entity/announcement.rs +++ b/packages/backend/native-utils/src/model/entity/announcement.rs @@ -15,6 +15,10 @@ pub struct Model { pub image_url: Option, #[sea_orm(column_name = "updatedAt")] pub updated_at: Option, + #[sea_orm(column_name = "showPopup")] + pub show_popup: bool, + #[sea_orm(column_name = "isGoodNews")] + pub is_good_news: bool, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/packages/backend/native-utils/src/model/entity/antenna.rs b/packages/backend/native-utils/src/model/entity/antenna.rs index 85bdfbfea..9d5a64f82 100644 --- a/packages/backend/native-utils/src/model/entity/antenna.rs +++ b/packages/backend/native-utils/src/model/entity/antenna.rs @@ -37,8 +37,6 @@ pub struct Model { #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] pub enum Relation { - #[sea_orm(has_many = "super::antenna_note::Entity")] - AntennaNote, #[sea_orm( belongs_to = "super::user::Entity", from = "Column::UserId", @@ -65,12 +63,6 @@ pub enum Relation { UserList, } -impl Related for Entity { - fn to() -> RelationDef { - Relation::AntennaNote.def() - } -} - impl Related for Entity { fn to() -> RelationDef { Relation::User.def() diff --git a/packages/backend/native-utils/src/model/entity/meta.rs b/packages/backend/native-utils/src/model/entity/meta.rs index 2c0dc315c..ac43a7842 100644 --- a/packages/backend/native-utils/src/model/entity/meta.rs +++ b/packages/backend/native-utils/src/model/entity/meta.rs @@ -189,6 +189,10 @@ pub struct Model { pub silenced_hosts: StringVec, #[sea_orm(column_name = "experimentalFeatures", column_type = "JsonBinary")] pub experimental_features: Json, + #[sea_orm(column_name = "enableServerMachineStats")] + pub enable_server_machine_stats: bool, + #[sea_orm(column_name = "enableIdenticonGeneration")] + pub enable_identicon_generation: bool, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/packages/backend/native-utils/src/model/entity/note.rs b/packages/backend/native-utils/src/model/entity/note.rs index 077841e48..5c97a20cf 100644 --- a/packages/backend/native-utils/src/model/entity/note.rs +++ b/packages/backend/native-utils/src/model/entity/note.rs @@ -67,8 +67,6 @@ pub struct Model { #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] pub enum Relation { - #[sea_orm(has_many = "super::antenna_note::Entity")] - AntennaNote, #[sea_orm( belongs_to = "super::channel::Entity", from = "Column::ChannelId", @@ -131,12 +129,6 @@ pub enum Relation { UserNotePining, } -impl Related for Entity { - fn to() -> RelationDef { - Relation::AntennaNote.def() - } -} - impl Related for Entity { fn to() -> RelationDef { Relation::Channel.def() diff --git a/packages/backend/native-utils/src/model/entity/prelude.rs b/packages/backend/native-utils/src/model/entity/prelude.rs index 8be696cb4..a859a9cc3 100644 --- a/packages/backend/native-utils/src/model/entity/prelude.rs +++ b/packages/backend/native-utils/src/model/entity/prelude.rs @@ -6,7 +6,6 @@ pub use super::ad::Entity as Ad; pub use super::announcement::Entity as Announcement; pub use super::announcement_read::Entity as AnnouncementRead; pub use super::antenna::Entity as Antenna; -pub use super::antenna_note::Entity as AntennaNote; pub use super::app::Entity as App; pub use super::attestation_challenge::Entity as AttestationChallenge; pub use super::auth_session::Entity as AuthSession; diff --git a/packages/backend/native-utils/src/model/repository/antenna.rs b/packages/backend/native-utils/src/model/repository/antenna.rs index 2b761173e..80f34fed1 100644 --- a/packages/backend/native-utils/src/model/repository/antenna.rs +++ b/packages/backend/native-utils/src/model/repository/antenna.rs @@ -1,9 +1,9 @@ use async_trait::async_trait; use cfg_if::cfg_if; -use sea_orm::{ColumnTrait, EntityTrait, QueryFilter}; +use sea_orm::EntityTrait; use crate::database; -use crate::model::entity::{antenna, antenna_note, user_group_joining}; +use crate::model::entity::{antenna, user_group_joining}; use crate::model::error::Error; use crate::model::schema::Antenna; @@ -14,12 +14,6 @@ use super::Repository; impl Repository for antenna::Model { async fn pack(self) -> Result { let db = database::get_database()?; - let has_unread_note = antenna_note::Entity::find() - .filter(antenna_note::Column::AntennaId.eq(self.id.to_owned())) - .filter(antenna_note::Column::Read.eq(false)) - .one(db) - .await? - .is_some(); let user_group_joining = match self.user_group_joining_id { None => None, Some(id) => user_group_joining::Entity::find_by_id(id).one(db).await?, @@ -52,7 +46,7 @@ impl Repository for antenna::Model { notify: self.notify, with_replies: self.with_replies, with_file: self.with_file, - has_unread_note, + has_unread_note: false, }) } diff --git a/packages/backend/native-utils/src/model/schema/app.rs b/packages/backend/native-utils/src/model/schema/app.rs index 9b5691154..5f859ecb3 100644 --- a/packages/backend/native-utils/src/model/schema/app.rs +++ b/packages/backend/native-utils/src/model/schema/app.rs @@ -105,9 +105,9 @@ mod unit_test { #[test] fn app_valid() { - init_id(12, ""); + init_id(16, ""); let instance = json!({ - "id": create_id().unwrap(), + "id": create_id(0).unwrap(), "name": "Test App", "secret": gen_string(24), "callbackUrl": "urn:ietf:wg:oauth:2.0:oob", @@ -119,9 +119,9 @@ mod unit_test { #[test] fn app_invalid() { - init_id(12, ""); + init_id(16, ""); let instance = json!({ - "id": create_id().unwrap(), + "id": create_id(0).unwrap(), // "name" is required "name": null, // "permission" must be one of the app permissions diff --git a/packages/backend/native-utils/src/util/id.rs b/packages/backend/native-utils/src/util/id.rs index d922518f9..b18637fdb 100644 --- a/packages/backend/native-utils/src/util/id.rs +++ b/packages/backend/native-utils/src/util/id.rs @@ -1,7 +1,10 @@ //! ID generation utility based on [cuid2] use cfg_if::cfg_if; +use chrono::Utc; use once_cell::sync::OnceCell; +use radix_fmt::radix_36; +use std::cmp; use crate::impl_into_napi_error; @@ -14,47 +17,56 @@ impl_into_napi_error!(ErrorUninitialized); static FINGERPRINT: OnceCell = OnceCell::new(); static GENERATOR: OnceCell = OnceCell::new(); +const TIME_2000: i64 = 946_684_800_000; +const TIMESTAMP_LENGTH: u16 = 8; + /// Initializes Cuid2 generator. Must be called before any [create_id]. -pub fn init_id(length: u16, fingerprint: impl Into) { - FINGERPRINT.get_or_init(move || format!("{}{}", fingerprint.into(), cuid2::create_id())); +pub fn init_id<'a>(length: u16, fingerprint: &'a str) { + FINGERPRINT.get_or_init(move || format!("{}{}", fingerprint, cuid2::create_id())); GENERATOR.get_or_init(move || { cuid2::CuidConstructor::new() - .with_length(length) + // length to pass shoule be greater than or equal to 8. + .with_length(cmp::max(length - TIMESTAMP_LENGTH, 8)) .with_fingerprinter(|| FINGERPRINT.get().unwrap().clone()) }); } /// Returns Cuid2 with the length specified by [init_id]. Must be called after /// [init_id], otherwise returns [ErrorUninitialized]. -pub fn create_id() -> Result { +/// The current timestamp via [chrono::Utc] is used if `date_num` is `0`. +pub fn create_id(date_num: i64) -> Result { match GENERATOR.get() { None => Err(ErrorUninitialized), - Some(gen) => Ok(gen.create_id()), + Some(gen) => { + let date_num = if date_num > 0 { + date_num + } else { + Utc::now().timestamp_millis() + }; + let time = cmp::max(date_num - TIME_2000, 0); + Ok(format!( + "{:0>8}{}", + radix_36(time).to_string(), + gen.create_id() + )) + } } } cfg_if! { if #[cfg(feature = "napi")] { - use radix_fmt::radix_36; - use std::cmp; - use napi::bindgen_prelude::BigInt; use napi_derive::napi; - const TIME_2000: u64 = 946_684_800_000; - const TIMESTAMP_LENGTH: u16 = 8; - /// Calls [init_id] inside. Must be called before [native_create_id]. #[napi] pub fn native_init_id_generator(length: u16, fingerprint: String) { - // length to pass init_id shoule be greater than or equal to 8. - init_id(cmp::max(length - TIMESTAMP_LENGTH, 8), fingerprint); + init_id(length, &fingerprint); } /// Generates #[napi] - pub fn native_create_id(date_num: BigInt) -> String { - let time = cmp::max(date_num.get_u64().1 - TIME_2000, 0); - format!("{:0>8}{}", radix_36(time).to_string(), create_id().unwrap()) + pub fn native_create_id(date_num: i64) -> String { + create_id(date_num).unwrap() } } } @@ -62,37 +74,17 @@ cfg_if! { #[cfg(test)] mod unit_test { use crate::util::id; - use cfg_if::cfg_if; use pretty_assertions::{assert_eq, assert_ne}; use std::thread; - cfg_if! { - if #[cfg(feature = "napi")] { - use chrono::Utc; - - #[test] - fn can_generate_aid_compat_ids() { - id::native_init_id_generator(20, "".to_string()); - let id1 = id::native_create_id(Utc::now().timestamp_millis().into()); - assert_eq!(id1.len(), 20); - let id1 = id::native_create_id(Utc::now().timestamp_millis().into()); - let id2 = id::native_create_id(Utc::now().timestamp_millis().into()); - assert_ne!(id1, id2); - let id1 = thread::spawn(|| id::native_create_id(Utc::now().timestamp_millis().into())); - let id2 = thread::spawn(|| id::native_create_id(Utc::now().timestamp_millis().into())); - assert_ne!(id1.join().unwrap(), id2.join().unwrap()); - } - } else { - #[test] - fn can_generate_unique_ids() { - assert_eq!(id::create_id(), Err(id::ErrorUninitialized)); - id::init_id(12, ""); - assert_eq!(id::create_id().unwrap().len(), 12); - assert_ne!(id::create_id().unwrap(), id::create_id().unwrap()); - let id1 = thread::spawn(|| id::create_id().unwrap()); - let id2 = thread::spawn(|| id::create_id().unwrap()); - assert_ne!(id1.join().unwrap(), id2.join().unwrap()); - } - } + #[test] + fn can_generate_unique_ids() { + assert_eq!(id::create_id(0), Err(id::ErrorUninitialized)); + id::init_id(16, ""); + assert_eq!(id::create_id(0).unwrap().len(), 16); + assert_ne!(id::create_id(0).unwrap(), id::create_id(0).unwrap()); + let id1 = thread::spawn(|| id::create_id(0).unwrap()); + let id2 = thread::spawn(|| id::create_id(0).unwrap()); + assert_ne!(id1.join().unwrap(), id2.join().unwrap()); } } diff --git a/packages/backend/native-utils/tests/common.rs b/packages/backend/native-utils/tests/common.rs index b134319ca..674297606 100644 --- a/packages/backend/native-utils/tests/common.rs +++ b/packages/backend/native-utils/tests/common.rs @@ -139,11 +139,11 @@ async fn cleanup() { } async fn setup_model(db: &DbConn) { - init_id(12, ""); + init_id(16, ""); db.transaction::<_, (), DbErr>(|txn| { Box::pin(async move { - let user_id = create_id().unwrap(); + let user_id = create_id(0).unwrap(); let name = "Alice"; let user_model = entity::user::Model { id: user_id.to_owned(), @@ -161,7 +161,7 @@ async fn setup_model(db: &DbConn) { .insert(txn) .await?; let antenna_model = entity::antenna::Model { - id: create_id().unwrap(), + id: create_id(0).unwrap(), created_at: Utc::now().into(), user_id: user_id.to_owned(), name: "Alice Antenna".to_string(), @@ -186,7 +186,7 @@ async fn setup_model(db: &DbConn) { .insert(txn) .await?; let note_model = entity::note::Model { - id: create_id().unwrap(), + id: create_id(0).unwrap(), created_at: Utc::now().into(), text: Some("Testing 123".to_string()), user_id: user_id.to_owned(), diff --git a/packages/backend/native-utils/tests/model/repository/antenna.rs b/packages/backend/native-utils/tests/model/repository/antenna.rs index 80eea6771..9b03d3652 100644 --- a/packages/backend/native-utils/tests/model/repository/antenna.rs +++ b/packages/backend/native-utils/tests/model/repository/antenna.rs @@ -93,7 +93,7 @@ mod int_test { .unwrap() .expect("note not found"); let antenna_note = antenna_note::Model { - id: util::id::create_id().unwrap(), + id: util::id::create_id(0).unwrap(), antenna_id: alice_antenna.id.to_owned(), note_id: note_model.id.to_owned(), read: false, diff --git a/packages/backend/package.json b/packages/backend/package.json index 8564ca832..99f1cadd5 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -59,12 +59,14 @@ "color-convert": "2.0.1", "content-disposition": "0.5.4", "date-fns": "2.30.0", + "decompress": "^4.2.1", "deep-email-validator": "0.1.21", "escape-regexp": "0.0.1", "feed": "4.2.2", "file-type": "17.1.6", "fluent-ffmpeg": "2.1.2", "got": "12.5.3", + "gunzip-maybe": "^1.4.2", "hpagent": "0.1.2", "ioredis": "5.3.2", "ip-cidr": "3.1.0", @@ -126,6 +128,7 @@ "summaly": "2.7.0", "syslog-pro": "1.0.0", "systeminformation": "5.17.17", + "tar-stream": "^3.1.6", "tesseract.js": "^3.0.3", "tinycolor2": "1.5.2", "tmp": "0.2.1", diff --git a/packages/backend/src/config/load.ts b/packages/backend/src/config/load.ts index fa9878955..df163a8ac 100644 --- a/packages/backend/src/config/load.ts +++ b/packages/backend/src/config/load.ts @@ -54,9 +54,9 @@ export default function load() { mixin.userAgent = `Calckey/${meta.version} (${config.url})`; mixin.clientEntry = clientManifest["src/init.ts"]; - if (!config.redis.prefix) config.redis.prefix = mixin.host; + if (!config.redis.prefix) config.redis.prefix = mixin.hostname; if (config.cacheServer && !config.cacheServer.prefix) - config.cacheServer.prefix = mixin.host; + config.cacheServer.prefix = mixin.hostname; return Object.assign(config, mixin); } diff --git a/packages/backend/src/db/postgre.ts b/packages/backend/src/db/postgre.ts index 89b7a7bf6..10ea5b15f 100644 --- a/packages/backend/src/db/postgre.ts +++ b/packages/backend/src/db/postgre.ts @@ -58,7 +58,6 @@ import { AnnouncementRead } from "@/models/entities/announcement-read.js"; import { Clip } from "@/models/entities/clip.js"; import { ClipNote } from "@/models/entities/clip-note.js"; import { Antenna } from "@/models/entities/antenna.js"; -import { AntennaNote } from "@/models/entities/antenna-note.js"; import { PromoNote } from "@/models/entities/promo-note.js"; import { PromoRead } from "@/models/entities/promo-read.js"; import { Relay } from "@/models/entities/relay.js"; @@ -168,7 +167,6 @@ export const entities = [ Clip, ClipNote, Antenna, - AntennaNote, PromoNote, PromoRead, Relay, diff --git a/packages/backend/src/misc/gen-id.ts b/packages/backend/src/misc/gen-id.ts index ea0d414e7..580c39c3c 100644 --- a/packages/backend/src/misc/gen-id.ts +++ b/packages/backend/src/misc/gen-id.ts @@ -17,5 +17,5 @@ nativeInitIdGenerator(length, fingerprint); * Ref: https://github.com/paralleldrive/cuid2#parameterized-length */ export function genId(date?: Date): string { - return nativeCreateId(BigInt((date ?? new Date()).getTime())); + return nativeCreateId((date ?? new Date()).getTime()); } diff --git a/packages/backend/src/misc/process-masto-notes.ts b/packages/backend/src/misc/process-masto-notes.ts new file mode 100644 index 000000000..b8c1c426d --- /dev/null +++ b/packages/backend/src/misc/process-masto-notes.ts @@ -0,0 +1,136 @@ +import * as fs from "node:fs"; +import Logger from "@/services/logger.js"; +import { createTemp, createTempDir } from "./create-temp.js"; +import { downloadUrl } from "./download-url.js"; +import { addFile } from "@/services/drive/add-file.js"; +import { Users } from "@/models/index.js"; +import * as tar from "tar-stream"; +import gunzip from "gunzip-maybe"; +import decompress from "decompress"; +import * as Path from "node:path"; + +const logger = new Logger("process-masto-notes"); + +export async function processMastoNotes( + fn: string, + url: string, + uid: string, +): Promise { + // Create temp file + const [path, cleanup] = await createTemp(); + + const [unzipPath, unzipCleanup] = await createTempDir(); + + logger.info(`Temp file is ${path}`); + + try { + // write content at URL to temp file + await downloadUrl(url, path); + return await processMastoFile(fn, path, unzipPath, uid); + } finally { + cleanup(); + //unzipCleanup(); + } +} + +function processMastoFile(fn: string, path: string, dir: string, uid: string) { + return new Promise(async (resolve, reject) => { + const user = await Users.findOneBy({ id: uid }); + try { + logger.info(`Start unzip ${path}`); + fn.endsWith("tar.gz") + ? await unzipTarGz(path, dir) + : await unzipZip(path, dir); + logger.info(`Unzip to ${dir}`); + const outbox = JSON.parse(fs.readFileSync(`${dir}/outbox.json`)); + for (const note of outbox.orderedItems) { + for (const attachment of note.object.attachment) { + const url = attachment.url.replaceAll("..", ""); + if (url.indexOf('\0') !== -1) { + logger.error(`Found Poison Null Bytes Attack: ${url}`); + reject(); + return; + } + try { + const fpath = Path.resolve(`${dir}${url}`); + if (!fpath.startsWith(dir)) { + logger.error(`Found Path Attack: ${url}`); + reject(); + return; + } + logger.info(fpath); + const driveFile = await addFile({ user: user, path: fpath }); + attachment.driveFile = driveFile; + } catch (e) { + logger.error(`Skipped adding file to drive: ${url}`); + } + } + } + resolve(outbox); + } catch (e) { + logger.error(`Error on extract masto note package: ${fn}`); + reject(e); + } + }); +} + +function createFileDir(fn: string) { + if (!fs.existsSync(fn)) { + fs.mkdirSync(fn, { recursive: true }); + fs.rmdirSync(fn); + } +} + +function unzipZip(fn: string, dir: string) { + return new Promise(async (resolve, reject) => { + try { + decompress(fn, dir).then((files: any) => { + resolve(files); + }); + } catch (e) { + reject(); + } + }); +} + +function unzipTarGz(fn: string, dir: string) { + return new Promise(async (resolve, reject) => { + const onErr = (err: any) => { + logger.error(`pipe broken: ${err}`); + reject(); + }; + try { + const extract = tar.extract().on("error", onErr); + dir = dir.endsWith("/") ? dir : dir + "/"; + const ls: string[] = []; + extract.on("entry", function (header: any, stream: any, next: any) { + try { + ls.push(dir + header.name); + createFileDir(dir + header.name); + stream + .on("error", onErr) + .pipe(fs.createWriteStream(dir + header.name)) + .on("error", onErr); + next(); + } catch (e) { + logger.error(`create dir error:${e}`); + reject(); + } + }); + + extract.on("finish", function () { + resolve(ls); + }); + + fs.createReadStream(fn) + .on("error", onErr) + .pipe(gunzip()) + .on("error", onErr) + .pipe(extract) + .on("error", onErr); + } catch (e) { + logger.error(`unzipTarGz error: ${e}`); + reject(); + } + }); +} diff --git a/packages/backend/src/models/entities/antenna-note.ts b/packages/backend/src/models/entities/antenna-note.ts deleted file mode 100644 index fe982c19e..000000000 --- a/packages/backend/src/models/entities/antenna-note.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { - Entity, - Index, - JoinColumn, - Column, - ManyToOne, - PrimaryColumn, -} from "typeorm"; -import { Note } from "./note.js"; -import { Antenna } from "./antenna.js"; -import { id } from "../id.js"; - -@Entity() -@Index(["noteId", "antennaId"], { unique: true }) -export class AntennaNote { - @PrimaryColumn(id()) - public id: string; - - @Index() - @Column({ - ...id(), - comment: "The note ID.", - }) - public noteId: Note["id"]; - - @ManyToOne((type) => Note, { - onDelete: "CASCADE", - }) - @JoinColumn() - public note: Note | null; - - @Index() - @Column({ - ...id(), - comment: "The antenna ID.", - }) - public antennaId: Antenna["id"]; - - @ManyToOne((type) => Antenna, { - onDelete: "CASCADE", - }) - @JoinColumn() - public antenna: Antenna | null; - - @Index() - @Column("boolean", { - default: false, - }) - public read: boolean; -} diff --git a/packages/backend/src/models/index.ts b/packages/backend/src/models/index.ts index cfc3b01c5..8782d5740 100644 --- a/packages/backend/src/models/index.ts +++ b/packages/backend/src/models/index.ts @@ -51,7 +51,6 @@ import { UsedUsername } from "./entities/used-username.js"; import { ClipRepository } from "./repositories/clip.js"; import { ClipNote } from "./entities/clip-note.js"; import { AntennaRepository } from "./repositories/antenna.js"; -import { AntennaNote } from "./entities/antenna-note.js"; import { PromoNote } from "./entities/promo-note.js"; import { PromoRead } from "./entities/promo-read.js"; import { EmojiRepository } from "./repositories/emoji.js"; @@ -123,7 +122,6 @@ export const ModerationLogs = ModerationLogRepository; export const Clips = ClipRepository; export const ClipNotes = db.getRepository(ClipNote); export const Antennas = AntennaRepository; -export const AntennaNotes = db.getRepository(AntennaNote); export const PromoNotes = db.getRepository(PromoNote); export const PromoReads = db.getRepository(PromoRead); export const Relays = RelayRepository; diff --git a/packages/backend/src/models/repositories/user.ts b/packages/backend/src/models/repositories/user.ts index e4aab896d..e7c4b6f00 100644 --- a/packages/backend/src/models/repositories/user.ts +++ b/packages/backend/src/models/repositories/user.ts @@ -18,7 +18,6 @@ import { createPerson } from "@/remote/activitypub/models/person.js"; import { AnnouncementReads, Announcements, - AntennaNotes, Blockings, ChannelFollowings, DriveFiles, @@ -258,23 +257,24 @@ export const UserRepository = db.getRepository(User).extend({ }, async getHasUnreadAntenna(userId: User["id"]): Promise { - try { - const myAntennas = (await getAntennas()).filter( - (a) => a.userId === userId, - ); + // try { + // const myAntennas = (await getAntennas()).filter( + // (a) => a.userId === userId, + // ); - const unread = - myAntennas.length > 0 - ? await AntennaNotes.findOneBy({ - antennaId: In(myAntennas.map((x) => x.id)), - read: false, - }) - : null; + // const unread = + // myAntennas.length > 0 + // ? await AntennaNotes.findOneBy({ + // antennaId: In(myAntennas.map((x) => x.id)), + // read: false, + // }) + // : null; - return unread != null; - } catch (e) { - return false; - } + // return unread != null; + // } catch (e) { + // return false; + // } + return false; // TODO }, async getHasUnreadChannel(userId: User["id"]): Promise { diff --git a/packages/backend/src/queue/processors/db/import-masto-post.ts b/packages/backend/src/queue/processors/db/import-masto-post.ts index 05166b085..1d18008a0 100644 --- a/packages/backend/src/queue/processors/db/import-masto-post.ts +++ b/packages/backend/src/queue/processors/db/import-masto-post.ts @@ -45,19 +45,26 @@ export async function importMastoPost( throw e; } job.progress(80); - const urls = post.object.attachment - .map((x: any) => x.url) - .filter((x: String) => x.startsWith("http")); - const files: DriveFile[] = []; - for (const url of urls) { - try { - const file = await uploadFromUrl({ - url: url, - user: user, - }); - files.push(file); - } catch (e) { - logger.error(`Skipped adding file to drive: ${url}`); + + let files: DriveFile[] = (post.object.attachment || []) + .map((x: any) => x?.driveFile) + .filter((x: any) => x); + + if (files.length == 0) { + const urls = post.object.attachment + .map((x: any) => x.url) + .filter((x: String) => x.startsWith("http")); + files = []; + for (const url of urls) { + try { + const file = await uploadFromUrl({ + url: url, + user: user, + }); + files.push(file); + } catch (e) { + logger.error(`Skipped adding file to drive: ${url}`); + } } } diff --git a/packages/backend/src/queue/processors/db/import-posts.ts b/packages/backend/src/queue/processors/db/import-posts.ts index f92a5f710..9bde7479e 100644 --- a/packages/backend/src/queue/processors/db/import-posts.ts +++ b/packages/backend/src/queue/processors/db/import-posts.ts @@ -1,4 +1,5 @@ import { downloadTextFile } from "@/misc/download-text-file.js"; +import { processMastoNotes } from "@/misc/process-masto-notes.js"; import { Users, DriveFiles } from "@/models/index.js"; import type { DbUserImportPostsJobData } from "@/queue/types.js"; import { queueLogger } from "../../logger.js"; @@ -30,6 +31,26 @@ export async function importPosts( return; } + if (file.name.endsWith("tar.gz") || file.name.endsWith("zip")) { + try { + logger.info("Reading Mastodon archive"); + const outbox = await processMastoNotes( + file.name, + file.url, + job.data.user.id, + ); + for (const post of outbox.orderedItems) { + createImportMastoPostJob(job.data.user, post, job.data.signatureCheck); + } + } catch (e) { + // handle error + logger.warn(`Failed reading Mastodon archive: ${e}`); + } + logger.succ("Mastodon archive imported"); + done(); + return; + } + const json = await downloadTextFile(file.url); try { diff --git a/packages/backend/src/server/api/endpoints/antennas/create.ts b/packages/backend/src/server/api/endpoints/antennas/create.ts index c2117d51a..ed1645012 100644 --- a/packages/backend/src/server/api/endpoints/antennas/create.ts +++ b/packages/backend/src/server/api/endpoints/antennas/create.ts @@ -30,7 +30,7 @@ export const meta = { id: "c3a5a51e-04d4-11ee-be56-0242ac120002", }, noKeywords: { - message: "No keywords", + message: "No keywords.", code: "NO_KEYWORDS", id: "aa975b74-1ddb-11ee-be56-0242ac120002", }, diff --git a/packages/backend/src/server/api/endpoints/antennas/markread.ts b/packages/backend/src/server/api/endpoints/antennas/markread.ts index e29e13bbb..db8e683e4 100644 --- a/packages/backend/src/server/api/endpoints/antennas/markread.ts +++ b/packages/backend/src/server/api/endpoints/antennas/markread.ts @@ -1,7 +1,6 @@ import define from "../../define.js"; -import { Antennas, AntennaNotes } from "@/models/index.js"; +import { Antennas } from "@/models/index.js"; import { FindOptionsWhere } from "typeorm"; -import { AntennaNote } from "@/models/entities/antenna-note.js"; export const meta = { tags: ["antennas", "account"], @@ -29,15 +28,15 @@ export default define(meta, paramDef, async (ps, me) => { return null; } - await AntennaNotes.update( - { - antennaId: antenna.id, - read: false, - }, - { - read: true, - }, - ); + // await AntennaNotes.update( + // { + // antennaId: antenna.id, + // read: false, + // }, + // { + // read: true, + // }, + // ); return true; }); diff --git a/packages/backend/src/server/api/endpoints/antennas/notes.ts b/packages/backend/src/server/api/endpoints/antennas/notes.ts index 29e2e085a..6ca71c08e 100644 --- a/packages/backend/src/server/api/endpoints/antennas/notes.ts +++ b/packages/backend/src/server/api/endpoints/antennas/notes.ts @@ -1,6 +1,8 @@ import define from "../../define.js"; import readNote from "@/services/note/read.js"; -import { Antennas, Notes, AntennaNotes } from "@/models/index.js"; +import { Antennas, Notes } from "@/models/index.js"; +import { redisClient } from "@/db/redis.js"; +import { genId } from "@/misc/gen-id.js"; import { makePaginationQuery } from "../../common/make-pagination-query.js"; import { generateVisibilityQuery } from "../../common/generate-visibility-query.js"; import { generateMutedUserQuery } from "../../common/generate-muted-user-query.js"; @@ -58,6 +60,26 @@ export default define(meta, paramDef, async (ps, user) => { throw new ApiError(meta.errors.noSuchAntenna); } + const noteIdsRes = await redisClient.xrevrange( + `antennaTimeline:${antenna.id}`, + ps.untilDate || "+", + "-", + "COUNT", + ps.limit + 1, + ); // untilIdに指定したものも含まれるため+1 + + if (noteIdsRes.length === 0) { + return []; + } + + const noteIds = noteIdsRes + .map((x) => x[1][1]) + .filter((x) => x !== ps.untilId); + + if (noteIds.length === 0) { + return []; + } + const query = makePaginationQuery( Notes.createQueryBuilder("note"), ps.sinceId, @@ -65,11 +87,7 @@ export default define(meta, paramDef, async (ps, user) => { ps.sinceDate, ps.untilDate, ) - .innerJoin( - AntennaNotes.metadata.targetName, - "antennaNote", - "antennaNote.noteId = note.id", - ) + .where("note.id IN (:...noteIds)", { noteIds: noteIds }) .innerJoinAndSelect("note.user", "user") .leftJoinAndSelect("user.avatar", "avatar") .leftJoinAndSelect("user.banner", "banner") @@ -81,7 +99,6 @@ export default define(meta, paramDef, async (ps, user) => { .leftJoinAndSelect("renote.user", "renoteUser") .leftJoinAndSelect("renoteUser.avatar", "renoteUserAvatar") .leftJoinAndSelect("renoteUser.banner", "renoteUserBanner") - .andWhere("antennaNote.antennaId = :antennaId", { antennaId: antenna.id }) .andWhere("note.visibility != 'home'"); generateVisibilityQuery(query, user); diff --git a/packages/backend/src/services/add-note-to-antenna.ts b/packages/backend/src/services/add-note-to-antenna.ts index 38979acb4..8d6d3e84d 100644 --- a/packages/backend/src/services/add-note-to-antenna.ts +++ b/packages/backend/src/services/add-note-to-antenna.ts @@ -1,61 +1,24 @@ import type { Antenna } from "@/models/entities/antenna.js"; import type { Note } from "@/models/entities/note.js"; -import { AntennaNotes, Mutings, Notes } from "@/models/index.js"; import { genId } from "@/misc/gen-id.js"; -import { isUserRelated } from "@/misc/is-user-related.js"; -import { publishAntennaStream, publishMainStream } from "@/services/stream.js"; +import { redisClient } from "@/db/redis.js"; +import { publishAntennaStream } from "@/services/stream.js"; import type { User } from "@/models/entities/user.js"; export async function addNoteToAntenna( antenna: Antenna, note: Note, - noteUser: { id: User["id"] }, + _noteUser: { id: User["id"] }, ) { - // 通知しない設定になっているか、自分自身の投稿なら既読にする - const read = !antenna.notify || antenna.userId === noteUser.id; - - AntennaNotes.insert({ - id: genId(), - antennaId: antenna.id, - noteId: note.id, - read: read, - }); + redisClient.xadd( + `antennaTimeline:${antenna.id}`, + "MAXLEN", + "~", + "200", + "*", + "note", + note.id, + ); publishAntennaStream(antenna.id, "note", note); - - if (!read) { - const mutings = await Mutings.find({ - where: { - muterId: antenna.userId, - }, - select: ["muteeId"], - }); - - // Copy - const _note: Note = { - ...note, - }; - - if (note.replyId != null) { - _note.reply = await Notes.findOneByOrFail({ id: note.replyId }); - } - if (note.renoteId != null) { - _note.renote = await Notes.findOneByOrFail({ id: note.renoteId }); - } - - if (isUserRelated(_note, new Set(mutings.map((x) => x.muteeId)))) { - return; - } - - // 2秒経っても既読にならなかったら通知 - setTimeout(async () => { - const unread = await AntennaNotes.findOneBy({ - antennaId: antenna.id, - read: false, - }); - if (unread) { - publishMainStream(antenna.userId, "unreadAntenna", antenna); - } - }, 2000); - } } diff --git a/packages/backend/src/services/note/read.ts b/packages/backend/src/services/note/read.ts index 53188f15f..5e61fe95d 100644 --- a/packages/backend/src/services/note/read.ts +++ b/packages/backend/src/services/note/read.ts @@ -3,7 +3,6 @@ import type { Note } from "@/models/entities/note.js"; import type { User } from "@/models/entities/user.js"; import { NoteUnreads, - AntennaNotes, Users, Followings, ChannelFollowings, @@ -51,11 +50,11 @@ export default async function ( ).map((x) => x.followeeId), ); - const myAntennas = (await getAntennas()).filter((a) => a.userId === userId); + // const myAntennas = (await getAntennas()).filter((a) => a.userId === userId); const readMentions: (Note | Packed<"Note">)[] = []; const readSpecifiedNotes: (Note | Packed<"Note">)[] = []; const readChannelNotes: (Note | Packed<"Note">)[] = []; - const readAntennaNotes: (Note | Packed<"Note">)[] = []; + // const readAntennaNotes: (Note | Packed<"Note">)[] = []; for (const note of notes) { if (note.mentions?.includes(userId)) { @@ -68,22 +67,22 @@ export default async function ( readChannelNotes.push(note); } - if (note.user != null) { - // たぶんnullになることは無いはずだけど一応 - for (const antenna of myAntennas) { - if ( - await checkHitAntenna( - antenna, - note, - note.user, - undefined, - Array.from(following), - ) - ) { - readAntennaNotes.push(note); - } - } - } + // if (note.user != null) { + // // たぶんnullになることは無いはずだけど一応 + // for (const antenna of myAntennas) { + // if ( + // await checkHitAntenna( + // antenna, + // note, + // note.user, + // undefined, + // Array.from(following), + // ) + // ) { + // readAntennaNotes.push(note); + // } + // } + // } } if ( @@ -141,33 +140,33 @@ export default async function ( }); } - if (readAntennaNotes.length > 0) { - await AntennaNotes.update( - { - antennaId: In(myAntennas.map((a) => a.id)), - noteId: In(readAntennaNotes.map((n) => n.id)), - }, - { - read: true, - }, - ); + // if (readAntennaNotes.length > 0) { + // await AntennaNotes.update( + // { + // antennaId: In(myAntennas.map((a) => a.id)), + // noteId: In(readAntennaNotes.map((n) => n.id)), + // }, + // { + // read: true, + // }, + // ); - // TODO: まとめてクエリしたい - for (const antenna of myAntennas) { - const count = await AntennaNotes.countBy({ - antennaId: antenna.id, - read: false, - }); + // // TODO: まとめてクエリしたい + // for (const antenna of myAntennas) { + // const count = await AntennaNotes.countBy({ + // antennaId: antenna.id, + // read: false, + // }); - if (count === 0) { - publishMainStream(userId, "readAntenna", antenna); - } - } + // if (count === 0) { + // publishMainStream(userId, "readAntenna", antenna); + // } + // } - Users.getHasUnreadAntenna(userId).then((unread) => { - if (!unread) { - publishMainStream(userId, "readAllAntennas"); - } - }); - } + // Users.getHasUnreadAntenna(userId).then((unread) => { + // if (!unread) { + // publishMainStream(userId, "readAllAntennas"); + // } + // }); + // } } diff --git a/packages/client/src/pages/antenna-timeline.vue b/packages/client/src/pages/antenna-timeline.vue index 825dbb96a..011e44ce6 100644 --- a/packages/client/src/pages/antenna-timeline.vue +++ b/packages/client/src/pages/antenna-timeline.vue @@ -100,11 +100,11 @@ const headerActions = $computed(() => text: i18n.ts.settings, handler: settings, }, - { - icon: "ph-check ph-bold ph-lg", - text: i18n.ts.markAllAsRead, - handler: markRead, - }, + // { + // icon: "ph-check ph-bold ph-lg", + // text: i18n.ts.markAllAsRead, + // handler: markRead, + // }, ] : [], ); diff --git a/packages/client/src/pages/settings/import-export.vue b/packages/client/src/pages/settings/import-export.vue index 4a9286748..b27bc6dd2 100644 --- a/packages/client/src/pages/settings/import-export.vue +++ b/packages/client/src/pages/settings/import-export.vue @@ -23,9 +23,7 @@ > - + diff --git a/packages/client/src/ui/deck.vue b/packages/client/src/ui/deck.vue index 9037ad156..6fdcfb28d 100644 --- a/packages/client/src/ui/deck.vue +++ b/packages/client/src/ui/deck.vue @@ -384,27 +384,6 @@ async function deleteProfile() { } - - -