From 879d499486f35c9ec420c4aac2238ff6e2a6149c Mon Sep 17 00:00:00 2001 From: naskya Date: Wed, 24 Apr 2024 07:37:16 +0900 Subject: [PATCH] refactor (backend-rs): never throw an error on ID generation --- packages/backend-rs/index.d.ts | 2 - packages/backend-rs/index.js | 3 +- .../src/misc/add_note_to_antenna.rs | 20 +++- packages/backend-rs/src/util/id.rs | 95 +++++++++++-------- packages/backend/src/boot/worker.ts | 6 -- 5 files changed, 71 insertions(+), 55 deletions(-) diff --git a/packages/backend-rs/index.d.ts b/packages/backend-rs/index.d.ts index 016932a092..49009c8b61 100644 --- a/packages/backend-rs/index.d.ts +++ b/packages/backend-rs/index.d.ts @@ -1129,8 +1129,6 @@ export enum ChatEvent { Typing = 'typing' } export function publishToChatStream(senderUserId: string, receiverUserId: string, kind: ChatEvent, object: any): void -/** Initializes Cuid2 generator. Must be called before any [create_id]. */ -export function initIdGenerator(length: number, fingerprint: string): void export function getTimestamp(id: string): number /** * The generated ID results in the form of `[8 chars timestamp] + [cuid2]`. diff --git a/packages/backend-rs/index.js b/packages/backend-rs/index.js index 9e15fa3658..563477d306 100644 --- a/packages/backend-rs/index.js +++ b/packages/backend-rs/index.js @@ -310,7 +310,7 @@ if (!nativeBinding) { throw new Error(`Failed to load native binding`) } -const { loadEnv, loadConfig, stringToAcct, acctToString, addNoteToAntenna, isBlockedServer, isSilencedServer, isAllowedServer, checkWordMute, getFullApAccount, isSelfHost, isSameOrigin, extractHost, toPuny, isUnicodeEmoji, sqlLikeEscape, safeForSql, formatMilliseconds, getNoteSummary, toMastodonId, fromMastodonId, fetchMeta, metaToPugArgs, nyaify, hashPassword, verifyPassword, isOldPasswordAlgorithm, decodeReaction, countReactions, toDbReaction, removeOldAttestationChallenges, AntennaSrcEnum, DriveFileUsageHintEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, ChatEvent, publishToChatStream, initIdGenerator, getTimestamp, genId, genIdAt, secureRndstr } = nativeBinding +const { loadEnv, loadConfig, stringToAcct, acctToString, addNoteToAntenna, isBlockedServer, isSilencedServer, isAllowedServer, checkWordMute, getFullApAccount, isSelfHost, isSameOrigin, extractHost, toPuny, isUnicodeEmoji, sqlLikeEscape, safeForSql, formatMilliseconds, getNoteSummary, toMastodonId, fromMastodonId, fetchMeta, metaToPugArgs, nyaify, hashPassword, verifyPassword, isOldPasswordAlgorithm, decodeReaction, countReactions, toDbReaction, removeOldAttestationChallenges, AntennaSrcEnum, DriveFileUsageHintEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, ChatEvent, publishToChatStream, getTimestamp, genId, genIdAt, secureRndstr } = nativeBinding module.exports.loadEnv = loadEnv module.exports.loadConfig = loadConfig @@ -356,7 +356,6 @@ module.exports.UserProfileFfvisibilityEnum = UserProfileFfvisibilityEnum module.exports.UserProfileMutingnotificationtypesEnum = UserProfileMutingnotificationtypesEnum module.exports.ChatEvent = ChatEvent module.exports.publishToChatStream = publishToChatStream -module.exports.initIdGenerator = initIdGenerator module.exports.getTimestamp = getTimestamp module.exports.genId = genId module.exports.genIdAt = genIdAt diff --git a/packages/backend-rs/src/misc/add_note_to_antenna.rs b/packages/backend-rs/src/misc/add_note_to_antenna.rs index 2ed698d7e6..2ce4e655d7 100644 --- a/packages/backend-rs/src/misc/add_note_to_antenna.rs +++ b/packages/backend-rs/src/misc/add_note_to_antenna.rs @@ -1,21 +1,31 @@ use crate::database::{redis_conn, redis_key}; use crate::model::entity::note; use crate::service::stream; -use crate::util::id::get_timestamp; -use redis::{streams::StreamMaxlen, Commands}; +use crate::util::id::{get_timestamp, InvalidIdErr}; +use redis::{streams::StreamMaxlen, Commands, RedisError}; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("Redis error: {0}")] + RedisErr(#[from] RedisError), + #[error("Invalid ID: {0}")] + InvalidIdErr(#[from] InvalidIdErr), + #[error("Stream error: {0}")] + StreamErr(#[from] stream::Error), +} type Note = note::Model; #[crate::export] -pub fn add_note_to_antenna(antenna_id: String, note: &Note) -> Result<(), stream::Error> { +pub fn add_note_to_antenna(antenna_id: String, note: &Note) -> Result<(), Error> { // for timeline API redis_conn()?.xadd_maxlen( redis_key(format!("antennaTimeline:{}", antenna_id)), StreamMaxlen::Approx(200), - format!("{}-*", get_timestamp(¬e.id)), + format!("{}-*", get_timestamp(¬e.id)?), &[("note", ¬e.id)], )?; // for streaming API - stream::antenna::publish(antenna_id, note) + Ok(stream::antenna::publish(antenna_id, note)?) } diff --git a/packages/backend-rs/src/util/id.rs b/packages/backend-rs/src/util/id.rs index 515c020ece..595a8ebb3a 100644 --- a/packages/backend-rs/src/util/id.rs +++ b/packages/backend-rs/src/util/id.rs @@ -1,54 +1,63 @@ //! ID generation utility based on [cuid2] +use crate::config::CONFIG; use basen::BASE36; use chrono::{DateTime, NaiveDateTime, Utc}; use once_cell::sync::OnceCell; use std::cmp; -#[derive(thiserror::Error, Debug, PartialEq, Eq)] -#[error("ID generator has not been initialized yet")] -pub struct ErrorUninitialized; - static FINGERPRINT: OnceCell = OnceCell::new(); static GENERATOR: OnceCell = OnceCell::new(); const TIME_2000: i64 = 946_684_800_000; -const TIMESTAMP_LENGTH: u16 = 8; +const TIMESTAMP_LENGTH: u8 = 8; -/// Initializes Cuid2 generator. Must be called before any [create_id]. -#[crate::export] -pub fn init_id_generator(length: u16, fingerprint: &str) { +/// Initializes Cuid2 generator. +fn init_id_generator(length: u8, fingerprint: &str) { FINGERPRINT.get_or_init(move || format!("{}{}", fingerprint, cuid2::create_id())); GENERATOR.get_or_init(move || { cuid2::CuidConstructor::new() // length to pass shoule be greater than or equal to 8. - .with_length(cmp::max(length - TIMESTAMP_LENGTH, 8)) + .with_length(cmp::max(length - TIMESTAMP_LENGTH, 8).into()) .with_fingerprinter(|| FINGERPRINT.get().unwrap().clone()) }); } -/// Returns Cuid2 with the length specified by [init_id]. Must be called after -/// [init_id], otherwise returns [ErrorUninitialized]. -pub fn create_id(datetime: &NaiveDateTime) -> Result { - match GENERATOR.get() { - None => Err(ErrorUninitialized), - Some(gen) => { - let date_num = cmp::max(0, datetime.and_utc().timestamp_millis() - TIME_2000) as u64; - Ok(format!( - "{:0>8}{}", - BASE36.encode_var_len(&date_num), - gen.create_id() - )) - } +/// Returns Cuid2 with the length specified by [init_id_generator]. +/// It automatically calls [init_id_generator], if the generator has not been initialized. +fn create_id(datetime: &NaiveDateTime) -> String { + if GENERATOR.get().is_none() { + let length = match &CONFIG.cuid { + Some(cuid) => cmp::min(cmp::max(cuid.length.unwrap_or(16), 16), 24), + None => 16, + }; + let fingerprint = match &CONFIG.cuid { + Some(cuid) => cuid.fingerprint.as_deref().unwrap_or_default(), + None => "", + }; + init_id_generator(length, fingerprint); } + let date_num = cmp::max(0, datetime.and_utc().timestamp_millis() - TIME_2000) as u64; + format!( + "{:0>8}{}", + BASE36.encode_var_len(&date_num), + GENERATOR.get().unwrap().create_id() + ) +} + +#[derive(thiserror::Error, Debug)] +#[error("Invalid ID: {id}")] +pub struct InvalidIdErr { + id: String, } #[crate::export] -pub fn get_timestamp(id: &str) -> i64 { +pub fn get_timestamp(id: &str) -> Result { let n: Option = BASE36.decode_var_len(&id[0..8]); - match n { - None => -1, - Some(n) => n as i64 + TIME_2000, + if let Some(n) = n { + Ok(n as i64 + TIME_2000) + } else { + Err(InvalidIdErr { id: id.to_string() }) } } @@ -60,35 +69,41 @@ pub fn get_timestamp(id: &str) -> i64 { /// Ref: https://github.com/paralleldrive/cuid2#parameterized-length #[crate::export] pub fn gen_id() -> String { - create_id(&Utc::now().naive_utc()).unwrap() + create_id(&Utc::now().naive_utc()) } /// Generate an ID using a specific datetime #[crate::export] pub fn gen_id_at(date: DateTime) -> String { - create_id(&date.naive_utc()).unwrap() + create_id(&date.naive_utc()) } #[cfg(test)] mod unit_test { - use crate::util::id; - use chrono::Utc; + use super::{gen_id, gen_id_at, get_timestamp}; + use chrono::{Duration, Utc}; use pretty_assertions::{assert_eq, assert_ne}; use std::thread; #[test] fn can_create_and_decode_id() { - let now = Utc::now().naive_utc(); - assert_eq!(id::create_id(&now), Err(id::ErrorUninitialized)); - id::init_id_generator(16, ""); - assert_eq!(id::create_id(&now).unwrap().len(), 16); - assert_ne!(id::create_id(&now).unwrap(), id::create_id(&now).unwrap()); - let id1 = thread::spawn(move || id::create_id(&now).unwrap()); - let id2 = thread::spawn(move || id::create_id(&now).unwrap()); + let now = Utc::now(); + assert_eq!(gen_id().len(), 16); + assert_ne!(gen_id_at(now), gen_id_at(now)); + assert_ne!(gen_id(), gen_id()); + + let id1 = thread::spawn(move || gen_id_at(now)); + let id2 = thread::spawn(move || gen_id_at(now)); assert_ne!(id1.join().unwrap(), id2.join().unwrap()); - let test_id = id::create_id(&now).unwrap(); - let timestamp = id::get_timestamp(&test_id); - assert_eq!(now.and_utc().timestamp_millis(), timestamp); + let test_id = gen_id_at(now); + let timestamp = get_timestamp(&test_id).unwrap(); + assert_eq!(now.timestamp_millis(), timestamp); + + let now_id = gen_id_at(now); + let old_id = gen_id_at(now - Duration::milliseconds(1)); + let future_id = gen_id_at(now + Duration::milliseconds(1)); + assert!(old_id < now_id); + assert!(now_id < future_id); } } diff --git a/packages/backend/src/boot/worker.ts b/packages/backend/src/boot/worker.ts index cae861230f..0acdcd97c6 100644 --- a/packages/backend/src/boot/worker.ts +++ b/packages/backend/src/boot/worker.ts @@ -1,17 +1,11 @@ import cluster from "node:cluster"; -import { config } from "@/config.js"; import { initDb } from "@/db/postgre.js"; -import { initIdGenerator } from "backend-rs"; import os from "node:os"; /** * Init worker process */ export async function workerMain() { - const length = Math.min(Math.max(config.cuid?.length ?? 16, 16), 24); - const fingerprint = config.cuid?.fingerprint ?? ""; - initIdGenerator(length, fingerprint); - await initDb(); if (!process.env.mode || process.env.mode === "web") {