mirrors/firefish
1
0
Fork 1
mirror of https://git.joinfirefish.org/firefish/firefish.git synced 2024-05-18 21:21:10 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'refactor/check-hit-antenna' into 'develop'

Browse source 
Draft: refactor (backend): port check-hit-antenna to backend-rs


See merge request firefish/firefish!10778
This commit is contained in:
naskya 2024-05-04 03:55:33 +00:00
parent f66ecd0759 2ddb2d7ae2
commit cedb711c31
35 changed files with 802 additions and 433 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
docs/downgrade.sql
View file

@ -1,6 +1,9 @@
BEGIN;
DELETE FROM "migrations" WHERE name IN (
'UserprofileJsonbToArray1714270605574',
'DropUnusedUserprofileColumns1714259023878',
'AntennaJsonbToArray1714192520471',
'DropUnusedIndexes1714643926317',
'AlterAkaType1714099399879',
'AddDriveFileUsage1713451569342',
@ -26,6 +29,45 @@ DELETE FROM "migrations" WHERE name IN (
'RemoveNativeUtilsMigration1705877093218'
);
-- userprofile-jsonb-to-array
ALTER TABLE "user_profile" RENAME COLUMN "mutedInstances" TO "mutedInstances_old";
ALTER TABLE "user_profile" ADD COLUMN "mutedInstances" jsonb NOT NULL DEFAULT '[]';
UPDATE "user_profile" SET "mutedInstances" = to_jsonb("mutedInstances_old");
ALTER TABLE "user_profile" DROP COLUMN "mutedInstances_old";
ALTER TABLE "user_profile" RENAME COLUMN "mutedWords" TO "mutedWords_old";
ALTER TABLE "user_profile" ADD COLUMN "mutedWords" jsonb NOT NULL DEFAULT '[]';
CREATE TEMP TABLE "BCrsGgLCUeMMLARy" ("userId" character varying(32), "kws" jsonb NOT NULL DEFAULT '[]');
INSERT INTO "BCrsGgLCUeMMLARy" ("userId", "kws") SELECT "userId", jsonb_agg("X"."w") FROM (SELECT "userId", to_jsonb(string_to_array(unnest("mutedWords_old"), ' ')) AS "w" FROM "user_profile") AS "X" GROUP BY "userId";
UPDATE "user_profile" SET "mutedWords" = "kws" FROM "BCrsGgLCUeMMLARy" WHERE "user_profile"."userId" = "BCrsGgLCUeMMLARy"."userId";
ALTER TABLE "user_profile" DROP COLUMN "mutedWords_old";
-- drop-unused-userprofile-columns
ALTER TABLE "user_profile" ADD "room" jsonb NOT NULL DEFAULT '{}';
COMMENT ON COLUMN "user_profile"."room" IS 'The room data of the User.';
ALTER TABLE "user_profile" ADD "clientData" jsonb NOT NULL DEFAULT '{}';
COMMENT ON COLUMN "user_profile"."clientData" IS 'The client-specific data of the User.';
-- antenna-jsonb-to-array
UPDATE "antenna" SET "instances" = '{""}' WHERE "instances" = '{}';
ALTER TABLE "antenna" RENAME COLUMN "instances" TO "instances_old";
ALTER TABLE "antenna" ADD COLUMN "instances" jsonb NOT NULL DEFAULT '[]';
UPDATE "antenna" SET "instances" = to_jsonb("instances_old");
ALTER TABLE "antenna" DROP COLUMN "instances_old";
UPDATE "antenna" SET "keywords" = '{""}' WHERE "keywords" = '{}';
ALTER TABLE "antenna" RENAME COLUMN "keywords" TO "keywords_old";
ALTER TABLE "antenna" ADD COLUMN "keywords" jsonb NOT NULL DEFAULT '[]';
CREATE TEMP TABLE "QvPNcMitBFkqqBgm" ("id" character varying(32), "kws" jsonb NOT NULL DEFAULT '[]');
INSERT INTO "QvPNcMitBFkqqBgm" ("id", "kws") SELECT "id", jsonb_agg("X"."w") FROM (SELECT "id", to_jsonb(string_to_array(unnest("keywords_old"), ' ')) AS "w" FROM "antenna") AS "X" GROUP BY "id";
UPDATE "antenna" SET "keywords" = "kws" FROM "QvPNcMitBFkqqBgm" WHERE "antenna"."id" = "QvPNcMitBFkqqBgm"."id";
ALTER TABLE "antenna" DROP COLUMN "keywords_old";
UPDATE "antenna" SET "excludeKeywords" = '{""}' WHERE "excludeKeywords" = '{}';
ALTER TABLE "antenna" RENAME COLUMN "excludeKeywords" TO "excludeKeywords_old";
ALTER TABLE "antenna" ADD COLUMN "excludeKeywords" jsonb NOT NULL DEFAULT '[]';
CREATE TEMP TABLE "MZvVSjHzYcGXmGmz" ("id" character varying(32), "kws" jsonb NOT NULL DEFAULT '[]');
INSERT INTO "MZvVSjHzYcGXmGmz" ("id", "kws") SELECT "id", jsonb_agg("X"."w") FROM (SELECT "id", to_jsonb(string_to_array(unnest("excludeKeywords_old"), ' ')) AS "w" FROM "antenna") AS "X" GROUP BY "id";
UPDATE "antenna" SET "excludeKeywords" = "kws" FROM "MZvVSjHzYcGXmGmz" WHERE "antenna"."id" = "MZvVSjHzYcGXmGmz"."id";
ALTER TABLE "antenna" DROP COLUMN "excludeKeywords_old";
-- drop-unused-indexes
CREATE INDEX "IDX_01f4581f114e0ebd2bbb876f0b" ON "note_reaction" ("createdAt");
CREATE INDEX "IDX_0610ebcfcfb4a18441a9bcdab2" ON "poll" ("userId");

34
packages/backend-rs/index.d.ts vendored
View file

@ -212,7 +212,6 @@ export interface Acct {
}
export function stringToAcct(acct: string): Acct
export function acctToString(acct: Acct): string
export function addNoteToAntenna(antennaId: string, note: Note): void
/**
* @param host punycoded instance host
* @returns whether the given host should be blocked
@ -228,16 +227,7 @@ export function isSilencedServer(host: string): Promise<boolean>
* @returns whether the given host is allowlisted (this is always true if private mode is disabled)
*/
export function isAllowedServer(host: string): Promise<boolean>
/** TODO: handle name collisions better */
export interface NoteLikeForCheckWordMute {
fileIds: Array<string>
userId: string | null
text: string | null
cw: string | null
renoteId: string | null
replyId: string | null
}
export function checkWordMute(note: NoteLikeForCheckWordMute, mutedWordLists: Array<Array<string>>, mutedPatterns: Array<string>): Promise<boolean>
export function checkWordMute(note: NoteLike, mutedWords: Array<string>, mutedPatterns: Array<string>): Promise<boolean>
export function getFullApAccount(username: string, host?: string | undefined | null): string
export function isSelfHost(host?: string | undefined | null): boolean
export function isSameOrigin(uri: string): boolean
@ -254,6 +244,15 @@ export interface ImageSize {
}
export function getImageSizeFromUrl(url: string): Promise<ImageSize>
/** TODO: handle name collisions better */
export interface NoteLikeForAllTexts {
fileIds: Array<string>
userId: string
text: string | null
cw: string | null
renoteId: string | null
replyId: string | null
}
/** TODO: handle name collisions better */
export interface NoteLikeForGetNoteSummary {
fileIds: Array<string>
text: string | null
@ -351,7 +350,6 @@ export interface Antenna {
name: string
src: AntennaSrcEnum
userListId: string | null
keywords: Json
withFile: boolean
expression: string | null
notify: boolean
@ -359,8 +357,9 @@ export interface Antenna {
withReplies: boolean
userGroupJoiningId: string | null
users: Array<string>
excludeKeywords: Json
instances: Json
instances: Array<string>
keywords: Array<string>
excludeKeywords: Array<string>
}
export interface App {
id: string
@ -1098,7 +1097,6 @@ export interface UserProfile {
twoFactorSecret: string | null
twoFactorEnabled: boolean
password: string | null
clientData: Json
autoAcceptFollowed: boolean
alwaysMarkNsfw: boolean
carefulBot: boolean
@ -1106,21 +1104,20 @@ export interface UserProfile {
securityKeysAvailable: boolean
usePasswordLessLogin: boolean
pinnedPageId: string | null
room: Json
injectFeaturedNote: boolean
enableWordMute: boolean
mutedWords: Json
mutingNotificationTypes: Array<UserProfileMutingnotificationtypesEnum>
noCrawle: boolean
receiveAnnouncementEmail: boolean
emailNotificationTypes: Json
mutedInstances: Json
publicReactions: boolean
ffVisibility: UserProfileFfvisibilityEnum
moderationNote: string
preventAiLearning: boolean
isIndexable: boolean
mutedPatterns: Array<string>
mutedInstances: Array<string>
mutedWords: Array<string>
}
export interface UserPublickey {
userId: string
@ -1146,6 +1143,7 @@ export interface Webhook {
latestSentAt: Date | null
latestStatus: number | null
}
export function updateAntennaOnCreateNote(antenna: Antenna, note: Note, noteAuthor: Acct): Promise<void>
export function initializeRustLogger(): void
export function watchNote(watcherId: string, noteAuthorId: string, noteId: string): Promise<void>
export function unwatchNote(watcherId: string, noteId: string): Promise<void>

4
packages/backend-rs/index.js
View file

@ -310,7 +310,7 @@ if (!nativeBinding) {
throw new Error(`Failed to load native binding`)
}
const { SECOND, MINUTE, HOUR, DAY, USER_ONLINE_THRESHOLD, USER_ACTIVE_THRESHOLD, FILE_TYPE_BROWSERSAFE, loadEnv, loadConfig, stringToAcct, acctToString, addNoteToAntenna, isBlockedServer, isSilencedServer, isAllowedServer, checkWordMute, getFullApAccount, isSelfHost, isSameOrigin, extractHost, toPuny, isUnicodeEmoji, sqlLikeEscape, safeForSql, formatMilliseconds, getImageSizeFromUrl, getNoteSummary, toMastodonId, fromMastodonId, fetchMeta, metaToPugArgs, nyaify, hashPassword, verifyPassword, isOldPasswordAlgorithm, decodeReaction, countReactions, toDbReaction, removeOldAttestationChallenges, AntennaSrcEnum, DriveFileUsageHintEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, initializeRustLogger, watchNote, unwatchNote, publishToChannelStream, ChatEvent, publishToChatStream, ChatIndexEvent, publishToChatIndexStream, publishToBroadcastStream, publishToGroupChatStream, publishToModerationStream, getTimestamp, genId, genIdAt, secureRndstr } = nativeBinding
const { SECOND, MINUTE, HOUR, DAY, USER_ONLINE_THRESHOLD, USER_ACTIVE_THRESHOLD, FILE_TYPE_BROWSERSAFE, loadEnv, loadConfig, stringToAcct, acctToString, isBlockedServer, isSilencedServer, isAllowedServer, checkWordMute, getFullApAccount, isSelfHost, isSameOrigin, extractHost, toPuny, isUnicodeEmoji, sqlLikeEscape, safeForSql, formatMilliseconds, getImageSizeFromUrl, getNoteSummary, toMastodonId, fromMastodonId, fetchMeta, metaToPugArgs, nyaify, hashPassword, verifyPassword, isOldPasswordAlgorithm, decodeReaction, countReactions, toDbReaction, removeOldAttestationChallenges, AntennaSrcEnum, DriveFileUsageHintEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, updateAntennaOnCreateNote, initializeRustLogger, watchNote, unwatchNote, publishToChannelStream, ChatEvent, publishToChatStream, ChatIndexEvent, publishToChatIndexStream, publishToBroadcastStream, publishToGroupChatStream, publishToModerationStream, getTimestamp, genId, genIdAt, secureRndstr } = nativeBinding
module.exports.SECOND = SECOND
module.exports.MINUTE = MINUTE
@ -323,7 +323,6 @@ module.exports.loadEnv = loadEnv
module.exports.loadConfig = loadConfig
module.exports.stringToAcct = stringToAcct
module.exports.acctToString = acctToString
module.exports.addNoteToAntenna = addNoteToAntenna
module.exports.isBlockedServer = isBlockedServer
module.exports.isSilencedServer = isSilencedServer
module.exports.isAllowedServer = isAllowedServer
@ -362,6 +361,7 @@ module.exports.RelayStatusEnum = RelayStatusEnum
module.exports.UserEmojimodpermEnum = UserEmojimodpermEnum
module.exports.UserProfileFfvisibilityEnum = UserProfileFfvisibilityEnum
module.exports.UserProfileMutingnotificationtypesEnum = UserProfileMutingnotificationtypesEnum
module.exports.updateAntennaOnCreateNote = updateAntennaOnCreateNote
module.exports.initializeRustLogger = initializeRustLogger
module.exports.watchNote = watchNote
module.exports.unwatchNote = unwatchNote

70
packages/backend-rs/src/misc/redis_cache.rs → packages/backend-rs/src/database/cache.rs
View file

@ -2,8 +2,20 @@ use crate::database::{redis_conn, redis_key};
use redis::{Commands, RedisError};
use serde::{Deserialize, Serialize};
#[derive(strum::Display)]
pub enum Category {
#[strum(serialize = "fetchUrl")]
FetchUrl,
#[strum(serialize = "blocking")]
Block,
#[strum(serialize = "following")]
Follow,
#[strum(serialize = "wordMute")]
WordMute,
}
#[derive(thiserror::Error, Debug)]
pub enum CacheError {
pub enum Error {
#[error("Redis error: {0}")]
RedisError(#[from] RedisError),
#[error("Data serialization error: {0}")]
@ -16,11 +28,15 @@ fn prefix_key(key: &str) -> String {
redis_key(format!("cache:{}", key))
}
pub fn set_cache<V: for<'a> Deserialize<'a> + Serialize>(
fn categorize(category: Category, key: &str) -> String {
format!("{}:{}", category, key)
}
pub fn set<V: for<'a> Deserialize<'a> + Serialize>(
key: &str,
value: &V,
expire_seconds: u64,
) -> Result<(), CacheError> {
) -> Result<(), Error> {
redis_conn()?.set_ex(
prefix_key(key),
rmp_serde::encode::to_vec(&value)?,
@ -29,9 +45,7 @@ pub fn set_cache<V: for<'a> Deserialize<'a> + Serialize>(
Ok(())
}
pub fn get_cache<V: for<'a> Deserialize<'a> + Serialize>(
key: &str,
) -> Result<Option<V>, CacheError> {
pub fn get<V: for<'a> Deserialize<'a> + Serialize>(key: &str) -> Result<Option<V>, Error> {
let serialized_value: Option<Vec<u8>> = redis_conn()?.get(prefix_key(key))?;
Ok(match serialized_value {
Some(v) => Some(rmp_serde::from_slice::<V>(v.as_ref())?),
@ -39,13 +53,35 @@ pub fn get_cache<V: for<'a> Deserialize<'a> + Serialize>(
})
}
pub fn delete_cache(key: &str) -> Result<(), CacheError> {
pub fn delete(key: &str) -> Result<(), Error> {
Ok(redis_conn()?.del(prefix_key(key))?)
}
pub fn set_one<V: for<'a> Deserialize<'a> + Serialize>(
category: Category,
key: &str,
value: &V,
expire_seconds: u64,
) -> Result<(), Error> {
set(&categorize(category, key), value, expire_seconds)
}
pub fn get_one<V: for<'a> Deserialize<'a> + Serialize>(
category: Category,
key: &str,
) -> Result<Option<V>, Error> {
get(&categorize(category, key))
}
pub fn delete_one(category: Category, key: &str) -> Result<(), Error> {
delete(&categorize(category, key))
}
// TODO: set_all(), get_all(), delete_all()
#[cfg(test)]
mod unit_test {
use super::{get_cache, set_cache};
use super::{get, set};
use pretty_assertions::assert_eq;
#[test]
@ -68,13 +104,13 @@ mod unit_test {
kind: "prime number".to_string(),
};
set_cache(key_1, &value_1, 1).unwrap();
set_cache(key_2, &value_2, 1).unwrap();
set_cache(key_3, &value_3, 1).unwrap();
set(key_1, &value_1, 1).unwrap();
set(key_2, &value_2, 1).unwrap();
set(key_3, &value_3, 1).unwrap();
let cached_value_1: Vec<i32> = get_cache(key_1).unwrap().unwrap();
let cached_value_2: String = get_cache(key_2).unwrap().unwrap();
let cached_value_3: Data = get_cache(key_3).unwrap().unwrap();
let cached_value_1: Vec<i32> = get(key_1).unwrap().unwrap();
let cached_value_2: String = get(key_2).unwrap().unwrap();
let cached_value_3: Data = get(key_3).unwrap().unwrap();
assert_eq!(value_1, cached_value_1);
assert_eq!(value_2, cached_value_2);
@ -83,9 +119,9 @@ mod unit_test {
// wait for the cache to expire
std::thread::sleep(std::time::Duration::from_millis(1100));
let expired_value_1: Option<Vec<i32>> = get_cache(key_1).unwrap();
let expired_value_2: Option<Vec<i32>> = get_cache(key_2).unwrap();
let expired_value_3: Option<Vec<i32>> = get_cache(key_3).unwrap();
let expired_value_1: Option<Vec<i32>> = get(key_1).unwrap();
let expired_value_2: Option<Vec<i32>> = get(key_2).unwrap();
let expired_value_3: Option<Vec<i32>> = get(key_3).unwrap();
assert!(expired_value_1.is_none());
assert!(expired_value_2.is_none());

1
packages/backend-rs/src/database/mod.rs
View file

@ -2,5 +2,6 @@ pub use postgresql::db_conn;
pub use redis::key as redis_key;
pub use redis::redis_conn;
pub mod cache;
pub mod postgresql;
pub mod redis;

31
packages/backend-rs/src/misc/add_note_to_antenna.rs
View file

@ -1,31 +0,0 @@
use crate::database::{redis_conn, redis_key};
use crate::model::entity::note;
use crate::service::stream;
use crate::util::id::{get_timestamp, InvalidIdErr};
use redis::{streams::StreamMaxlen, Commands, RedisError};
#[derive(thiserror::Error, Debug)]
pub enum Error {
#[error("Redis error: {0}")]
RedisErr(#[from] RedisError),
#[error("Invalid ID: {0}")]
InvalidIdErr(#[from] InvalidIdErr),
#[error("Stream error: {0}")]
StreamErr(#[from] stream::Error),
}
type Note = note::Model;
#[crate::export]
pub fn add_note_to_antenna(antenna_id: String, note: &Note) -> Result<(), Error> {
// for timeline API
redis_conn()?.xadd_maxlen(
redis_key(format!("antennaTimeline:{}", antenna_id)),
StreamMaxlen::Approx(200),
format!("{}-*", get_timestamp(&note.id)?),
&[("note", &note.id)],
)?;
// for streaming API
Ok(stream::antenna::publish(antenna_id, note)?)
}

203
packages/backend-rs/src/misc/check_hit_antenna.rs Normal file
View file

@ -0,0 +1,203 @@
use crate::config::CONFIG;
use crate::database::{cache, db_conn};
use crate::misc::acct::{string_to_acct, Acct};
use crate::misc::check_word_mute::check_word_mute_bare;
use crate::misc::get_note_all_texts::{all_texts, NoteLike};
use crate::model::entity::{
antenna, blocking, following, note, sea_orm_active_enums::*, user_profile,
};
use sea_orm::{ColumnTrait, DbErr, EntityTrait, QueryFilter, QuerySelect};
#[derive(thiserror::Error, Debug)]
pub enum AntennaCheckError {
#[error("Database error: {0}")]
DbErr(#[from] DbErr),
#[error("Cache error: {0}")]
CacheErr(#[from] cache::Error),
#[error("User profile not found: {0}")]
UserProfileNotFoundErr(String),
}
fn match_all(space_separated_words: &str, text: &str, case_sensitive: bool) -> bool {
if case_sensitive {
space_separated_words
.split_whitespace()
.all(|word| text.contains(word))
} else {
space_separated_words
.to_lowercase()
.split_whitespace()
.all(|word| text.to_lowercase().contains(word))
}
}
pub async fn check_hit_antenna(
antenna: &antenna::Model,
note: note::Model,
note_author: &Acct,
) -> Result<bool, AntennaCheckError> {
if note.visibility == NoteVisibilityEnum::Specified {
return Ok(false);
}
if antenna.with_file && note.file_ids.is_empty() {
return Ok(false);
}
if !antenna.with_replies && note.reply_id.is_some() {
return Ok(false);
}
if antenna.src == AntennaSrcEnum::Users {
let is_from_one_of_specified_authors = antenna
.users
.iter()
.map(|s| string_to_acct(s))
.any(|acct| acct.username == note_author.username && acct.host == note_author.host);
if !is_from_one_of_specified_authors {
return Ok(false);
}
} else if antenna.src == AntennaSrcEnum::Instances {
let is_from_one_of_specified_servers = !antenna.instances.iter().any(|host| {
host.to_ascii_lowercase()
== note_author
.host
.clone()
.unwrap_or(CONFIG.host.clone())
.to_ascii_lowercase()
});
if !is_from_one_of_specified_servers {
return Ok(false);
}
}
// "Home", "Group", "List" sources are currently disabled
let note_texts = all_texts(NoteLike {
file_ids: note.file_ids,
user_id: note.user_id.clone(),
text: note.text,
cw: note.cw,
renote_id: note.renote_id,
reply_id: note.reply_id,
})
.await?;
let has_keyword = antenna.keywords.iter().any(|words| {
note_texts
.iter()
.any(|text| match_all(words, text, antenna.case_sensitive))
});
if !has_keyword {
return Ok(false);
}
let has_excluded_word = antenna.exclude_keywords.iter().any(|words| {
note_texts
.iter()
.any(|text| match_all(words, text, antenna.case_sensitive))
});
if has_excluded_word {
return Ok(false);
}
let db = db_conn().await?;
let blocked_user_ids: Vec<String> = cache::get_one(cache::Category::Block, &note.user_id)?
.unwrap_or({
// cache miss
let blocks = blocking::Entity::find()
.select_only()
.column(blocking::Column::BlockeeId)
.filter(blocking::Column::BlockerId.eq(&note.user_id))
.into_tuple::<String>()
.all(db)
.await?;
cache::set_one(cache::Category::Block, &note.user_id, &blocks, 10 * 60)?;
blocks
});
// if the antenna owner is blocked by the note author, return false
if blocked_user_ids.contains(&antenna.user_id) {
return Ok(false);
}
if [NoteVisibilityEnum::Home, NoteVisibilityEnum::Followers].contains(&note.visibility) {
let following_user_ids: Vec<String> =
cache::get_one(cache::Category::Follow, &antenna.user_id)?.unwrap_or({
// cache miss
let following = following::Entity::find()
.select_only()
.column(following::Column::FolloweeId)
.filter(following::Column::FollowerId.eq(&antenna.user_id))
.into_tuple::<String>()
.all(db)
.await?;
cache::set_one(
cache::Category::Follow,
&antenna.user_id,
&following,
10 * 60,
)?;
following
});
// if the antenna owner is not following the note author, return false
if !following_user_ids.contains(&note.user_id) {
return Ok(false);
}
}
type WordMute = (
Vec<String>, // muted words
Vec<String>, // muted patterns
);
let word_mute: WordMute = cache::get_one(cache::Category::WordMute, &antenna.user_id)?
.unwrap_or({
// cache miss
let mute = user_profile::Entity::find()
.select_only()
.columns([
user_profile::Column::MutedWords,
user_profile::Column::MutedPatterns,
])
.into_tuple::<WordMute>()
.one(db)
.await?
.ok_or({
tracing::warn!("there is no user_profile for user {}", &antenna.user_id);
AntennaCheckError::UserProfileNotFoundErr(antenna.user_id.clone())
})?;
cache::set_one(cache::Category::WordMute, &antenna.user_id, &mute, 10 * 60)?;
mute
});
if check_word_mute_bare(&note_texts, &word_mute.0, &word_mute.1) {
return Ok(false);
}
Ok(true)
}
#[cfg(test)]
mod unit_test {
use super::match_all;
use pretty_assertions::assert_eq;
#[test]
fn test_match_all() {
assert_eq!(match_all("Apple", "apple and banana", false), true);
assert_eq!(match_all("Apple", "apple and banana", true), false);
assert_eq!(match_all("Apple Banana", "apple and banana", false), true);
assert_eq!(match_all("Apple Banana", "apple and cinnamon", true), false);
assert_eq!(
match_all("Apple Banana", "apple and cinnamon", false),
false
);
}
}

121
packages/backend-rs/src/misc/check_word_mute.rs
View file

@ -1,122 +1,47 @@
use crate::database::db_conn;
use crate::model::entity::{drive_file, note};
use crate::misc::get_note_all_texts::{all_texts, NoteLike};
use once_cell::sync::Lazy;
use regex::Regex;
use sea_orm::{prelude::*, QuerySelect};
/// TODO: handle name collisions better
#[crate::export(object, js_name = "NoteLikeForCheckWordMute")]
pub struct NoteLike {
pub file_ids: Vec<String>,
pub user_id: Option<String>,
pub text: Option<String>,
pub cw: Option<String>,
pub renote_id: Option<String>,
pub reply_id: Option<String>,
}
async fn all_texts(note: NoteLike) -> Result<Vec<String>, DbErr> {
let db = db_conn().await?;
let mut texts: Vec<String> = vec![];
if let Some(text) = note.text {
texts.push(text);
}
if let Some(cw) = note.cw {
texts.push(cw);
}
texts.extend(
drive_file::Entity::find()
.select_only()
.column(drive_file::Column::Comment)
.filter(drive_file::Column::Id.is_in(note.file_ids))
.into_tuple::<Option<String>>()
.all(db)
.await?
.into_iter()
.flatten(),
);
if let Some(renote_id) = &note.renote_id {
if let Some((text, cw)) = note::Entity::find_by_id(renote_id)
.select_only()
.columns([note::Column::Text, note::Column::Cw])
.into_tuple::<(Option<String>, Option<String>)>()
.one(db)
.await?
{
if let Some(t) = text {
texts.push(t);
}
if let Some(c) = cw {
texts.push(c);
}
} else {
tracing::warn!("nonexistent renote id: {:#?}", renote_id);
}
}
if let Some(reply_id) = &note.reply_id {
if let Some((text, cw)) = note::Entity::find_by_id(reply_id)
.select_only()
.columns([note::Column::Text, note::Column::Cw])
.into_tuple::<(Option<String>, Option<String>)>()
.one(db)
.await?
{
if let Some(t) = text {
texts.push(t);
}
if let Some(c) = cw {
texts.push(c);
}
} else {
tracing::warn!("nonexistent reply id: {:#?}", reply_id);
}
}
Ok(texts)
}
use sea_orm::DbErr;
fn convert_regex(js_regex: &str) -> String {
static RE: Lazy<Regex> = Lazy::new(|| Regex::new(r"^/(.+)/(.*)$").unwrap());
RE.replace(js_regex, "(?$2)$1").to_string()
}
fn check_word_mute_impl(
pub fn check_word_mute_bare(
texts: &[String],
muted_word_lists: &[Vec<String>],
muted_words: &[String],
muted_patterns: &[String],
) -> bool {
muted_word_lists.iter().any(|muted_word_list| {
texts.iter().any(|text| {
let text_lower = text.to_lowercase();
muted_word_list
.iter()
.all(|muted_word| text_lower.contains(&muted_word.to_lowercase()))
muted_words
.iter()
.map(|row| row.split_whitespace())
.any(|mut words| {
texts.iter().any(|text| {
let text_lower = text.to_lowercase();
words.all(|muted_word| text_lower.contains(&muted_word.to_lowercase()))
})
})
|| muted_patterns.iter().any(|muted_pattern| {
Regex::new(convert_regex(muted_pattern).as_str())
.map(|re| texts.iter().any(|text| re.is_match(text)))
.unwrap_or(false)
})
}) || muted_patterns.iter().any(|muted_pattern| {
Regex::new(convert_regex(muted_pattern).as_str())
.map(|re| texts.iter().any(|text| re.is_match(text)))
.unwrap_or(false)
})
}
#[crate::export]
pub async fn check_word_mute(
note: NoteLike,
muted_word_lists: Vec<Vec<String>>,
muted_patterns: Vec<String>,
muted_words: &[String],
muted_patterns: &[String],
) -> Result<bool, DbErr> {
if muted_word_lists.is_empty() && muted_patterns.is_empty() {
if muted_words.is_empty() && muted_patterns.is_empty() {
Ok(false)
} else {
Ok(check_word_mute_impl(
Ok(check_word_mute_bare(
&all_texts(note).await?,
&muted_word_lists,
&muted_patterns,
muted_words,
muted_patterns,
))
}
}

29
packages/backend-rs/src/misc/get_image_size.rs
View file

@ -1,4 +1,4 @@
use crate::misc::redis_cache::{get_cache, set_cache, CacheError};
use crate::database::cache;
use crate::util::http_client;
use image::{io::Reader, ImageError, ImageFormat};
use nom_exif::{parse_jpeg_exif, EntryValue, ExifTag};
@ -8,7 +8,7 @@ use tokio::sync::Mutex;
#[derive(thiserror::Error, Debug)]
pub enum Error {
#[error("Redis cache error: {0}")]
CacheErr(#[from] CacheError),
CacheErr(#[from] cache::Error),
#[error("Reqwest error: {0}")]
ReqwestErr(#[from] reqwest::Error),
#[error("Image decoding error: {0}")]
@ -50,11 +50,10 @@ pub async fn get_image_size_from_url(url: &str) -> Result<ImageSize, Error> {
{
let _ = MTX_GUARD.lock().await;
let key = format!("fetchImage:{}", url);
attempted = get_cache::<bool>(&key)?.is_some();
attempted = cache::get_one::<bool>(cache::Category::FetchUrl, url)?.is_some();
if !attempted {
set_cache(&key, &true, 10 * 60)?;
cache::set_one(cache::Category::FetchUrl, url, &true, 10 * 60)?;
}
}
@ -109,7 +108,7 @@ pub async fn get_image_size_from_url(url: &str) -> Result<ImageSize, Error> {
#[cfg(test)]
mod unit_test {
use super::{get_image_size_from_url, ImageSize};
use crate::misc::redis_cache::delete_cache;
use crate::database::cache;
use pretty_assertions::assert_eq;
#[tokio::test]
@ -126,15 +125,15 @@ mod unit_test {
// Delete caches in case you run this test multiple times
// (should be disabled in CI tasks)
delete_cache(&format!("fetchImage:{}", png_url_1)).unwrap();
delete_cache(&format!("fetchImage:{}", png_url_2)).unwrap();
delete_cache(&format!("fetchImage:{}", png_url_3)).unwrap();
delete_cache(&format!("fetchImage:{}", rotated_jpeg_url)).unwrap();
delete_cache(&format!("fetchImage:{}", webp_url_1)).unwrap();
delete_cache(&format!("fetchImage:{}", webp_url_2)).unwrap();
delete_cache(&format!("fetchImage:{}", ico_url)).unwrap();
delete_cache(&format!("fetchImage:{}", gif_url)).unwrap();
delete_cache(&format!("fetchImage:{}", mp3_url)).unwrap();
cache::delete_one(cache::Category::FetchUrl, png_url_1).unwrap();
cache::delete_one(cache::Category::FetchUrl, png_url_2).unwrap();
cache::delete_one(cache::Category::FetchUrl, png_url_3).unwrap();
cache::delete_one(cache::Category::FetchUrl, rotated_jpeg_url).unwrap();
cache::delete_one(cache::Category::FetchUrl, webp_url_1).unwrap();
cache::delete_one(cache::Category::FetchUrl, webp_url_2).unwrap();
cache::delete_one(cache::Category::FetchUrl, ico_url).unwrap();
cache::delete_one(cache::Category::FetchUrl, gif_url).unwrap();
cache::delete_one(cache::Category::FetchUrl, mp3_url).unwrap();
let png_size_1 = ImageSize {
width: 1024,

79
packages/backend-rs/src/misc/get_note_all_texts.rs Normal file
View file

@ -0,0 +1,79 @@
use crate::database::db_conn;
use crate::model::entity::{drive_file, note};
use sea_orm::{prelude::*, QuerySelect};
/// TODO: handle name collisions better
#[crate::export(object, js_name = "NoteLikeForAllTexts")]
pub struct NoteLike {
pub file_ids: Vec<String>,
pub user_id: String,
pub text: Option<String>,
pub cw: Option<String>,
pub renote_id: Option<String>,
pub reply_id: Option<String>,
}
pub async fn all_texts(note: NoteLike) -> Result<Vec<String>, DbErr> {
let db = db_conn().await?;
let mut texts: Vec<String> = vec![];
if let Some(text) = note.text {
texts.push(text);
}
if let Some(cw) = note.cw {
texts.push(cw);
}
texts.extend(
drive_file::Entity::find()
.select_only()
.column(drive_file::Column::Comment)
.filter(drive_file::Column::Id.is_in(note.file_ids))
.into_tuple::<Option<String>>()
.all(db)
.await?
.into_iter()
.flatten(),
);
if let Some(renote_id) = &note.renote_id {
if let Some((text, cw)) = note::Entity::find_by_id(renote_id)
.select_only()
.columns([note::Column::Text, note::Column::Cw])
.into_tuple::<(Option<String>, Option<String>)>()
.one(db)
.await?
{
if let Some(t) = text {
texts.push(t);
}
if let Some(c) = cw {
texts.push(c);
}
} else {
tracing::warn!("nonexistent renote id: {:#?}", renote_id);
}
}
if let Some(reply_id) = &note.reply_id {
if let Some((text, cw)) = note::Entity::find_by_id(reply_id)
.select_only()
.columns([note::Column::Text, note::Column::Cw])
.into_tuple::<(Option<String>, Option<String>)>()
.one(db)
.await?
{
if let Some(t) = text {
texts.push(t);
}
if let Some(c) = cw {
texts.push(c);
}
} else {
tracing::warn!("nonexistent reply id: {:#?}", reply_id);
}
}
Ok(texts)
}

4
packages/backend-rs/src/misc/mod.rs
View file

@ -1,5 +1,5 @@
pub mod acct;
pub mod add_note_to_antenna;
pub mod check_hit_antenna;
pub mod check_server_block;
pub mod check_word_mute;
pub mod convert_host;
@ -7,11 +7,11 @@ pub mod emoji;
pub mod escape_sql;
pub mod format_milliseconds;
pub mod get_image_size;
pub mod get_note_all_texts;
pub mod get_note_summary;
pub mod mastodon_id;
pub mod meta;
pub mod nyaify;
pub mod password;
pub mod reaction;
pub mod redis_cache;
pub mod remove_old_attestation_challenges;

10
packages/backend-rs/src/model/entity/antenna.rs
View file

@ -21,8 +21,6 @@ pub struct Model {
pub src: AntennaSrcEnum,
#[sea_orm(column_name = "userListId")]
pub user_list_id: Option<String>,
#[sea_orm(column_type = "JsonBinary")]
pub keywords: Json,
#[sea_orm(column_name = "withFile")]
pub with_file: bool,
pub expression: Option<String>,
@ -34,10 +32,10 @@ pub struct Model {
#[sea_orm(column_name = "userGroupJoiningId")]
pub user_group_joining_id: Option<String>,
pub users: Vec<String>,
#[sea_orm(column_name = "excludeKeywords", column_type = "JsonBinary")]
pub exclude_keywords: Json,
#[sea_orm(column_type = "JsonBinary")]
pub instances: Json,
pub instances: Vec<String>,
pub keywords: Vec<String>,
#[sea_orm(column_name = "excludeKeywords")]
pub exclude_keywords: Vec<String>,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]

12
packages/backend-rs/src/model/entity/user_profile.rs
View file

@ -32,8 +32,6 @@ pub struct Model {
#[sea_orm(column_name = "twoFactorEnabled")]
pub two_factor_enabled: bool,
pub password: Option<String>,
#[sea_orm(column_name = "clientData", column_type = "JsonBinary")]
pub client_data: Json,
#[sea_orm(column_name = "autoAcceptFollowed")]
pub auto_accept_followed: bool,
#[sea_orm(column_name = "alwaysMarkNsfw")]
@ -48,14 +46,10 @@ pub struct Model {
pub use_password_less_login: bool,
#[sea_orm(column_name = "pinnedPageId", unique)]
pub pinned_page_id: Option<String>,
#[sea_orm(column_type = "JsonBinary")]
pub room: Json,
#[sea_orm(column_name = "injectFeaturedNote")]
pub inject_featured_note: bool,
#[sea_orm(column_name = "enableWordMute")]
pub enable_word_mute: bool,
#[sea_orm(column_name = "mutedWords", column_type = "JsonBinary")]
pub muted_words: Json,
#[sea_orm(column_name = "mutingNotificationTypes")]
pub muting_notification_types: Vec<UserProfileMutingnotificationtypesEnum>,
#[sea_orm(column_name = "noCrawle")]
@ -64,8 +58,6 @@ pub struct Model {
pub receive_announcement_email: bool,
#[sea_orm(column_name = "emailNotificationTypes", column_type = "JsonBinary")]
pub email_notification_types: Json,
#[sea_orm(column_name = "mutedInstances", column_type = "JsonBinary")]
pub muted_instances: Json,
#[sea_orm(column_name = "publicReactions")]
pub public_reactions: bool,
#[sea_orm(column_name = "ffVisibility")]
@ -78,6 +70,10 @@ pub struct Model {
pub is_indexable: bool,
#[sea_orm(column_name = "mutedPatterns")]
pub muted_patterns: Vec<String>,
#[sea_orm(column_name = "mutedInstances")]
pub muted_instances: Vec<String>,
#[sea_orm(column_name = "mutedWords")]
pub muted_words: Vec<String>,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]

48
packages/backend-rs/src/service/antenna.rs Normal file
View file

@ -0,0 +1,48 @@
use crate::database::{redis_conn, redis_key};
use crate::misc::acct::Acct;
use crate::misc::check_hit_antenna::{check_hit_antenna, AntennaCheckError};
use crate::model::entity::{antenna, note};
use crate::service::stream;
use crate::util::id::{get_timestamp, InvalidIdErr};
use redis::{streams::StreamMaxlen, Commands, RedisError};
#[derive(thiserror::Error, Debug)]
pub enum Error {
#[error("Redis error: {0}")]
RedisErr(#[from] RedisError),
#[error("Invalid ID: {0}")]
InvalidIdErr(#[from] InvalidIdErr),
#[error("Stream error: {0}")]
StreamErr(#[from] stream::Error),
#[error("Failed to check if the note should be added to antenna: {0}")]
AntennaCheckErr(#[from] AntennaCheckError),
}
// https://github.com/napi-rs/napi-rs/issues/2060
type Antenna = antenna::Model;
type Note = note::Model;
#[crate::export]
pub async fn update_antenna_on_create_note(
antenna: &Antenna,
note: Note,
note_author: &Acct,
) -> Result<(), Error> {
if check_hit_antenna(antenna, note.clone(), note_author).await? {
add_note_to_antenna(&antenna.id, &note)?;
}
Ok(())
}
fn add_note_to_antenna(antenna_id: &str, note: &Note) -> Result<(), Error> {
// for timeline API
redis_conn()?.xadd_maxlen(
redis_key(format!("antennaTimeline:{}", antenna_id)),
StreamMaxlen::Approx(200),
format!("{}-*", get_timestamp(&note.id)?),
&[("note", &note.id)],
)?;
// for streaming API
Ok(stream::antenna::publish(antenna_id.to_string(), note)?)
}

1
packages/backend-rs/src/service/mod.rs
View file

@ -1,3 +1,4 @@
pub mod antenna;
pub mod log;
pub mod note;
pub mod stream;

118
packages/backend/src/migration/1714192520471-antenna-jsonb-to-array.ts Normal file
View file

@ -0,0 +1,118 @@
import type { MigrationInterface, QueryRunner } from "typeorm";
export class AntennaJsonbToArray1714192520471 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "antenna" RENAME COLUMN "instances" TO "instances_old"`,
);
await queryRunner.query(
`ALTER TABLE "antenna" ADD COLUMN "instances" character varying(512)[] NOT NULL DEFAULT '{}'`,
);
await queryRunner.query(
`UPDATE "antenna" SET "instances" = ARRAY(SELECT jsonb_array_elements_text("instances_old"))::character varying(512)[]`,
);
await queryRunner.query(
`UPDATE "antenna" SET "instances" = '{}' WHERE "instances" = '{""}'`,
);
await queryRunner.query(
`ALTER TABLE "antenna" DROP COLUMN "instances_old"`,
);
await queryRunner.query(
`ALTER TABLE "antenna" RENAME COLUMN "keywords" TO "keywords_old"`,
);
await queryRunner.query(
`ALTER TABLE "antenna" ADD COLUMN "keywords" text[] NOT NULL DEFAULT '{}'`,
);
await queryRunner.query(
`CREATE TEMP TABLE "HMyeXPcdtQYGsSrf" ("id" character varying(32), "kws" text[])`,
);
await queryRunner.query(
`INSERT INTO "HMyeXPcdtQYGsSrf" ("id", "kws") SELECT "id", array_agg("X"."w") FROM (SELECT "id", array_to_string(ARRAY(SELECT jsonb_array_elements_text("kw")), ' ') AS "w" FROM (SELECT "id", jsonb_array_elements("keywords_old") AS "kw" FROM "antenna") AS "a") AS "X" GROUP BY "id"`,
);
await queryRunner.query(
`UPDATE "antenna" SET "keywords" = "kws" FROM "HMyeXPcdtQYGsSrf" WHERE "antenna"."id" = "HMyeXPcdtQYGsSrf"."id"`,
);
await queryRunner.query(
`UPDATE "antenna" SET "keywords" = '{}' WHERE "keywords" = '{""}'`,
);
await queryRunner.query(`ALTER TABLE "antenna" DROP COLUMN "keywords_old"`);
await queryRunner.query(
`ALTER TABLE "antenna" RENAME COLUMN "excludeKeywords" TO "excludeKeywords_old"`,
);
await queryRunner.query(
`ALTER TABLE "antenna" ADD COLUMN "excludeKeywords" text[] NOT NULL DEFAULT '{}'`,
);
await queryRunner.query(
`CREATE TEMP TABLE "kpdsACdZTRYqLkfK" ("id" character varying(32), "kws" text[])`,
);
await queryRunner.query(
`INSERT INTO "kpdsACdZTRYqLkfK" ("id", "kws") SELECT "id", array_agg("X"."w") FROM (SELECT "id", array_to_string(ARRAY(SELECT jsonb_array_elements_text("kw")), ' ') AS "w" FROM (SELECT "id", jsonb_array_elements("excludeKeywords_old") AS "kw" FROM "antenna") AS "a") AS "X" GROUP BY "id"`,
);
await queryRunner.query(
`UPDATE "antenna" SET "excludeKeywords" = "kws" FROM "kpdsACdZTRYqLkfK" WHERE "antenna"."id" = "kpdsACdZTRYqLkfK"."id"`,
);
await queryRunner.query(
`UPDATE "antenna" SET "excludeKeywords" = '{}' WHERE "excludeKeywords" = '{""}'`,
);
await queryRunner.query(
`ALTER TABLE "antenna" DROP COLUMN "excludeKeywords_old"`,
);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`UPDATE "antenna" SET "instances" = '{""}' WHERE "instances" = '{}'`,
);
await queryRunner.query(
`ALTER TABLE "antenna" RENAME COLUMN "instances" TO "instances_old"`,
);
await queryRunner.query(
`ALTER TABLE "antenna" ADD COLUMN "instances" jsonb NOT NULL DEFAULT '[]'`,
);
await queryRunner.query(
`UPDATE "antenna" SET "instances" = to_jsonb("instances_old")`,
);
await queryRunner.query(
`ALTER TABLE "antenna" DROP COLUMN "instances_old"`,
);
await queryRunner.query(
`UPDATE "antenna" SET "keywords" = '{""}' WHERE "keywords" = '{}'`,
);
await queryRunner.query(
`ALTER TABLE "antenna" RENAME COLUMN "keywords" TO "keywords_old"`,
);
await queryRunner.query(
`ALTER TABLE "antenna" ADD COLUMN "keywords" jsonb NOT NULL DEFAULT '[]'`,
);
await queryRunner.query(
`CREATE TEMP TABLE "QvPNcMitBFkqqBgm" ("id" character varying(32), "kws" jsonb NOT NULL DEFAULT '[]')`,
);
await queryRunner.query(
`INSERT INTO "QvPNcMitBFkqqBgm" ("id", "kws") SELECT "id", jsonb_agg("X"."w") FROM (SELECT "id", to_jsonb(string_to_array(unnest("keywords_old"), ' ')) AS "w" FROM "antenna") AS "X" GROUP BY "id"`,
);
await queryRunner.query(
`UPDATE "antenna" SET "keywords" = "kws" FROM "QvPNcMitBFkqqBgm" WHERE "antenna"."id" = "QvPNcMitBFkqqBgm"."id"`,
);
await queryRunner.query(`ALTER TABLE "antenna" DROP COLUMN "keywords_old"`);
await queryRunner.query(
`UPDATE "antenna" SET "excludeKeywords" = '{""}' WHERE "excludeKeywords" = '{}'`,
);
await queryRunner.query(
`ALTER TABLE "antenna" RENAME COLUMN "excludeKeywords" TO "excludeKeywords_old"`,
);
await queryRunner.query(
`ALTER TABLE "antenna" ADD COLUMN "excludeKeywords" jsonb NOT NULL DEFAULT '[]'`,
);
await queryRunner.query(
`CREATE TEMP TABLE "MZvVSjHzYcGXmGmz" ("id" character varying(32), "kws" jsonb NOT NULL DEFAULT '[]')`,
);
await queryRunner.query(
`INSERT INTO "MZvVSjHzYcGXmGmz" ("id", "kws") SELECT "id", jsonb_agg("X"."w") FROM (SELECT "id", to_jsonb(string_to_array(unnest("excludeKeywords_old"), ' ')) AS "w" FROM "antenna") AS "X" GROUP BY "id"`,
);
await queryRunner.query(
`UPDATE "antenna" SET "excludeKeywords" = "kws" FROM "MZvVSjHzYcGXmGmz" WHERE "antenna"."id" = "MZvVSjHzYcGXmGmz"."id"`,
);
await queryRunner.query(
`ALTER TABLE "antenna" DROP COLUMN "excludeKeywords_old"`,
);
}
}

27
packages/backend/src/migration/1714259023878-drop-unused-userprofile-columns.ts Normal file
View file

@ -0,0 +1,27 @@
import type { MigrationInterface, QueryRunner } from "typeorm";
export class DropUnusedUserprofileColumns1714259023878
implements MigrationInterface
{
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "user_profile" DROP COLUMN "clientData"`,
);
await queryRunner.query(`ALTER TABLE "user_profile" DROP COLUMN "room"`);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "user_profile" ADD "room" jsonb NOT NULL DEFAULT '{}'`,
);
await queryRunner.query(
`COMMENT ON COLUMN "user_profile"."room" IS 'The room data of the User.'`,
);
await queryRunner.query(
`ALTER TABLE "user_profile" ADD "clientData" jsonb NOT NULL DEFAULT '{}'`,
);
await queryRunner.query(
`COMMENT ON COLUMN "user_profile"."clientData" IS 'The client-specific data of the User.'`,
);
}
}

71
packages/backend/src/migration/1714270605574-userprofile-jsonb-to-array.ts Normal file
View file

@ -0,0 +1,71 @@
import type { MigrationInterface, QueryRunner } from "typeorm";
export class UserprofileJsonbToArray1714270605574
implements MigrationInterface
{
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "user_profile" RENAME COLUMN "mutedInstances" TO "mutedInstances_old"`,
);
await queryRunner.query(
`ALTER TABLE "user_profile" ADD COLUMN "mutedInstances" character varying(512)[] NOT NULL DEFAULT '{}'`,
);
await queryRunner.query(
`UPDATE "user_profile" SET "mutedInstances" = ARRAY(SELECT jsonb_array_elements_text("mutedInstances_old"))::character varying(512)[]`,
);
await queryRunner.query(
`ALTER TABLE "user_profile" DROP COLUMN "mutedInstances_old"`,
);
await queryRunner.query(
`ALTER TABLE "user_profile" RENAME COLUMN "mutedWords" TO "mutedWords_old"`,
);
await queryRunner.query(
`ALTER TABLE "user_profile" ADD COLUMN "mutedWords" text[] NOT NULL DEFAULT '{}'`,
);
await queryRunner.query(
`CREATE TEMP TABLE "MmVqAUUgpshTCQcw" ("userId" character varying(32), "kws" text[])`,
);
await queryRunner.query(
`INSERT INTO "MmVqAUUgpshTCQcw" ("userId", "kws") SELECT "userId", array_agg("X"."w") FROM (SELECT "userId", array_to_string(ARRAY(SELECT jsonb_array_elements_text("kw")), ' ') AS "w" FROM (SELECT "userId", jsonb_array_elements("mutedWords_old") AS "kw" FROM "user_profile") AS "a") AS "X" GROUP BY "userId"`,
);
await queryRunner.query(
`UPDATE "user_profile" SET "mutedWords" = "kws" FROM "MmVqAUUgpshTCQcw" WHERE "user_profile"."userId" = "MmVqAUUgpshTCQcw"."userId"`,
);
await queryRunner.query(
`ALTER TABLE "user_profile" DROP COLUMN "mutedWords_old"`,
);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "user_profile" RENAME COLUMN "mutedInstances" TO "mutedInstances_old"`,
);
await queryRunner.query(
`ALTER TABLE "user_profile" ADD COLUMN "mutedInstances" jsonb NOT NULL DEFAULT '[]'`,
);
await queryRunner.query(
`UPDATE "user_profile" SET "mutedInstances" = to_jsonb("mutedInstances_old")`,
);
await queryRunner.query(
`ALTER TABLE "user_profile" DROP COLUMN "mutedInstances_old"`,
);
await queryRunner.query(
`ALTER TABLE "user_profile" RENAME COLUMN "mutedWords" TO "mutedWords_old"`,
);
await queryRunner.query(
`ALTER TABLE "user_profile" ADD COLUMN "mutedWords" jsonb NOT NULL DEFAULT '[]'`,
);
await queryRunner.query(
`CREATE TEMP TABLE "BCrsGgLCUeMMLARy" ("userId" character varying(32), "kws" jsonb NOT NULL DEFAULT '[]')`,
);
await queryRunner.query(
`INSERT INTO "BCrsGgLCUeMMLARy" ("userId", "kws") SELECT "userId", jsonb_agg("X"."w") FROM (SELECT "userId", to_jsonb(string_to_array(unnest("mutedWords_old"), ' ')) AS "w" FROM "user_profile") AS "X" GROUP BY "userId"`,
);
await queryRunner.query(
`UPDATE "user_profile" SET "mutedWords" = "kws" FROM "BCrsGgLCUeMMLARy" WHERE "user_profile"."userId" = "BCrsGgLCUeMMLARy"."userId"`,
);
await queryRunner.query(
`ALTER TABLE "user_profile" DROP COLUMN "mutedWords_old"`,
);
}
}

132
packages/backend/src/misc/check-hit-antenna.ts
View file

@ -1,132 +0,0 @@
import type { Antenna } from "@/models/entities/antenna.js";
import type { Note } from "@/models/entities/note.js";
import type { User } from "@/models/entities/user.js";
import type { UserProfile } from "@/models/entities/user-profile.js";
import { Blockings, Followings, UserProfiles } from "@/models/index.js";
import { checkWordMute, getFullApAccount, stringToAcct } from "backend-rs";
import type { Packed } from "@/misc/schema.js";
import { Cache } from "@/misc/cache.js";
const blockingCache = new Cache<User["id"][]>("blocking", 60 * 5);
const hardMutesCache = new Cache<{
userId: UserProfile["userId"];
mutedWords: UserProfile["mutedWords"];
mutedPatterns: UserProfile["mutedPatterns"];
}>("hardMutes", 60 * 5);
const followingCache = new Cache<User["id"][]>("following", 60 * 5);
export async function checkHitAntenna(
antenna: Antenna,
note: Note | Packed<"Note">,
noteUser: { id: User["id"]; username: string; host: string | null },
): Promise<boolean> {
if (note.visibility === "specified") return false;
if (antenna.withFile) {
if (note.fileIds && note.fileIds.length === 0) return false;
}
if (!antenna.withReplies && note.replyId != null) return false;
if (antenna.src === "users") {
const accts = antenna.users.map((x) => {
const { username, host } = stringToAcct(x);
return getFullApAccount(username, host).toLowerCase();
});
if (
!accts.includes(
getFullApAccount(noteUser.username, noteUser.host).toLowerCase(),
)
)
return false;
} else if (antenna.src === "instances") {
const instances = antenna.instances
.filter((x) => x !== "")
.map((host) => {
return host.toLowerCase();
});
if (!instances.includes(noteUser.host?.toLowerCase() ?? "")) return false;
}
const keywords = antenna.keywords
// Clean up
.map((xs) => xs.filter((x) => x !== ""))
.filter((xs) => xs.length > 0);
let text = `${note.text ?? ""} ${note.cw ?? ""}`;
if (note.files != null)
text += ` ${note.files.map((f) => f.comment ?? "").join(" ")}`;
text = text.trim();
if (keywords.length > 0) {
if (note.text == null) return false;
const matched = keywords.some((and) =>
and.every((keyword) =>
antenna.caseSensitive
? text.includes(keyword)
: text.toLowerCase().includes(keyword.toLowerCase()),
),
);
if (!matched) return false;
}
const excludeKeywords = antenna.excludeKeywords
// Clean up
.map((xs) => xs.filter((x) => x !== ""))
.filter((xs) => xs.length > 0);
if (excludeKeywords.length > 0) {
if (note.text == null) return false;
const matched = excludeKeywords.some((and) =>
and.every((keyword) =>
antenna.caseSensitive
? note.text?.includes(keyword)
: note.text?.toLowerCase().includes(keyword.toLowerCase()),
),
);
if (matched) return false;
}
// アンテナ作成者がノート作成者にブロックされていたらスキップ
const blockings = await blockingCache.fetch(noteUser.id, () =>
Blockings.findBy({ blockerId: noteUser.id }).then((res) =>
res.map((x) => x.blockeeId),
),
);
if (blockings.includes(antenna.userId)) return false;
if (note.visibility === "followers" || note.visibility === "home") {
const following = await followingCache.fetch(antenna.userId, () =>
Followings.find({
where: { followerId: antenna.userId },
select: ["followeeId"],
}).then((relations) => relations.map((relation) => relation.followeeId)),
);
if (!following.includes(note.userId)) return false;
}
const mutes = await hardMutesCache.fetch(antenna.userId, () =>
UserProfiles.findOneByOrFail({
userId: antenna.userId,
}).then((profile) => {
return {
userId: antenna.userId,
mutedWords: profile.mutedWords,
mutedPatterns: profile.mutedPatterns,
};
}),
);
if (
mutes.mutedWords != null &&
mutes.mutedPatterns != null &&
antenna.userId !== note.userId &&
(await checkWordMute(note, mutes.mutedWords, mutes.mutedPatterns))
)
return false;
// TODO: eval expression
return true;
}

20
packages/backend/src/models/entities/antenna.ts
View file

@ -59,20 +59,24 @@ export class Antenna {
})
public users: string[];
@Column("jsonb", {
default: [],
@Column("varchar", {
length: 512,
array: true,
default: "{}",
})
public instances: string[];
@Column("jsonb", {
default: [],
@Column("text", {
array: true,
default: "{}",
})
public keywords: string[][];
public keywords: string[];
@Column("jsonb", {
default: [],
@Column("text", {
array: true,
default: "{}",
})
public excludeKeywords: string[][];
public excludeKeywords: string[];
@Column("boolean", {
default: false,

40
packages/backend/src/models/entities/user-profile.ts
View file

@ -132,20 +132,6 @@ export class UserProfile {
})
public moderationNote: string | null;
// TODO: そのうち消す
@Column("jsonb", {
default: {},
comment: "The client-specific data of the User.",
})
public clientData: Record<string, any>;
// TODO: そのうち消す
@Column("jsonb", {
default: {},
comment: "The room data of the User.",
})
public room: Record<string, any>;
@Column("boolean", {
default: false,
})
@ -194,12 +180,6 @@ export class UserProfile {
})
public pinnedPageId: Page["id"] | null;
@OneToOne((type) => Page, {
onDelete: "SET NULL",
})
@JoinColumn()
public pinnedPage: Page | null;
@Index()
@Column("boolean", {
default: false,
@ -207,10 +187,11 @@ export class UserProfile {
})
public enableWordMute: boolean;
@Column("jsonb", {
default: [],
@Column("text", {
array: true,
default: "{}",
})
public mutedWords: string[][];
public mutedWords: string[];
@Column("text", {
array: true,
@ -218,8 +199,10 @@ export class UserProfile {
})
public mutedPatterns: string[];
@Column("jsonb", {
default: [],
@Column("varchar", {
length: 512,
array: true,
default: "{}",
comment: "List of instances muted by the user.",
})
public mutedInstances: string[];
@ -247,6 +230,13 @@ export class UserProfile {
})
@JoinColumn()
public user: Relation<User>;
@OneToOne(() => Page, {
onDelete: "SET NULL",
nullable: true,
})
@JoinColumn()
public pinnedPage: Page | null;
//#endregion
constructor(data: Partial<UserProfile>) {

4
packages/backend/src/models/repositories/antenna.ts
View file

@ -16,8 +16,8 @@ export const AntennaRepository = db.getRepository(Antenna).extend({
id: antenna.id,
createdAt: antenna.createdAt.toISOString(),
name: antenna.name,
keywords: antenna.keywords,
excludeKeywords: antenna.excludeKeywords,
keywords: antenna.keywords.map((row) => row.split(" ")),
excludeKeywords: antenna.excludeKeywords.map((row) => row.split(" ")),
src: antenna.src,
userListId: antenna.userListId,
userGroupId: userGroupJoining ? userGroupJoining.userGroupId : null,

4
packages/backend/src/models/repositories/drive-file.ts
View file

@ -68,9 +68,7 @@ export const DriveFileRepository = db.getRepository(DriveFile).extend({
}
}
const isImage =
file.type &&
[
const isImage = [
"image/png",
"image/apng",
"image/gif",

2
packages/backend/src/models/repositories/user.ts
View file

@ -572,7 +572,7 @@ export const UserRepository = db.getRepository(User).extend({
hasUnreadNotification: this.getHasUnreadNotification(user.id),
hasPendingReceivedFollowRequest:
this.getHasPendingReceivedFollowRequest(user.id),
mutedWords: profile?.mutedWords,
mutedWords: profile?.mutedWords.map((row) => row.split(" ")),
mutedPatterns: profile?.mutedPatterns,
mutedInstances: profile?.mutedInstances,
mutingNotificationTypes: profile?.mutingNotificationTypes,

22
packages/backend/src/prelude/undefined-to-null.ts → packages/backend/src/prelude/as-rust-object.ts
View file

@ -29,24 +29,6 @@ type UndefinedToNull<T> = T extends undefined
? { [K in keyof T]: UndefinedToNull<T[K]> }
: unknown;
function _nullToUndefined<T>(obj: T): NullToUndefined<T> {
if (obj === null) {
return undefined as any;
}
if (typeof obj === "object") {
if (obj instanceof Map) {
obj.forEach((value, key) => obj.set(key, _nullToUndefined(value)));
} else {
for (const key in obj) {
obj[key] = _nullToUndefined(obj[key]) as any;
}
}
}
return obj as any;
}
function _undefinedToNull<T>(obj: T): UndefinedToNull<T> {
if (obj === undefined) {
return null as any;
@ -71,6 +53,6 @@ function _undefinedToNull<T>(obj: T): UndefinedToNull<T> {
* @param obj object to convert
* @returns a copy of the object with all its undefined values converted to null
*/
export function undefinedToNull<T>(obj: T) {
return _undefinedToNull(structuredClone(obj));
export function asRustObject<T>(obj: object): T {
return _undefinedToNull(structuredClone(obj)) as T;
}

2
packages/backend/src/remote/activitypub/models/note.ts
View file

@ -657,7 +657,7 @@ export async function updateNote(value: string | IObject, resolver?: Resolver) {
const fileTypes = driveFiles.map((file) => file.type);
const apEmojis = (
await extractEmojis(post.tag || [], actor.host).catch((e) => [])
await extractEmojis(post.tag || [], actor.host).catch((_) => [])
).map((emoji) => emoji.name);
const apMentions = await extractApMentions(post.tag);
const apHashtags = await extractApHashtags(post.tag);

6
packages/backend/src/server/api/common/generate-muted-user-query.ts
View file

@ -44,21 +44,21 @@ export function generateMutedUserQuery(
.andWhere(
new Brackets((qb) => {
qb.andWhere("note.userHost IS NULL").orWhere(
`NOT ((${mutingInstanceQuery.getQuery()})::jsonb ? note.userHost)`,
`NOT EXISTS (SELECT 1 FROM "user_profile" WHERE note.userHost = ANY("mutedInstances"))`,
);
}),
)
.andWhere(
new Brackets((qb) => {
qb.where("note.replyUserHost IS NULL").orWhere(
`NOT ((${mutingInstanceQuery.getQuery()})::jsonb ? note.replyUserHost)`,
`NOT EXISTS (SELECT 1 FROM "user_profile" WHERE note.replyUserHost = ANY("mutedInstances"))`,
);
}),
)
.andWhere(
new Brackets((qb) => {
qb.where("note.renoteUserHost IS NULL").orWhere(
`NOT ((${mutingInstanceQuery.getQuery()})::jsonb ? note.renoteUserHost)`,
`NOT EXISTS (SELECT 1 FROM "user_profile" WHERE note.renoteUserHost = ANY("mutedInstances"))`,
);
}),
);

2
packages/backend/src/server/api/endpoints/admin/show-user.ts
View file

@ -59,7 +59,7 @@ export default define(meta, paramDef, async (ps, me) => {
carefulBot: profile.carefulBot,
injectFeaturedNote: profile.injectFeaturedNote,
receiveAnnouncementEmail: profile.receiveAnnouncementEmail,
mutedWords: profile.mutedWords,
mutedWords: profile.mutedWords.map((row) => row.split(" ")),
mutedPatterns: profile.mutedPatterns,
mutedInstances: profile.mutedInstances,
mutingNotificationTypes: profile.mutingNotificationTypes,

22
packages/backend/src/server/api/endpoints/antennas/create.ts
View file

@ -104,8 +104,22 @@ export const paramDef = {
} as const;
export default define(meta, paramDef, async (ps, user) => {
const flatten = (arr: string[][]) =>
JSON.stringify(arr) === "[[]]"
? ([] as string[])
: arr.map((row) => row.join(" "));
const keywords = flatten(
ps.keywords.map((row) => row.filter((word) => word.trim().length > 0)),
);
const excludedWords = flatten(
ps.excludeKeywords.map((row) =>
row.filter((word) => word.trim().length > 0),
),
);
if (user.movedToUri != null) throw new ApiError(meta.errors.noSuchUserGroup);
if (ps.keywords.length === 0) throw new ApiError(meta.errors.noKeywords);
if (keywords.length === 0) throw new ApiError(meta.errors.noKeywords);
let userList;
let userGroupJoining;
@ -146,10 +160,10 @@ export default define(meta, paramDef, async (ps, user) => {
src: ps.src,
userListId: userList ? userList.id : null,
userGroupJoiningId: userGroupJoining ? userGroupJoining.id : null,
keywords: ps.keywords,
excludeKeywords: ps.excludeKeywords,
keywords: keywords,
excludeKeywords: excludedWords,
users: ps.users,
instances: ps.instances,
instances: ps.instances.filter((instance) => instance.trim().length > 0),
caseSensitive: ps.caseSensitive,
withReplies: ps.withReplies,
withFile: ps.withFile,

20
packages/backend/src/server/api/endpoints/antennas/update.ts
View file

@ -100,6 +100,20 @@ export const paramDef = {
} as const;
export default define(meta, paramDef, async (ps, user) => {
const flatten = (arr: string[][]) =>
JSON.stringify(arr) === "[[]]"
? ([] as string[])
: arr.map((row) => row.join(" "));
const keywords = flatten(
ps.keywords.map((row) => row.filter((word) => word.trim().length > 0)),
);
const excludedWords = flatten(
ps.excludeKeywords.map((row) =>
row.filter((word) => word.trim().length > 0),
),
);
// Fetch the antenna
const antenna = await Antennas.findOneBy({
id: ps.antennaId,
@ -138,10 +152,10 @@ export default define(meta, paramDef, async (ps, user) => {
src: ps.src,
userListId: userList ? userList.id : null,
userGroupJoiningId: userGroupJoining ? userGroupJoining.id : null,
keywords: ps.keywords,
excludeKeywords: ps.excludeKeywords,
keywords: keywords,
excludeKeywords: excludedWords,
users: ps.users,
instances: ps.instances,
instances: ps.instances.filter((instance) => instance.trim().length > 0),
caseSensitive: ps.caseSensitive,
withReplies: ps.withReplies,
withFile: ps.withFile,

3
packages/backend/src/server/api/endpoints/i/notifications.ts
View file

@ -125,7 +125,8 @@ export default define(meta, paramDef, async (ps, user) => {
query.andWhere(
new Brackets((qb) => {
qb.andWhere("notifier.host IS NULL").orWhere(
`NOT (( ${mutingInstanceQuery.getQuery()} )::jsonb ? notifier.host)`,
`NOT EXISTS (SELECT 1 FROM "user_profile" WHERE "userId" = :muterId AND notifier.host = ANY("mutedInstances"))`,
{ muterId: user.id },
);
}),
);

24
packages/backend/src/server/api/endpoints/i/update.ts
View file

@ -176,26 +176,12 @@ export default define(meta, paramDef, async (ps, _user, token) => {
}
}
if (ps.mutedWords !== undefined) {
// for backward compatibility
for (const item of ps.mutedWords) {
if (Array.isArray(item)) continue;
const flatten = (arr: string[][]) =>
JSON.stringify(arr) === "[[]]"
? ([] as string[])
: arr.map((row) => row.join(" "));
const regexp = item.match(/^\/(.+)\/(.*)$/);
if (!regexp) throw new ApiError(meta.errors.invalidRegexp);
try {
new RegExp(regexp[1], regexp[2]);
} catch (err) {
throw new ApiError(meta.errors.invalidRegexp);
}
profileUpdates.mutedPatterns = profileUpdates.mutedPatterns ?? [];
profileUpdates.mutedPatterns.push(item);
}
profileUpdates.mutedWords = ps.mutedWords.filter((item) =>
Array.isArray(item),
);
profileUpdates.mutedWords = flatten(ps.mutedWords);
}
if (
profileUpdates.mutedWords !== undefined ||

2
packages/backend/src/services/create-notification.ts
View file

@ -62,7 +62,7 @@ export async function createNotification(
id: genId(),
createdAt: new Date(),
notifieeId: notifieeId,
type: type,
type,
// 相手がこの通知をミュートしているようなら、既読を予めつけておく
isRead: isMuted,
...data,

2
packages/backend/src/services/insert-moderation-log.ts
View file

@ -11,7 +11,7 @@ export async function insertModerationLog(
id: genId(),
createdAt: new Date(),
userId: moderator.id,
type: type,
type,
info: info || {},
});
}

23
packages/backend/src/services/note/create.ts
View file

@ -42,10 +42,9 @@ import type { IPoll } from "@/models/entities/poll.js";
import { Poll } from "@/models/entities/poll.js";
import { createNotification } from "@/services/create-notification.js";
import { isDuplicateKeyValueError } from "@/misc/is-duplicate-key-value-error.js";
import { checkHitAntenna } from "@/misc/check-hit-antenna.js";
import {
addNoteToAntenna,
checkWordMute,
updateAntennaOnCreateNote,
genId,
genIdAt,
isSilencedServer,
@ -66,7 +65,7 @@ import { Mutex } from "redis-semaphore";
import { langmap } from "@/misc/langmap.js";
import Logger from "@/services/logger.js";
import { inspect } from "node:util";
import { undefinedToNull } from "@/prelude/undefined-to-null.js";
import { asRustObject } from "@/prelude/as-rust-object.js";
const logger = new Logger("create-note");
@ -400,13 +399,13 @@ export default async (
});
// Antenna
for (const antenna of await getAntennas()) {
checkHitAntenna(antenna, note, user).then((hit) => {
if (hit) {
// TODO: do this more sanely
addNoteToAntenna(antenna.id, undefinedToNull(note) as Note);
}
});
const antennas = await getAntennas();
for await (const antenna of antennas) {
await updateAntennaOnCreateNote(
asRustObject(antenna),
asRustObject(note),
user,
);
}
// Channel
@ -740,7 +739,9 @@ async function insertNote(
: []
: [],
attachedFileTypes: data.files ? data.files.map((file) => file.type) : [],
attachedFileTypes: data.files
? data.files.map((file) => file.mimeType)
: [],
// 以下非正規化データ
replyUserId: data.reply ? data.reply.userId : null,