Merge branch 'develop' into 'main'

release: v20240430

Co-authored-by: Laura Hausmann <laura@hausmann.dev>
Co-authored-by: Hosted Weblate <hosted@weblate.org>
Co-authored-by: Linerly <linerly@proton.me>
Co-authored-by: jolupa <jolupameister@gmail.com>
Co-authored-by: Lhcfl <Lhcfl@outlook.com>
Co-authored-by: Salif Mehmed <mail@salif.eu>
Co-authored-by: 老周部落 <laozhoubuluo@gmail.com>
Co-authored-by: Sal Rahman <salehen.rahman@gmail.com>

See merge request firefish/firefish!10781
This commit is contained in:
naskya 2024-04-29 21:45:39 +00:00
commit 72e3c0ca06
200 changed files with 4228 additions and 2119 deletions

View file

@ -13,16 +13,8 @@ redis:
host: firefish_redis
port: 6379
id: 'aid'
#allowedPrivateNetworks: [
# '10.69.1.0/24'
#]
logLevel: [
'error',
'success',
'warning',
'debug',
'info'
]
maxLogLevel: 'debug'

View file

@ -145,16 +145,11 @@ reservedUsernames: [
# IP address family used for outgoing request (ipv4, ipv6 or dual)
#outgoingAddressFamily: ipv4
# Log Option
# Production env: ['error', 'success', 'warning', 'info']
# Debug/Test env or Troubleshooting: ['error', 'success', 'warning', 'debug' ,'info']
# Production env which storage space or IO is tight: ['error', 'warning']
logLevel: [
'error',
'success',
'warning',
'info'
]
# Log level (error, warning, info, debug, trace)
# Production env: info
# Production env whose storage space or IO is tight: warning
# Debug/Test env or Troubleshooting: debug (or trace)
maxLogLevel: info
# Syslog option
#syslog:

1337
Cargo.lock generated

File diff suppressed because it is too large Load diff

View file

@ -5,34 +5,40 @@ resolver = "2"
[workspace.dependencies]
macro_rs = { path = "packages/macro-rs" }
napi = { version = "2.16.2", default-features = false }
napi-derive = "2.16.2"
napi = { version = "2.16.4", default-features = false }
napi-derive = "2.16.3"
napi-build = "2.1.3"
argon2 = "0.5.3"
basen = "0.1.0"
bcrypt = "0.15.1"
chrono = "0.4.37"
chrono = "0.4.38"
convert_case = "0.6.0"
cuid2 = "0.1.2"
emojis = "0.6.1"
emojis = "0.6.2"
idna = "0.5.0"
image = "0.25.1"
nom-exif = "1.2.0"
once_cell = "1.19.0"
openssl = "0.10.64"
pretty_assertions = "1.4.0"
proc-macro2 = "1.0.79"
proc-macro2 = "1.0.81"
quote = "1.0.36"
rand = "0.8.5"
redis = "0.25.3"
regex = "1.10.4"
reqwest = "0.12.4"
rmp-serde = "1.2.0"
sea-orm = "0.12.15"
serde = "1.0.197"
serde_json = "1.0.115"
serde = "1.0.198"
serde_json = "1.0.116"
serde_yaml = "0.9.34"
strum = "0.26.2"
syn = "2.0.58"
thiserror = "1.0.58"
syn = "2.0.60"
thiserror = "1.0.59"
tokio = "1.37.0"
tracing = "0.1.40"
tracing-subscriber = "0.3.18"
url = "2.5.0"
urlencoding = "2.1.3"

View file

@ -3,7 +3,7 @@ FROM docker.io/node:20-alpine as build
WORKDIR /firefish
# Install compilation dependencies
RUN apk update && apk add --no-cache build-base linux-headers curl ca-certificates python3
RUN apk update && apk add --no-cache build-base linux-headers curl ca-certificates python3 perl
RUN curl --proto '=https' --tlsv1.2 --silent --show-error --fail https://sh.rustup.rs | sh -s -- -y
ENV PATH="/root/.cargo/bin:${PATH}"
@ -34,6 +34,7 @@ RUN corepack enable && corepack prepare pnpm@latest --activate && pnpm install -
# Copy in the rest of the rust files
COPY packages/backend-rs packages/backend-rs/
# COPY packages/macro-rs packages/macro-rs/
# Compile backend-rs
RUN NODE_ENV='production' pnpm run --filter backend-rs build

View file

@ -1,4 +1,4 @@
COMPOSE='docker compose'
COMPOSE='podman-compose'
POSTGRES_PASSWORD=password
POSTGRES_USER=firefish
POSTGRES_DB=firefish_db

View file

@ -5,11 +5,20 @@ Critical security updates are indicated by the :warning: icon.
- Server administrators should check [notice-for-admins.md](./notice-for-admins.md) as well.
- Third-party client/bot developers may want to check [api-change.md](./api-change.md) as well.
## :warning: [v20240430](https://firefish.dev/firefish/firefish/-/merge_requests/10781/commits)
- Add ability to group similar notifications
- Add features to share links to an account in the three dots menu on the profile page
- Improve server logs
- Fix bugs (including a critical security issue)
- We are very thankful to @tesaguri and Laura Hausmann for helping to fix the security issue.
## [v20240424](https://firefish.dev/firefish/firefish/-/merge_requests/10765/commits)
- Improve the usability of the feature to prevent forgetting to write alt texts
- Add a server-wide setting for the maximum number of antennas each user can create
- Fix bugs
- Fix bugs (including a medium sevirity security issue)
- We are very thankful to @mei23 for kindly sharing the information about the security issue.
## [v20240421](https://firefish.dev/firefish/firefish/-/merge_requests/10756/commits)

View file

@ -1,6 +1,7 @@
BEGIN;
DELETE FROM "migrations" WHERE name IN (
'AlterAkaType1714099399879',
'AddDriveFileUsage1713451569342',
'ConvertCwVarcharToText1713225866247',
'FixChatFileConstraint1712855579316',
@ -24,6 +25,13 @@ DELETE FROM "migrations" WHERE name IN (
'RemoveNativeUtilsMigration1705877093218'
);
-- alter-aka-type
ALTER TABLE "user" RENAME COLUMN "alsoKnownAs" TO "alsoKnownAsOld";
ALTER TABLE "user" ADD COLUMN "alsoKnownAs" text;
UPDATE "user" SET "alsoKnownAs" = array_to_string("alsoKnownAsOld", ',');
COMMENT ON COLUMN "user"."alsoKnownAs" IS 'URIs the user is known as too';
ALTER TABLE "user" DROP COLUMN "alsoKnownAsOld";
-- AddDriveFileUsage
ALTER TABLE "drive_file" DROP COLUMN "usageHint";
DROP TYPE "drive_file_usage_hint_enum";

View file

@ -2,6 +2,33 @@
You can skip intermediate versions when upgrading from an old version, but please read the notices and follow the instructions for each intermediate version before [upgrading](./upgrade.md).
## v20240430
### For all users
You can control the verbosity of the server log by adding `maxLogLevel` in `.config/default.yml`. `logLevels` has been deprecated in favor of this setting. (see also: <https://firefish.dev/firefish/firefish/-/blob/eac0c1c47cd23789dcc395ab08b074934409fd96/.config/example.yml#L152>)
### For systemd/pm2 users
Not only Firefish but also Node.js has recently fixed a few security issues:
- https://nodejs.org/en/blog/vulnerability/april-2024-security-releases
- https://nodejs.org/en/blog/vulnerability/april-2024-security-releases-2
So, it is highly recommended that you upgrade your Node.js version as well. The new versions are
- Node v18.20.2 (v18.x LTS)
- Node v20.12.2 (v20.x LTS)
- Node v21.7.3 (v21.x)
You can check your Node.js version by this command:
```sh
node --version
```
[Node v22](https://nodejs.org/en/blog/announcements/v22-release-announce) was also released several days ago, but we have not yet tested Firefish with this version.
## v20240413
### For all users

View file

@ -19,7 +19,7 @@ deleteAndEditConfirm: Сигурни ли сте, че искате да изт
copyUsername: Копиране на потребителското име
searchUser: Търсене на потребител
reply: Отговор
showMore: Покажи още
showMore: Показване на повече
loadMore: Зареди още
followRequestAccepted: Заявката за последване е приета
importAndExport: Импорт/експорт на данни
@ -336,6 +336,10 @@ _pages:
title: Заглавие
my: Моите страници
pageSetting: Настройки на страницата
url: Адрес на страницата
summary: Кратко обобщение
alignCenter: Центриране на елементите
variables: Променливи
_deck:
_columns:
notifications: Известия
@ -398,7 +402,7 @@ sendMessage: Изпращане на съобщение
jumpToPrevious: Премини към предишно
newer: по-ново
older: по-старо
showLess: Покажи по-малко
showLess: Показване на по-малко
youGotNewFollower: те последва
receiveFollowRequest: Заявка за последване получена
mention: Споменаване
@ -754,7 +758,7 @@ _feeds:
general: Общи
metadata: Метаданни
disk: Диск
featured: Препоръчани
featured: Препоръчано
yearsOld: на {age} години
reload: Опресняване
invites: Покани
@ -940,3 +944,11 @@ showGapBetweenNotesInTimeline: Показване на празнина межд
lookup: Поглеждане
media: Мултимедия
welcomeBackWithName: Добре дошли отново, {name}
reduceUiAnimation: Намаляване на UI анимациите
clickToFinishEmailVerification: Моля, натиснете [{ok}], за да завършите потвърждаването
на ел. поща.
_cw:
show: Показване на съдържанието
remoteFollow: Отдалечено последване
messagingUnencryptedInfo: Чатовете във Firefish не са шифровани от край до край. Не
споделяйте чувствителна информация през Firefish.

View file

@ -738,6 +738,7 @@ _notification:
reacted: Ha reaccionat a la teva publicació
renoted: Ha impulsat la teva publicació
voted: Ha votat a la teva enquesta
andCountUsers: I {count} usuaris més {acted}
_deck:
_columns:
notifications: "Notificacions"
@ -2292,3 +2293,11 @@ media: Multimèdia
antennaLimit: El nombre màxim d'antenes que pot crear un usuari
showAddFileDescriptionAtFirstPost: Obra de forma automàtica un formulari per escriure
una descripció quant intentes publicar un fitxer que no en té
remoteFollow: Seguiment remot
cannotEditVisibility: No pots canviar la visibilitat
useThisAccountConfirm: Vols continuar amb aquest compte?
inputAccountId: Sisplau introdueix el teu compte (per exemple @firefish@info.firefish.dev)
getQrCode: Mostrar el codi QR
copyRemoteFollowUrl: Còpia la adreça URL del seguidor remot
foldNotification: Agrupar les notificacions similars
slashQuote: Cita encadenada

View file

@ -645,6 +645,7 @@ deletedNote: "Deleted post"
invisibleNote: "Invisible post"
enableInfiniteScroll: "Automatically load more"
visibility: "Visiblility"
cannotEditVisibility: "You can't edit the visibility"
poll: "Poll"
useCw: "Hide content"
enablePlayer: "Open video player"
@ -1010,6 +1011,8 @@ isSystemAccount: "This account is created and automatically operated by the syst
Please do not moderate, edit, delete, or otherwise tamper with this account, or
it may break your server."
typeToConfirm: "Please enter {x} to confirm"
useThisAccountConfirm: "Do you want to continue with this account?"
inputAccountId: "Please input your account (e.g., @firefish@info.firefish.dev)"
deleteAccount: "Delete account"
document: "Documentation"
numberOfPageCache: "Number of cached pages"
@ -1156,6 +1159,9 @@ addRe: "Add \"re:\" at the beginning of comment in reply to a post with a conten
confirm: "Confirm"
importZip: "Import ZIP"
exportZip: "Export ZIP"
getQrCode: "Show QR code"
remoteFollow: "Remote follow"
copyRemoteFollowUrl: "Copy remote follow URL"
emojiPackCreator: "Emoji pack creator"
indexable: "Indexable"
indexableDescription: "Allow built-in search to show your public posts"
@ -2146,6 +2152,7 @@ _notification:
reacted: "reacted to your post"
renoted: "boosted your post"
voted: "voted on your poll"
andCountUsers: "and {count} more users {acted}"
_types:
all: "All"
follow: "New followers"
@ -2232,3 +2239,5 @@ autocorrectNoteLanguage: "Show a warning if the post language does not match the
incorrectLanguageWarning: "It looks like your post is in {detected}, but you selected
{current}.\nWould you like to set the language to {detected} instead?"
noteEditHistory: "Post edit history"
slashQuote: "Chain quote"
foldNotification: "Group similar notifications"

View file

@ -928,6 +928,8 @@ colored: "Coloré"
label: "Étiquette"
localOnly: "Local seulement"
account: "Comptes"
getQrCode: "Obtenir le code QR"
_emailUnavailable:
used: "Adresse non disponible"
format: "Le format de cette adresse de courriel est invalide"

View file

@ -1825,6 +1825,7 @@ _notification:
reacted: mereaksi postinganmu
renoted: memposting ulang postinganmu
voted: memilih di angketmu
andCountUsers: dan {count} lebih banyak pengguna {acted}
_deck:
alwaysShowMainColumn: "Selalu tampilkan kolom utama"
columnAlign: "Luruskan kolom"
@ -2267,3 +2268,14 @@ markLocalFilesNsfwByDefaultDescription: Terlepas dari pengaturan ini, pengguna d
menghapus sendiri tanda NSFW. Berkas yang ada tidak berpengaruh.
noteEditHistory: Riwayat penyuntingan kiriman
media: Media
antennaLimit: Jumlah antena maksimum yang dapat dibuat oleh setiap pengguna
showAddFileDescriptionAtFirstPost: Buka formulir secara otomatis untuk menulis deskripsi
ketika mencoba mengirim berkas tanpa deskripsi
remoteFollow: Ikuti jarak jauh
foldNotification: Kelompokkan notifikasi yang sama
getQrCode: Tampilkan kode QR
cannotEditVisibility: Kamu tidak bisa menyunting keterlihatan
useThisAccountConfirm: Apakah kamu ingin melanjutkan dengan akun ini?
inputAccountId: Silakan memasukkan akunmu (misalnya, @firefish@info.firefish.dev)
copyRemoteFollowUrl: Salin URL ikuti jarak jauh
slashQuote: Kutipan rantai

View file

@ -1902,6 +1902,7 @@ _notification:
reacted: がリアクションしました
renoted: がブーストしました
voted: が投票しました
andCountUsers: と{count}人{acted}
_deck:
alwaysShowMainColumn: "常にメインカラムを表示"
columnAlign: "カラムの寄せ"
@ -2059,3 +2060,11 @@ markLocalFilesNsfwByDefaultDescription: この設定が有効でも、ユーザ
noteEditHistory: 編集履歴
showAddFileDescriptionAtFirstPost: 説明の無い添付ファイルを投稿しようとした際に説明を書く画面を自動で開く
antennaLimit: 各ユーザーが作れるアンテナの最大数
inputAccountId: 'あなたのアカウントを入力してください(例: @firefish@info.firefish.dev'
remoteFollow: リモートフォロー
cannotEditVisibility: 公開範囲は変更できません
useThisAccountConfirm: このアカウントで操作を続けますか?
getQrCode: QRコードを表示
copyRemoteFollowUrl: リモートからフォローするURLをコピー
foldNotification: 同じ種類の通知をまとめて表示する
slashQuote: 繋げて引用

View file

@ -564,6 +564,7 @@ deletedNote: "已删除的帖子"
invisibleNote: "隐藏的帖子"
enableInfiniteScroll: "滚动页面以载入更多内容"
visibility: "可见性"
cannotEditVisibility: "不能编辑帖子的可见性"
poll: "调查问卷"
useCw: "隐藏内容"
enablePlayer: "打开播放器"
@ -878,6 +879,8 @@ driveCapOverrideCaption: "输入 0 或以下的值将容量重置为默认值。
requireAdminForView: "您需要使用管理员账号登录才能查看。"
isSystemAccount: "该账号由系统自动创建。请不要修改、编辑、删除或以其它方式篡改这个账号,否则可能会破坏您的服务器。"
typeToConfirm: "输入 {x} 以确认操作"
useThisAccountConfirm: "您想使用此帐户继续执行此操作吗?"
inputAccountId: "请输入您的帐户(例如 @firefish@info.firefish.dev "
deleteAccount: "删除账号"
document: "文档"
numberOfPageCache: "缓存页数"
@ -1787,6 +1790,7 @@ _notification:
reacted: 回应了您的帖子
voted: 在您的问卷调查中投了票
renoted: 转发了您的帖子
andCountUsers: "和其他 {count} 名用户{acted}"
_deck:
alwaysShowMainColumn: "总是显示主列"
columnAlign: "列对齐"
@ -1972,6 +1976,9 @@ origin: 起源
confirm: 确认
importZip: 导入 ZIP
exportZip: 导出 ZIP
getQrCode: "获取二维码"
remoteFollow: "远程关注"
copyRemoteFollowUrl: "复制远程关注 URL"
emojiPackCreator: 表情包创建工具
objectStorageS3ForcePathStyleDesc: 打开此选项可构建格式为 "s3.amazonaws.com/<bucket>/" 而非 "<bucket>.s3.amazonaws.com"
的端点 URL。
@ -2059,3 +2066,5 @@ autocorrectNoteLanguage: 当帖子语言不符合自动检测的结果的时候
incorrectLanguageWarning: "看上去您帖子使用的语言是{detected},但您选择的语言是{current}。\n要改为以{detected}发帖吗?"
noteEditHistory: "帖子编辑历史"
media: 媒体
slashQuote: "斜杠引用"
foldNotification: "将通知按同类型分组"

View file

@ -1,6 +1,6 @@
{
"name": "firefish",
"version": "20240424",
"version": "20240430",
"repository": {
"type": "git",
"url": "https://firefish.dev/firefish/firefish.git"
@ -45,11 +45,11 @@
"js-yaml": "4.1.0"
},
"devDependencies": {
"@biomejs/biome": "1.6.4",
"@biomejs/cli-darwin-arm64": "^1.6.4",
"@biomejs/cli-darwin-x64": "^1.6.4",
"@biomejs/cli-linux-arm64": "^1.6.4",
"@biomejs/cli-linux-x64": "^1.6.4",
"@biomejs/biome": "1.7.1",
"@biomejs/cli-darwin-arm64": "^1.7.1",
"@biomejs/cli-darwin-x64": "^1.7.1",
"@biomejs/cli-linux-arm64": "^1.7.1",
"@biomejs/cli-linux-x64": "^1.7.1",
"@types/node": "20.12.7",
"execa": "8.0.1",
"pnpm": "8.15.7",

View file

@ -1,12 +0,0 @@
target
Cargo.lock
.cargo
.github
npm
.eslintrc
rustfmt.toml
yarn.lock
*.node
.yarn
__test__
renovate.json

View file

@ -24,10 +24,14 @@ chrono = { workspace = true }
cuid2 = { workspace = true }
emojis = { workspace = true }
idna = { workspace = true }
image = { workspace = true }
nom-exif = { workspace = true }
once_cell = { workspace = true }
openssl = { workspace = true, features = ["vendored"] }
rand = { workspace = true }
redis = { workspace = true }
regex = { workspace = true }
reqwest = { workspace = true, features = ["blocking"] }
rmp-serde = { workspace = true }
sea-orm = { workspace = true, features = ["sqlx-postgres", "runtime-tokio-rustls"] }
serde = { workspace = true, features = ["derive"] }
@ -36,6 +40,8 @@ serde_yaml = { workspace = true }
strum = { workspace = true, features = ["derive"] }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["full"] }
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
url = { workspace = true }
urlencoding = { workspace = true }

View file

@ -1,32 +0,0 @@
import test from "ava";
import {
convertId,
IdConvertType,
nativeInitIdGenerator,
nativeCreateId,
nativeRandomStr,
} from "../built/index.js";
test("convert to mastodon id", (t) => {
t.is(convertId("9gf61ehcxv", IdConvertType.MastodonId), "960365976481219");
t.is(
convertId("9fbr9z0wbrjqyd3u", IdConvertType.MastodonId),
"2083785058661759970208986",
);
t.is(
convertId("9fbs680oyviiqrol9md73p8g", IdConvertType.MastodonId),
"5878598648988104013828532260828151168",
);
});
test("create cuid2 with timestamp prefix", (t) => {
nativeInitIdGenerator(16, "");
t.not(nativeCreateId(Date.now()), nativeCreateId(Date.now()));
t.is(nativeCreateId(Date.now()).length, 16);
});
test("create random string", (t) => {
t.not(nativeRandomStr(16), nativeRandomStr(16));
t.is(nativeRandomStr(24).length, 24);
});

View file

@ -3,6 +3,21 @@
/* auto-generated by NAPI-RS */
export const SECOND: number
export const MINUTE: number
export const HOUR: number
export const DAY: number
export const USER_ONLINE_THRESHOLD: number
export const USER_ACTIVE_THRESHOLD: number
/**
* List of file types allowed to be viewed directly in the browser
* Anything not included here will be responded as application/octet-stream
* SVG is not allowed because it generates XSS <- we need to fix this and later allow it to be viewed directly
* https://github.com/sindresorhus/file-type/blob/main/supported.js
* https://github.com/sindresorhus/file-type/blob/main/core.js
* https://developer.mozilla.org/en-US/docs/Web/Media/Formats/Containers
*/
export const FILE_TYPE_BROWSERSAFE: string[]
export interface EnvConfig {
onlyQueue: boolean
onlyServer: boolean
@ -38,7 +53,9 @@ export interface ServerConfig {
inboxJobPerSec?: number
deliverJobMaxAttempts?: number
inboxJobMaxAttempts?: number
/** deprecated */
logLevel?: Array<string>
maxLogLevel?: string
syslog?: SysLogConfig
proxyRemoteFiles?: boolean
mediaProxy?: string
@ -148,7 +165,9 @@ export interface Config {
inboxJobPerSec?: number
deliverJobMaxAttempts?: number
inboxJobMaxAttempts?: number
/** deprecated */
logLevel?: Array<string>
maxLogLevel?: string
syslog?: SysLogConfig
proxyRemoteFiles?: boolean
mediaProxy?: string
@ -156,8 +175,8 @@ export interface Config {
reservedUsernames?: Array<string>
maxUserSignups?: number
isManagedHosting?: boolean
maxNoteLength?: number
maxCaptionLength?: number
maxNoteLength: number
maxCaptionLength: number
deepl?: DeepLConfig
libreTranslate?: LibreTranslateConfig
email?: EmailConfig
@ -229,6 +248,11 @@ export function sqlLikeEscape(src: string): string
export function safeForSql(src: string): boolean
/** Convert milliseconds to a human readable string */
export function formatMilliseconds(milliseconds: number): string
export interface ImageSize {
width: number
height: number
}
export function getImageSizeFromUrl(url: string): Promise<ImageSize>
/** TODO: handle name collisions better */
export interface NoteLikeForGetNoteSummary {
fileIds: Array<string>
@ -993,10 +1017,10 @@ export interface User {
isDeleted: boolean
driveCapacityOverrideMb: number | null
movedToUri: string | null
alsoKnownAs: string | null
speakAsCat: boolean
emojiModPerm: UserEmojimodpermEnum
isIndexable: boolean
alsoKnownAs: Array<string> | null
}
export interface UserGroup {
id: string
@ -1122,8 +1146,10 @@ export interface Webhook {
latestSentAt: Date | null
latestStatus: number | null
}
export function initializeRustLogger(): void
export function watchNote(watcherId: string, noteAuthorId: string, noteId: string): Promise<void>
export function unwatchNote(watcherId: string, noteId: string): Promise<void>
export function publishToChannelStream(channelId: string, userId: string): void
export enum ChatEvent {
Message = 'message',
Read = 'read',
@ -1131,6 +1157,31 @@ export enum ChatEvent {
Typing = 'typing'
}
export function publishToChatStream(senderUserId: string, receiverUserId: string, kind: ChatEvent, object: any): void
export enum ChatIndexEvent {
Message = 'message',
Read = 'read'
}
export function publishToChatIndexStream(userId: string, kind: ChatIndexEvent, object: any): void
export interface PackedEmoji {
id: string
aliases: Array<string>
name: string
category: string | null
host: string | null
url: string
license: string | null
width: number | null
height: number | null
}
export function publishToBroadcastStream(emoji: PackedEmoji): void
export function publishToGroupChatStream(groupId: string, kind: ChatEvent, object: any): void
export interface AbuseUserReportLike {
id: string
targetUserId: string
reporterId: string
comment: string
}
export function publishToModerationStream(moderatorId: string, report: AbuseUserReportLike): void
export function getTimestamp(id: string): number
/**
* The generated ID results in the form of `[8 chars timestamp] + [cuid2]`.

View file

@ -310,8 +310,15 @@ if (!nativeBinding) {
throw new Error(`Failed to load native binding`)
}
const { loadEnv, loadConfig, stringToAcct, acctToString, addNoteToAntenna, isBlockedServer, isSilencedServer, isAllowedServer, checkWordMute, getFullApAccount, isSelfHost, isSameOrigin, extractHost, toPuny, isUnicodeEmoji, sqlLikeEscape, safeForSql, formatMilliseconds, getNoteSummary, toMastodonId, fromMastodonId, fetchMeta, metaToPugArgs, nyaify, hashPassword, verifyPassword, isOldPasswordAlgorithm, decodeReaction, countReactions, toDbReaction, removeOldAttestationChallenges, AntennaSrcEnum, DriveFileUsageHintEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, watchNote, unwatchNote, ChatEvent, publishToChatStream, getTimestamp, genId, genIdAt, secureRndstr } = nativeBinding
const { SECOND, MINUTE, HOUR, DAY, USER_ONLINE_THRESHOLD, USER_ACTIVE_THRESHOLD, FILE_TYPE_BROWSERSAFE, loadEnv, loadConfig, stringToAcct, acctToString, addNoteToAntenna, isBlockedServer, isSilencedServer, isAllowedServer, checkWordMute, getFullApAccount, isSelfHost, isSameOrigin, extractHost, toPuny, isUnicodeEmoji, sqlLikeEscape, safeForSql, formatMilliseconds, getImageSizeFromUrl, getNoteSummary, toMastodonId, fromMastodonId, fetchMeta, metaToPugArgs, nyaify, hashPassword, verifyPassword, isOldPasswordAlgorithm, decodeReaction, countReactions, toDbReaction, removeOldAttestationChallenges, AntennaSrcEnum, DriveFileUsageHintEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, initializeRustLogger, watchNote, unwatchNote, publishToChannelStream, ChatEvent, publishToChatStream, ChatIndexEvent, publishToChatIndexStream, publishToBroadcastStream, publishToGroupChatStream, publishToModerationStream, getTimestamp, genId, genIdAt, secureRndstr } = nativeBinding
module.exports.SECOND = SECOND
module.exports.MINUTE = MINUTE
module.exports.HOUR = HOUR
module.exports.DAY = DAY
module.exports.USER_ONLINE_THRESHOLD = USER_ONLINE_THRESHOLD
module.exports.USER_ACTIVE_THRESHOLD = USER_ACTIVE_THRESHOLD
module.exports.FILE_TYPE_BROWSERSAFE = FILE_TYPE_BROWSERSAFE
module.exports.loadEnv = loadEnv
module.exports.loadConfig = loadConfig
module.exports.stringToAcct = stringToAcct
@ -330,6 +337,7 @@ module.exports.isUnicodeEmoji = isUnicodeEmoji
module.exports.sqlLikeEscape = sqlLikeEscape
module.exports.safeForSql = safeForSql
module.exports.formatMilliseconds = formatMilliseconds
module.exports.getImageSizeFromUrl = getImageSizeFromUrl
module.exports.getNoteSummary = getNoteSummary
module.exports.toMastodonId = toMastodonId
module.exports.fromMastodonId = fromMastodonId
@ -354,10 +362,17 @@ module.exports.RelayStatusEnum = RelayStatusEnum
module.exports.UserEmojimodpermEnum = UserEmojimodpermEnum
module.exports.UserProfileFfvisibilityEnum = UserProfileFfvisibilityEnum
module.exports.UserProfileMutingnotificationtypesEnum = UserProfileMutingnotificationtypesEnum
module.exports.initializeRustLogger = initializeRustLogger
module.exports.watchNote = watchNote
module.exports.unwatchNote = unwatchNote
module.exports.publishToChannelStream = publishToChannelStream
module.exports.ChatEvent = ChatEvent
module.exports.publishToChatStream = publishToChatStream
module.exports.ChatIndexEvent = ChatIndexEvent
module.exports.publishToChatIndexStream = publishToChatIndexStream
module.exports.publishToBroadcastStream = publishToBroadcastStream
module.exports.publishToGroupChatStream = publishToGroupChatStream
module.exports.publishToModerationStream = publishToModerationStream
module.exports.getTimestamp = getTimestamp
module.exports.genId = genId
module.exports.genIdAt = genIdAt

View file

@ -22,11 +22,7 @@
}
},
"devDependencies": {
"@napi-rs/cli": "2.18.1",
"ava": "6.1.2"
},
"ava": {
"timeout": "3m"
"@napi-rs/cli": "2.18.1"
},
"engines": {
"node": ">= 10"
@ -36,11 +32,7 @@
"build": "napi build --features napi --no-const-enum --platform --release ./built/",
"build:debug": "napi build --features napi --no-const-enum --platform ./built/",
"prepublishOnly": "napi prepublish -t npm",
"test": "pnpm run cargo:test && pnpm run build:debug && ava",
"universal": "napi universal",
"version": "napi version",
"cargo:test": "pnpm run cargo:unit && pnpm run cargo:integration",
"cargo:unit": "cargo test unit_test && cargo test -F napi unit_test",
"cargo:integration": "cargo test int_test"
"version": "napi version"
}
}

View file

@ -0,0 +1,67 @@
#[crate::export]
pub const SECOND: i32 = 1000;
#[crate::export]
pub const MINUTE: i32 = 60 * SECOND;
#[crate::export]
pub const HOUR: i32 = 60 * MINUTE;
#[crate::export]
pub const DAY: i32 = 24 * HOUR;
#[crate::export]
pub const USER_ONLINE_THRESHOLD: i32 = 10 * MINUTE;
#[crate::export]
pub const USER_ACTIVE_THRESHOLD: i32 = 3 * DAY;
/// List of file types allowed to be viewed directly in the browser
/// Anything not included here will be responded as application/octet-stream
/// SVG is not allowed because it generates XSS <- we need to fix this and later allow it to be viewed directly
/// https://github.com/sindresorhus/file-type/blob/main/supported.js
/// https://github.com/sindresorhus/file-type/blob/main/core.js
/// https://developer.mozilla.org/en-US/docs/Web/Media/Formats/Containers
#[crate::export]
pub const FILE_TYPE_BROWSERSAFE: [&str; 41] = [
// Images
"image/png",
"image/gif", // TODO: deprecated, but still used by old posts, new gifs should be converted to webp in the future
"image/jpeg",
"image/webp", // TODO: make this the default image format
"image/apng",
"image/bmp",
"image/tiff",
"image/x-icon",
"image/avif", // not as good supported now, but its good to introduce initial support for the future
// OggS
"audio/opus",
"video/ogg",
"audio/ogg",
"application/ogg",
// ISO/IEC base media file format
"video/quicktime",
"video/mp4", // TODO: we need to check for av1 later
"video/vnd.avi", // also av1
"audio/mp4",
"video/x-m4v",
"audio/x-m4a",
"video/3gpp",
"video/3gpp2",
"video/3gp2",
"audio/3gpp",
"audio/3gpp2",
"audio/3gp2",
"video/mpeg",
"audio/mpeg",
"video/webm",
"audio/webm",
"audio/aac",
"audio/x-flac",
"audio/flac",
"audio/vnd.wave",
"audio/mod",
"audio/x-mod",
"audio/s3m",
"audio/x-s3m",
"audio/xm",
"audio/x-xm",
"audio/it",
"audio/x-it",
];

View file

@ -1,4 +1,5 @@
pub use server::CONFIG;
pub mod constant;
pub mod environment;
pub mod server;

View file

@ -36,8 +36,11 @@ struct ServerConfig {
pub deliver_job_max_attempts: Option<u32>,
pub inbox_job_max_attempts: Option<u32>,
/// deprecated
pub log_level: Option<Vec<String>>,
pub max_log_level: Option<String>,
pub syslog: Option<SysLogConfig>,
pub proxy_remote_files: Option<bool>,
@ -197,7 +200,11 @@ pub struct Config {
pub inbox_job_per_sec: Option<u32>,
pub deliver_job_max_attempts: Option<u32>,
pub inbox_job_max_attempts: Option<u32>,
/// deprecated
pub log_level: Option<Vec<String>>,
pub max_log_level: Option<String>,
pub syslog: Option<SysLogConfig>,
pub proxy_remote_files: Option<bool>,
pub media_proxy: Option<String>,
@ -205,8 +212,8 @@ pub struct Config {
pub reserved_usernames: Option<Vec<String>>,
pub max_user_signups: Option<u32>,
pub is_managed_hosting: Option<bool>,
pub max_note_length: Option<u32>,
pub max_caption_length: Option<u32>,
pub max_note_length: u32,
pub max_caption_length: u32,
pub deepl: Option<DeepLConfig>,
pub libre_translate: Option<LibreTranslateConfig>,
pub email: Option<EmailConfig>,
@ -346,6 +353,7 @@ fn load_config() -> Config {
deliver_job_max_attempts: server_config.deliver_job_max_attempts,
inbox_job_max_attempts: server_config.inbox_job_max_attempts,
log_level: server_config.log_level,
max_log_level: server_config.max_log_level,
syslog: server_config.syslog,
proxy_remote_files: server_config.proxy_remote_files,
media_proxy: server_config.media_proxy,
@ -353,8 +361,8 @@ fn load_config() -> Config {
reserved_usernames: server_config.reserved_usernames,
max_user_signups: server_config.max_user_signups,
is_managed_hosting: server_config.is_managed_hosting,
max_note_length: server_config.max_note_length,
max_caption_length: server_config.max_caption_length,
max_note_length: server_config.max_note_length.unwrap_or(3000),
max_caption_length: server_config.max_caption_length.unwrap_or(1500),
deepl: server_config.deepl,
libre_translate: server_config.libre_translate,
email: server_config.email,

View file

@ -1,5 +1,6 @@
use crate::config::CONFIG;
use sea_orm::{Database, DbConn, DbErr};
use sea_orm::{ConnectOptions, Database, DbConn, DbErr};
use tracing::log::LevelFilter;
static DB_CONN: once_cell::sync::OnceCell<DbConn> = once_cell::sync::OnceCell::new();
@ -12,7 +13,13 @@ async fn init_database() -> Result<&'static DbConn, DbErr> {
CONFIG.db.port,
CONFIG.db.db,
);
let conn = Database::connect(database_uri).await?;
let option: ConnectOptions = ConnectOptions::new(database_uri)
.sqlx_logging_level(LevelFilter::Trace)
.to_owned();
tracing::info!("Initializing PostgreSQL connection");
let conn = Database::connect(option).await?;
Ok(DB_CONN.get_or_init(move || conn))
}

View file

@ -26,6 +26,8 @@ fn init_redis() -> Result<Client, RedisError> {
params.concat()
};
tracing::info!("Initializing Redis connection");
Client::open(redis_url)
}

View file

@ -39,7 +39,7 @@ async fn all_texts(note: NoteLike) -> Result<Vec<String>, DbErr> {
.flatten(),
);
if let Some(renote_id) = note.renote_id {
if let Some(renote_id) = &note.renote_id {
if let Some((text, cw)) = note::Entity::find_by_id(renote_id)
.select_only()
.columns([note::Column::Text, note::Column::Cw])
@ -53,10 +53,12 @@ async fn all_texts(note: NoteLike) -> Result<Vec<String>, DbErr> {
if let Some(c) = cw {
texts.push(c);
}
} else {
tracing::warn!("nonexistent renote id: {:#?}", renote_id);
}
}
if let Some(reply_id) = note.reply_id {
if let Some(reply_id) = &note.reply_id {
if let Some((text, cw)) = note::Entity::find_by_id(reply_id)
.select_only()
.columns([note::Column::Text, note::Column::Cw])
@ -70,6 +72,8 @@ async fn all_texts(note: NoteLike) -> Result<Vec<String>, DbErr> {
if let Some(c) = cw {
texts.push(c);
}
} else {
tracing::warn!("nonexistent reply id: {:#?}", reply_id);
}
}

View file

@ -0,0 +1,200 @@
use crate::misc::redis_cache::{get_cache, set_cache, CacheError};
use crate::util::http_client;
use image::{io::Reader, ImageError, ImageFormat};
use nom_exif::{parse_jpeg_exif, EntryValue, ExifTag};
use std::io::Cursor;
use tokio::sync::Mutex;
#[derive(thiserror::Error, Debug)]
pub enum Error {
#[error("Redis cache error: {0}")]
CacheErr(#[from] CacheError),
#[error("Reqwest error: {0}")]
ReqwestErr(#[from] reqwest::Error),
#[error("Image decoding error: {0}")]
ImageErr(#[from] ImageError),
#[error("Image decoding error: {0}")]
IoErr(#[from] std::io::Error),
#[error("Exif extraction error: {0}")]
ExifErr(#[from] nom_exif::Error),
#[error("Emoji meta attempt limit exceeded: {0}")]
TooManyAttempts(String),
#[error("Unsupported image type: {0}")]
UnsupportedImageErr(String),
}
const BROWSER_SAFE_IMAGE_TYPES: [ImageFormat; 8] = [
ImageFormat::Png,
ImageFormat::Jpeg,
ImageFormat::Gif,
ImageFormat::WebP,
ImageFormat::Tiff,
ImageFormat::Bmp,
ImageFormat::Ico,
ImageFormat::Avif,
];
static MTX_GUARD: Mutex<()> = Mutex::const_new(());
#[derive(Debug, PartialEq)]
#[crate::export(object)]
pub struct ImageSize {
pub width: u32,
pub height: u32,
}
#[crate::export]
pub async fn get_image_size_from_url(url: &str) -> Result<ImageSize, Error> {
let attempted: bool;
{
let _ = MTX_GUARD.lock().await;
let key = format!("fetchImage:{}", url);
attempted = get_cache::<bool>(&key)?.is_some();
if !attempted {
set_cache(&key, &true, 10 * 60)?;
}
}
if attempted {
tracing::warn!("attempt limit exceeded: {}", url);
return Err(Error::TooManyAttempts(url.to_string()));
}
tracing::info!("retrieving image size from {}", url);
let image_bytes = http_client()?.get(url).send().await?.bytes().await?;
let reader = Reader::new(Cursor::new(&image_bytes)).with_guessed_format()?;
let format = reader.format();
if format.is_none() || !BROWSER_SAFE_IMAGE_TYPES.contains(&format.unwrap()) {
return Err(Error::UnsupportedImageErr(format!("{:?}", format)));
}
let size = reader.into_dimensions()?;
let res = ImageSize {
width: size.0,
height: size.1,
};
if format.unwrap() != ImageFormat::Jpeg {
return Ok(res);
}
// handle jpeg orientation
// https://magnushoff.com/articles/jpeg-orientation/
let exif = parse_jpeg_exif(&*image_bytes)?;
if exif.is_none() {
return Ok(res);
}
let orientation = exif.unwrap().get_value(&ExifTag::Orientation)?;
let rotated =
orientation.is_some() && matches!(orientation.unwrap(), EntryValue::U32(v) if v >= 5);
if !rotated {
return Ok(res);
}
Ok(ImageSize {
width: size.1,
height: size.0,
})
}
#[cfg(test)]
mod unit_test {
use super::{get_image_size_from_url, ImageSize};
use crate::misc::redis_cache::delete_cache;
use pretty_assertions::assert_eq;
#[tokio::test]
async fn test_get_image_size() {
let png_url_1 = "https://firefish.dev/firefish/firefish/-/raw/5891a90f71a8b9d5ea99c683ade7e485c685d642/packages/backend/assets/splash.png";
let png_url_2 = "https://firefish.dev/firefish/firefish/-/raw/5891a90f71a8b9d5ea99c683ade7e485c685d642/packages/backend/assets/notification-badges/at.png";
let png_url_3 = "https://firefish.dev/firefish/firefish/-/raw/5891a90f71a8b9d5ea99c683ade7e485c685d642/packages/backend/assets/api-doc.png";
let rotated_jpeg_url = "https://firefish.dev/firefish/firefish/-/raw/5891a90f71a8b9d5ea99c683ade7e485c685d642/packages/backend/test/resources/rotate.jpg";
let webp_url_1 = "https://firefish.dev/firefish/firefish/-/raw/5891a90f71a8b9d5ea99c683ade7e485c685d642/custom/assets/badges/error.webp";
let webp_url_2 = "https://firefish.dev/firefish/firefish/-/raw/5891a90f71a8b9d5ea99c683ade7e485c685d642/packages/backend/assets/screenshots/1.webp";
let ico_url = "https://firefish.dev/firefish/firefish/-/raw/5891a90f71a8b9d5ea99c683ade7e485c685d642/packages/backend/assets/favicon.ico";
let gif_url = "https://firefish.dev/firefish/firefish/-/raw/b9c3dfbd3d473cb2cee20c467eeae780bc401271/packages/backend/test/resources/anime.gif";
let mp3_url = "https://firefish.dev/firefish/firefish/-/blob/5891a90f71a8b9d5ea99c683ade7e485c685d642/packages/backend/assets/sounds/aisha/1.mp3";
// Delete caches in case you run this test multiple times
// (should be disabled in CI tasks)
delete_cache(&format!("fetchImage:{}", png_url_1)).unwrap();
delete_cache(&format!("fetchImage:{}", png_url_2)).unwrap();
delete_cache(&format!("fetchImage:{}", png_url_3)).unwrap();
delete_cache(&format!("fetchImage:{}", rotated_jpeg_url)).unwrap();
delete_cache(&format!("fetchImage:{}", webp_url_1)).unwrap();
delete_cache(&format!("fetchImage:{}", webp_url_2)).unwrap();
delete_cache(&format!("fetchImage:{}", ico_url)).unwrap();
delete_cache(&format!("fetchImage:{}", gif_url)).unwrap();
delete_cache(&format!("fetchImage:{}", mp3_url)).unwrap();
let png_size_1 = ImageSize {
width: 1024,
height: 1024,
};
let png_size_2 = ImageSize {
width: 96,
height: 96,
};
let png_size_3 = ImageSize {
width: 1024,
height: 354,
};
let rotated_jpeg_size = ImageSize {
width: 256,
height: 512,
};
let webp_size_1 = ImageSize {
width: 256,
height: 256,
};
let webp_size_2 = ImageSize {
width: 1080,
height: 2340,
};
let ico_size = ImageSize {
width: 256,
height: 256,
};
let gif_size = ImageSize {
width: 256,
height: 256,
};
assert_eq!(
png_size_1,
get_image_size_from_url(png_url_1).await.unwrap()
);
assert_eq!(
png_size_2,
get_image_size_from_url(png_url_2).await.unwrap()
);
assert_eq!(
png_size_3,
get_image_size_from_url(png_url_3).await.unwrap()
);
assert_eq!(
rotated_jpeg_size,
get_image_size_from_url(rotated_jpeg_url).await.unwrap()
);
assert_eq!(
webp_size_1,
get_image_size_from_url(webp_url_1).await.unwrap()
);
assert_eq!(
webp_size_2,
get_image_size_from_url(webp_url_2).await.unwrap()
);
assert_eq!(ico_size, get_image_size_from_url(ico_url).await.unwrap());
assert_eq!(gif_size, get_image_size_from_url(gif_url).await.unwrap());
assert!(get_image_size_from_url(mp3_url).await.is_err());
}
}

View file

@ -1,6 +1,6 @@
#[crate::export]
pub fn to_mastodon_id(firefish_id: &str) -> Option<String> {
let decoded: [u8; 16] = basen::BASE36.decode_var_len(&firefish_id.to_ascii_lowercase())?;
let decoded: [u8; 16] = basen::BASE36.decode_var_len(firefish_id)?;
Some(basen::BASE10.encode_var_len(&decoded))
}

View file

@ -6,6 +6,7 @@ pub mod convert_host;
pub mod emoji;
pub mod escape_sql;
pub mod format_milliseconds;
pub mod get_image_size;
pub mod get_note_summary;
pub mod mastodon_id;
pub mod meta;

View file

@ -97,6 +97,8 @@ pub async fn to_db_reaction(reaction: Option<&str>, host: Option<&str>) -> Resul
{
return Ok(format!(":{name}@{ascii_host}:"));
}
tracing::info!("nonexistent remote custom emoji: :{name}@{ascii_host}:");
} else {
// local emoji
// TODO: Does SeaORM have the `exists` method?
@ -109,6 +111,8 @@ pub async fn to_db_reaction(reaction: Option<&str>, host: Option<&str>) -> Resul
{
return Ok(format!(":{name}:"));
}
tracing::info!("nonexistent local custom emoji: :{name}:");
}
};
};

View file

@ -3,7 +3,7 @@ use redis::{Commands, RedisError};
use serde::{Deserialize, Serialize};
#[derive(thiserror::Error, Debug)]
pub enum Error {
pub enum CacheError {
#[error("Redis error: {0}")]
RedisError(#[from] RedisError),
#[error("Data serialization error: {0}")]
@ -12,27 +12,37 @@ pub enum Error {
DeserializeError(#[from] rmp_serde::decode::Error),
}
fn prefix_key(key: &str) -> String {
redis_key(format!("cache:{}", key))
}
pub fn set_cache<V: for<'a> Deserialize<'a> + Serialize>(
key: &str,
value: &V,
expire_seconds: u64,
) -> Result<(), Error> {
) -> Result<(), CacheError> {
redis_conn()?.set_ex(
redis_key(key),
prefix_key(key),
rmp_serde::encode::to_vec(&value)?,
expire_seconds,
)?;
Ok(())
}
pub fn get_cache<V: for<'a> Deserialize<'a> + Serialize>(key: &str) -> Result<Option<V>, Error> {
let serialized_value: Option<Vec<u8>> = redis_conn()?.get(redis_key(key))?;
pub fn get_cache<V: for<'a> Deserialize<'a> + Serialize>(
key: &str,
) -> Result<Option<V>, CacheError> {
let serialized_value: Option<Vec<u8>> = redis_conn()?.get(prefix_key(key))?;
Ok(match serialized_value {
Some(v) => Some(rmp_serde::from_slice::<V>(v.as_ref())?),
None => None,
})
}
pub fn delete_cache(key: &str) -> Result<(), CacheError> {
Ok(redis_conn()?.del(prefix_key(key))?)
}
#[cfg(test)]
mod unit_test {
use super::{get_cache, set_cache};

View file

@ -8,10 +8,12 @@ use sea_orm::{ColumnTrait, DbErr, EntityTrait, QueryFilter};
/// Delete all entries in the "attestation_challenge" table created at more than 5 minutes ago
#[crate::export]
pub async fn remove_old_attestation_challenges() -> Result<(), DbErr> {
attestation_challenge::Entity::delete_many()
let res = attestation_challenge::Entity::delete_many()
.filter(attestation_challenge::Column::CreatedAt.lt(Local::now() - Duration::minutes(5)))
.exec(db_conn().await?)
.await?;
tracing::info!("{} attestation challenges are removed", res.rows_affected);
Ok(())
}

View file

@ -71,14 +71,14 @@ pub struct Model {
pub drive_capacity_override_mb: Option<i32>,
#[sea_orm(column_name = "movedToUri")]
pub moved_to_uri: Option<String>,
#[sea_orm(column_name = "alsoKnownAs", column_type = "Text", nullable)]
pub also_known_as: Option<String>,
#[sea_orm(column_name = "speakAsCat")]
pub speak_as_cat: bool,
#[sea_orm(column_name = "emojiModPerm")]
pub emoji_mod_perm: UserEmojimodpermEnum,
#[sea_orm(column_name = "isIndexable")]
pub is_indexable: bool,
#[sea_orm(column_name = "alsoKnownAs")]
pub also_known_as: Option<Vec<String>>,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]

View file

@ -0,0 +1,51 @@
use crate::config::CONFIG;
use tracing::Level;
use tracing_subscriber::FmtSubscriber;
#[crate::export(js_name = "initializeRustLogger")]
pub fn initialize_logger() {
let mut builder = FmtSubscriber::builder();
if let Some(max_level) = &CONFIG.max_log_level {
builder = builder.with_max_level(match max_level.as_str() {
"error" => Level::ERROR,
"warning" => Level::WARN,
"info" => Level::INFO,
"debug" => Level::DEBUG,
"trace" => Level::TRACE,
_ => Level::INFO, // Fallback
});
} else if let Some(levels) = &CONFIG.log_level {
// `logLevel` config is Deprecated
if levels.contains(&"trace".to_string()) {
builder = builder.with_max_level(Level::TRACE);
} else if levels.contains(&"debug".to_string()) {
builder = builder.with_max_level(Level::DEBUG);
} else if levels.contains(&"info".to_string()) {
builder = builder.with_max_level(Level::INFO);
} else if levels.contains(&"warning".to_string()) {
builder = builder.with_max_level(Level::WARN);
} else if levels.contains(&"error".to_string()) {
builder = builder.with_max_level(Level::ERROR);
} else {
// Fallback
builder = builder.with_max_level(Level::INFO);
}
} else {
// Fallback
builder = builder.with_max_level(Level::INFO);
};
let subscriber = builder
.without_time()
.with_level(true)
.with_ansi(true)
.with_target(true)
.with_thread_names(true)
.with_line_number(true)
.log_internal_errors(true)
.compact()
.finish();
tracing::subscriber::set_global_default(subscriber).expect("Failed to initialize the logger");
}

View file

@ -1,2 +1,3 @@
pub mod log;
pub mod note;
pub mod stream;

View file

@ -1,5 +1,10 @@
pub mod antenna;
pub mod channel;
pub mod chat;
pub mod chat_index;
pub mod custom_emoji;
pub mod group_chat;
pub mod moderation;
use crate::config::CONFIG;
use crate::database::redis_conn;
@ -10,9 +15,9 @@ pub enum Stream {
#[strum(serialize = "internal")]
Internal,
#[strum(serialize = "broadcast")]
Broadcast,
#[strum(to_string = "adminStream:{user_id}")]
Admin { user_id: String },
CustomEmoji,
#[strum(to_string = "adminStream:{moderator_id}")]
Moderation { moderator_id: String },
#[strum(to_string = "user:{user_id}")]
User { user_id: String },
#[strum(to_string = "channelStream:{channel_id}")]
@ -37,7 +42,7 @@ pub enum Stream {
#[strum(to_string = "messagingStream:{group_id}")]
GroupChat { group_id: String },
#[strum(to_string = "messagingIndexStream:{user_id}")]
MessagingIndex { user_id: String },
ChatIndex { user_id: String },
}
#[derive(thiserror::Error, Debug)]
@ -57,7 +62,7 @@ pub fn publish_to_stream(
) -> Result<(), Error> {
let message = if let Some(kind) = kind {
format!(
"{{ \"type\": \"{}\", \"body\": {} }}",
"{{\"type\":\"{}\",\"body\":{}}}",
kind,
value.unwrap_or("null".to_string()),
)
@ -67,10 +72,7 @@ pub fn publish_to_stream(
redis_conn()?.publish(
&CONFIG.host,
format!(
"{{ \"channel\": \"{}\", \"message\": {} }}",
stream, message,
),
format!("{{\"channel\":\"{}\",\"message\":{}}}", stream, message),
)?;
Ok(())
@ -84,10 +86,10 @@ mod unit_test {
#[test]
fn channel_to_string() {
assert_eq!(Stream::Internal.to_string(), "internal");
assert_eq!(Stream::Broadcast.to_string(), "broadcast");
assert_eq!(Stream::CustomEmoji.to_string(), "broadcast");
assert_eq!(
Stream::Admin {
user_id: "9tb42br63g5apjcq".to_string()
Stream::Moderation {
moderator_id: "9tb42br63g5apjcq".to_string()
}
.to_string(),
"adminStream:9tb42br63g5apjcq"

View file

@ -0,0 +1,10 @@
use crate::service::stream::{publish_to_stream, Error, Stream};
#[crate::export(js_name = "publishToChannelStream")]
pub fn publish(channel_id: String, user_id: String) -> Result<(), Error> {
publish_to_stream(
&Stream::Channel { channel_id },
Some("typing".to_string()),
Some(format!("\"{}\"", user_id)),
)
}

View file

@ -13,12 +13,15 @@ pub enum ChatEvent {
Typing,
}
// We want to merge `kind` and `object` into a single enum
// https://github.com/napi-rs/napi-rs/issues/2036
#[crate::export(js_name = "publishToChatStream")]
pub fn publish(
sender_user_id: String,
receiver_user_id: String,
kind: ChatEvent,
object: &serde_json::Value, // TODO?: change this to enum
object: &serde_json::Value,
) -> Result<(), Error> {
publish_to_stream(
&Stream::Chat {

View file

@ -0,0 +1,26 @@
use crate::service::stream::{publish_to_stream, Error, Stream};
#[derive(strum::Display)]
#[crate::export(string_enum = "camelCase")]
pub enum ChatIndexEvent {
#[strum(serialize = "message")]
Message,
#[strum(serialize = "read")]
Read,
}
// We want to merge `kind` and `object` into a single enum
// https://github.com/napi-rs/napi-rs/issues/2036
#[crate::export(js_name = "publishToChatIndexStream")]
pub fn publish(
user_id: String,
kind: ChatIndexEvent,
object: &serde_json::Value,
) -> Result<(), Error> {
publish_to_stream(
&Stream::ChatIndex { user_id },
Some(kind.to_string()),
Some(serde_json::to_string(object)?),
)
}

View file

@ -0,0 +1,27 @@
use crate::service::stream::{publish_to_stream, Error, Stream};
use serde::{Deserialize, Serialize};
// TODO: define schema type in other place
#[derive(Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
#[crate::export(object)]
pub struct PackedEmoji {
pub id: String,
pub aliases: Vec<String>,
pub name: String,
pub category: Option<String>,
pub host: Option<String>,
pub url: String,
pub license: Option<String>,
pub width: Option<i32>,
pub height: Option<i32>,
}
#[crate::export(js_name = "publishToBroadcastStream")]
pub fn publish(emoji: &PackedEmoji) -> Result<(), Error> {
publish_to_stream(
&Stream::CustomEmoji,
Some("emojiAdded".to_string()),
Some(format!("{{\"emoji\":{}}}", serde_json::to_string(emoji)?)),
)
}

View file

@ -0,0 +1,13 @@
use crate::service::stream::{chat::ChatEvent, publish_to_stream, Error, Stream};
// We want to merge `kind` and `object` into a single enum
// https://github.com/napi-rs/napi-rs/issues/2036
#[crate::export(js_name = "publishToGroupChatStream")]
pub fn publish(group_id: String, kind: ChatEvent, object: &serde_json::Value) -> Result<(), Error> {
publish_to_stream(
&Stream::GroupChat { group_id },
Some(kind.to_string()),
Some(serde_json::to_string(object)?),
)
}

View file

@ -0,0 +1,21 @@
use crate::service::stream::{publish_to_stream, Error, Stream};
use serde::{Deserialize, Serialize};
#[derive(Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
#[crate::export(object)]
pub struct AbuseUserReportLike {
pub id: String,
pub target_user_id: String,
pub reporter_id: String,
pub comment: String,
}
#[crate::export(js_name = "publishToModerationStream")]
pub fn publish(moderator_id: String, report: &AbuseUserReportLike) -> Result<(), Error> {
publish_to_stream(
&Stream::Moderation { moderator_id },
Some("newAbuseUserReport".to_string()),
Some(serde_json::to_string(report)?),
)
}

View file

@ -0,0 +1,24 @@
use crate::config::CONFIG;
use once_cell::sync::OnceCell;
use reqwest::{Client, Error, NoProxy, Proxy};
use std::time::Duration;
static CLIENT: OnceCell<Client> = OnceCell::new();
pub fn http_client() -> Result<Client, Error> {
CLIENT
.get_or_try_init(|| {
let mut builder = Client::builder().timeout(Duration::from_secs(5));
if let Some(proxy_url) = &CONFIG.proxy {
let mut proxy = Proxy::all(proxy_url)?;
if let Some(proxy_bypass_hosts) = &CONFIG.proxy_bypass_hosts {
proxy = proxy.no_proxy(NoProxy::from_string(&proxy_bypass_hosts.join(",")));
}
builder = builder.proxy(proxy);
}
builder.build()
})
.cloned()
}

View file

@ -1,2 +1,5 @@
pub use http_client::http_client;
pub mod http_client;
pub mod id;
pub mod random;

View file

@ -22,21 +22,21 @@
"@swc/core-android-arm64": "1.3.11"
},
"dependencies": {
"@bull-board/api": "5.15.5",
"@bull-board/koa": "5.15.5",
"@bull-board/ui": "5.15.5",
"@bull-board/api": "5.16.0",
"@bull-board/koa": "5.16.0",
"@bull-board/ui": "5.16.0",
"@discordapp/twemoji": "^15.0.3",
"@koa/cors": "5.0.0",
"@koa/multer": "3.0.2",
"@koa/router": "12.0.1",
"@ladjs/koa-views": "9.0.0",
"@peertube/http-signature": "1.7.0",
"@redocly/openapi-core": "1.11.0",
"@redocly/openapi-core": "1.12.0",
"@sinonjs/fake-timers": "11.2.2",
"adm-zip": "0.5.10",
"ajv": "8.12.0",
"archiver": "7.0.1",
"aws-sdk": "2.1599.0",
"aws-sdk": "2.1608.0",
"axios": "^1.6.8",
"backend-rs": "workspace:*",
"blurhash": "2.0.5",
@ -61,7 +61,7 @@
"gunzip-maybe": "^1.4.2",
"happy-dom": "^14.7.1",
"hpagent": "1.2.0",
"ioredis": "5.3.2",
"ioredis": "5.4.1",
"ip-cidr": "4.0.0",
"is-svg": "5.0.0",
"json5": "2.2.3",
@ -125,7 +125,7 @@
},
"devDependencies": {
"@swc/cli": "0.3.12",
"@swc/core": "1.4.13",
"@swc/core": "1.5.0",
"@types/adm-zip": "^0.5.5",
"@types/color-convert": "^2.0.3",
"@types/content-disposition": "^0.5.8",
@ -155,7 +155,7 @@
"@types/pug": "2.0.10",
"@types/punycode": "2.1.4",
"@types/qrcode": "1.5.5",
"@types/qs": "6.9.14",
"@types/qs": "6.9.15",
"@types/random-seed": "0.3.5",
"@types/ratelimiter": "3.4.6",
"@types/rename": "1.0.7",
@ -170,7 +170,7 @@
"@types/websocket": "1.0.10",
"@types/ws": "8.5.10",
"cross-env": "7.0.3",
"eslint": "^9.0.0",
"eslint": "^9.1.1",
"mocha": "10.4.0",
"pug": "3.0.2",
"strict-event-emitter-types": "2.0.0",
@ -178,7 +178,7 @@
"ts-loader": "9.5.1",
"ts-node": "10.9.2",
"tsconfig-paths": "4.2.0",
"type-fest": "4.15.0",
"type-fest": "4.17.0",
"typescript": "5.4.5",
"webpack": "^5.91.0",
"ws": "8.16.0"

View file

@ -9,6 +9,7 @@ import semver from "semver";
import Logger from "@/services/logger.js";
import type { Config } from "backend-rs";
import { initializeRustLogger } from "backend-rs";
import { fetchMeta, removeOldAttestationChallenges } from "backend-rs";
import { config, envOption } from "@/config.js";
import { showMachineInfo } from "@/misc/show-machine-info.js";
@ -94,6 +95,7 @@ export async function masterMain() {
await showMachineInfo(bootLogger);
showNodejsVersion();
await connectDb();
initializeRustLogger();
} catch (e) {
bootLogger.error(
`Fatal error occurred during initialization:\n${inspect(e)}`,
@ -103,13 +105,13 @@ export async function masterMain() {
process.exit(1);
}
bootLogger.succ("Firefish initialized");
bootLogger.info("Firefish initialized");
if (!envOption.disableClustering) {
await spawnWorkers(config.clusterLimits);
}
bootLogger.succ(
bootLogger.info(
`Now listening on port ${config.port} on ${config.url}`,
null,
true,
@ -160,7 +162,7 @@ async function connectDb(): Promise<void> {
const v = await db
.query("SHOW server_version")
.then((x) => x[0].server_version);
dbLogger.succ(`Connected: v${v}`);
dbLogger.info(`Connected: v${v}`);
} catch (e) {
dbLogger.error("Failed to connect to the database", null, true);
dbLogger.error(inspect(e));
@ -196,7 +198,7 @@ async function spawnWorkers(
`Starting ${clusterLimits.web} web workers and ${clusterLimits.queue} queue workers (total ${total})...`,
);
await Promise.all(workers.map((mode) => spawnWorker(mode)));
bootLogger.succ("All workers started");
bootLogger.info("All workers started");
}
function spawnWorker(mode: "web" | "queue"): Promise<void> {

View file

@ -1,83 +0,0 @@
import { config } from "@/config.js";
import {
DB_MAX_IMAGE_COMMENT_LENGTH,
DB_MAX_NOTE_TEXT_LENGTH,
} from "@/misc/hard-limits.js";
export const MAX_NOTE_TEXT_LENGTH = Math.min(
config.maxNoteLength ?? 3000,
DB_MAX_NOTE_TEXT_LENGTH,
);
export const MAX_CAPTION_TEXT_LENGTH = Math.min(
config.maxCaptionLength ?? 1500,
DB_MAX_IMAGE_COMMENT_LENGTH,
);
export const SECOND = 1000;
export const MINUTE = 60 * SECOND;
export const HOUR = 60 * MINUTE;
export const DAY = 24 * HOUR;
export const USER_ONLINE_THRESHOLD = 10 * MINUTE;
export const USER_ACTIVE_THRESHOLD = 3 * DAY;
// List of file types allowed to be viewed directly in the browser
// Anything not included here will be responded as application/octet-stream
// SVG is not allowed because it generates XSS <- we need to fix this and later allow it to be viewed directly
export const FILE_TYPE_BROWSERSAFE = [
// Images
"image/png",
"image/gif", // TODO: deprecated, but still used by old notes, new gifs should be converted to webp in the future
"image/jpeg",
"image/webp", // TODO: make this the default image format
"image/apng",
"image/bmp",
"image/tiff",
"image/x-icon",
"image/avif", // not as good supported now, but its good to introduce initial support for the future
// OggS
"audio/opus",
"video/ogg",
"audio/ogg",
"application/ogg",
// ISO/IEC base media file format
"video/quicktime",
"video/mp4", // TODO: we need to check for av1 later
"video/vnd.avi", // also av1
"audio/mp4",
"video/x-m4v",
"audio/x-m4a",
"video/3gpp",
"video/3gpp2",
"video/3gp2",
"audio/3gpp",
"audio/3gpp2",
"audio/3gp2",
"video/mpeg",
"audio/mpeg",
"video/webm",
"audio/webm",
"audio/aac",
"audio/x-flac",
"audio/flac",
"audio/vnd.wave",
"audio/mod",
"audio/x-mod",
"audio/s3m",
"audio/x-s3m",
"audio/xm",
"audio/x-xm",
"audio/it",
"audio/x-it",
];
/*
https://github.com/sindresorhus/file-type/blob/main/supported.js
https://github.com/sindresorhus/file-type/blob/main/core.js
https://developer.mozilla.org/en-US/docs/Web/Media/Formats/Containers
*/

View file

@ -80,7 +80,7 @@ import { dbLogger } from "./logger.js";
const sqlLogger = dbLogger.createSubLogger("sql", "gray", false);
class MyCustomLogger implements Logger {
class DbLogger implements Logger {
private highlight(sql: string) {
return highlight.highlight(sql, {
language: "sql",
@ -89,15 +89,16 @@ class MyCustomLogger implements Logger {
}
public logQuery(query: string, parameters?: any[]) {
sqlLogger.info(this.highlight(query).substring(0, 100));
sqlLogger.trace(this.highlight(query).substring(0, 100));
}
public logQueryError(error: string, query: string, parameters?: any[]) {
sqlLogger.error(this.highlight(query));
sqlLogger.error(error);
sqlLogger.trace(this.highlight(query));
}
public logQuerySlow(time: number, query: string, parameters?: any[]) {
sqlLogger.warn(this.highlight(query));
sqlLogger.trace(this.highlight(query));
}
public logSchemaBuild(message: string) {
@ -215,7 +216,7 @@ export const db = new DataSource({
}
: false,
logging: log,
logger: log ? new MyCustomLogger() : undefined,
logger: log ? new DbLogger() : undefined,
maxQueryExecutionTime: 300,
entities: entities,
migrations: ["../../migration/*.js"],

View file

@ -0,0 +1,36 @@
import type { MigrationInterface, QueryRunner } from "typeorm";
export class AlterAkaType1714099399879 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "user" RENAME COLUMN "alsoKnownAs" TO "alsoKnownAsOld"`,
);
await queryRunner.query(
`ALTER TABLE "user" ADD COLUMN "alsoKnownAs" character varying(512)[]`,
);
await queryRunner.query(
`UPDATE "user" SET "alsoKnownAs" = string_to_array("alsoKnownAsOld", ',')::character varying[]`,
);
await queryRunner.query(
`UPDATE "user" SET "alsoKnownAs" = NULL WHERE "alsoKnownAs" = '{}'`,
);
await queryRunner.query(
`COMMENT ON COLUMN "user"."alsoKnownAs" IS 'URIs the user is known as too'`,
);
await queryRunner.query(`ALTER TABLE "user" DROP COLUMN "alsoKnownAsOld"`);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "user" RENAME COLUMN "alsoKnownAs" TO "alsoKnownAsOld"`,
);
await queryRunner.query(`ALTER TABLE "user" ADD COLUMN "alsoKnownAs" text`);
await queryRunner.query(
`UPDATE "user" SET "alsoKnownAs" = array_to_string("alsoKnownAsOld", ',')`,
);
await queryRunner.query(
`COMMENT ON COLUMN "user"."alsoKnownAs" IS 'URIs the user is known as too'`,
);
await queryRunner.query(`ALTER TABLE "user" DROP COLUMN "alsoKnownAsOld"`);
}
}

View file

@ -14,9 +14,9 @@ export async function downloadUrl(url: string, path: string): Promise<void> {
throw new StatusError("Invalid URL", 400);
}
const logger = new Logger("download");
const downloadLogger = new Logger("download");
logger.info(`Downloading ${chalk.cyan(url)} ...`);
downloadLogger.debug(`Downloading ${chalk.cyan(url)} ...`);
const timeout = 30 * 1000;
const operationTimeout = 60 * 1000;
@ -45,7 +45,7 @@ export async function downloadUrl(url: string, path: string): Promise<void> {
})
.on("redirect", (res: Got.Response, opts: Got.NormalizedOptions) => {
if (!isValidUrl(opts.url)) {
logger.warn(`Invalid URL: ${opts.url}`);
downloadLogger.warn(`Invalid URL: ${opts.url}`);
req.destroy();
}
})
@ -57,7 +57,7 @@ export async function downloadUrl(url: string, path: string): Promise<void> {
res.ip
) {
if (isPrivateIp(res.ip)) {
logger.warn(`Blocked address: ${res.ip}`);
downloadLogger.warn(`Blocked address: ${res.ip}`);
req.destroy();
}
}
@ -66,14 +66,16 @@ export async function downloadUrl(url: string, path: string): Promise<void> {
if (contentLength != null) {
const size = Number(contentLength);
if (size > maxSize) {
logger.warn(`maxSize exceeded (${size} > ${maxSize}) on response`);
downloadLogger.warn(
`maxSize exceeded (${size} > ${maxSize}) on response`,
);
req.destroy();
}
}
})
.on("downloadProgress", (progress: Got.Progress) => {
if (progress.transferred > maxSize) {
logger.warn(
downloadLogger.warn(
`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`,
);
req.destroy();
@ -94,7 +96,7 @@ export async function downloadUrl(url: string, path: string): Promise<void> {
}
}
logger.succ(`Download finished: ${chalk.cyan(url)}`);
downloadLogger.debug(`Download finished: ${chalk.cyan(url)}`);
}
export function isPrivateIp(ip: string): boolean {

View file

@ -1,59 +0,0 @@
import probeImageSize from "probe-image-size";
import { Mutex } from "redis-semaphore";
import { FILE_TYPE_BROWSERSAFE } from "@/const.js";
import Logger from "@/services/logger.js";
import { Cache } from "./cache.js";
import { redisClient } from "@/db/redis.js";
import { inspect } from "node:util";
export type Size = {
width: number;
height: number;
};
const cache = new Cache<boolean>("emojiMeta", 60 * 10); // once every 10 minutes for the same url
const logger = new Logger("emoji");
export async function getEmojiSize(url: string): Promise<Size> {
let attempted = true;
const lock = new Mutex(redisClient, "getEmojiSize");
await lock.acquire();
try {
attempted = (await cache.get(url)) === true;
if (!attempted) {
await cache.set(url, true);
}
} finally {
await lock.release();
}
if (attempted) {
logger.warn(`Attempt limit exceeded: ${url}`);
throw new Error("Too many attempts");
}
try {
logger.debug(`Retrieving emoji size from ${url}`);
const { width, height, mime } = await probeImageSize(url, {
timeout: 5000,
});
if (!(mime.startsWith("image/") && FILE_TYPE_BROWSERSAFE.includes(mime))) {
throw new Error("Unsupported image type");
}
return { width, height };
} catch (e) {
throw new Error(`Unable to retrieve metadata:\n${inspect(e)}`);
}
}
export function getNormalSize(
{ width, height }: Size,
orientation?: number,
): Size {
return (orientation || 0) >= 5
? { width: height, height: width }
: { width, height };
}

View file

@ -7,20 +7,16 @@ import probeImageSize from "probe-image-size";
import isSvg from "is-svg";
import sharp from "sharp";
import { encode } from "blurhash";
import { inspect } from "node:util";
export type FileInfo = {
type FileInfo = {
size: number;
md5: string;
type: {
mime: string;
ext: string | null;
};
mime: string;
fileExtension: string | null;
width?: number;
height?: number;
orientation?: number;
blurhash?: string;
warnings: string[];
};
const TYPE_OCTET_STREAM = {
@ -37,8 +33,6 @@ const TYPE_SVG = {
* Get file information
*/
export async function getFileInfo(path: string): Promise<FileInfo> {
const warnings = [] as string[];
const size = await getFileSize(path);
const md5 = await calcHash(path);
@ -63,14 +57,12 @@ export async function getFileInfo(path: string): Promise<FileInfo> {
"image/avif",
].includes(type.mime)
) {
const imageSize = await detectImageSize(path).catch((e) => {
warnings.push(`detectImageSize failed:\n${inspect(e)}`);
const imageSize = await detectImageSize(path).catch((_) => {
return undefined;
});
// うまく判定できない画像は octet-stream にする
if (!imageSize) {
warnings.push("cannot detect image dimensions");
type = TYPE_OCTET_STREAM;
} else if (imageSize.wUnits === "px") {
width = imageSize.width;
@ -79,11 +71,8 @@ export async function getFileInfo(path: string): Promise<FileInfo> {
// 制限を超えている画像は octet-stream にする
if (imageSize.width > 16383 || imageSize.height > 16383) {
warnings.push("image dimensions exceeds limits");
type = TYPE_OCTET_STREAM;
}
} else {
warnings.push(`unsupported unit type: ${imageSize.wUnits}`);
}
}
@ -100,8 +89,7 @@ export async function getFileInfo(path: string): Promise<FileInfo> {
"image/avif",
].includes(type.mime)
) {
blurhash = await getBlurhash(path).catch((e) => {
warnings.push(`getBlurhash failed:\n${inspect(e)}`);
blurhash = await getBlurhash(path).catch((_) => {
return undefined;
});
}
@ -109,12 +97,12 @@ export async function getFileInfo(path: string): Promise<FileInfo> {
return {
size,
md5,
type,
mime: type.mime,
fileExtension: type.ext,
width,
height,
orientation,
blurhash,
warnings,
};
}

View file

@ -1,18 +0,0 @@
// If you change DB_* values, you must also change the DB schema.
/**
* Maximum note text length that can be stored in DB.
* Surrogate pairs count as one
*
* NOTE: this can hypothetically be pushed further
* (up to 250000000), but will likely cause truncations
* and incompatibilities with other servers,
* as well as potential performance issues.
*/
export const DB_MAX_NOTE_TEXT_LENGTH = 100000;
/**
* Maximum image description length that can be stored in DB.
* Surrogate pairs count as one
*/
export const DB_MAX_IMAGE_COMMENT_LENGTH = 8192;

View file

@ -1,4 +1,4 @@
import { FILE_TYPE_BROWSERSAFE } from "@/const.js";
import { FILE_TYPE_BROWSERSAFE } from "backend-rs";
const dictionary = {
"safe-file": FILE_TYPE_BROWSERSAFE,

View file

@ -2,7 +2,7 @@ import { Brackets } from "typeorm";
import { isBlockedServer } from "backend-rs";
import { Instances } from "@/models/index.js";
import type { Instance } from "@/models/entities/instance.js";
import { DAY } from "@/const.js";
import { DAY } from "backend-rs";
// Threshold from last contact after which an instance will be considered
// "dead" and should no longer get activities delivered to it.

View file

@ -13,7 +13,6 @@ import { id } from "../id.js";
import { Note } from "./note.js";
import { User } from "./user.js";
import { DriveFolder } from "./drive-folder.js";
import { DB_MAX_IMAGE_COMMENT_LENGTH } from "@/misc/hard-limits.js";
import { NoteFile } from "./note-file.js";
export type DriveFileUsageHint = "userAvatar" | "userBanner" | null;
@ -73,7 +72,7 @@ export class DriveFile {
@Index() // USING pgroonga pgroonga_varchar_full_text_search_ops_v2
@Column("varchar", {
length: DB_MAX_IMAGE_COMMENT_LENGTH,
length: 8192,
nullable: true,
comment: "The comment of the DriveFile.",
})

View file

@ -88,7 +88,9 @@ export class User {
})
public movedToUri: string | null;
@Column("simple-array", {
@Column("varchar", {
length: 512,
array: true,
nullable: true,
comment: "URIs the user is known as too",
})

View file

@ -7,7 +7,7 @@ import type { Packed } from "@/misc/schema.js";
import type { Promiseable } from "@/prelude/await-all.js";
import { awaitAll } from "@/prelude/await-all.js";
import { populateEmojis } from "@/misc/populate-emojis.js";
import { USER_ACTIVE_THRESHOLD, USER_ONLINE_THRESHOLD } from "@/const.js";
import { USER_ACTIVE_THRESHOLD, USER_ONLINE_THRESHOLD } from "backend-rs";
import { Cache } from "@/misc/cache.js";
import { db } from "@/db/postgre.js";
import { isActor, getApId } from "@/remote/activitypub/type.js";

View file

@ -70,10 +70,10 @@ deliverQueue
),
)
.on("failed", (job, err) =>
deliverLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job.data.to}`),
deliverLogger.info(`failed(${err}) ${getJobInfo(job)} to=${job.data.to}`),
)
.on("error", (job: any, err: Error) =>
deliverLogger.error(`error ${err}`, { job, e: renderError(err) }),
deliverLogger.warn(`error ${err}`, { job, e: renderError(err) }),
)
.on("stalled", (job) =>
deliverLogger.warn(`stalled ${getJobInfo(job)} to=${job.data.to}`),
@ -564,12 +564,12 @@ export default function () {
export function destroy() {
deliverQueue.once("cleaned", (jobs, status) => {
deliverLogger.succ(`Cleaned ${jobs.length} ${status} jobs`);
deliverLogger.info(`Cleaned ${jobs.length} ${status} jobs`);
});
deliverQueue.clean(0, "delayed");
inboxQueue.once("cleaned", (jobs, status) => {
inboxLogger.succ(`Cleaned ${jobs.length} ${status} jobs`);
inboxLogger.info(`Cleaned ${jobs.length} ${status} jobs`);
});
inboxQueue.clean(0, "delayed");
}

View file

@ -13,7 +13,7 @@ const logger = queueLogger.createSubLogger("delete-account");
export async function deleteAccount(
job: Bull.Job<DbUserDeleteJobData>,
): Promise<string | void> {
logger.info(`Deleting account of ${job.data.user.id} ...`);
logger.info(`Deleting account ${job.data.user.id} ...`);
const user = await Users.findOneBy({ id: job.data.user.id });
if (!user) return;
@ -43,7 +43,7 @@ export async function deleteAccount(
await Notes.delete(notes.map((note) => note.id));
}
logger.succ("All of notes deleted");
logger.info(`All posts of user ${job.data.user.id} were deleted`);
}
{
@ -73,7 +73,7 @@ export async function deleteAccount(
}
}
logger.succ("All of files deleted");
logger.info(`All files of user ${job.data.user.id} were deleted`);
}
{

View file

@ -54,8 +54,6 @@ export async function deleteDriveFiles(
job.progress(deletedCount / total);
}
logger.succ(
`All drive files (${deletedCount}) of ${user.id} has been deleted.`,
);
logger.info(`${deletedCount} drive files of user ${user.id} were deleted.`);
done();
}

View file

@ -9,6 +9,7 @@ import { createTemp } from "@/misc/create-temp.js";
import { Users, Blockings } from "@/models/index.js";
import { MoreThan } from "typeorm";
import type { DbUserJobData } from "@/queue/types.js";
import { inspect } from "node:util";
const logger = queueLogger.createSubLogger("export-blocking");
@ -27,7 +28,7 @@ export async function exportBlocking(
// Create temp file
const [path, cleanup] = await createTemp();
logger.info(`Temp file is ${path}`);
logger.info(`temp file created: ${path}`);
try {
const stream = fs.createWriteStream(path, { flags: "a" });
@ -63,9 +64,10 @@ export async function exportBlocking(
const content = getFullApAccount(u.username, u.host);
await new Promise<void>((res, rej) => {
stream.write(content + "\n", (err) => {
stream.write(`${content}\n`, (err) => {
if (err) {
logger.error(err);
logger.warn("failed");
logger.info(inspect(err));
rej(err);
} else {
res();
@ -83,7 +85,7 @@ export async function exportBlocking(
}
stream.end();
logger.succ(`Exported to: ${path}`);
logger.info(`Exported to: ${path}`);
const fileName = `blocking-${dateFormat(
new Date(),
@ -96,7 +98,7 @@ export async function exportBlocking(
force: true,
});
logger.succ(`Exported to: ${driveFile.id}`);
logger.info(`Exported to: ${driveFile.id}`);
} finally {
cleanup();
}

View file

@ -29,7 +29,7 @@ export async function exportCustomEmojis(
const [path, cleanup] = await createTempDir();
logger.info(`Temp dir is ${path}`);
logger.info(`temp dir created: ${path}`);
const metaPath = `${path}/meta.json`;
@ -41,7 +41,8 @@ export async function exportCustomEmojis(
return new Promise<void>((res, rej) => {
metaStream.write(text, (err) => {
if (err) {
logger.error(err);
logger.warn("Failed to export custom emojis");
logger.info(inspect(err));
rej(err);
} else {
res();
@ -105,7 +106,7 @@ export async function exportCustomEmojis(
zlib: { level: 0 },
});
archiveStream.on("close", async () => {
logger.succ(`Exported to: ${archivePath}`);
logger.info(`Exported to: ${archivePath}`);
const fileName = `custom-emojis-${dateFormat(
new Date(),
@ -118,7 +119,7 @@ export async function exportCustomEmojis(
force: true,
});
logger.succ(`Exported to: ${driveFile.id}`);
logger.info(`Exported to: ${driveFile.id}`);
cleanup();
archiveCleanup();
done();

View file

@ -10,6 +10,7 @@ import { Users, Followings, Mutings } from "@/models/index.js";
import { In, MoreThan, Not } from "typeorm";
import type { DbUserJobData } from "@/queue/types.js";
import type { Following } from "@/models/entities/following.js";
import { inspect } from "node:util";
const logger = queueLogger.createSubLogger("export-following");
@ -28,7 +29,7 @@ export async function exportFollowing(
// Create temp file
const [path, cleanup] = await createTemp();
logger.info(`Temp file is ${path}`);
logger.info(`temp file created: ${path}`);
try {
const stream = fs.createWriteStream(path, { flags: "a" });
@ -78,9 +79,12 @@ export async function exportFollowing(
const content = getFullApAccount(u.username, u.host);
await new Promise<void>((res, rej) => {
stream.write(content + "\n", (err) => {
stream.write(`${content}\n`, (err) => {
if (err) {
logger.error(err);
logger.warn(
`failed to export following users of ${job.data.user.id}`,
);
logger.info(inspect(err));
rej(err);
} else {
res();
@ -91,7 +95,7 @@ export async function exportFollowing(
}
stream.end();
logger.succ(`Exported to: ${path}`);
logger.info(`Exported to: ${path}`);
const fileName = `following-${dateFormat(
new Date(),
@ -104,7 +108,7 @@ export async function exportFollowing(
force: true,
});
logger.succ(`Exported to: ${driveFile.id}`);
logger.info(`Exported to: ${driveFile.id}`);
} finally {
cleanup();
}

View file

@ -9,6 +9,7 @@ import { createTemp } from "@/misc/create-temp.js";
import { Users, Mutings } from "@/models/index.js";
import { IsNull, MoreThan } from "typeorm";
import type { DbUserJobData } from "@/queue/types.js";
import { inspect } from "node:util";
const logger = queueLogger.createSubLogger("export-mute");
@ -16,7 +17,7 @@ export async function exportMute(
job: Bull.Job<DbUserJobData>,
done: any,
): Promise<void> {
logger.info(`Exporting mute of ${job.data.user.id} ...`);
logger.info(`Exporting mutes of ${job.data.user.id} ...`);
const user = await Users.findOneBy({ id: job.data.user.id });
if (user == null) {
@ -27,7 +28,7 @@ export async function exportMute(
// Create temp file
const [path, cleanup] = await createTemp();
logger.info(`Temp file is ${path}`);
logger.info(`temp file created: ${path}`);
try {
const stream = fs.createWriteStream(path, { flags: "a" });
@ -64,9 +65,10 @@ export async function exportMute(
const content = getFullApAccount(u.username, u.host);
await new Promise<void>((res, rej) => {
stream.write(content + "\n", (err) => {
stream.write(`${content}\n`, (err) => {
if (err) {
logger.error(err);
logger.warn("failed");
logger.info(inspect(err));
rej(err);
} else {
res();
@ -84,7 +86,7 @@ export async function exportMute(
}
stream.end();
logger.succ(`Exported to: ${path}`);
logger.info(`Exported to: ${path}`);
const fileName = `mute-${dateFormat(
new Date(),
@ -97,7 +99,7 @@ export async function exportMute(
force: true,
});
logger.succ(`Exported to: ${driveFile.id}`);
logger.info(`Exported to: ${driveFile.id}`);
} finally {
cleanup();
}

View file

@ -10,6 +10,7 @@ import type { Note } from "@/models/entities/note.js";
import type { Poll } from "@/models/entities/poll.js";
import type { DbUserJobData } from "@/queue/types.js";
import { createTemp } from "@/misc/create-temp.js";
import { inspect } from "node:util";
const logger = queueLogger.createSubLogger("export-notes");
@ -28,7 +29,7 @@ export async function exportNotes(
// Create temp file
const [path, cleanup] = await createTemp();
logger.info(`Temp file is ${path}`);
logger.info(`temp file created: ${path}`);
try {
const stream = fs.createWriteStream(path, { flags: "a" });
@ -37,7 +38,8 @@ export async function exportNotes(
return new Promise<void>((res, rej) => {
stream.write(text, (err) => {
if (err) {
logger.error(err);
logger.warn(`failed to export posts of ${job.data.user.id}`);
logger.info(inspect(err));
rej(err);
} else {
res();
@ -91,7 +93,7 @@ export async function exportNotes(
await write("]");
stream.end();
logger.succ(`Exported to: ${path}`);
logger.info(`Exported to: ${path}`);
const fileName = `notes-${dateFormat(
new Date(),
@ -104,7 +106,7 @@ export async function exportNotes(
force: true,
});
logger.succ(`Exported to: ${driveFile.id}`);
logger.info(`Exported to: ${driveFile.id}`);
} finally {
cleanup();
}

View file

@ -9,6 +9,7 @@ import { createTemp } from "@/misc/create-temp.js";
import { Users, UserLists, UserListJoinings } from "@/models/index.js";
import { In } from "typeorm";
import type { DbUserJobData } from "@/queue/types.js";
import { inspect } from "node:util";
const logger = queueLogger.createSubLogger("export-user-lists");
@ -31,7 +32,7 @@ export async function exportUserLists(
// Create temp file
const [path, cleanup] = await createTemp();
logger.info(`Temp file is ${path}`);
logger.info(`temp file created: ${path}`);
try {
const stream = fs.createWriteStream(path, { flags: "a" });
@ -46,9 +47,10 @@ export async function exportUserLists(
const acct = getFullApAccount(u.username, u.host);
const content = `${list.name},${acct}`;
await new Promise<void>((res, rej) => {
stream.write(content + "\n", (err) => {
stream.write(`${content}\n`, (err) => {
if (err) {
logger.error(err);
logger.warn(`failed to export ${list.id}`);
logger.info(inspect(err));
rej(err);
} else {
res();
@ -59,7 +61,7 @@ export async function exportUserLists(
}
stream.end();
logger.succ(`Exported to: ${path}`);
logger.info(`Exported to: ${path}`);
const fileName = `user-lists-${dateFormat(
new Date(),
@ -72,7 +74,7 @@ export async function exportUserLists(
force: true,
});
logger.succ(`Exported to: ${driveFile.id}`);
logger.info(`Exported to: ${driveFile.id}`);
} finally {
cleanup();
}

View file

@ -66,14 +66,15 @@ export async function importBlocking(
// skip myself
if (target.id === job.data.user.id) continue;
logger.info(`Block[${linenum}] ${target.id} ...`);
logger.debug(`Block[${linenum}] ${target.id} ...`);
await block(user, target);
} catch (e) {
logger.warn(`Error in line ${linenum}:\n${inspect(e)}`);
logger.warn(`failed: error in line ${linenum}`);
logger.info(inspect(e));
}
}
logger.succ("Imported");
logger.info("Imported");
done();
}

View file

@ -11,14 +11,28 @@ import { addFile } from "@/services/drive/add-file.js";
import { genId } from "backend-rs";
import { db } from "@/db/postgre.js";
import probeImageSize from "probe-image-size";
import * as path from "path";
import * as path from "node:path";
const logger = queueLogger.createSubLogger("import-custom-emojis");
// probeImageSize acceptable extensions
// JPG, GIF, PNG, WebP, BMP, TIFF, SVG, PSD.
const acceptableExtensions = [
".jpeg",
".jpg",
".gif",
".png",
".webp",
".bmp",
// ".tiff", // Cannot be used as emoji
// ".svg", // Disable for secure issues
// ".psd", // Cannot be used as emoji
];
// TODO: 名前衝突時の動作を選べるようにする
export async function importCustomEmojis(
job: Bull.Job<DbUserImportJobData>,
done: any,
done: () => void,
): Promise<void> {
logger.info("Importing custom emojis ...");
@ -32,7 +46,7 @@ export async function importCustomEmojis(
const [tempPath, cleanup] = await createTempDir();
logger.info(`Temp dir is ${tempPath}`);
logger.debug(`temp dir created: ${tempPath}`);
const destPath = `${tempPath}/emojis.zip`;
@ -62,6 +76,14 @@ export async function importCustomEmojis(
if (!record.downloaded) continue;
const emojiInfo = record.emoji;
const emojiPath = `${outputPath}/${record.fileName}`;
const extname = path.extname(record.fileName);
// Skip non-support files
if (!acceptableExtensions.includes(extname.toLowerCase())) {
continue;
}
await Emojis.delete({
name: emojiInfo.name,
});
@ -92,7 +114,7 @@ export async function importCustomEmojis(
} else {
logger.info("starting emoji import without metadata");
// Since we lack metadata, we import into a randomized category name instead
let categoryName = genId();
const categoryName = genId();
let containedEmojis = fs.readdirSync(outputPath);
@ -103,7 +125,14 @@ export async function importCustomEmojis(
for (const emojiFilename of containedEmojis) {
// strip extension and get filename to use as name
const name = path.basename(emojiFilename, path.extname(emojiFilename));
const extname = path.extname(emojiFilename);
// Skip non-emoji files, such as LICENSE
if (!acceptableExtensions.includes(extname.toLowerCase())) {
continue;
}
const name = path.basename(emojiFilename, extname);
const emojiPath = `${outputPath}/${emojiFilename}`;
logger.info(`importing ${name}`);
@ -143,8 +172,8 @@ export async function importCustomEmojis(
cleanup();
logger.succ("Imported");
logger.info("Imported");
done();
});
logger.succ(`Unzipping to ${outputPath}`);
logger.info(`Unzipping to ${outputPath}`);
}

View file

@ -1,6 +1,6 @@
import * as Post from "@/misc/post.js";
import create from "@/services/note/create.js";
import { Users } from "@/models/index.js";
import { NoteFiles, Users } from "@/models/index.js";
import type { DbUserImportMastoPostJobData } from "@/queue/types.js";
import { queueLogger } from "../../logger.js";
import { uploadFromUrl } from "@/services/drive/upload-from-url.js";
@ -49,7 +49,7 @@ export async function importCkPost(
});
files.push(file);
} catch (e) {
logger.error(`Skipped adding file to drive: ${url}`);
logger.info(`Skipped adding file to drive: ${url}`);
}
}
const { text, cw, localOnly, createdAt, visibility } = Post.parse(post);
@ -59,9 +59,18 @@ export async function importCkPost(
userId: user.id,
});
if (note && (note?.fileIds?.length || 0) < files.length) {
// FIXME: What is this condition?
if (note != null && (note.fileIds?.length || 0) < files.length) {
const update: Partial<Note> = {};
update.fileIds = files.map((x) => x.id);
if (update.fileIds != null) {
await NoteFiles.delete({ noteId: note.id });
await NoteFiles.insert(
update.fileIds.map((fileId) => ({ noteId: note?.id, fileId })),
);
}
await Notes.update(note.id, update);
await NoteEdits.insert({
id: genId(),
@ -71,12 +80,12 @@ export async function importCkPost(
fileIds: note.fileIds,
updatedAt: new Date(),
});
logger.info(`Note file updated`);
logger.info("Post updated");
}
if (!note) {
if (note == null) {
note = await create(user, {
createdAt: createdAt,
files: files.length == 0 ? undefined : files,
files: files.length === 0 ? undefined : files,
poll: undefined,
text: text || undefined,
reply: post.replyId ? job.data.parent : null,
@ -90,11 +99,11 @@ export async function importCkPost(
apHashtags: undefined,
apEmojis: undefined,
});
logger.info(`Create new note`);
logger.debug("New post has been created");
} else {
logger.info(`Note exist`);
logger.info("This post already exists");
}
logger.succ("Imported");
logger.info("Imported");
if (post.childNotes) {
for (const child of post.childNotes) {
createImportCkPostJob(

View file

@ -64,11 +64,12 @@ export async function importFollowing(
// skip myself
if (target.id === job.data.user.id) continue;
logger.info(`Follow[${linenum}] ${target.id} ...`);
logger.debug(`Follow[${linenum}] ${target.id} ...`);
follow(user, target);
} catch (e) {
logger.warn(`Error in line ${linenum}:\n${inspect(e)}`);
logger.warn(`Error in line ${linenum}`);
logger.info(inspect(e));
}
}
} else {
@ -102,15 +103,16 @@ export async function importFollowing(
// skip myself
if (target.id === job.data.user.id) continue;
logger.info(`Follow[${linenum}] ${target.id} ...`);
logger.debug(`Follow[${linenum}] ${target.id} ...`);
follow(user, target);
} catch (e) {
logger.warn(`Error in line ${linenum}:\n${inspect(e)}`);
logger.warn(`Error in line ${linenum}`);
logger.info(inspect(e));
}
}
}
logger.succ("Imported");
logger.info("Imported");
done();
}

View file

@ -1,5 +1,5 @@
import create from "@/services/note/create.js";
import { Users } from "@/models/index.js";
import { NoteFiles, Users } from "@/models/index.js";
import type { DbUserImportMastoPostJobData } from "@/queue/types.js";
import { queueLogger } from "../../logger.js";
import type Bull from "bull";
@ -73,7 +73,7 @@ export async function importMastoPost(
});
files.push(file);
} catch (e) {
logger.error(`Skipped adding file to drive: ${url}`);
logger.warn(`Skipped adding file to drive: ${url}`);
}
}
}
@ -85,9 +85,18 @@ export async function importMastoPost(
userId: user.id,
});
if (note && (note?.fileIds?.length || 0) < files.length) {
// FIXME: What is this condition?
if (note != null && (note.fileIds?.length || 0) < files.length) {
const update: Partial<Note> = {};
update.fileIds = files.map((x) => x.id);
if (update.fileIds != null) {
await NoteFiles.delete({ noteId: note.id });
await NoteFiles.insert(
update.fileIds.map((fileId) => ({ noteId: note?.id, fileId })),
);
}
await Notes.update(note.id, update);
await NoteEdits.insert({
id: genId(),
@ -97,14 +106,14 @@ export async function importMastoPost(
fileIds: note.fileIds,
updatedAt: new Date(),
});
logger.info(`Note file updated`);
logger.info("Post updated");
}
if (!note) {
if (note == null) {
note = await create(user, {
createdAt: isRenote
? new Date(post.published)
: new Date(post.object.published),
files: files.length == 0 ? undefined : files,
files: files.length === 0 ? undefined : files,
poll: undefined,
text: text || undefined,
reply,
@ -118,12 +127,12 @@ export async function importMastoPost(
apHashtags: undefined,
apEmojis: undefined,
});
logger.info(`Create new note`);
logger.debug("New post has been created");
} else {
logger.info(`Note exist`);
logger.info("This post already exists");
}
job.progress(100);
done();
logger.succ("Imported");
logger.info("Imported");
}

View file

@ -66,15 +66,16 @@ export async function importMuting(
// skip myself
if (target.id === job.data.user.id) continue;
logger.info(`Mute[${linenum}] ${target.id} ...`);
logger.debug(`Mute[${linenum}] ${target.id} ...`);
await mute(user, target);
} catch (e) {
logger.warn(`Error in line ${linenum}: ${inspect(e)}`);
logger.warn(`Error in line ${linenum}`);
logger.info(inspect(e));
}
}
logger.succ("Imported");
logger.info("Imported");
done();
}

View file

@ -45,9 +45,10 @@ export async function importPosts(
}
} catch (e) {
// handle error
logger.warn(`Failed to read Mastodon archive:\n${inspect(e)}`);
logger.warn("Failed to read Mastodon archive");
logger.info(inspect(e));
}
logger.succ("Mastodon archive imported");
logger.info("Mastodon archive imported");
done();
return;
}
@ -56,24 +57,25 @@ export async function importPosts(
try {
const parsed = JSON.parse(json);
if (parsed instanceof Array) {
logger.info("Parsing key style posts");
if (Array.isArray(parsed)) {
logger.info("Parsing *key posts");
const arr = recreateChain(parsed);
for (const post of arr) {
createImportCkPostJob(job.data.user, post, job.data.signatureCheck);
}
} else if (parsed instanceof Object) {
logger.info("Parsing animal style posts");
logger.info("Parsing Mastodon posts");
for (const post of parsed.orderedItems) {
createImportMastoPostJob(job.data.user, post, job.data.signatureCheck);
}
}
} catch (e) {
// handle error
logger.warn(`Error occured while reading:\n${inspect(e)}`);
logger.warn("an error occured while reading");
logger.info(inspect(e));
}
logger.succ("Imported");
logger.info("Imported");
done();
}

View file

@ -86,10 +86,11 @@ export async function importUserLists(
pushUserToUserList(target, list!);
} catch (e) {
logger.warn(`Error in line ${linenum}:\n${inspect(e)}`);
logger.warn(`Error in line ${linenum}`);
logger.info(inspect(e));
}
}
logger.succ("Imported");
logger.info("Imported");
done();
}

View file

@ -24,7 +24,7 @@ const logger = new Logger("inbox");
// Processing when an activity arrives in the user's inbox
export default async (job: Bull.Job<InboxJobData>): Promise<string> => {
const signature = job.data.signature; // HTTP-signature
const activity = job.data.activity;
let activity = job.data.activity;
//#region Log
const info = Object.assign({}, activity) as any;
@ -149,6 +149,8 @@ export default async (job: Bull.Job<InboxJobData>): Promise<string> => {
return "skip: LD-Signatureの検証に失敗しました";
}
activity = await ldSignature.compactToWellKnown(activity);
// もう一度actorチェック
if (authUser.user.uri !== activity.actor) {
return `skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`;

View file

@ -48,6 +48,6 @@ export default async function cleanRemoteFiles(
job.progress(deletedCount / total);
}
logger.succ("All cached remote files has been deleted.");
logger.info("All cached remote files are deleted.");
done();
}

View file

@ -28,6 +28,6 @@ export async function checkExpiredMutings(
}
}
logger.succ("All expired mutings checked.");
logger.info("All expired mutings checked.");
done();
}

View file

@ -11,6 +11,6 @@ export async function cleanCharts(
): Promise<void> {
logger.info("Cleaning active users chart...");
await activeUsersChart.clean();
logger.succ("Active users chart has been cleaned.");
logger.info("Active users chart has been cleaned.");
done();
}

View file

@ -4,7 +4,7 @@ import { UserIps } from "@/models/index.js";
import { queueLogger } from "../../logger.js";
const logger = queueLogger.createSubLogger("clean");
const logger = queueLogger.createSubLogger("clean-user-ip-log");
export async function clean(
job: Bull.Job<Record<string, unknown>>,
@ -16,6 +16,6 @@ export async function clean(
createdAt: LessThan(new Date(Date.now() - 1000 * 60 * 60 * 24 * 90)),
});
logger.succ("Cleaned.");
logger.info("Cleaned.");
done();
}

View file

@ -3,7 +3,7 @@ import { IsNull } from "typeorm";
import { Emojis } from "@/models/index.js";
import { queueLogger } from "../../logger.js";
import { getEmojiSize } from "@/misc/emoji-meta.js";
import { getImageSizeFromUrl } from "backend-rs";
import { inspect } from "node:util";
const logger = queueLogger.createSubLogger("local-emoji-size");
@ -21,23 +21,22 @@ export async function setLocalEmojiSizes(
for (let i = 0; i < emojis.length; i++) {
try {
const size = await getEmojiSize(emojis[i].publicUrl);
const size = await getImageSizeFromUrl(emojis[i].publicUrl);
await Emojis.update(emojis[i].id, {
width: size.width || null,
height: size.height || null,
});
} catch (e) {
logger.error(
`Unable to set emoji size (${i + 1}/${emojis.length}):\n${inspect(e)}`,
);
logger.warn(`Unable to set emoji size (${i + 1}/${emojis.length})`);
logger.info(inspect(e));
/* skip if any error happens */
} finally {
// wait for 1sec so that this would not overwhelm the object storage.
await new Promise((resolve) => setTimeout(resolve, 1000));
if (i % 10 === 9) logger.succ(`fetched ${i + 1}/${emojis.length} emojis`);
if (i % 10 === 9) logger.info(`fetched ${i + 1}/${emojis.length} emojis`);
}
}
logger.succ("Done.");
logger.info("Done.");
done();
}

View file

@ -33,12 +33,13 @@ export async function verifyLinks(
fields: user.fields,
});
} catch (e) {
logger.error(`Failed to update user ${user.userId}:\n${inspect(e)}`);
logger.error(`Failed to update user ${user.userId}`);
logger.info(inspect(e));
done(e);
}
}
}
logger.succ("All links successfully verified.");
logger.info("All links successfully verified.");
done();
}

View file

@ -133,7 +133,8 @@ export default class DeliverManager {
host: new URL(inbox).host,
});
} catch (error) {
apLogger.error(`Invalid Inbox ${inbox}:\n${inspect(error)}`);
apLogger.info(`Invalid Inbox ${inbox}`);
apLogger.debug(inspect(error));
}
}

View file

@ -6,20 +6,19 @@ import { isFollow, getApType } from "../../type.js";
import { apLogger } from "../../logger.js";
import { inspect } from "node:util";
const logger = apLogger;
export default async (
actor: CacheableRemoteUser,
activity: IAccept,
): Promise<string> => {
const uri = activity.id || activity;
logger.info(`Accept: ${uri}`);
apLogger.info(`Accept: ${uri}`);
const resolver = new Resolver();
const object = await resolver.resolve(activity.object).catch((e) => {
logger.error(`Resolution failed:\n${inspect(e)}`);
apLogger.info(`Failed to resolve AP object: ${e}`);
apLogger.debug(inspect(e));
throw e;
});

View file

@ -5,15 +5,13 @@ import type { IAnnounce } from "../../type.js";
import { getApId } from "../../type.js";
import { apLogger } from "../../logger.js";
const logger = apLogger;
export default async (
actor: CacheableRemoteUser,
activity: IAnnounce,
): Promise<void> => {
const uri = getApId(activity);
logger.info(`Announce: ${uri}`);
apLogger.info(`Announce: ${uri}`);
const resolver = new Resolver();

View file

@ -13,8 +13,6 @@ import { Notes } from "@/models/index.js";
import { isBlockedServer } from "backend-rs";
import { inspect } from "node:util";
const logger = apLogger;
/**
* Handle announcement activities
*/
@ -50,11 +48,14 @@ export default async function (
// Skip if target is 4xx
if (e instanceof StatusError) {
if (e.isClientError) {
logger.warn(`Ignored announce target ${targetUri} - ${e.statusCode}`);
apLogger.info(
`Ignored announce target ${targetUri} - ${e.statusCode}`,
);
return;
}
logger.warn(`Error in announce target ${targetUri}:\n${inspect(e)}`);
apLogger.warn(`Error in announce target ${targetUri}`);
apLogger.debug(inspect(e));
}
throw e;
}
@ -63,7 +64,7 @@ export default async function (
console.log("skip: invalid actor for this activity");
return;
}
logger.info(`Creating the (Re)Note: ${uri}`);
apLogger.info(`Creating (re)note: ${uri}`);
const activityAudience = await parseAudience(
actor,

View file

@ -7,15 +7,13 @@ import { apLogger } from "../../logger.js";
import { toArray, concat, unique } from "@/prelude/array.js";
import { inspect } from "node:util";
const logger = apLogger;
export default async (
actor: CacheableRemoteUser,
activity: ICreate,
): Promise<void> => {
const uri = getApId(activity);
logger.info(`Create: ${uri}`);
apLogger.info(`Create: ${uri}`);
// copy audiences between activity <=> object.
if (typeof activity.object === "object") {
@ -40,13 +38,14 @@ export default async (
const resolver = new Resolver();
const object = await resolver.resolve(activity.object).catch((e) => {
logger.error(`Resolution failed:\n${inspect(e)}`);
apLogger.info(`Failed to resolve AP object: ${e}`);
apLogger.debug(inspect(e));
throw e;
});
if (isPost(object)) {
createNote(resolver, actor, object, false, activity);
} else {
logger.warn(`Unknown type: ${getApType(object)}`);
apLogger.info(`Unknown type: ${getApType(object)}`);
}
};

View file

@ -3,13 +3,11 @@ import { createDeleteAccountJob } from "@/queue/index.js";
import type { CacheableRemoteUser } from "@/models/entities/user.js";
import { Users } from "@/models/index.js";
const logger = apLogger;
export async function deleteActor(
actor: CacheableRemoteUser,
uri: string,
): Promise<string> {
logger.info(`Deleting the Actor: ${uri}`);
apLogger.info(`Deleting Actor: ${uri}`);
if (actor.uri !== uri) {
return `skip: delete actor ${actor.uri} !== ${uri}`;

View file

@ -5,13 +5,11 @@ import DbResolver from "../../db-resolver.js";
import { getApLock } from "@/misc/app-lock.js";
import { deleteMessage } from "@/services/messages/delete.js";
const logger = apLogger;
export default async function (
actor: CacheableRemoteUser,
uri: string,
): Promise<string> {
logger.info(`Deleting the Note: ${uri}`);
apLogger.info(`Deleting note: ${uri}`);
const lock = await getApLock(uri);

View file

@ -54,7 +54,8 @@ export async function performActivity(
try {
await performOneActivity(actor, act);
} catch (err) {
apLogger.error(inspect(err));
apLogger.info(`Failed to perform activity: ${err}`);
apLogger.debug(inspect(err));
}
}
} else {
@ -88,9 +89,15 @@ async function performOneActivity(
} else if (isReject(activity)) {
await reject(actor, activity);
} else if (isAdd(activity)) {
await add(actor, activity).catch((err) => apLogger.error(inspect(err)));
await add(actor, activity).catch((err) => {
apLogger.warn(`Failed to perform 'add' activity: ${err}`);
apLogger.debug(inspect(err));
});
} else if (isRemove(activity)) {
await remove(actor, activity).catch((err) => apLogger.error(inspect(err)));
await remove(actor, activity).catch((err) => {
apLogger.warn(`Failed to perform 'remove' activity: ${err}`);
apLogger.debug(inspect(err));
});
} else if (isAnnounce(activity)) {
await announce(actor, activity);
} else if (isLike(activity)) {
@ -104,7 +111,7 @@ async function performOneActivity(
} else if (isMove(activity)) {
await move(actor, activity);
} else {
apLogger.warn(
apLogger.info(
`Unrecognized activity type: ${(activity as IActivity).type}`,
);
}

View file

@ -14,12 +14,13 @@ export default async (
): Promise<string> => {
const uri = activity.id || activity;
logger.info(`Reject: ${uri}`);
apLogger.info(`Reject: ${uri}`);
const resolver = new Resolver();
const object = await resolver.resolve(activity.object).catch((e) => {
logger.error(`Resolution failed:\n${inspect(e)}`);
apLogger.info(`Failed to resolve AP object: ${e}`);
apLogger.debug(inspect(e));
throw e;
});

View file

@ -17,8 +17,6 @@ import Resolver from "../../resolver.js";
import { apLogger } from "../../logger.js";
import { inspect } from "node:util";
const logger = apLogger;
export default async (
actor: CacheableRemoteUser,
activity: IUndo,
@ -29,12 +27,13 @@ export default async (
const uri = activity.id || activity;
logger.info(`Undo: ${uri}`);
apLogger.info(`Undo: ${uri}`);
const resolver = new Resolver();
const object = await resolver.resolve(activity.object).catch((e) => {
logger.error(`Resolution failed:\n${inspect(e)}`);
apLogger.info(`Failed to resolve AP object: ${e}`);
apLogger.debug(inspect(e));
throw e;
});

View file

@ -18,12 +18,13 @@ export default async (
return "skip: invalid actor";
}
apLogger.debug("Update");
apLogger.info("Update");
const resolver = new Resolver();
const object = await resolver.resolve(activity.object).catch((e) => {
apLogger.error(`Resolution failed:\n${inspect(e)}`);
apLogger.info(`Failed to resolve AP object: ${e}`);
apLogger.debug(inspect(e));
throw e;
});

View file

@ -518,6 +518,54 @@ const activitystreams = {
},
};
export const WellKnownContext = {
"@context": [
"https://www.w3.org/ns/activitystreams",
"https://w3id.org/security/v1",
{
// as non-standards
manuallyApprovesFollowers: "as:manuallyApprovesFollowers",
movedTo: {
"@id": "https://www.w3.org/ns/activitystreams#movedTo",
"@type": "@id"
},
movedToUri: "as:movedTo",
sensitive: "as:sensitive",
Hashtag: "as:Hashtag",
quoteUri: "fedibird:quoteUri",
quoteUrl: "as:quoteUrl",
// Mastodon
toot: "http://joinmastodon.org/ns#",
Emoji: "toot:Emoji",
featured: "toot:featured",
discoverable: "toot:discoverable",
indexable: "toot:indexable",
// schema
schema: "http://schema.org#",
PropertyValue: "schema:PropertyValue",
value: "schema:value",
// Firefish
firefish: "https://firefish.dev/ns#",
speakAsCat: "firefish:speakAsCat",
// Misskey
misskey: "https://misskey-hub.net/ns#",
_misskey_talk: "misskey:_misskey_talk",
_misskey_reaction: "misskey:_misskey_reaction",
_misskey_votes: "misskey:_misskey_votes",
_misskey_summary: "misskey:_misskey_summary",
isCat: "misskey:isCat",
// Fedibird
fedibird: "http://fedibird.com/ns#",
// vcard
vcard: "http://www.w3.org/2006/vcard/ns#",
// litepub
litepub: "http://litepub.social/ns#",
ChatMessage: "litepub:ChatMessage",
directMessage: "litepub:directMessage",
},
],
};
export const CONTEXTS: Record<string, unknown> = {
"https://w3id.org/identity/v1": id_v1,
"https://w3id.org/security/v1": security_v1,

View file

@ -1,6 +1,6 @@
import * as crypto from "node:crypto";
import jsonld from "jsonld";
import { CONTEXTS } from "./contexts.js";
import { CONTEXTS, WellKnownContext } from "./contexts.js";
import fetch from "node-fetch";
import { httpAgent, httpsAgent } from "@/misc/fetch.js";
@ -89,6 +89,13 @@ export class LdSignature {
});
}
public async compactToWellKnown(data: any): Promise<any> {
const options = { documentLoader: this.getLoader() };
const context = WellKnownContext as any;
delete data["signature"];
return await jsonld.compact(data, context, options);
}
private getLoader() {
return async (url: string): Promise<any> => {
if (!url.match("^https?://")) throw new Error(`Invalid URL ${url}`);

View file

@ -9,9 +9,7 @@ import type {
} from "@/models/entities/drive-file.js";
import { DriveFiles } from "@/models/index.js";
import { truncate } from "@/misc/truncate.js";
import { DB_MAX_IMAGE_COMMENT_LENGTH } from "@/misc/hard-limits.js";
const logger = apLogger;
import { config } from "@/config.js";
/**
* create an Image.
@ -36,7 +34,7 @@ export async function createImage(
throw new Error(`Invalid image, unexpected schema: ${image.url}`);
}
logger.info(`Creating the Image: ${image.url}`);
apLogger.info(`Creating an image: ${image.url}`);
const instance = await fetchMeta(true);
@ -46,7 +44,7 @@ export async function createImage(
uri: image.url,
sensitive: image.sensitive,
isLink: !instance.cacheRemoteFiles,
comment: truncate(image.name, DB_MAX_IMAGE_COMMENT_LENGTH),
comment: truncate(image.name, config.maxCaptionLength),
usageHint: usage,
});

View file

@ -13,7 +13,13 @@ import { extractPollFromQuestion } from "./question.js";
import vote from "@/services/note/polls/vote.js";
import { apLogger } from "../logger.js";
import type { DriveFile } from "@/models/entities/drive-file.js";
import { extractHost, isSameOrigin, toPuny } from "backend-rs";
import {
type ImageSize,
extractHost,
getImageSizeFromUrl,
isSameOrigin,
toPuny,
} from "backend-rs";
import {
Emojis,
Polls,
@ -44,14 +50,11 @@ import { publishNoteStream } from "@/services/stream.js";
import { extractHashtags } from "@/misc/extract-hashtags.js";
import { UserProfiles } from "@/models/index.js";
import { In } from "typeorm";
import { DB_MAX_IMAGE_COMMENT_LENGTH } from "@/misc/hard-limits.js";
import { config } from "@/config.js";
import { truncate } from "@/misc/truncate.js";
import { type Size, getEmojiSize } from "@/misc/emoji-meta.js";
import { langmap } from "@/misc/langmap.js";
import { inspect } from "node:util";
const logger = apLogger;
export function validateNote(object: any, uri: string) {
const expectHost = extractHost(uri);
@ -112,13 +115,16 @@ export async function createNote(
const entryUri = getApId(value);
const err = validateNote(object, entryUri);
if (err) {
logger.error(`${err.message}`, {
resolver: {
history: resolver.getHistory(),
},
value: value,
object: object,
});
apLogger.info(`${err.message}`);
apLogger.debug(
inspect({
resolver: {
history: resolver.getHistory(),
},
value: value,
object: object,
}),
);
throw new Error("invalid note");
}
@ -140,8 +146,8 @@ export async function createNote(
throw new Error(`unexpected schema of note url: ${url}`);
}
logger.debug(`Note fetched: ${JSON.stringify(note, null, 2)}`);
logger.info(`Creating the Note: ${note.id}`);
apLogger.trace(`Note fetched: ${JSON.stringify(note, null, 2)}`);
apLogger.info(`Creating the Note: ${note.id}`);
// Skip if note is made before 2007 (1yr before Fedi was created)
// OR skip if note is made 3 days in advance
@ -150,13 +156,13 @@ export async function createNote(
const FutureCheck = new Date();
FutureCheck.setDate(FutureCheck.getDate() + 3); // Allow some wiggle room for misconfigured hosts
if (DateChecker.getFullYear() < 2007) {
logger.warn(
apLogger.info(
"Note somehow made before Activitypub was created; discarding",
);
return null;
}
if (DateChecker > FutureCheck) {
logger.warn("Note somehow made after today; discarding");
apLogger.info("Note somehow made after today; discarding");
return null;
}
}
@ -169,8 +175,8 @@ export async function createNote(
// Skip if author is suspended.
if (actor.isSuspended) {
logger.debug(
`User ${actor.usernameLower}@${actor.host} suspended; discarding.`,
apLogger.info(
`User ${actor.usernameLower}@${actor.host} is suspended; discarding.`,
);
return null;
}
@ -224,7 +230,7 @@ export async function createNote(
? await resolveNote(note.inReplyTo, resolver)
.then((x) => {
if (x == null) {
logger.warn("Specified inReplyTo, but nout found");
apLogger.info(`Specified inReplyTo not found: ${note.inReplyTo}`);
throw new Error("inReplyTo not found");
} else {
return x;
@ -242,7 +248,8 @@ export async function createNote(
}
}
logger.warn(`Error in inReplyTo ${note.inReplyTo}:\n${inspect(e)}`);
apLogger.info(`Error in inReplyTo ${note.inReplyTo}`);
apLogger.debug(inspect(e));
throw e;
})
: null;
@ -336,11 +343,11 @@ export async function createNote(
index: number,
): Promise<null> => {
if (poll.expiresAt && Date.now() > new Date(poll.expiresAt).getTime()) {
logger.warn(
`vote to expired poll from AP: actor=${actor.username}@${actor.host}, note=${note.id}, choice=${name}`,
apLogger.info(
`discarding vote to expired poll: actor=${actor.username}@${actor.host}, note=${note.id}, choice=${name}`,
);
} else if (index >= 0) {
logger.info(
apLogger.info(
`vote from AP: actor=${actor.username}@${actor.host}, note=${note.id}, choice=${name}`,
);
await vote(actor, reply, index);
@ -357,7 +364,8 @@ export async function createNote(
}
const emojis = await extractEmojis(note.tag || [], actor.host).catch((e) => {
logger.info(`extractEmojis:\n${inspect(e)}`);
apLogger.info("Failed to extract emojis");
apLogger.debug(inspect(e));
return [] as Emoji[];
});
@ -485,11 +493,16 @@ export async function extractEmojis(
tag.icon!.url !== exists.originalUrl ||
!(exists.width && exists.height)
) {
let size: Size = { width: 0, height: 0 };
try {
size = await getEmojiSize(tag.icon!.url);
} catch {
/* skip if any error happens */
let size: ImageSize | null = null;
if (tag.icon?.url != null) {
try {
size = await getImageSizeFromUrl(tag.icon.url);
} catch (err) {
apLogger.info(
`Failed to determine the size of the image: ${tag.icon.url}`,
);
apLogger.debug(inspect(err));
}
}
await Emojis.update(
{
@ -501,8 +514,8 @@ export async function extractEmojis(
originalUrl: tag.icon!.url,
publicUrl: tag.icon!.url,
updatedAt: new Date(),
width: size.width || null,
height: size.height || null,
width: size?.width || null,
height: size?.height || null,
},
);
@ -515,11 +528,11 @@ export async function extractEmojis(
return exists;
}
logger.info(`register emoji host=${host}, name=${name}`);
apLogger.info(`register emoji host=${host}, name=${name}`);
let size: Size = { width: 0, height: 0 };
let size: ImageSize = { width: 0, height: 0 };
try {
size = await getEmojiSize(tag.icon!.url);
size = await getImageSizeFromUrl(tag.icon!.url);
} catch {
/* skip if any error happens */
}
@ -619,7 +632,7 @@ export async function updateNote(value: string | IObject, resolver?: Resolver) {
const file = await resolveImage(actor, x, null);
const update: Partial<DriveFile> = {};
const altText = truncate(x.name, DB_MAX_IMAGE_COMMENT_LENGTH);
const altText = truncate(x.name, config.maxCaptionLength);
if (file.comment !== altText) {
update.comment = altText;
}

Some files were not shown because too many files have changed in this diff Show more