From fd1bc109d9a74a7892d1365069571353efc4fe46 Mon Sep 17 00:00:00 2001 From: Syuilo Date: Sat, 8 Jul 2023 15:39:42 -0700 Subject: [PATCH 01/26] =?UTF-8?q?refactor:=20=E2=9A=A1=20antenna=20notes?= =?UTF-8?q?=20in=20cache?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Kainoa Kanter --- .../migration/1680491187535-cleanup.js | 9 ++ packages/backend/src/models/index.ts | 1 - .../backend/src/models/repositories/user.ts | 32 +++---- .../server/api/endpoints/antennas/markread.ts | 20 ++--- .../server/api/endpoints/antennas/notes.ts | 31 +++++-- .../src/services/add-note-to-antenna.ts | 56 +++--------- packages/backend/src/services/note/read.ts | 87 +++++++++---------- 7 files changed, 113 insertions(+), 123 deletions(-) create mode 100644 packages/backend/migration/1680491187535-cleanup.js diff --git a/packages/backend/migration/1680491187535-cleanup.js b/packages/backend/migration/1680491187535-cleanup.js new file mode 100644 index 000000000..021a368c0 --- /dev/null +++ b/packages/backend/migration/1680491187535-cleanup.js @@ -0,0 +1,9 @@ +export class cleanup1680491187535 { + name = "cleanup1680491187535"; + + async up(queryRunner) { + await queryRunner.query(`DROP TABLE "antenna_note" `); + } + + async down(queryRunner) {} +} diff --git a/packages/backend/src/models/index.ts b/packages/backend/src/models/index.ts index cfc3b01c5..ca35b7708 100644 --- a/packages/backend/src/models/index.ts +++ b/packages/backend/src/models/index.ts @@ -123,7 +123,6 @@ export const ModerationLogs = ModerationLogRepository; export const Clips = ClipRepository; export const ClipNotes = db.getRepository(ClipNote); export const Antennas = AntennaRepository; -export const AntennaNotes = db.getRepository(AntennaNote); export const PromoNotes = db.getRepository(PromoNote); export const PromoReads = db.getRepository(PromoRead); export const Relays = RelayRepository; diff --git a/packages/backend/src/models/repositories/user.ts b/packages/backend/src/models/repositories/user.ts index 5ca36e3d3..1f9cd80fb 100644 --- a/packages/backend/src/models/repositories/user.ts +++ b/packages/backend/src/models/repositories/user.ts @@ -18,7 +18,6 @@ import { createPerson } from "@/remote/activitypub/models/person.js"; import { AnnouncementReads, Announcements, - AntennaNotes, Blockings, ChannelFollowings, DriveFiles, @@ -258,23 +257,24 @@ export const UserRepository = db.getRepository(User).extend({ }, async getHasUnreadAntenna(userId: User["id"]): Promise { - try { - const myAntennas = (await getAntennas()).filter( - (a) => a.userId === userId, - ); + // try { + // const myAntennas = (await getAntennas()).filter( + // (a) => a.userId === userId, + // ); - const unread = - myAntennas.length > 0 - ? await AntennaNotes.findOneBy({ - antennaId: In(myAntennas.map((x) => x.id)), - read: false, - }) - : null; + // const unread = + // myAntennas.length > 0 + // ? await AntennaNotes.findOneBy({ + // antennaId: In(myAntennas.map((x) => x.id)), + // read: false, + // }) + // : null; - return unread != null; - } catch (e) { - return false; - } + // return unread != null; + // } catch (e) { + // return false; + // } + return false; // TODO }, async getHasUnreadChannel(userId: User["id"]): Promise { diff --git a/packages/backend/src/server/api/endpoints/antennas/markread.ts b/packages/backend/src/server/api/endpoints/antennas/markread.ts index e29e13bbb..fedb48c91 100644 --- a/packages/backend/src/server/api/endpoints/antennas/markread.ts +++ b/packages/backend/src/server/api/endpoints/antennas/markread.ts @@ -1,5 +1,5 @@ import define from "../../define.js"; -import { Antennas, AntennaNotes } from "@/models/index.js"; +import { Antennas } from "@/models/index.js"; import { FindOptionsWhere } from "typeorm"; import { AntennaNote } from "@/models/entities/antenna-note.js"; @@ -29,15 +29,15 @@ export default define(meta, paramDef, async (ps, me) => { return null; } - await AntennaNotes.update( - { - antennaId: antenna.id, - read: false, - }, - { - read: true, - }, - ); + // await AntennaNotes.update( + // { + // antennaId: antenna.id, + // read: false, + // }, + // { + // read: true, + // }, + // ); return true; }); diff --git a/packages/backend/src/server/api/endpoints/antennas/notes.ts b/packages/backend/src/server/api/endpoints/antennas/notes.ts index 29e2e085a..6ca71c08e 100644 --- a/packages/backend/src/server/api/endpoints/antennas/notes.ts +++ b/packages/backend/src/server/api/endpoints/antennas/notes.ts @@ -1,6 +1,8 @@ import define from "../../define.js"; import readNote from "@/services/note/read.js"; -import { Antennas, Notes, AntennaNotes } from "@/models/index.js"; +import { Antennas, Notes } from "@/models/index.js"; +import { redisClient } from "@/db/redis.js"; +import { genId } from "@/misc/gen-id.js"; import { makePaginationQuery } from "../../common/make-pagination-query.js"; import { generateVisibilityQuery } from "../../common/generate-visibility-query.js"; import { generateMutedUserQuery } from "../../common/generate-muted-user-query.js"; @@ -58,6 +60,26 @@ export default define(meta, paramDef, async (ps, user) => { throw new ApiError(meta.errors.noSuchAntenna); } + const noteIdsRes = await redisClient.xrevrange( + `antennaTimeline:${antenna.id}`, + ps.untilDate || "+", + "-", + "COUNT", + ps.limit + 1, + ); // untilIdに指定したものも含まれるため+1 + + if (noteIdsRes.length === 0) { + return []; + } + + const noteIds = noteIdsRes + .map((x) => x[1][1]) + .filter((x) => x !== ps.untilId); + + if (noteIds.length === 0) { + return []; + } + const query = makePaginationQuery( Notes.createQueryBuilder("note"), ps.sinceId, @@ -65,11 +87,7 @@ export default define(meta, paramDef, async (ps, user) => { ps.sinceDate, ps.untilDate, ) - .innerJoin( - AntennaNotes.metadata.targetName, - "antennaNote", - "antennaNote.noteId = note.id", - ) + .where("note.id IN (:...noteIds)", { noteIds: noteIds }) .innerJoinAndSelect("note.user", "user") .leftJoinAndSelect("user.avatar", "avatar") .leftJoinAndSelect("user.banner", "banner") @@ -81,7 +99,6 @@ export default define(meta, paramDef, async (ps, user) => { .leftJoinAndSelect("renote.user", "renoteUser") .leftJoinAndSelect("renoteUser.avatar", "renoteUserAvatar") .leftJoinAndSelect("renoteUser.banner", "renoteUserBanner") - .andWhere("antennaNote.antennaId = :antennaId", { antennaId: antenna.id }) .andWhere("note.visibility != 'home'"); generateVisibilityQuery(query, user); diff --git a/packages/backend/src/services/add-note-to-antenna.ts b/packages/backend/src/services/add-note-to-antenna.ts index 38979acb4..131c0348c 100644 --- a/packages/backend/src/services/add-note-to-antenna.ts +++ b/packages/backend/src/services/add-note-to-antenna.ts @@ -1,9 +1,8 @@ import type { Antenna } from "@/models/entities/antenna.js"; import type { Note } from "@/models/entities/note.js"; -import { AntennaNotes, Mutings, Notes } from "@/models/index.js"; import { genId } from "@/misc/gen-id.js"; -import { isUserRelated } from "@/misc/is-user-related.js"; -import { publishAntennaStream, publishMainStream } from "@/services/stream.js"; +import { redisClient } from "@/db/redis.js"; +import { publishAntennaStream } from "@/services/stream.js"; import type { User } from "@/models/entities/user.js"; export async function addNoteToAntenna( @@ -14,48 +13,15 @@ export async function addNoteToAntenna( // 通知しない設定になっているか、自分自身の投稿なら既読にする const read = !antenna.notify || antenna.userId === noteUser.id; - AntennaNotes.insert({ - id: genId(), - antennaId: antenna.id, - noteId: note.id, - read: read, - }); + redisClient.xadd( + `antennaTimeline:${antenna.id}`, + "MAXLEN", + "~", + "200", + `${genId(note.createdAt)}-*`, + "note", + note.id, + ); publishAntennaStream(antenna.id, "note", note); - - if (!read) { - const mutings = await Mutings.find({ - where: { - muterId: antenna.userId, - }, - select: ["muteeId"], - }); - - // Copy - const _note: Note = { - ...note, - }; - - if (note.replyId != null) { - _note.reply = await Notes.findOneByOrFail({ id: note.replyId }); - } - if (note.renoteId != null) { - _note.renote = await Notes.findOneByOrFail({ id: note.renoteId }); - } - - if (isUserRelated(_note, new Set(mutings.map((x) => x.muteeId)))) { - return; - } - - // 2秒経っても既読にならなかったら通知 - setTimeout(async () => { - const unread = await AntennaNotes.findOneBy({ - antennaId: antenna.id, - read: false, - }); - if (unread) { - publishMainStream(antenna.userId, "unreadAntenna", antenna); - } - }, 2000); - } } diff --git a/packages/backend/src/services/note/read.ts b/packages/backend/src/services/note/read.ts index 53188f15f..2dcf49235 100644 --- a/packages/backend/src/services/note/read.ts +++ b/packages/backend/src/services/note/read.ts @@ -3,7 +3,6 @@ import type { Note } from "@/models/entities/note.js"; import type { User } from "@/models/entities/user.js"; import { NoteUnreads, - AntennaNotes, Users, Followings, ChannelFollowings, @@ -55,7 +54,7 @@ export default async function ( const readMentions: (Note | Packed<"Note">)[] = []; const readSpecifiedNotes: (Note | Packed<"Note">)[] = []; const readChannelNotes: (Note | Packed<"Note">)[] = []; - const readAntennaNotes: (Note | Packed<"Note">)[] = []; + // const readAntennaNotes: (Note | Packed<"Note">)[] = []; for (const note of notes) { if (note.mentions?.includes(userId)) { @@ -68,22 +67,22 @@ export default async function ( readChannelNotes.push(note); } - if (note.user != null) { - // たぶんnullになることは無いはずだけど一応 - for (const antenna of myAntennas) { - if ( - await checkHitAntenna( - antenna, - note, - note.user, - undefined, - Array.from(following), - ) - ) { - readAntennaNotes.push(note); - } - } - } + // if (note.user != null) { + // // たぶんnullになることは無いはずだけど一応 + // for (const antenna of myAntennas) { + // if ( + // await checkHitAntenna( + // antenna, + // note, + // note.user, + // undefined, + // Array.from(following), + // ) + // ) { + // readAntennaNotes.push(note); + // } + // } + // } } if ( @@ -141,33 +140,33 @@ export default async function ( }); } - if (readAntennaNotes.length > 0) { - await AntennaNotes.update( - { - antennaId: In(myAntennas.map((a) => a.id)), - noteId: In(readAntennaNotes.map((n) => n.id)), - }, - { - read: true, - }, - ); + // if (readAntennaNotes.length > 0) { + // await AntennaNotes.update( + // { + // antennaId: In(myAntennas.map((a) => a.id)), + // noteId: In(readAntennaNotes.map((n) => n.id)), + // }, + // { + // read: true, + // }, + // ); - // TODO: まとめてクエリしたい - for (const antenna of myAntennas) { - const count = await AntennaNotes.countBy({ - antennaId: antenna.id, - read: false, - }); + // // TODO: まとめてクエリしたい + // for (const antenna of myAntennas) { + // const count = await AntennaNotes.countBy({ + // antennaId: antenna.id, + // read: false, + // }); - if (count === 0) { - publishMainStream(userId, "readAntenna", antenna); - } - } + // if (count === 0) { + // publishMainStream(userId, "readAntenna", antenna); + // } + // } - Users.getHasUnreadAntenna(userId).then((unread) => { - if (!unread) { - publishMainStream(userId, "readAllAntennas"); - } - }); - } + // Users.getHasUnreadAntenna(userId).then((unread) => { + // if (!unread) { + // publishMainStream(userId, "readAllAntennas"); + // } + // }); + // } } From 01064dd3c6dac2cbe362541a578245d2ba2147fd Mon Sep 17 00:00:00 2001 From: ThatOneCalculator Date: Sat, 8 Jul 2023 15:59:47 -0700 Subject: [PATCH 02/26] fix --- packages/backend/src/db/postgre.ts | 2 - .../src/models/entities/antenna-note.ts | 50 ------------------- packages/backend/src/models/index.ts | 1 - .../server/api/endpoints/antennas/markread.ts | 1 - 4 files changed, 54 deletions(-) delete mode 100644 packages/backend/src/models/entities/antenna-note.ts diff --git a/packages/backend/src/db/postgre.ts b/packages/backend/src/db/postgre.ts index 89b7a7bf6..10ea5b15f 100644 --- a/packages/backend/src/db/postgre.ts +++ b/packages/backend/src/db/postgre.ts @@ -58,7 +58,6 @@ import { AnnouncementRead } from "@/models/entities/announcement-read.js"; import { Clip } from "@/models/entities/clip.js"; import { ClipNote } from "@/models/entities/clip-note.js"; import { Antenna } from "@/models/entities/antenna.js"; -import { AntennaNote } from "@/models/entities/antenna-note.js"; import { PromoNote } from "@/models/entities/promo-note.js"; import { PromoRead } from "@/models/entities/promo-read.js"; import { Relay } from "@/models/entities/relay.js"; @@ -168,7 +167,6 @@ export const entities = [ Clip, ClipNote, Antenna, - AntennaNote, PromoNote, PromoRead, Relay, diff --git a/packages/backend/src/models/entities/antenna-note.ts b/packages/backend/src/models/entities/antenna-note.ts deleted file mode 100644 index fe982c19e..000000000 --- a/packages/backend/src/models/entities/antenna-note.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { - Entity, - Index, - JoinColumn, - Column, - ManyToOne, - PrimaryColumn, -} from "typeorm"; -import { Note } from "./note.js"; -import { Antenna } from "./antenna.js"; -import { id } from "../id.js"; - -@Entity() -@Index(["noteId", "antennaId"], { unique: true }) -export class AntennaNote { - @PrimaryColumn(id()) - public id: string; - - @Index() - @Column({ - ...id(), - comment: "The note ID.", - }) - public noteId: Note["id"]; - - @ManyToOne((type) => Note, { - onDelete: "CASCADE", - }) - @JoinColumn() - public note: Note | null; - - @Index() - @Column({ - ...id(), - comment: "The antenna ID.", - }) - public antennaId: Antenna["id"]; - - @ManyToOne((type) => Antenna, { - onDelete: "CASCADE", - }) - @JoinColumn() - public antenna: Antenna | null; - - @Index() - @Column("boolean", { - default: false, - }) - public read: boolean; -} diff --git a/packages/backend/src/models/index.ts b/packages/backend/src/models/index.ts index ca35b7708..8782d5740 100644 --- a/packages/backend/src/models/index.ts +++ b/packages/backend/src/models/index.ts @@ -51,7 +51,6 @@ import { UsedUsername } from "./entities/used-username.js"; import { ClipRepository } from "./repositories/clip.js"; import { ClipNote } from "./entities/clip-note.js"; import { AntennaRepository } from "./repositories/antenna.js"; -import { AntennaNote } from "./entities/antenna-note.js"; import { PromoNote } from "./entities/promo-note.js"; import { PromoRead } from "./entities/promo-read.js"; import { EmojiRepository } from "./repositories/emoji.js"; diff --git a/packages/backend/src/server/api/endpoints/antennas/markread.ts b/packages/backend/src/server/api/endpoints/antennas/markread.ts index fedb48c91..db8e683e4 100644 --- a/packages/backend/src/server/api/endpoints/antennas/markread.ts +++ b/packages/backend/src/server/api/endpoints/antennas/markread.ts @@ -1,7 +1,6 @@ import define from "../../define.js"; import { Antennas } from "@/models/index.js"; import { FindOptionsWhere } from "typeorm"; -import { AntennaNote } from "@/models/entities/antenna-note.js"; export const meta = { tags: ["antennas", "account"], From c33313f7975e7f58b896a6ff566ab20ff150a9a0 Mon Sep 17 00:00:00 2001 From: ThatOneCalculator Date: Sat, 8 Jul 2023 16:10:12 -0700 Subject: [PATCH 03/26] typo --- packages/backend/src/server/api/endpoints/antennas/create.ts | 2 +- packages/backend/src/services/note/read.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/server/api/endpoints/antennas/create.ts b/packages/backend/src/server/api/endpoints/antennas/create.ts index c2117d51a..ed1645012 100644 --- a/packages/backend/src/server/api/endpoints/antennas/create.ts +++ b/packages/backend/src/server/api/endpoints/antennas/create.ts @@ -30,7 +30,7 @@ export const meta = { id: "c3a5a51e-04d4-11ee-be56-0242ac120002", }, noKeywords: { - message: "No keywords", + message: "No keywords.", code: "NO_KEYWORDS", id: "aa975b74-1ddb-11ee-be56-0242ac120002", }, diff --git a/packages/backend/src/services/note/read.ts b/packages/backend/src/services/note/read.ts index 2dcf49235..5e61fe95d 100644 --- a/packages/backend/src/services/note/read.ts +++ b/packages/backend/src/services/note/read.ts @@ -50,7 +50,7 @@ export default async function ( ).map((x) => x.followeeId), ); - const myAntennas = (await getAntennas()).filter((a) => a.userId === userId); + // const myAntennas = (await getAntennas()).filter((a) => a.userId === userId); const readMentions: (Note | Packed<"Note">)[] = []; const readSpecifiedNotes: (Note | Packed<"Note">)[] = []; const readChannelNotes: (Note | Packed<"Note">)[] = []; From 1f5cf72ae77082198e83562aee7c7d31bd4531da Mon Sep 17 00:00:00 2001 From: ThatOneCalculator Date: Sat, 8 Jul 2023 16:19:55 -0700 Subject: [PATCH 04/26] fix: :bug: server stats setting, meta fetching --- packages/backend/src/daemons/server-stats.ts | 6 ++++-- packages/client/src/pages/admin/settings.vue | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/daemons/server-stats.ts b/packages/backend/src/daemons/server-stats.ts index ba7427876..42e19d813 100644 --- a/packages/backend/src/daemons/server-stats.ts +++ b/packages/backend/src/daemons/server-stats.ts @@ -21,8 +21,10 @@ export default function () { ev.emit(`serverStatsLog:${x.id}`, log.slice(0, x.length || 50)); }); - const meta = fetchMeta(); - if (!meta.enableServerMachineStats) return; + fetchMeta().then((meta) => { + if (!meta.enableServerMachineStats) return; + } + ); async function tick() { const cpu = await cpuUsage(); diff --git a/packages/client/src/pages/admin/settings.vue b/packages/client/src/pages/admin/settings.vue index 5108c0696..b03a98ae5 100644 --- a/packages/client/src/pages/admin/settings.vue +++ b/packages/client/src/pages/admin/settings.vue @@ -468,6 +468,7 @@ let enableIdenticonGeneration: boolean = $ref(false); async function init() { const meta = await os.api("admin/meta"); + if (!meta) throw new Error("No meta"); name = meta.name; description = meta.description; tosUrl = meta.tosUrl; From 61f0f52d42165fbe306292807ed81969bd629a73 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Mon, 10 Jul 2023 01:39:33 -0400 Subject: [PATCH 05/26] copy existing posts in antenna to redis at migration --- .../migration/1680491187535-cleanup.js | 4 +- packages/backend/native-utils/Cargo.lock | 145 ++++++++++- packages/backend/native-utils/Cargo.toml | 4 +- .../native-utils/__test__/index.spec.mjs | 8 +- .../backend/native-utils/migration/Cargo.toml | 9 +- .../backend/native-utils/migration/src/lib.rs | 2 + .../m20230709_000510_move_antenna_to_cache.rs | 240 ++++++++++++++++++ .../native-utils/migration/src/main.rs | 81 +++++- .../native-utils/src/model/schema/app.rs | 8 +- packages/backend/native-utils/src/util/id.rs | 82 +++--- packages/backend/native-utils/tests/common.rs | 8 +- .../tests/model/repository/antenna.rs | 2 +- packages/backend/src/misc/gen-id.ts | 2 +- .../src/services/add-note-to-antenna.ts | 7 +- 14 files changed, 516 insertions(+), 86 deletions(-) create mode 100644 packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs diff --git a/packages/backend/migration/1680491187535-cleanup.js b/packages/backend/migration/1680491187535-cleanup.js index 021a368c0..671f7521e 100644 --- a/packages/backend/migration/1680491187535-cleanup.js +++ b/packages/backend/migration/1680491187535-cleanup.js @@ -1,9 +1,7 @@ export class cleanup1680491187535 { name = "cleanup1680491187535"; - async up(queryRunner) { - await queryRunner.query(`DROP TABLE "antenna_note" `); - } + async up(queryRunner) {} async down(queryRunner) {} } diff --git a/packages/backend/native-utils/Cargo.lock b/packages/backend/native-utils/Cargo.lock index 1a20b792c..e5f8af37a 100644 --- a/packages/backend/native-utils/Cargo.lock +++ b/packages/backend/native-utils/Cargo.lock @@ -458,6 +458,20 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "combine" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" +dependencies = [ + "bytes", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util", +] + [[package]] name = "console" version = "0.15.7" @@ -486,6 +500,16 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.4" @@ -1278,11 +1302,14 @@ dependencies = [ "futures", "indicatif", "native-utils", + "redis", + "sea-orm", "sea-orm-migration", "serde", "serde_json", "serde_yaml", "tokio", + "url", "urlencoding", ] @@ -1509,6 +1536,12 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + [[package]] name = "os_str_bytes" version = "6.5.0" @@ -1843,6 +1876,29 @@ dependencies = [ "rand_core", ] +[[package]] +name = "redis" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ea8c51b5dc1d8e5fd3350ec8167f464ec0995e79f2e90a075b63371500d557f" +dependencies = [ + "async-trait", + "bytes", + "combine", + "futures-util", + "itoa", + "percent-encoding", + "pin-project-lite", + "rustls 0.21.3", + "rustls-native-certs", + "ryu", + "sha1_smol", + "tokio", + "tokio-rustls 0.24.1", + "tokio-util", + "url", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -2043,6 +2099,30 @@ dependencies = [ "webpki", ] +[[package]] +name = "rustls" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b19faa85ecb5197342b54f987b142fb3e30d0c90da40f80ef4fa9a726e6676ed" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "1.0.2" @@ -2052,6 +2132,16 @@ dependencies = [ "base64 0.21.2", ] +[[package]] +name = "rustls-webpki" +version = "0.101.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f36a6828982f422756984e47912a7a51dcbc2a197aa791158f8ca61cd8204e" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.12" @@ -2076,6 +2166,15 @@ version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +[[package]] +name = "schannel" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +dependencies = [ + "windows-sys 0.48.0", +] + [[package]] name = "schemars" version = "0.8.12" @@ -2286,6 +2385,29 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "security-framework" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.17" @@ -2370,6 +2492,12 @@ dependencies = [ "digest", ] +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + [[package]] name = "sha2" version = "0.10.6" @@ -2518,7 +2646,7 @@ dependencies = [ "percent-encoding", "rand", "rust_decimal", - "rustls", + "rustls 0.20.8", "rustls-pemfile", "serde", "serde_json", @@ -2564,7 +2692,7 @@ checksum = "804d3f245f894e61b1e6263c84b23ca675d96753b5abfd5cc8597d86806e8024" dependencies = [ "once_cell", "tokio", - "tokio-rustls", + "tokio-rustls 0.23.4", ] [[package]] @@ -2778,11 +2906,21 @@ version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ - "rustls", + "rustls 0.20.8", "tokio", "webpki", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.3", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.14" @@ -2949,6 +3087,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] diff --git a/packages/backend/native-utils/Cargo.toml b/packages/backend/native-utils/Cargo.toml index f93180fe4..6f4dd9175 100644 --- a/packages/backend/native-utils/Cargo.toml +++ b/packages/backend/native-utils/Cargo.toml @@ -9,7 +9,7 @@ members = ["migration"] [features] default = [] noarray = [] -napi = ["dep:napi", "dep:napi-derive", "dep:radix_fmt"] +napi = ["dep:napi", "dep:napi-derive"] [lib] crate-type = ["cdylib", "lib"] @@ -31,11 +31,11 @@ serde_json = "1.0.96" thiserror = "1.0.40" tokio = { version = "1.28.1", features = ["full"] } utoipa = "3.3.0" +radix_fmt = "1.0.0" # Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix napi = { version = "2.13.1", default-features = false, features = ["napi6", "tokio_rt"], optional = true } napi-derive = { version = "2.12.0", optional = true } -radix_fmt = { version = "1.0.0", optional = true } [dev-dependencies] pretty_assertions = "1.3.0" diff --git a/packages/backend/native-utils/__test__/index.spec.mjs b/packages/backend/native-utils/__test__/index.spec.mjs index 6e6a91858..9ff8ead6c 100644 --- a/packages/backend/native-utils/__test__/index.spec.mjs +++ b/packages/backend/native-utils/__test__/index.spec.mjs @@ -12,18 +12,18 @@ test("convert to mastodon id", (t) => { t.is(convertId("9gf61ehcxv", IdConvertType.MastodonId), "960365976481219"); t.is( convertId("9fbr9z0wbrjqyd3u", IdConvertType.MastodonId), - "3954607381600562394", + "2083785058661759970208986", ); t.is( convertId("9fbs680oyviiqrol9md73p8g", IdConvertType.MastodonId), - "3494513243013053824", + "5878598648988104013828532260828151168", ); }); test("create cuid2 with timestamp prefix", (t) => { nativeInitIdGenerator(16, ""); - t.not(nativeCreateId(BigInt(Date.now())), nativeCreateId(BigInt(Date.now()))); - t.is(nativeCreateId(BigInt(Date.now())).length, 16); + t.not(nativeCreateId(Date.now()), nativeCreateId(Date.now())); + t.is(nativeCreateId(Date.now()).length, 16); }); test("create random string", (t) => { diff --git a/packages/backend/native-utils/migration/Cargo.toml b/packages/backend/native-utils/migration/Cargo.toml index 7ed9fd5f0..1e5fa57c3 100644 --- a/packages/backend/native-utils/migration/Cargo.toml +++ b/packages/backend/native-utils/migration/Cargo.toml @@ -10,17 +10,20 @@ path = "src/lib.rs" [features] default = [] -convert = ["dep:native-utils", "dep:indicatif", "dep:futures"] +convert = ["dep:indicatif"] [dependencies] serde_json = "1.0.96" -native-utils = { path = "../", optional = true } +native-utils = { path = "../" } indicatif = { version = "0.17.4", features = ["tokio"], optional = true } tokio = { version = "1.28.2", features = ["full"] } -futures = { version = "0.3.28", optional = true } +futures = "0.3.28" serde_yaml = "0.9.21" serde = { version = "1.0.163", features = ["derive"] } urlencoding = "2.1.2" +redis = { version = "0.23.0", features = ["tokio-rustls-comp"] } +sea-orm = "0.11.3" +url = { version = "2.4.0", features = ["serde"] } [dependencies.sea-orm-migration] version = "0.11.0" diff --git a/packages/backend/native-utils/migration/src/lib.rs b/packages/backend/native-utils/migration/src/lib.rs index 94e2b08cc..5ad23f162 100644 --- a/packages/backend/native-utils/migration/src/lib.rs +++ b/packages/backend/native-utils/migration/src/lib.rs @@ -2,6 +2,7 @@ pub use sea_orm_migration::prelude::*; mod m20230531_180824_drop_reversi; mod m20230627_185451_index_note_url; +mod m20230709_000510_move_antenna_to_cache; pub struct Migrator; @@ -11,6 +12,7 @@ impl MigratorTrait for Migrator { vec![ Box::new(m20230531_180824_drop_reversi::Migration), Box::new(m20230627_185451_index_note_url::Migration), + Box::new(m20230709_000510_move_antenna_to_cache::Migration), ] } } diff --git a/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs b/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs new file mode 100644 index 000000000..7c38fd1bd --- /dev/null +++ b/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs @@ -0,0 +1,240 @@ +use redis::streams::StreamMaxlen; +use sea_orm::Statement; +use sea_orm_migration::prelude::*; +use std::env; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + let cache_url = env::var("CACHE_URL").unwrap(); + let copy_limit = env::var("ANTENNA_MIGRATION_LIMIT").unwrap_or_default(); + let copy_limit: i64 = match copy_limit.parse() { + Ok(limit) => limit, + Err(_) => 0, + }; + + if cache_url != "no" { + let prefix = env::var("CACHE_PREFIX").unwrap(); + + let db = manager.get_connection(); + let bk = manager.get_database_backend(); + + let count_stmt = + Statement::from_string(bk, "SELECT COUNT(1) FROM antenna_note".to_owned()); + let total_num = db + .query_one(count_stmt) + .await? + .unwrap() + .try_get_by_index::(0)?; + let copy_limit = if copy_limit == 0 { + total_num + } else { + copy_limit + }; + println!( + "Copying {} out of {} entries in antenna_note.", + copy_limit, total_num + ); + + let stmt_base = Query::select() + .column((AntennaNote::Table, AntennaNote::Id)) + .column(AntennaNote::AntennaId) + .column(AntennaNote::NoteId) + .from(AntennaNote::Table) + .order_by((AntennaNote::Table, AntennaNote::Id), Order::Asc) + .limit(1000) + .to_owned(); + + let mut stmt = stmt_base.clone(); + + let client = redis::Client::open(cache_url).unwrap(); + let mut redis_conn = client.get_connection().unwrap(); + + let mut remaining = total_num; + let mut pagination: i64 = 0; + + loop { + let res = db.query_all(bk.build(&stmt)).await?; + if res.len() == 0 { + break; + } + let val: Vec<(String, String, String)> = res + .iter() + .filter_map(|q| q.try_get_many_by_index().ok()) + .collect(); + + remaining -= val.len() as i64; + if remaining <= copy_limit { + let mut pipe = redis::pipe(); + for v in &val { + pipe.xadd_maxlen( + format!("{}:antennaTimeline:{}", prefix, v.1), + StreamMaxlen::Approx(200), + "*", + &[("note", v.2.to_owned())], + ) + .ignore(); + } + pipe.query::<()>(&mut redis_conn).unwrap(); + } + + let copied = total_num - remaining; + let copied = std::cmp::min(copied, total_num); + pagination += 1; + if pagination % 100 == 0 { + println!( + "Migrating antenna [{:.2}%]", + (copied as f64 / total_num as f64) * 100_f64, + ); + } + + if let Some((last_id, _, _)) = val.last() { + stmt = stmt_base + .clone() + .and_where( + Expr::col((AntennaNote::Table, AntennaNote::Id)).gt(last_id.to_owned()), + ) + .to_owned(); + } else { + break; + } + } + } + println!("Migrating antenna [100.00%]"); + + manager + .drop_table( + Table::drop() + .table(AntennaNote::Table) + .if_exists() + .to_owned(), + ) + .await?; + + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(AntennaNote::Table) + .if_not_exists() + .col( + ColumnDef::new(AntennaNote::Id) + .string_len(32) + .not_null() + .primary_key(), + ) + .col( + ColumnDef::new(AntennaNote::NoteId) + .string_len(32) + .not_null(), + ) + .col( + ColumnDef::new(AntennaNote::AntennaId) + .string_len(32) + .not_null(), + ) + .col( + ColumnDef::new(AntennaNote::Read) + .boolean() + .default(false) + .not_null(), + ) + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .name("IDX_0d775946662d2575dfd2068a5f") + .table(AntennaNote::Table) + .col(AntennaNote::AntennaId) + .if_not_exists() + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .name("IDX_bd0397be22147e17210940e125") + .table(AntennaNote::Table) + .col(AntennaNote::NoteId) + .if_not_exists() + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .name("IDX_335a0bf3f904406f9ef3dd51c2") + .table(AntennaNote::Table) + .col(AntennaNote::NoteId) + .col(AntennaNote::AntennaId) + .unique() + .if_not_exists() + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .name("IDX_9937ea48d7ae97ffb4f3f063a4") + .table(AntennaNote::Table) + .col(AntennaNote::Read) + .if_not_exists() + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_0d775946662d2575dfd2068a5f5") + .from(AntennaNote::Table, AntennaNote::AntennaId) + .to(Antenna::Table, Antenna::Id) + .on_delete(ForeignKeyAction::Cascade) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_bd0397be22147e17210940e125b") + .from(AntennaNote::Table, AntennaNote::NoteId) + .to(Note::Table, Note::Id) + .on_delete(ForeignKeyAction::Cascade) + .to_owned(), + ) + .await?; + + Ok(()) + } +} + +/// Learn more at https://docs.rs/sea-query#iden +#[derive(Iden)] +enum AntennaNote { + Table, + Id, + #[iden = "noteId"] + NoteId, + #[iden = "antennaId"] + AntennaId, + Read, +} + +#[derive(Iden)] +enum Antenna { + Table, + Id, +} + +#[derive(Iden)] +enum Note { + Table, + Id, +} diff --git a/packages/backend/native-utils/migration/src/main.rs b/packages/backend/native-utils/migration/src/main.rs index f0f761f65..fb9920b67 100644 --- a/packages/backend/native-utils/migration/src/main.rs +++ b/packages/backend/native-utils/migration/src/main.rs @@ -5,6 +5,10 @@ use urlencoding::encode; use sea_orm_migration::prelude::*; +const DB_URL_ENV: &str = "DATABASE_URL"; +const CACHE_URL_ENV: &str = "CACHE_URL"; +const CACHE_PREFIX_ENV: &str = "CACHE_PREFIX"; + #[cfg(feature = "convert")] mod vec_to_json; @@ -15,17 +19,48 @@ async fn main() { .expect("Failed to open '.config/default.yml'"); let config: Config = serde_yaml::from_reader(yml).expect("Failed to parse yaml"); - env::set_var( - "DATABASE_URL", - format!( - "postgres://{}:{}@{}:{}/{}", - config.db.user, - encode(&config.db.pass), - config.db.host, - config.db.port, - config.db.db, - ), - ); + if env::var_os(DB_URL_ENV).is_none() { + env::set_var( + DB_URL_ENV, + format!( + "postgres://{}:{}@{}:{}/{}", + config.db.user, + encode(&config.db.pass), + config.db.host, + config.db.port, + config.db.db, + ), + ); + }; + + if env::var_os(CACHE_URL_ENV).is_none() { + let redis_conf = match config.cache_server { + None => config.redis, + Some(conf) => conf, + }; + let redis_proto = match redis_conf.tls { + None => "redis", + Some(_) => "rediss", + }; + let redis_uri_userpass = match redis_conf.user { + None => "".to_string(), + Some(user) => format!("{}:{}@", user, redis_conf.pass.unwrap_or_default()), + }; + let redis_uri_hostport = format!("{}:{}", redis_conf.host, redis_conf.port); + let redis_uri = format!( + "{}://{}{}", + redis_proto, redis_uri_userpass, redis_uri_hostport + ); + env::set_var(CACHE_URL_ENV, redis_uri); + env::set_var( + CACHE_PREFIX_ENV, + if redis_conf.prefix.is_empty() { + config.url.host_str().unwrap() + } else { + &redis_conf.prefix + }, + ); + } cli::run_cli(migration::Migrator).await; @@ -36,7 +71,10 @@ async fn main() { #[derive(Debug, PartialEq, Deserialize)] #[serde(rename = "camelCase")] pub struct Config { + pub url: url::Url, pub db: DbConfig, + pub redis: RedisConfig, + pub cache_server: Option, } #[derive(Debug, PartialEq, Deserialize)] @@ -48,3 +86,24 @@ pub struct DbConfig { pub user: String, pub pass: String, } + +#[derive(Debug, PartialEq, Deserialize)] +#[serde(rename = "camelCase")] +pub struct RedisConfig { + pub host: String, + pub port: u32, + pub user: Option, + pub pass: Option, + pub tls: Option, + #[serde(default)] + pub db: u32, + #[serde(default)] + pub prefix: String, +} + +#[derive(Debug, PartialEq, Deserialize)] +#[serde(rename = "camelCase")] +pub struct TlsConfig { + pub host: String, + pub reject_unauthorized: bool, +} diff --git a/packages/backend/native-utils/src/model/schema/app.rs b/packages/backend/native-utils/src/model/schema/app.rs index 682b82ec0..d13e9ef36 100644 --- a/packages/backend/native-utils/src/model/schema/app.rs +++ b/packages/backend/native-utils/src/model/schema/app.rs @@ -105,9 +105,9 @@ mod unit_test { #[test] fn app_valid() { - init_id(12, ""); + init_id(16, ""); let instance = json!({ - "id": create_id().unwrap(), + "id": create_id(0).unwrap(), "name": "Test App", "secret": gen_string(24), "callbackUrl": "urn:ietf:wg:oauth:2.0:oob", @@ -119,9 +119,9 @@ mod unit_test { #[test] fn app_invalid() { - init_id(12, ""); + init_id(16, ""); let instance = json!({ - "id": create_id().unwrap(), + "id": create_id(0).unwrap(), // "name" is required "name": null, // "permission" must be one of the app permissions diff --git a/packages/backend/native-utils/src/util/id.rs b/packages/backend/native-utils/src/util/id.rs index d922518f9..b18637fdb 100644 --- a/packages/backend/native-utils/src/util/id.rs +++ b/packages/backend/native-utils/src/util/id.rs @@ -1,7 +1,10 @@ //! ID generation utility based on [cuid2] use cfg_if::cfg_if; +use chrono::Utc; use once_cell::sync::OnceCell; +use radix_fmt::radix_36; +use std::cmp; use crate::impl_into_napi_error; @@ -14,47 +17,56 @@ impl_into_napi_error!(ErrorUninitialized); static FINGERPRINT: OnceCell = OnceCell::new(); static GENERATOR: OnceCell = OnceCell::new(); +const TIME_2000: i64 = 946_684_800_000; +const TIMESTAMP_LENGTH: u16 = 8; + /// Initializes Cuid2 generator. Must be called before any [create_id]. -pub fn init_id(length: u16, fingerprint: impl Into) { - FINGERPRINT.get_or_init(move || format!("{}{}", fingerprint.into(), cuid2::create_id())); +pub fn init_id<'a>(length: u16, fingerprint: &'a str) { + FINGERPRINT.get_or_init(move || format!("{}{}", fingerprint, cuid2::create_id())); GENERATOR.get_or_init(move || { cuid2::CuidConstructor::new() - .with_length(length) + // length to pass shoule be greater than or equal to 8. + .with_length(cmp::max(length - TIMESTAMP_LENGTH, 8)) .with_fingerprinter(|| FINGERPRINT.get().unwrap().clone()) }); } /// Returns Cuid2 with the length specified by [init_id]. Must be called after /// [init_id], otherwise returns [ErrorUninitialized]. -pub fn create_id() -> Result { +/// The current timestamp via [chrono::Utc] is used if `date_num` is `0`. +pub fn create_id(date_num: i64) -> Result { match GENERATOR.get() { None => Err(ErrorUninitialized), - Some(gen) => Ok(gen.create_id()), + Some(gen) => { + let date_num = if date_num > 0 { + date_num + } else { + Utc::now().timestamp_millis() + }; + let time = cmp::max(date_num - TIME_2000, 0); + Ok(format!( + "{:0>8}{}", + radix_36(time).to_string(), + gen.create_id() + )) + } } } cfg_if! { if #[cfg(feature = "napi")] { - use radix_fmt::radix_36; - use std::cmp; - use napi::bindgen_prelude::BigInt; use napi_derive::napi; - const TIME_2000: u64 = 946_684_800_000; - const TIMESTAMP_LENGTH: u16 = 8; - /// Calls [init_id] inside. Must be called before [native_create_id]. #[napi] pub fn native_init_id_generator(length: u16, fingerprint: String) { - // length to pass init_id shoule be greater than or equal to 8. - init_id(cmp::max(length - TIMESTAMP_LENGTH, 8), fingerprint); + init_id(length, &fingerprint); } /// Generates #[napi] - pub fn native_create_id(date_num: BigInt) -> String { - let time = cmp::max(date_num.get_u64().1 - TIME_2000, 0); - format!("{:0>8}{}", radix_36(time).to_string(), create_id().unwrap()) + pub fn native_create_id(date_num: i64) -> String { + create_id(date_num).unwrap() } } } @@ -62,37 +74,17 @@ cfg_if! { #[cfg(test)] mod unit_test { use crate::util::id; - use cfg_if::cfg_if; use pretty_assertions::{assert_eq, assert_ne}; use std::thread; - cfg_if! { - if #[cfg(feature = "napi")] { - use chrono::Utc; - - #[test] - fn can_generate_aid_compat_ids() { - id::native_init_id_generator(20, "".to_string()); - let id1 = id::native_create_id(Utc::now().timestamp_millis().into()); - assert_eq!(id1.len(), 20); - let id1 = id::native_create_id(Utc::now().timestamp_millis().into()); - let id2 = id::native_create_id(Utc::now().timestamp_millis().into()); - assert_ne!(id1, id2); - let id1 = thread::spawn(|| id::native_create_id(Utc::now().timestamp_millis().into())); - let id2 = thread::spawn(|| id::native_create_id(Utc::now().timestamp_millis().into())); - assert_ne!(id1.join().unwrap(), id2.join().unwrap()); - } - } else { - #[test] - fn can_generate_unique_ids() { - assert_eq!(id::create_id(), Err(id::ErrorUninitialized)); - id::init_id(12, ""); - assert_eq!(id::create_id().unwrap().len(), 12); - assert_ne!(id::create_id().unwrap(), id::create_id().unwrap()); - let id1 = thread::spawn(|| id::create_id().unwrap()); - let id2 = thread::spawn(|| id::create_id().unwrap()); - assert_ne!(id1.join().unwrap(), id2.join().unwrap()); - } - } + #[test] + fn can_generate_unique_ids() { + assert_eq!(id::create_id(0), Err(id::ErrorUninitialized)); + id::init_id(16, ""); + assert_eq!(id::create_id(0).unwrap().len(), 16); + assert_ne!(id::create_id(0).unwrap(), id::create_id(0).unwrap()); + let id1 = thread::spawn(|| id::create_id(0).unwrap()); + let id2 = thread::spawn(|| id::create_id(0).unwrap()); + assert_ne!(id1.join().unwrap(), id2.join().unwrap()); } } diff --git a/packages/backend/native-utils/tests/common.rs b/packages/backend/native-utils/tests/common.rs index 186e862bd..31c6ef053 100644 --- a/packages/backend/native-utils/tests/common.rs +++ b/packages/backend/native-utils/tests/common.rs @@ -139,11 +139,11 @@ async fn cleanup() { } async fn setup_model(db: &DbConn) { - init_id(12, ""); + init_id(16, ""); db.transaction::<_, (), DbErr>(|txn| { Box::pin(async move { - let user_id = create_id().unwrap(); + let user_id = create_id(0).unwrap(); let name = "Alice"; let user_model = entity::user::Model { id: user_id.to_owned(), @@ -161,7 +161,7 @@ async fn setup_model(db: &DbConn) { .insert(txn) .await?; let antenna_model = entity::antenna::Model { - id: create_id().unwrap(), + id: create_id(0).unwrap(), created_at: Utc::now().into(), user_id: user_id.to_owned(), name: "Alice Antenna".to_string(), @@ -186,7 +186,7 @@ async fn setup_model(db: &DbConn) { .insert(txn) .await?; let note_model = entity::note::Model { - id: create_id().unwrap(), + id: create_id(0).unwrap(), created_at: Utc::now().into(), text: Some("Testing 123".to_string()), user_id: user_id.to_owned(), diff --git a/packages/backend/native-utils/tests/model/repository/antenna.rs b/packages/backend/native-utils/tests/model/repository/antenna.rs index 3bda2ca18..02ef8f5be 100644 --- a/packages/backend/native-utils/tests/model/repository/antenna.rs +++ b/packages/backend/native-utils/tests/model/repository/antenna.rs @@ -95,7 +95,7 @@ mod int_test { .unwrap() .expect("note not found"); let antenna_note = antenna_note::Model { - id: util::id::create_id().unwrap(), + id: util::id::create_id(0).unwrap(), antenna_id: alice_antenna.id.to_owned(), note_id: note_model.id.to_owned(), read: false, diff --git a/packages/backend/src/misc/gen-id.ts b/packages/backend/src/misc/gen-id.ts index ea0d414e7..580c39c3c 100644 --- a/packages/backend/src/misc/gen-id.ts +++ b/packages/backend/src/misc/gen-id.ts @@ -17,5 +17,5 @@ nativeInitIdGenerator(length, fingerprint); * Ref: https://github.com/paralleldrive/cuid2#parameterized-length */ export function genId(date?: Date): string { - return nativeCreateId(BigInt((date ?? new Date()).getTime())); + return nativeCreateId((date ?? new Date()).getTime()); } diff --git a/packages/backend/src/services/add-note-to-antenna.ts b/packages/backend/src/services/add-note-to-antenna.ts index 131c0348c..8d6d3e84d 100644 --- a/packages/backend/src/services/add-note-to-antenna.ts +++ b/packages/backend/src/services/add-note-to-antenna.ts @@ -8,17 +8,14 @@ import type { User } from "@/models/entities/user.js"; export async function addNoteToAntenna( antenna: Antenna, note: Note, - noteUser: { id: User["id"] }, + _noteUser: { id: User["id"] }, ) { - // 通知しない設定になっているか、自分自身の投稿なら既読にする - const read = !antenna.notify || antenna.userId === noteUser.id; - redisClient.xadd( `antennaTimeline:${antenna.id}`, "MAXLEN", "~", "200", - `${genId(note.createdAt)}-*`, + "*", "note", note.id, ); From 4d36c7f4ed7eba991167018c6ae49097bf3472cc Mon Sep 17 00:00:00 2001 From: Namekuji Date: Mon, 10 Jul 2023 02:17:06 -0400 Subject: [PATCH 06/26] fix: renew entity models --- packages/backend/native-utils/src/model/entity.rs | 1 - .../native-utils/src/model/entity/announcement.rs | 4 ++++ .../backend/native-utils/src/model/entity/antenna.rs | 8 -------- .../backend/native-utils/src/model/entity/meta.rs | 4 ++++ .../backend/native-utils/src/model/entity/note.rs | 8 -------- .../backend/native-utils/src/model/entity/prelude.rs | 1 - .../native-utils/src/model/repository/antenna.rs | 12 +++--------- 7 files changed, 11 insertions(+), 27 deletions(-) diff --git a/packages/backend/native-utils/src/model/entity.rs b/packages/backend/native-utils/src/model/entity.rs index d71057fde..ccf3f0060 100644 --- a/packages/backend/native-utils/src/model/entity.rs +++ b/packages/backend/native-utils/src/model/entity.rs @@ -8,7 +8,6 @@ pub mod ad; pub mod announcement; pub mod announcement_read; pub mod antenna; -pub mod antenna_note; pub mod app; pub mod attestation_challenge; pub mod auth_session; diff --git a/packages/backend/native-utils/src/model/entity/announcement.rs b/packages/backend/native-utils/src/model/entity/announcement.rs index e8a2a28aa..5cdb690d2 100644 --- a/packages/backend/native-utils/src/model/entity/announcement.rs +++ b/packages/backend/native-utils/src/model/entity/announcement.rs @@ -15,6 +15,10 @@ pub struct Model { pub image_url: Option, #[sea_orm(column_name = "updatedAt")] pub updated_at: Option, + #[sea_orm(column_name = "showPopup")] + pub show_popup: bool, + #[sea_orm(column_name = "isGoodNews")] + pub is_good_news: bool, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/packages/backend/native-utils/src/model/entity/antenna.rs b/packages/backend/native-utils/src/model/entity/antenna.rs index 85bdfbfea..9d5a64f82 100644 --- a/packages/backend/native-utils/src/model/entity/antenna.rs +++ b/packages/backend/native-utils/src/model/entity/antenna.rs @@ -37,8 +37,6 @@ pub struct Model { #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] pub enum Relation { - #[sea_orm(has_many = "super::antenna_note::Entity")] - AntennaNote, #[sea_orm( belongs_to = "super::user::Entity", from = "Column::UserId", @@ -65,12 +63,6 @@ pub enum Relation { UserList, } -impl Related for Entity { - fn to() -> RelationDef { - Relation::AntennaNote.def() - } -} - impl Related for Entity { fn to() -> RelationDef { Relation::User.def() diff --git a/packages/backend/native-utils/src/model/entity/meta.rs b/packages/backend/native-utils/src/model/entity/meta.rs index 2c0dc315c..ac43a7842 100644 --- a/packages/backend/native-utils/src/model/entity/meta.rs +++ b/packages/backend/native-utils/src/model/entity/meta.rs @@ -189,6 +189,10 @@ pub struct Model { pub silenced_hosts: StringVec, #[sea_orm(column_name = "experimentalFeatures", column_type = "JsonBinary")] pub experimental_features: Json, + #[sea_orm(column_name = "enableServerMachineStats")] + pub enable_server_machine_stats: bool, + #[sea_orm(column_name = "enableIdenticonGeneration")] + pub enable_identicon_generation: bool, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/packages/backend/native-utils/src/model/entity/note.rs b/packages/backend/native-utils/src/model/entity/note.rs index 077841e48..5c97a20cf 100644 --- a/packages/backend/native-utils/src/model/entity/note.rs +++ b/packages/backend/native-utils/src/model/entity/note.rs @@ -67,8 +67,6 @@ pub struct Model { #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] pub enum Relation { - #[sea_orm(has_many = "super::antenna_note::Entity")] - AntennaNote, #[sea_orm( belongs_to = "super::channel::Entity", from = "Column::ChannelId", @@ -131,12 +129,6 @@ pub enum Relation { UserNotePining, } -impl Related for Entity { - fn to() -> RelationDef { - Relation::AntennaNote.def() - } -} - impl Related for Entity { fn to() -> RelationDef { Relation::Channel.def() diff --git a/packages/backend/native-utils/src/model/entity/prelude.rs b/packages/backend/native-utils/src/model/entity/prelude.rs index 8be696cb4..a859a9cc3 100644 --- a/packages/backend/native-utils/src/model/entity/prelude.rs +++ b/packages/backend/native-utils/src/model/entity/prelude.rs @@ -6,7 +6,6 @@ pub use super::ad::Entity as Ad; pub use super::announcement::Entity as Announcement; pub use super::announcement_read::Entity as AnnouncementRead; pub use super::antenna::Entity as Antenna; -pub use super::antenna_note::Entity as AntennaNote; pub use super::app::Entity as App; pub use super::attestation_challenge::Entity as AttestationChallenge; pub use super::auth_session::Entity as AuthSession; diff --git a/packages/backend/native-utils/src/model/repository/antenna.rs b/packages/backend/native-utils/src/model/repository/antenna.rs index 7c614b954..36c641cb0 100644 --- a/packages/backend/native-utils/src/model/repository/antenna.rs +++ b/packages/backend/native-utils/src/model/repository/antenna.rs @@ -1,9 +1,9 @@ use async_trait::async_trait; use cfg_if::cfg_if; -use sea_orm::{ColumnTrait, EntityTrait, QueryFilter}; +use sea_orm::EntityTrait; use crate::database; -use crate::model::entity::{antenna, antenna_note, user_group_joining}; +use crate::model::entity::{antenna, user_group_joining}; use crate::model::error::Error; use crate::model::schema::Antenna; @@ -14,12 +14,6 @@ use super::Repository; impl Repository for antenna::Model { async fn pack(self) -> Result { let db = database::get_database()?; - let has_unread_note = antenna_note::Entity::find() - .filter(antenna_note::Column::AntennaId.eq(self.id.to_owned())) - .filter(antenna_note::Column::Read.eq(false)) - .one(db) - .await? - .is_some(); let user_group_joining = match self.user_group_joining_id { None => None, Some(id) => user_group_joining::Entity::find_by_id(id).one(db).await?, @@ -52,7 +46,7 @@ impl Repository for antenna::Model { notify: self.notify, with_replies: self.with_replies, with_file: self.with_file, - has_unread_note, + has_unread_note: false, }) } From 74eb19acdd7353a04ef3057478fa0454f5159d14 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Mon, 10 Jul 2023 02:31:50 -0400 Subject: [PATCH 07/26] add environment variable to skip copying antenna --- .../src/m20230709_000510_move_antenna_to_cache.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs b/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs index 7c38fd1bd..d285678c9 100644 --- a/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs +++ b/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs @@ -10,13 +10,16 @@ pub struct Migration; impl MigrationTrait for Migration { async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { let cache_url = env::var("CACHE_URL").unwrap(); + let skip_copy = env::var("ANTENNA_MIGRATION_SKIP").unwrap_or_default(); let copy_limit = env::var("ANTENNA_MIGRATION_LIMIT").unwrap_or_default(); let copy_limit: i64 = match copy_limit.parse() { Ok(limit) => limit, Err(_) => 0, }; - if cache_url != "no" { + if skip_copy == "true" { + println!("Skipped antenna migration"); + } else { let prefix = env::var("CACHE_PREFIX").unwrap(); let db = manager.get_connection(); @@ -102,8 +105,9 @@ impl MigrationTrait for Migration { break; } } + + println!("Migrating antenna [100.00%]"); } - println!("Migrating antenna [100.00%]"); manager .drop_table( From a9ac5fcfeca57182edeb88a99ac721c5f46dd491 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Mon, 10 Jul 2023 02:51:54 -0400 Subject: [PATCH 08/26] add environment variable of read size --- .../src/m20230709_000510_move_antenna_to_cache.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs b/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs index d285678c9..a514e9b8e 100644 --- a/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs +++ b/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs @@ -11,7 +11,11 @@ impl MigrationTrait for Migration { async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { let cache_url = env::var("CACHE_URL").unwrap(); let skip_copy = env::var("ANTENNA_MIGRATION_SKIP").unwrap_or_default(); - let copy_limit = env::var("ANTENNA_MIGRATION_LIMIT").unwrap_or_default(); + let copy_limit = env::var("ANTENNA_MIGRATION_COPY_LIMIT").unwrap_or_default(); + let read_limit: u64 = env::var("ANTENNA_MIGRATION_READ_LIMIT") + .unwrap_or("10000".to_string()) + .parse() + .unwrap(); let copy_limit: i64 = match copy_limit.parse() { Ok(limit) => limit, Err(_) => 0, @@ -48,7 +52,7 @@ impl MigrationTrait for Migration { .column(AntennaNote::NoteId) .from(AntennaNote::Table) .order_by((AntennaNote::Table, AntennaNote::Id), Order::Asc) - .limit(1000) + .limit(read_limit) .to_owned(); let mut stmt = stmt_base.clone(); @@ -87,7 +91,7 @@ impl MigrationTrait for Migration { let copied = total_num - remaining; let copied = std::cmp::min(copied, total_num); pagination += 1; - if pagination % 100 == 0 { + if pagination % 10 == 0 { println!( "Migrating antenna [{:.2}%]", (copied as f64 / total_num as f64) * 100_f64, From b6d909f79d4203456ae30d220af086fbb97e3266 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Mon, 10 Jul 2023 03:23:25 -0400 Subject: [PATCH 09/26] fix config parser --- packages/backend/native-utils/migration/src/main.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/backend/native-utils/migration/src/main.rs b/packages/backend/native-utils/migration/src/main.rs index fb9920b67..a7b649c51 100644 --- a/packages/backend/native-utils/migration/src/main.rs +++ b/packages/backend/native-utils/migration/src/main.rs @@ -69,7 +69,7 @@ async fn main() { } #[derive(Debug, PartialEq, Deserialize)] -#[serde(rename = "camelCase")] +#[serde(rename_all = "camelCase")] pub struct Config { pub url: url::Url, pub db: DbConfig, @@ -78,7 +78,6 @@ pub struct Config { } #[derive(Debug, PartialEq, Deserialize)] -#[serde(rename = "camelCase")] pub struct DbConfig { pub host: String, pub port: u32, @@ -88,7 +87,6 @@ pub struct DbConfig { } #[derive(Debug, PartialEq, Deserialize)] -#[serde(rename = "camelCase")] pub struct RedisConfig { pub host: String, pub port: u32, @@ -102,7 +100,7 @@ pub struct RedisConfig { } #[derive(Debug, PartialEq, Deserialize)] -#[serde(rename = "camelCase")] +#[serde(rename_all = "camelCase")] pub struct TlsConfig { pub host: String, pub reject_unauthorized: bool, From 4f0c6c94aa7b6d0635adcfa21449836b6ab624eb Mon Sep 17 00:00:00 2001 From: Namekuji Date: Mon, 10 Jul 2023 03:43:44 -0400 Subject: [PATCH 10/26] remove unnecessary migration file --- packages/backend/migration/1680491187535-cleanup.js | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 packages/backend/migration/1680491187535-cleanup.js diff --git a/packages/backend/migration/1680491187535-cleanup.js b/packages/backend/migration/1680491187535-cleanup.js deleted file mode 100644 index 671f7521e..000000000 --- a/packages/backend/migration/1680491187535-cleanup.js +++ /dev/null @@ -1,7 +0,0 @@ -export class cleanup1680491187535 { - name = "cleanup1680491187535"; - - async up(queryRunner) {} - - async down(queryRunner) {} -} From b9794dd62557ebf41030534ac302ab38acac1495 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Mon, 10 Jul 2023 03:48:46 -0400 Subject: [PATCH 11/26] make unused dependencies optional --- .../backend/native-utils/migration/Cargo.toml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/backend/native-utils/migration/Cargo.toml b/packages/backend/native-utils/migration/Cargo.toml index 1e5fa57c3..9bf793e04 100644 --- a/packages/backend/native-utils/migration/Cargo.toml +++ b/packages/backend/native-utils/migration/Cargo.toml @@ -10,14 +10,14 @@ path = "src/lib.rs" [features] default = [] -convert = ["dep:indicatif"] +convert = ["dep:native-utils", "dep:indicatif", "dep:futures"] [dependencies] serde_json = "1.0.96" -native-utils = { path = "../" } +native-utils = { path = "../", optional = true } indicatif = { version = "0.17.4", features = ["tokio"], optional = true } tokio = { version = "1.28.2", features = ["full"] } -futures = "0.3.28" +futures = { version = "0.3.28", optional = true } serde_yaml = "0.9.21" serde = { version = "1.0.163", features = ["derive"] } urlencoding = "2.1.2" @@ -28,10 +28,10 @@ url = { version = "2.4.0", features = ["serde"] } [dependencies.sea-orm-migration] version = "0.11.0" features = [ - # Enable at least one `ASYNC_RUNTIME` and `DATABASE_DRIVER` feature if you want to run migration via CLI. - # View the list of supported features at https://www.sea-ql.org/SeaORM/docs/install-and-config/database-and-async-runtime. - # e.g. - "runtime-tokio-rustls", # `ASYNC_RUNTIME` feature - "sqlx-postgres", # `DATABASE_DRIVER` feature + # Enable at least one `ASYNC_RUNTIME` and `DATABASE_DRIVER` feature if you want to run migration via CLI. + # View the list of supported features at https://www.sea-ql.org/SeaORM/docs/install-and-config/database-and-async-runtime. + # e.g. + "runtime-tokio-rustls", # `ASYNC_RUNTIME` feature + "sqlx-postgres", # `DATABASE_DRIVER` feature "sqlx-sqlite", ] From 5291fd4f10f032ce00a423773f2f7fc9a29404a2 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Mon, 10 Jul 2023 03:54:28 -0400 Subject: [PATCH 12/26] fix: add db to redis uri --- packages/backend/native-utils/migration/src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/backend/native-utils/migration/src/main.rs b/packages/backend/native-utils/migration/src/main.rs index a7b649c51..896f1ed59 100644 --- a/packages/backend/native-utils/migration/src/main.rs +++ b/packages/backend/native-utils/migration/src/main.rs @@ -44,12 +44,12 @@ async fn main() { }; let redis_uri_userpass = match redis_conf.user { None => "".to_string(), - Some(user) => format!("{}:{}@", user, redis_conf.pass.unwrap_or_default()), + Some(user) => format!("{}:{}@", user, encode(&redis_conf.pass.unwrap_or_default())), }; let redis_uri_hostport = format!("{}:{}", redis_conf.host, redis_conf.port); let redis_uri = format!( - "{}://{}{}", - redis_proto, redis_uri_userpass, redis_uri_hostport + "{}://{}{}/{}", + redis_proto, redis_uri_userpass, redis_uri_hostport, redis_conf.db ); env::set_var(CACHE_URL_ENV, redis_uri); env::set_var( From 3fc381c05089392f404855818281d6fcde900b49 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Thu, 13 Jul 2023 18:19:08 -0400 Subject: [PATCH 13/26] refactor: use copy_limit if greater than 0 --- .../migration/src/m20230709_000510_move_antenna_to_cache.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs b/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs index a514e9b8e..83bd0448a 100644 --- a/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs +++ b/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs @@ -36,10 +36,10 @@ impl MigrationTrait for Migration { .await? .unwrap() .try_get_by_index::(0)?; - let copy_limit = if copy_limit == 0 { - total_num - } else { + let copy_limit = if copy_limit > 0 { copy_limit + } else { + total_num }; println!( "Copying {} out of {} entries in antenna_note.", From 324d1a932d313f524181df89603d95cf254d9629 Mon Sep 17 00:00:00 2001 From: CGsama Date: Sat, 15 Jul 2023 23:02:00 -0400 Subject: [PATCH 14/26] handle import mastodon package --- .../backend/src/misc/process-masto-notes.ts | 58 +++++++++++++++++++ .../queue/processors/db/import-masto-post.ts | 33 ++++++----- .../src/queue/processors/db/import-posts.ts | 17 ++++++ 3 files changed, 95 insertions(+), 13 deletions(-) create mode 100644 packages/backend/src/misc/process-masto-notes.ts diff --git a/packages/backend/src/misc/process-masto-notes.ts b/packages/backend/src/misc/process-masto-notes.ts new file mode 100644 index 000000000..1310748d2 --- /dev/null +++ b/packages/backend/src/misc/process-masto-notes.ts @@ -0,0 +1,58 @@ +import * as fs from "node:fs"; +import Logger from "@/services/logger.js"; +import { createTemp, createTempDir } from "./create-temp.js"; +import { downloadUrl } from "./download-url.js"; +import { addFile } from "@/services/drive/add-file.js"; +import { exec } from "node:child_process"; +import { Users } from "@/models/index.js"; + +const logger = new Logger("download-text-file"); + +export async function processMastoNotes( + url: string, + uid: string, +): Promise { + // Create temp file + const [path, cleanup] = await createTemp(); + + const [unzipPath, unzipCleanup] = await createTempDir(); + + logger.info(`Temp file is ${path}`); + + try { + // write content at URL to temp file + await downloadUrl(url, path); + return await processMastoFile(path, unzipPath, uid); + } finally { + cleanup(); + unzipCleanup(); + } +} + +function processMastoFile(fn: string, dir: string, uid: string) { + return new Promise(async (resolve, reject) => { + const user = await Users.findOneBy({ id: uid }); + exec( + `tar -xf ${fn} -C ${dir}`, + async (error: any, stdout: string, stderr: string) => { + if (error) { + reject(error); + } + const outbox = JSON.parse(fs.readFileSync(`${dir}/outbox.json`)); + for (const note of outbox.orderedItems) { + for (const attachment of note.object.attachment) { + const url = attachment.url.replace("..", ""); + try { + const fpath = `${dir}${url}`; + const driveFile = await addFile({ user: user, path: fpath }); + attachment.driveFile = driveFile; + } catch (e) { + logger.error(`Skipped adding file to drive: ${url}`); + } + } + } + resolve(outbox); + }, + ); + }); +} diff --git a/packages/backend/src/queue/processors/db/import-masto-post.ts b/packages/backend/src/queue/processors/db/import-masto-post.ts index 05166b085..1d18008a0 100644 --- a/packages/backend/src/queue/processors/db/import-masto-post.ts +++ b/packages/backend/src/queue/processors/db/import-masto-post.ts @@ -45,19 +45,26 @@ export async function importMastoPost( throw e; } job.progress(80); - const urls = post.object.attachment - .map((x: any) => x.url) - .filter((x: String) => x.startsWith("http")); - const files: DriveFile[] = []; - for (const url of urls) { - try { - const file = await uploadFromUrl({ - url: url, - user: user, - }); - files.push(file); - } catch (e) { - logger.error(`Skipped adding file to drive: ${url}`); + + let files: DriveFile[] = (post.object.attachment || []) + .map((x: any) => x?.driveFile) + .filter((x: any) => x); + + if (files.length == 0) { + const urls = post.object.attachment + .map((x: any) => x.url) + .filter((x: String) => x.startsWith("http")); + files = []; + for (const url of urls) { + try { + const file = await uploadFromUrl({ + url: url, + user: user, + }); + files.push(file); + } catch (e) { + logger.error(`Skipped adding file to drive: ${url}`); + } } } diff --git a/packages/backend/src/queue/processors/db/import-posts.ts b/packages/backend/src/queue/processors/db/import-posts.ts index f92a5f710..54061f8bf 100644 --- a/packages/backend/src/queue/processors/db/import-posts.ts +++ b/packages/backend/src/queue/processors/db/import-posts.ts @@ -1,4 +1,5 @@ import { downloadTextFile } from "@/misc/download-text-file.js"; +import { processMastoNotes } from "@/misc/process-masto-notes.js"; import { Users, DriveFiles } from "@/models/index.js"; import type { DbUserImportPostsJobData } from "@/queue/types.js"; import { queueLogger } from "../../logger.js"; @@ -30,6 +31,22 @@ export async function importPosts( return; } + if (file.name.endsWith("tar.gz")) { + try { + logger.info("Parsing animal style posts in package"); + const outbox = await processMastoNotes(file.url, job.data.user.id); + for (const post of outbox.orderedItems) { + createImportMastoPostJob(job.data.user, post, job.data.signatureCheck); + } + } catch (e) { + // handle error + logger.warn(`Error reading: ${e}`); + } + logger.succ("Imported"); + done(); + return; + } + const json = await downloadTextFile(file.url); try { From 809d4180185a12d9eff48562e81caea864b4f5b3 Mon Sep 17 00:00:00 2001 From: CGsama Date: Sun, 16 Jul 2023 01:41:57 -0400 Subject: [PATCH 15/26] use gunzip-maybe tar-stream to replace exec --- packages/backend/package.json | 2 + .../backend/src/misc/process-masto-notes.ts | 87 ++++++++++++++----- 2 files changed, 67 insertions(+), 22 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index fe8c078a0..6e9f82edd 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -65,6 +65,7 @@ "file-type": "17.1.6", "fluent-ffmpeg": "2.1.2", "got": "12.5.3", + "gunzip-maybe": "^1.4.2", "hpagent": "0.1.2", "ioredis": "5.3.2", "ip-cidr": "3.1.0", @@ -125,6 +126,7 @@ "summaly": "2.7.0", "syslog-pro": "1.0.0", "systeminformation": "5.17.17", + "tar-stream": "^3.1.6", "tesseract.js": "^3.0.3", "tinycolor2": "1.5.2", "tmp": "0.2.1", diff --git a/packages/backend/src/misc/process-masto-notes.ts b/packages/backend/src/misc/process-masto-notes.ts index 1310748d2..2fe7c7a58 100644 --- a/packages/backend/src/misc/process-masto-notes.ts +++ b/packages/backend/src/misc/process-masto-notes.ts @@ -3,10 +3,11 @@ import Logger from "@/services/logger.js"; import { createTemp, createTempDir } from "./create-temp.js"; import { downloadUrl } from "./download-url.js"; import { addFile } from "@/services/drive/add-file.js"; -import { exec } from "node:child_process"; import { Users } from "@/models/index.js"; +import * as tar from 'tar-stream'; +import gunzip from "gunzip-maybe"; -const logger = new Logger("download-text-file"); +const logger = new Logger("process-masto-notes"); export async function processMastoNotes( url: string, @@ -32,27 +33,69 @@ export async function processMastoNotes( function processMastoFile(fn: string, dir: string, uid: string) { return new Promise(async (resolve, reject) => { const user = await Users.findOneBy({ id: uid }); - exec( - `tar -xf ${fn} -C ${dir}`, - async (error: any, stdout: string, stderr: string) => { - if (error) { - reject(error); - } - const outbox = JSON.parse(fs.readFileSync(`${dir}/outbox.json`)); - for (const note of outbox.orderedItems) { - for (const attachment of note.object.attachment) { - const url = attachment.url.replace("..", ""); - try { - const fpath = `${dir}${url}`; - const driveFile = await addFile({ user: user, path: fpath }); - attachment.driveFile = driveFile; - } catch (e) { - logger.error(`Skipped adding file to drive: ${url}`); - } + try{ + logger.info(`Start unzip ${fn}`); + await unzipTarGz(fn, dir); + logger.info(`Unzip to ${dir}`); + const outbox = JSON.parse(fs.readFileSync(`${dir}/outbox.json`)); + for (const note of outbox.orderedItems) { + for (const attachment of note.object.attachment) { + const url = attachment.url.replace("..", ""); + try { + const fpath = `${dir}${url}`; + const driveFile = await addFile({ user: user, path: fpath }); + attachment.driveFile = driveFile; + } catch (e) { + logger.error(`Skipped adding file to drive: ${url}`); } } - resolve(outbox); - }, - ); + } + resolve(outbox); + }catch(e){ + logger.error(`Error on extract masto note package: ${fn}`); + reject(e); + } }); } + +function createFileDir(fn: string){ + if(!fs.existsSync(fn)){ + fs.mkdirSync(fn, {recursive: true}); + fs.rmdirSync(fn); + } +} + +function unzipTarGz(fn: string, dir: string){ + return new Promise(async (resolve, reject) => { + const onErr = (err: any) => { + logger.error(`pipe broken: ${err}`); + reject(); + } + try{ + const extract = tar.extract().on('error', onErr); + dir = dir.endsWith("/") ? dir : dir + "/"; + const ls: string[] = []; + extract.on('entry', function (header: any, stream: any, next: any) { + try{ + ls.push(dir + header.name); + createFileDir(dir + header.name); + stream.on('error', onErr).pipe(fs.createWriteStream(dir + header.name)).on('error', onErr); + next(); + }catch(e){ + logger.error(`create dir error:${e}`); + reject(); + } + }); + + extract.on('finish', function () { + resolve(ls); + }); + + fs.createReadStream(fn).on('error', onErr).pipe(gunzip()).on('error', onErr).pipe(extract).on('error', onErr); + + }catch(e){ + logger.error(`unzipTarGz error: ${e}`); + reject(); + } + }); +} \ No newline at end of file From 0c9ab9fdfa5ea1e0be71b6abb680e5c7ef97c116 Mon Sep 17 00:00:00 2001 From: CGsama Date: Sun, 16 Jul 2023 01:47:06 -0400 Subject: [PATCH 16/26] fix format --- .../backend/src/misc/process-masto-notes.ts | 49 +++++++++++-------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/packages/backend/src/misc/process-masto-notes.ts b/packages/backend/src/misc/process-masto-notes.ts index 2fe7c7a58..f51764ae8 100644 --- a/packages/backend/src/misc/process-masto-notes.ts +++ b/packages/backend/src/misc/process-masto-notes.ts @@ -4,7 +4,7 @@ import { createTemp, createTempDir } from "./create-temp.js"; import { downloadUrl } from "./download-url.js"; import { addFile } from "@/services/drive/add-file.js"; import { Users } from "@/models/index.js"; -import * as tar from 'tar-stream'; +import * as tar from "tar-stream"; import gunzip from "gunzip-maybe"; const logger = new Logger("process-masto-notes"); @@ -33,7 +33,7 @@ export async function processMastoNotes( function processMastoFile(fn: string, dir: string, uid: string) { return new Promise(async (resolve, reject) => { const user = await Users.findOneBy({ id: uid }); - try{ + try { logger.info(`Start unzip ${fn}`); await unzipTarGz(fn, dir); logger.info(`Unzip to ${dir}`); @@ -51,51 +51,58 @@ function processMastoFile(fn: string, dir: string, uid: string) { } } resolve(outbox); - }catch(e){ + } catch (e) { logger.error(`Error on extract masto note package: ${fn}`); reject(e); } }); } -function createFileDir(fn: string){ - if(!fs.existsSync(fn)){ - fs.mkdirSync(fn, {recursive: true}); +function createFileDir(fn: string) { + if (!fs.existsSync(fn)) { + fs.mkdirSync(fn, { recursive: true }); fs.rmdirSync(fn); } } -function unzipTarGz(fn: string, dir: string){ +function unzipTarGz(fn: string, dir: string) { return new Promise(async (resolve, reject) => { const onErr = (err: any) => { logger.error(`pipe broken: ${err}`); reject(); - } - try{ - const extract = tar.extract().on('error', onErr); + }; + try { + const extract = tar.extract().on("error", onErr); dir = dir.endsWith("/") ? dir : dir + "/"; const ls: string[] = []; - extract.on('entry', function (header: any, stream: any, next: any) { - try{ + extract.on("entry", function (header: any, stream: any, next: any) { + try { ls.push(dir + header.name); createFileDir(dir + header.name); - stream.on('error', onErr).pipe(fs.createWriteStream(dir + header.name)).on('error', onErr); + stream + .on("error", onErr) + .pipe(fs.createWriteStream(dir + header.name)) + .on("error", onErr); next(); - }catch(e){ + } catch (e) { logger.error(`create dir error:${e}`); reject(); } }); - extract.on('finish', function () { - resolve(ls); + extract.on("finish", function () { + resolve(ls); }); - - fs.createReadStream(fn).on('error', onErr).pipe(gunzip()).on('error', onErr).pipe(extract).on('error', onErr); - - }catch(e){ + + fs.createReadStream(fn) + .on("error", onErr) + .pipe(gunzip()) + .on("error", onErr) + .pipe(extract) + .on("error", onErr); + } catch (e) { logger.error(`unzipTarGz error: ${e}`); reject(); } }); -} \ No newline at end of file +} From 35f09d00674ea742cfa833e6dd0ba81b4726ab76 Mon Sep 17 00:00:00 2001 From: CGsama Date: Sun, 16 Jul 2023 03:24:37 -0400 Subject: [PATCH 17/26] change wording --- packages/backend/src/queue/processors/db/import-posts.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/queue/processors/db/import-posts.ts b/packages/backend/src/queue/processors/db/import-posts.ts index 54061f8bf..0c5280ae1 100644 --- a/packages/backend/src/queue/processors/db/import-posts.ts +++ b/packages/backend/src/queue/processors/db/import-posts.ts @@ -33,16 +33,16 @@ export async function importPosts( if (file.name.endsWith("tar.gz")) { try { - logger.info("Parsing animal style posts in package"); + logger.info("Reading Mastodon archive"); const outbox = await processMastoNotes(file.url, job.data.user.id); for (const post of outbox.orderedItems) { createImportMastoPostJob(job.data.user, post, job.data.signatureCheck); } } catch (e) { // handle error - logger.warn(`Error reading: ${e}`); + logger.warn(`Failed reading Mastodon archive: ${e}`); } - logger.succ("Imported"); + logger.succ("Mastodon archive imported"); done(); return; } From ffb7fb5309946e42b8e9af9fa34ff7d60c9b1cd5 Mon Sep 17 00:00:00 2001 From: CGsama Date: Sun, 16 Jul 2023 04:07:55 -0400 Subject: [PATCH 18/26] Update pnpm-lock.yaml --- pnpm-lock.yaml | 165 +++++++++++++++++++++++++------------------------ 1 file changed, 85 insertions(+), 80 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index adead9c6f..4fbb6870d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -201,6 +201,9 @@ importers: got: specifier: 12.5.3 version: 12.5.3 + gunzip-maybe: + specifier: ^1.4.2 + version: 1.4.2 hpagent: specifier: 0.1.2 version: 0.1.2 @@ -381,6 +384,9 @@ importers: systeminformation: specifier: 5.17.17 version: 5.17.17 + tar-stream: + specifier: ^3.1.6 + version: 3.1.6 tesseract.js: specifier: ^3.0.3 version: 3.0.3(eslint@8.44.0) @@ -595,7 +601,7 @@ importers: version: 5.1.6 webpack: specifier: ^5.88.1 - version: 5.88.1(@swc/core@1.3.68) + version: 5.88.1(@swc/core@1.3.68)(webpack-cli@5.1.3) ws: specifier: 8.13.0 version: 8.13.0 @@ -780,7 +786,7 @@ importers: version: 2.30.0 emojilib: specifier: github:thatonecalculator/emojilib - version: github.com/thatonecalculator/emojilib/1d6adc1af8105b4937d2f0e7479acf5bd565bd9b + version: github.com/thatonecalculator/emojilib/d3c8c6a77d4362b3b3180099f1d2eac344ce245c escape-regexp: specifier: 0.0.1 version: 0.0.1 @@ -5138,7 +5144,7 @@ packages: /axios@0.24.0: resolution: {integrity: sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==} dependencies: - follow-redirects: 1.15.2 + follow-redirects: 1.15.2(debug@4.3.4) transitivePeerDependencies: - debug dev: false @@ -5164,13 +5170,17 @@ packages: /axios@1.4.0: resolution: {integrity: sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==} dependencies: - follow-redirects: 1.15.2 + follow-redirects: 1.15.2(debug@4.3.4) form-data: 4.0.0 proxy-from-env: 1.1.0 transitivePeerDependencies: - debug dev: false + /b4a@1.6.4: + resolution: {integrity: sha512-fpWrvyVHEKyeEvbKZTVOeZF3VSKKWtJxFIxX/jaVPf+cLbGUSitjb49pHLqPV2BUNNZ0LcoeEGfE/YCpyDYHIw==} + dev: false + /babel-eslint@10.1.0(eslint@8.44.0): resolution: {integrity: sha512-ifWaTHQ0ce+448CYop8AdrQiBsGrnC+bMgfyKFdi6EsPLTAWG+QfyDeM6OH+FmWnKvEq5NnBMLvlBUPKQZoDSg==} engines: {node: '>=6'} @@ -5479,6 +5489,12 @@ packages: resolution: {integrity: sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==} dev: true + /browserify-zlib@0.1.4: + resolution: {integrity: sha512-19OEpq7vWgsH6WkvkBJQDFvJS1uPcbFOQ4v9CU839dO+ZZXUZO6XpE6hNCqvlIIj+4fZvRiJ6DsAQ382GwiyTQ==} + dependencies: + pako: 0.2.9 + dev: false + /browserslist@1.7.7: resolution: {integrity: sha512-qHJblDE2bXVRYzuDetv/wAeHOJyO97+9wxC1cdCtyzgNuSozOyRCiiLaCR1f71AN66lQdVVBipWm63V+a7bPOw==} deprecated: Browserslist 2 could fail on reading Browserslist >3.0 config used in other tools. @@ -6843,17 +6859,6 @@ packages: dependencies: ms: 2.0.0 - /debug@3.2.7: - resolution: {integrity: sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==} - peerDependencies: - supports-color: '*' - peerDependenciesMeta: - supports-color: - optional: true - dependencies: - ms: 2.1.3 - dev: false - /debug@3.2.7(supports-color@8.1.1): resolution: {integrity: sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==} peerDependencies: @@ -6864,7 +6869,6 @@ packages: dependencies: ms: 2.1.3 supports-color: 8.1.1 - dev: true /debug@4.3.3: resolution: {integrity: sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==} @@ -7204,7 +7208,6 @@ packages: inherits: 2.0.4 readable-stream: 2.3.8 stream-shift: 1.0.1 - dev: true /each-props@1.3.2: resolution: {integrity: sha512-vV0Hem3zAGkJAyU7JSjixeU66rwdynTAa1vofCrSA5fEln+m67Az9CcnkVD776/fsN/UjIWmBDoNRS6t6G9RfA==} @@ -8106,6 +8109,10 @@ packages: resolution: {integrity: sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw==} dev: true + /fast-fifo@1.3.0: + resolution: {integrity: sha512-IgfweLvEpwyA4WgiQe9Nx6VV2QkML2NkvZnk1oKnIzXgXdWxuhF7zw4DvLTPZJn6PIUneiAXPF24QmoEqHTjyw==} + dev: false + /fast-glob@3.3.0: resolution: {integrity: sha512-ChDuvbOypPuNjO8yIDf36x7BlZX1smcUMTTcyoIjycexOxd6DFsKsg21qVBzEmr3G7fUKIRy2/psii+CIUt7FA==} engines: {node: '>=8.6.0'} @@ -8377,16 +8384,6 @@ packages: tabbable: 6.2.0 dev: true - /follow-redirects@1.15.2: - resolution: {integrity: sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==} - engines: {node: '>=4.0'} - peerDependencies: - debug: '*' - peerDependenciesMeta: - debug: - optional: true - dev: false - /follow-redirects@1.15.2(debug@4.3.4): resolution: {integrity: sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==} engines: {node: '>=4.0'} @@ -9057,6 +9054,18 @@ packages: glogg: 1.0.2 dev: true + /gunzip-maybe@1.4.2: + resolution: {integrity: sha512-4haO1M4mLO91PW57BMsDFf75UmwoRX0GkdD+Faw+Lr+r/OZrOCS0pIBwOL1xCKQqnQzbNFGgK2V2CpBUPeFNTw==} + hasBin: true + dependencies: + browserify-zlib: 0.1.4 + is-deflate: 1.0.0 + is-gzip: 1.0.0 + peek-stream: 1.1.3 + pumpify: 1.5.1 + through2: 2.0.5 + dev: false + /hammerjs@2.0.8: resolution: {integrity: sha512-tSQXBXS/MWQOn/RKckawJ61vvsDpCom87JgxiYdGwHdOa0ht0vzUWDlfioofFCRU0L+6NGDt6XzbgoJvZkMeRQ==} engines: {node: '>=0.8.0'} @@ -9355,7 +9364,7 @@ packages: engines: {node: '>= 4.5.0'} dependencies: agent-base: 4.3.0 - debug: 3.2.7 + debug: 3.2.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color dev: false @@ -9716,6 +9725,10 @@ packages: has-tostringtag: 1.0.0 dev: true + /is-deflate@1.0.0: + resolution: {integrity: sha512-YDoFpuZWu1VRXlsnlYMzKyVRITXj7Ej/V9gXQ2/pAe7X1J7M/RNOqaIYi6qUn+B7nGyB9pDXrv02dsB58d2ZAQ==} + dev: false + /is-descriptor@0.1.6: resolution: {integrity: sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==} engines: {node: '>=0.10.0'} @@ -9804,6 +9817,11 @@ packages: dependencies: is-extglob: 2.1.1 + /is-gzip@1.0.0: + resolution: {integrity: sha512-rcfALRIb1YewtnksfRIHGcIY93QnK8BIQ/2c9yDYcG/Y6+vRoJuTWBmmSEbyLLYtXm7q35pHOHbZFQBaLrhlWQ==} + engines: {node: '>=0.10.0'} + dev: false + /is-installed-globally@0.4.0: resolution: {integrity: sha512-iwGqO3J21aaSkC7jWnHP/difazwS7SFeIqxv6wEtLU8Y5KlzFTjyqcSIT0d8s4+dDhKytsk9PJZ2BkS5eZwQRQ==} engines: {node: '>=10'} @@ -11280,7 +11298,7 @@ packages: json5: 2.2.3 loader-utils: 2.0.4 schema-utils: 3.3.0 - webpack: 5.88.1(@swc/core@1.3.68) + webpack: 5.88.1(@swc/core@1.3.68)(webpack-cli@5.1.3) dev: true /json5@1.0.2: @@ -11535,7 +11553,7 @@ packages: resolution: {integrity: sha512-UqyYyH5YEXaJrf9S8E23GoJFQZXkBVJ9zYYMPGz919MSX1KuvAcycIuS0ci150HCoPf4XQVhQ84Qf8xRPWxFaQ==} engines: {node: '>= 7.6.0'} dependencies: - debug: 3.2.7 + debug: 3.2.7(supports-color@8.1.1) koa-send: 5.0.1 transitivePeerDependencies: - supports-color @@ -12579,7 +12597,7 @@ packages: engines: {node: '>= 4.4.x'} hasBin: true dependencies: - debug: 3.2.7 + debug: 3.2.7(supports-color@8.1.1) iconv-lite: 0.4.24 sax: 1.2.4 transitivePeerDependencies: @@ -13135,6 +13153,10 @@ packages: resolution: {integrity: sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==} dev: false + /pako@0.2.9: + resolution: {integrity: sha512-NUcwaKxUxWrZLpDG+z/xZaCgQITkA/Dv4V/T6bw7VON6l1Xz/VnrBqrYjZQ12TamKHzITTfOEIYUj48y2KXImA==} + dev: false + /parent-module@1.0.1: resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} engines: {node: '>=6'} @@ -13319,6 +13341,14 @@ packages: resolution: {integrity: sha512-YtCKvLUOvwtMGmrniQPdO7MwPjgkFBtFIrmfSbYmYuq3tKDV/mcfAhBth1+C3ru7uXIZasc/pHnb+YDYNkkj4A==} engines: {node: '>=14.16'} + /peek-stream@1.1.3: + resolution: {integrity: sha512-FhJ+YbOSBb9/rIl2ZeE/QHEsWn7PqNYt8ARAY3kIgNGOk13g9FGyIY6JIl/xB/3TFRVoTv5as0l11weORrTekA==} + dependencies: + buffer-from: 1.1.2 + duplexify: 3.7.1 + through2: 2.0.5 + dev: false + /pend@1.2.0: resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==} dev: true @@ -14021,7 +14051,6 @@ packages: dependencies: end-of-stream: 1.4.4 once: 1.4.0 - dev: true /pump@3.0.0: resolution: {integrity: sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==} @@ -14035,7 +14064,6 @@ packages: duplexify: 3.7.1 inherits: 2.0.4 pump: 2.0.1 - dev: true /punycode@1.3.2: resolution: {integrity: sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==} @@ -14128,6 +14156,10 @@ packages: /queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} + /queue-tick@1.0.1: + resolution: {integrity: sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==} + dev: false + /quick-lru@4.0.1: resolution: {integrity: sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g==} engines: {node: '>=8'} @@ -15278,13 +15310,19 @@ packages: /stream-shift@1.0.1: resolution: {integrity: sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==} - dev: true /streamsearch@1.1.0: resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} engines: {node: '>=10.0.0'} dev: false + /streamx@2.15.0: + resolution: {integrity: sha512-HcxY6ncGjjklGs1xsP1aR71INYcsXFJet5CU1CHqihQ2J5nOsbd4OjgjHO42w/4QNv9gZb3BueV+Vxok5pLEXg==} + dependencies: + fast-fifo: 1.3.0 + queue-tick: 1.0.1 + dev: false + /strict-event-emitter-types@2.0.0: resolution: {integrity: sha512-Nk/brWYpD85WlOgzw5h173aci0Teyv8YdIAEtV+N88nDB0dLlazZyJMIsN6eo1/AR61l+p6CJTG1JIyFaoNEEA==} dev: true @@ -15558,7 +15596,7 @@ packages: webpack: '>=2' dependencies: '@swc/core': 1.3.68 - webpack: 5.88.1(@swc/core@1.3.68) + webpack: 5.88.1(@swc/core@1.3.68)(webpack-cli@5.1.3) dev: true /swiper@10.0.4: @@ -15616,6 +15654,14 @@ packages: readable-stream: 3.6.2 dev: false + /tar-stream@3.1.6: + resolution: {integrity: sha512-B/UyjYwPpMBv+PaFSWAmtYjwdrlEaZQEhMIBFNC5oEG8lpiW8XjcSdmEaClj28ArfKScKHs2nshz3k2le6crsg==} + dependencies: + b4a: 1.6.4 + fast-fifo: 1.3.0 + streamx: 2.15.0 + dev: false + /tar@4.4.19: resolution: {integrity: sha512-a20gEsvHnWe0ygBY8JbxoM4w3SJdhc7ZAuxkLqh+nvNQN2IOt0B5lLgM490X5Hl8FF0dl0tOf2ewFYAlIFgzVA==} engines: {node: '>=4.5'} @@ -15677,7 +15723,7 @@ packages: schema-utils: 3.3.0 serialize-javascript: 6.0.1 terser: 5.19.0 - webpack: 5.88.1(@swc/core@1.3.68) + webpack: 5.88.1(@swc/core@1.3.68)(webpack-cli@5.1.3) dev: true /terser@5.19.0: @@ -15779,7 +15825,6 @@ packages: dependencies: readable-stream: 2.3.8 xtend: 4.0.2 - dev: true /through2@4.0.2: resolution: {integrity: sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==} @@ -16016,7 +16061,7 @@ packages: micromatch: 4.0.5 semver: 7.5.4 typescript: 5.1.6 - webpack: 5.88.1(@swc/core@1.3.68) + webpack: 5.88.1(@swc/core@1.3.68)(webpack-cli@5.1.3) dev: true /ts-node@10.4.0(@swc/core@1.3.68)(@types/node@20.3.1)(typescript@5.1.3): @@ -16933,46 +16978,6 @@ packages: engines: {node: '>=10.13.0'} dev: true - /webpack@5.88.1(@swc/core@1.3.68): - resolution: {integrity: sha512-FROX3TxQnC/ox4N+3xQoWZzvGXSuscxR32rbzjpXgEzWudJFEJBpdlkkob2ylrv5yzzufD1zph1OoFsLtm6stQ==} - engines: {node: '>=10.13.0'} - hasBin: true - peerDependencies: - webpack-cli: '*' - peerDependenciesMeta: - webpack-cli: - optional: true - dependencies: - '@types/eslint-scope': 3.7.4 - '@types/estree': 1.0.1 - '@webassemblyjs/ast': 1.11.6 - '@webassemblyjs/wasm-edit': 1.11.6 - '@webassemblyjs/wasm-parser': 1.11.6 - acorn: 8.10.0 - acorn-import-assertions: 1.9.0(acorn@8.10.0) - browserslist: 4.21.9 - chrome-trace-event: 1.0.3 - enhanced-resolve: 5.15.0 - es-module-lexer: 1.3.0 - eslint-scope: 5.1.1 - events: 3.3.0 - glob-to-regexp: 0.4.1 - graceful-fs: 4.2.11 - json-parse-even-better-errors: 2.3.1 - loader-runner: 4.3.0 - mime-types: 2.1.35 - neo-async: 2.6.2 - schema-utils: 3.3.0 - tapable: 2.2.1 - terser-webpack-plugin: 5.3.9(@swc/core@1.3.68)(webpack@5.88.1) - watchpack: 2.4.0 - webpack-sources: 3.2.3 - transitivePeerDependencies: - - '@swc/core' - - esbuild - - uglify-js - dev: true - /webpack@5.88.1(@swc/core@1.3.68)(webpack-cli@5.1.3): resolution: {integrity: sha512-FROX3TxQnC/ox4N+3xQoWZzvGXSuscxR32rbzjpXgEzWudJFEJBpdlkkob2ylrv5yzzufD1zph1OoFsLtm6stQ==} engines: {node: '>=10.13.0'} @@ -17486,8 +17491,8 @@ packages: url-polyfill: 1.1.12 dev: true - github.com/thatonecalculator/emojilib/1d6adc1af8105b4937d2f0e7479acf5bd565bd9b: - resolution: {tarball: https://codeload.github.com/thatonecalculator/emojilib/tar.gz/1d6adc1af8105b4937d2f0e7479acf5bd565bd9b} + github.com/thatonecalculator/emojilib/d3c8c6a77d4362b3b3180099f1d2eac344ce245c: + resolution: {tarball: https://codeload.github.com/thatonecalculator/emojilib/tar.gz/d3c8c6a77d4362b3b3180099f1d2eac344ce245c} name: emojilib version: 3.0.10 dev: true From 4bb6d7105a8b0509d6579af98bdb08fd7f04370c Mon Sep 17 00:00:00 2001 From: CGsama Date: Sun, 16 Jul 2023 06:24:37 -0400 Subject: [PATCH 19/26] support new zip file by mastodon --- packages/backend/package.json | 1 + .../backend/src/misc/process-masto-notes.ts | 42 ++++- .../src/queue/processors/db/import-posts.ts | 8 +- pnpm-lock.yaml | 161 +++++++++++++++++- 4 files changed, 196 insertions(+), 16 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 6e9f82edd..d05583845 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -59,6 +59,7 @@ "color-convert": "2.0.1", "content-disposition": "0.5.4", "date-fns": "2.30.0", + "decompress": "^4.2.1", "deep-email-validator": "0.1.21", "escape-regexp": "0.0.1", "feed": "4.2.2", diff --git a/packages/backend/src/misc/process-masto-notes.ts b/packages/backend/src/misc/process-masto-notes.ts index f51764ae8..b8c1c426d 100644 --- a/packages/backend/src/misc/process-masto-notes.ts +++ b/packages/backend/src/misc/process-masto-notes.ts @@ -6,10 +6,13 @@ import { addFile } from "@/services/drive/add-file.js"; import { Users } from "@/models/index.js"; import * as tar from "tar-stream"; import gunzip from "gunzip-maybe"; +import decompress from "decompress"; +import * as Path from "node:path"; const logger = new Logger("process-masto-notes"); export async function processMastoNotes( + fn: string, url: string, uid: string, ): Promise { @@ -23,26 +26,39 @@ export async function processMastoNotes( try { // write content at URL to temp file await downloadUrl(url, path); - return await processMastoFile(path, unzipPath, uid); + return await processMastoFile(fn, path, unzipPath, uid); } finally { cleanup(); - unzipCleanup(); + //unzipCleanup(); } } -function processMastoFile(fn: string, dir: string, uid: string) { +function processMastoFile(fn: string, path: string, dir: string, uid: string) { return new Promise(async (resolve, reject) => { const user = await Users.findOneBy({ id: uid }); try { - logger.info(`Start unzip ${fn}`); - await unzipTarGz(fn, dir); + logger.info(`Start unzip ${path}`); + fn.endsWith("tar.gz") + ? await unzipTarGz(path, dir) + : await unzipZip(path, dir); logger.info(`Unzip to ${dir}`); const outbox = JSON.parse(fs.readFileSync(`${dir}/outbox.json`)); for (const note of outbox.orderedItems) { for (const attachment of note.object.attachment) { - const url = attachment.url.replace("..", ""); + const url = attachment.url.replaceAll("..", ""); + if (url.indexOf('\0') !== -1) { + logger.error(`Found Poison Null Bytes Attack: ${url}`); + reject(); + return; + } try { - const fpath = `${dir}${url}`; + const fpath = Path.resolve(`${dir}${url}`); + if (!fpath.startsWith(dir)) { + logger.error(`Found Path Attack: ${url}`); + reject(); + return; + } + logger.info(fpath); const driveFile = await addFile({ user: user, path: fpath }); attachment.driveFile = driveFile; } catch (e) { @@ -65,6 +81,18 @@ function createFileDir(fn: string) { } } +function unzipZip(fn: string, dir: string) { + return new Promise(async (resolve, reject) => { + try { + decompress(fn, dir).then((files: any) => { + resolve(files); + }); + } catch (e) { + reject(); + } + }); +} + function unzipTarGz(fn: string, dir: string) { return new Promise(async (resolve, reject) => { const onErr = (err: any) => { diff --git a/packages/backend/src/queue/processors/db/import-posts.ts b/packages/backend/src/queue/processors/db/import-posts.ts index 0c5280ae1..9bde7479e 100644 --- a/packages/backend/src/queue/processors/db/import-posts.ts +++ b/packages/backend/src/queue/processors/db/import-posts.ts @@ -31,10 +31,14 @@ export async function importPosts( return; } - if (file.name.endsWith("tar.gz")) { + if (file.name.endsWith("tar.gz") || file.name.endsWith("zip")) { try { logger.info("Reading Mastodon archive"); - const outbox = await processMastoNotes(file.url, job.data.user.id); + const outbox = await processMastoNotes( + file.name, + file.url, + job.data.user.id, + ); for (const post of outbox.orderedItems) { createImportMastoPostJob(job.data.user, post, job.data.signatureCheck); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4fbb6870d..8ad191710 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -183,6 +183,9 @@ importers: date-fns: specifier: 2.30.0 version: 2.30.0 + decompress: + specifier: ^4.2.1 + version: 4.2.1 deep-email-validator: specifier: 0.1.21 version: 0.1.21 @@ -5402,6 +5405,13 @@ packages: engines: {node: '>=0.8'} dev: true + /bl@1.2.3: + resolution: {integrity: sha512-pvcNpa0UU69UT341rO6AYy4FVAIkUHuZXRIWbq+zHnsVcRzDDjIAhGuuYoi0d//cwIwtt4pkpKycWEfjdV+vww==} + dependencies: + readable-stream: 2.3.8 + safe-buffer: 5.2.1 + dev: false + /bl@4.1.0: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} dependencies: @@ -5527,6 +5537,17 @@ packages: node-int64: 0.4.0 dev: true + /buffer-alloc-unsafe@1.1.0: + resolution: {integrity: sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==} + dev: false + + /buffer-alloc@1.2.0: + resolution: {integrity: sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==} + dependencies: + buffer-alloc-unsafe: 1.1.0 + buffer-fill: 1.0.0 + dev: false + /buffer-crc32@0.2.13: resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==} @@ -5539,6 +5560,10 @@ packages: engines: {node: '>=0.4'} dev: true + /buffer-fill@1.0.0: + resolution: {integrity: sha512-T7zexNBwiiaCOGDg9xNX9PBmjrubblRkENuptryuI64URkXDFum9il/JGL8Lm8wYfAXpredVXXZz7eMHilimiQ==} + dev: false + /buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} @@ -6930,6 +6955,59 @@ packages: dependencies: mimic-response: 3.1.0 + /decompress-tar@4.1.1: + resolution: {integrity: sha512-JdJMaCrGpB5fESVyxwpCx4Jdj2AagLmv3y58Qy4GE6HMVjWz1FeVQk1Ct4Kye7PftcdOo/7U7UKzYBJgqnGeUQ==} + engines: {node: '>=4'} + dependencies: + file-type: 5.2.0 + is-stream: 1.1.0 + tar-stream: 1.6.2 + dev: false + + /decompress-tarbz2@4.1.1: + resolution: {integrity: sha512-s88xLzf1r81ICXLAVQVzaN6ZmX4A6U4z2nMbOwobxkLoIIfjVMBg7TeguTUXkKeXni795B6y5rnvDw7rxhAq9A==} + engines: {node: '>=4'} + dependencies: + decompress-tar: 4.1.1 + file-type: 6.2.0 + is-stream: 1.1.0 + seek-bzip: 1.0.6 + unbzip2-stream: 1.4.3 + dev: false + + /decompress-targz@4.1.1: + resolution: {integrity: sha512-4z81Znfr6chWnRDNfFNqLwPvm4db3WuZkqV+UgXQzSngG3CEKdBkw5jrv3axjjL96glyiiKjsxJG3X6WBZwX3w==} + engines: {node: '>=4'} + dependencies: + decompress-tar: 4.1.1 + file-type: 5.2.0 + is-stream: 1.1.0 + dev: false + + /decompress-unzip@4.0.1: + resolution: {integrity: sha512-1fqeluvxgnn86MOh66u8FjbtJpAFv5wgCT9Iw8rcBqQcCo5tO8eiJw7NNTrvt9n4CRBVq7CstiS922oPgyGLrw==} + engines: {node: '>=4'} + dependencies: + file-type: 3.9.0 + get-stream: 2.3.1 + pify: 2.3.0 + yauzl: 2.10.0 + dev: false + + /decompress@4.2.1: + resolution: {integrity: sha512-e48kc2IjU+2Zw8cTb6VZcJQ3lgVbS4uuB1TfCHbiZIP/haNXm+SVyhu+87jts5/3ROpd82GSVCoNs/z8l4ZOaQ==} + engines: {node: '>=4'} + dependencies: + decompress-tar: 4.1.1 + decompress-tarbz2: 4.1.1 + decompress-targz: 4.1.1 + decompress-unzip: 4.0.1 + graceful-fs: 4.2.11 + make-dir: 1.3.0 + pify: 2.3.0 + strip-dirs: 2.1.0 + dev: false + /dedent@0.7.0: resolution: {integrity: sha512-Q6fKUPqnAHAyhiUgFU7BUzLiv0kd8saH9al7tnu5Q/okj6dnupxyTgFIBjVzJATdfIAm9NAsvXNzjaKa+bxVyA==} dev: true @@ -8161,7 +8239,6 @@ packages: resolution: {integrity: sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==} dependencies: pend: 1.2.0 - dev: true /feed@4.2.2: resolution: {integrity: sha512-u5/sxGfiMfZNtJ3OvQpXcvotFpYkL0n9u9mM2vkui2nGo8b4wvDkJ8gAkYqbA8QpGyFCv3RK0Z+Iv+9veCS9bQ==} @@ -8211,6 +8288,21 @@ packages: strtok3: 7.0.0 token-types: 5.0.1 + /file-type@3.9.0: + resolution: {integrity: sha512-RLoqTXE8/vPmMuTI88DAzhMYC99I8BWv7zYP4A1puo5HIjEJ5EX48ighy4ZyKMG9EDXxBgW6e++cn7d1xuFghA==} + engines: {node: '>=0.10.0'} + dev: false + + /file-type@5.2.0: + resolution: {integrity: sha512-Iq1nJ6D2+yIO4c8HHg4fyVb8mAJieo1Oloy1mLLaB2PvezNedhBVm+QU7g0qM42aiMbRXTxKKwGD17rjKNJYVQ==} + engines: {node: '>=4'} + dev: false + + /file-type@6.2.0: + resolution: {integrity: sha512-YPcTBDV+2Tm0VqjybVd32MHdlEGAtuxS3VAYsumFokDSMG+ROT5wawGlnHDoz7bfMcMDt9hxuXvXwoKUx2fkOg==} + engines: {node: '>=4'} + dev: false + /filelist@1.0.4: resolution: {integrity: sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==} dependencies: @@ -8679,6 +8771,14 @@ packages: engines: {node: '>=8'} dev: false + /get-stream@2.3.1: + resolution: {integrity: sha512-AUGhbbemXxrZJRD5cDvKtQxLuYaIbNtDTK8YqupCI393Q2KSTreEsLUN3ZxAWFGiKTzL6nKuzfcIvieflUX9qA==} + engines: {node: '>=0.10.0'} + dependencies: + object-assign: 4.1.1 + pinkie-promise: 2.0.1 + dev: false + /get-stream@3.0.0: resolution: {integrity: sha512-GlhdIUuVakc8SJ6kK0zAFbiGzRFzNnY4jUuEbV9UROo4Y+0Ny4fjvcZFVTeDA4odpFyOQzaw6hXukJSq/f28sQ==} engines: {node: '>=4'} @@ -9841,6 +9941,10 @@ packages: resolution: {integrity: sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ==} dev: false + /is-natural-number@4.0.1: + resolution: {integrity: sha512-Y4LTamMe0DDQIIAlaer9eKebAlDSV6huy+TWhJVPlzZh2o4tRP5SQWFlLn5N0To4mDD22/qdOq+veo1cSISLgQ==} + dev: false + /is-negated-glob@1.0.0: resolution: {integrity: sha512-czXVVn/QEmgvej1f50BZ648vUI+em0xqMq2Sn+QncCLN4zj1UAxlT+kw/6ggQTOaZPd1HqKQGEqbpQVtJucWug==} engines: {node: '>=0.10.0'} @@ -9942,7 +10046,6 @@ packages: /is-stream@1.1.0: resolution: {integrity: sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==} engines: {node: '>=0.10.0'} - dev: true /is-stream@2.0.1: resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} @@ -12023,6 +12126,13 @@ packages: resolution: {integrity: sha512-3WjL8+ZDouZwKlyJBMp/4LeziLFXgleOdsYu87piGcMLqhBzCsy2QFdbtAwv757TFC/rtqd738fgJw1tFQCSgA==} dev: false + /make-dir@1.3.0: + resolution: {integrity: sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==} + engines: {node: '>=4'} + dependencies: + pify: 3.0.0 + dev: false + /make-dir@3.1.0: resolution: {integrity: sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==} engines: {node: '>=8'} @@ -13351,7 +13461,6 @@ packages: /pend@1.2.0: resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==} - dev: true /performance-now@2.1.0: resolution: {integrity: sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==} @@ -13435,7 +13544,11 @@ packages: /pify@2.3.0: resolution: {integrity: sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==} engines: {node: '>=0.10.0'} - dev: true + + /pify@3.0.0: + resolution: {integrity: sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg==} + engines: {node: '>=4'} + dev: false /pify@4.0.1: resolution: {integrity: sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==} @@ -13446,12 +13559,10 @@ packages: engines: {node: '>=0.10.0'} dependencies: pinkie: 2.0.4 - dev: true /pinkie@2.0.4: resolution: {integrity: sha512-MnUuEycAemtSaeFSjXKW/aroV7akBbY+Sv+RkyqFjgAe73F+MR0TBWKBRDkmfWq/HiFmdavfZ1G7h4SPZXaCSg==} engines: {node: '>=0.10.0'} - dev: true /pirates@4.0.6: resolution: {integrity: sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==} @@ -14818,6 +14929,13 @@ packages: /seedrandom@3.0.5: resolution: {integrity: sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==} + /seek-bzip@1.0.6: + resolution: {integrity: sha512-e1QtP3YL5tWww8uKaOCQ18UxIT2laNBXHjV/S2WYCiK4udiv8lkG89KRIoCjUagnAmCBurjF4zEVX2ByBbnCjQ==} + hasBin: true + dependencies: + commander: 2.20.3 + dev: false + /semver-greatest-satisfied-range@1.1.0: resolution: {integrity: sha512-Ny/iyOzSSa8M5ML46IAx3iXc6tfOsYU2R4AXi2UpHk60Zrgyq6eqPj/xiOfS0rRl/iiQ/rdJkVjw/5cdUyCntQ==} engines: {node: '>= 0.10'} @@ -15450,6 +15568,12 @@ packages: engines: {node: '>=8'} dev: true + /strip-dirs@2.1.0: + resolution: {integrity: sha512-JOCxOeKLm2CAS73y/U4ZeZPTkE+gNVCzKt7Eox84Iej1LT/2pTWYpZKJuxwQpvX1LiZb1xokNR7RLfuBAa7T3g==} + dependencies: + is-natural-number: 4.0.1 + dev: false + /strip-eof@1.0.0: resolution: {integrity: sha512-7FCwGGmx8mD5xQd3RPUvnSpUXHM3BWuzjtpD4TXsfcZ9EL4azvVVUscFYwD9nx8Kh+uCBC00XBtAykoMHwTh8Q==} engines: {node: '>=0.10.0'} @@ -15643,6 +15767,19 @@ packages: tar-stream: 2.2.0 dev: false + /tar-stream@1.6.2: + resolution: {integrity: sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==} + engines: {node: '>= 0.8.0'} + dependencies: + bl: 1.2.3 + buffer-alloc: 1.2.0 + end-of-stream: 1.4.4 + fs-constants: 1.0.0 + readable-stream: 2.3.8 + to-buffer: 1.1.1 + xtend: 4.0.2 + dev: false + /tar-stream@2.2.0: resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} engines: {node: '>=6'} @@ -15875,6 +16012,10 @@ packages: is-negated-glob: 1.0.0 dev: true + /to-buffer@1.1.1: + resolution: {integrity: sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==} + dev: false + /to-fast-properties@2.0.0: resolution: {integrity: sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==} engines: {node: '>=4'} @@ -16413,6 +16554,13 @@ packages: which-boxed-primitive: 1.0.2 dev: true + /unbzip2-stream@1.4.3: + resolution: {integrity: sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==} + dependencies: + buffer: 5.7.1 + through: 2.3.8 + dev: false + /unc-path-regex@0.1.2: resolution: {integrity: sha512-eXL4nmJT7oCpkZsHZUOJo8hcX3GbsiDOa0Qu9F646fi8dT3XuSVopVqAcEiVzSKKH7UoDti23wNX3qGFxcW5Qg==} engines: {node: '>=0.10.0'} @@ -17424,7 +17572,6 @@ packages: dependencies: buffer-crc32: 0.2.13 fd-slicer: 1.1.0 - dev: true /ylru@1.3.2: resolution: {integrity: sha512-RXRJzMiK6U2ye0BlGGZnmpwJDPgakn6aNQ0A7gHRbD4I0uvK4TW6UqkK1V0pp9jskjJBAXd3dRrbzWkqJ+6cxA==} From 32faf81637318eab0f9c513cb7270765322dcd18 Mon Sep 17 00:00:00 2001 From: CGsama Date: Sun, 16 Jul 2023 20:40:40 -0400 Subject: [PATCH 20/26] fix lock --- pnpm-lock.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9a5e94571..0e2c3552e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10745,6 +10745,10 @@ packages: resolution: {integrity: sha512-RGdriMmQQvZ2aqaQq3awNA6dCGtKpiDFcOzrTWrDAT2MiWrKQVPmxLGHl7Y2nNu6led0kEyoX0enY0qXYsv9zw==} dev: true + /is-deflate@1.0.0: + resolution: {integrity: sha512-YDoFpuZWu1VRXlsnlYMzKyVRITXj7Ej/V9gXQ2/pAe7X1J7M/RNOqaIYi6qUn+B7nGyB9pDXrv02dsB58d2ZAQ==} + dev: false + /is-descriptor@0.1.6: resolution: {integrity: sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==} engines: {node: '>=0.10.0'} @@ -10833,16 +10837,14 @@ packages: dependencies: is-extglob: 2.1.1 -<<<<<<< HEAD /is-gzip@1.0.0: resolution: {integrity: sha512-rcfALRIb1YewtnksfRIHGcIY93QnK8BIQ/2c9yDYcG/Y6+vRoJuTWBmmSEbyLLYtXm7q35pHOHbZFQBaLrhlWQ==} engines: {node: '>=0.10.0'} dev: false -======= + /is-hexadecimal@1.0.4: resolution: {integrity: sha512-gyPJuv83bHMpocVYoqof5VDiZveEoGoFL8m3BXNb2VW8Xs+rz9kqO8LOQ5DH6EsuvilT1ApazU0pyl+ytbPtlw==} dev: true ->>>>>>> 97a0127dbf0b96203902fb075b1d51659a921bfc /is-installed-globally@0.4.0: resolution: {integrity: sha512-iwGqO3J21aaSkC7jWnHP/difazwS7SFeIqxv6wEtLU8Y5KlzFTjyqcSIT0d8s4+dDhKytsk9PJZ2BkS5eZwQRQ==} @@ -14271,11 +14273,10 @@ packages: resolution: {integrity: sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==} dev: false -<<<<<<< HEAD /pako@0.2.9: resolution: {integrity: sha512-NUcwaKxUxWrZLpDG+z/xZaCgQITkA/Dv4V/T6bw7VON6l1Xz/VnrBqrYjZQ12TamKHzITTfOEIYUj48y2KXImA==} dev: false -======= + /paralint@1.2.1: resolution: {integrity: sha512-HPGVDd5eUNYBhtftRypBHlUTMy5UAnHBzFO601oLEODZ0uvXUJge1y43GZQM2CnyOOUyDQlNMK/9vcjjAxbfcA==} hasBin: true @@ -14286,7 +14287,6 @@ packages: minimist: 1.2.8 tslib: 2.6.0 dev: true ->>>>>>> 97a0127dbf0b96203902fb075b1d51659a921bfc /parent-module@1.0.1: resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} From 0775ad96f9476c9389026a379c28c23def17ad86 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Mon, 17 Jul 2023 01:48:53 -0400 Subject: [PATCH 21/26] fix: use hostname as prefix instead of host --- packages/backend/src/config/load.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/config/load.ts b/packages/backend/src/config/load.ts index fa9878955..df163a8ac 100644 --- a/packages/backend/src/config/load.ts +++ b/packages/backend/src/config/load.ts @@ -54,9 +54,9 @@ export default function load() { mixin.userAgent = `Calckey/${meta.version} (${config.url})`; mixin.clientEntry = clientManifest["src/init.ts"]; - if (!config.redis.prefix) config.redis.prefix = mixin.host; + if (!config.redis.prefix) config.redis.prefix = mixin.hostname; if (config.cacheServer && !config.cacheServer.prefix) - config.cacheServer.prefix = mixin.host; + config.cacheServer.prefix = mixin.hostname; return Object.assign(config, mixin); } From 3b56117fabf56b918c3de730d822012216c18a02 Mon Sep 17 00:00:00 2001 From: Kainoa Kanter Date: Mon, 17 Jul 2023 16:26:07 +0000 Subject: [PATCH 22/26] revert 00d1c034d513e28ace63b0ce074c7bd04a846196 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit revert fix: 🐛 Scrolling Issue in Safari for Top and Bottom Bars --- packages/client/src/ui/deck.vue | 21 --------------------- packages/client/src/ui/universal.vue | 21 --------------------- 2 files changed, 42 deletions(-) diff --git a/packages/client/src/ui/deck.vue b/packages/client/src/ui/deck.vue index 9037ad156..6fdcfb28d 100644 --- a/packages/client/src/ui/deck.vue +++ b/packages/client/src/ui/deck.vue @@ -384,27 +384,6 @@ async function deleteProfile() { } - - -