From 9c2264fb8e476dcb686a7751e75ec0f3ce40d3b3 Mon Sep 17 00:00:00 2001 From: ThatOneCalculator Date: Sat, 15 Jul 2023 00:19:09 -0700 Subject: [PATCH 1/5] =?UTF-8?q?fix:=20=F0=9F=90=9B=20allow=20up=20to=20102?= =?UTF-8?q?4=20chars=20for=20SMTP=20login?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #10472 --- .../migration/1678426061773-tweak-varchar-length.js | 10 ++++++++++ packages/backend/src/models/entities/meta.ts | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 packages/backend/migration/1678426061773-tweak-varchar-length.js diff --git a/packages/backend/migration/1678426061773-tweak-varchar-length.js b/packages/backend/migration/1678426061773-tweak-varchar-length.js new file mode 100644 index 000000000..883374599 --- /dev/null +++ b/packages/backend/migration/1678426061773-tweak-varchar-length.js @@ -0,0 +1,10 @@ +export class tweakVarcharLength1678426061773 { + name = 'tweakVarcharLength1678426061773' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "smtpUser" TYPE character varying(1024)`, undefined); + await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "smtpPass" TYPE character varying(1024)`, undefined); + } + + async down(queryRunner) {} +} diff --git a/packages/backend/src/models/entities/meta.ts b/packages/backend/src/models/entities/meta.ts index d3de7ba53..e90a5269a 100644 --- a/packages/backend/src/models/entities/meta.ts +++ b/packages/backend/src/models/entities/meta.ts @@ -326,13 +326,13 @@ export class Meta { public smtpPort: number | null; @Column("varchar", { - length: 128, + length: 1024, nullable: true, }) public smtpUser: string | null; @Column("varchar", { - length: 128, + length: 1024, nullable: true, }) public smtpPass: string | null; From 247c5ec760c37968c8f35867861cba3459a996fd Mon Sep 17 00:00:00 2001 From: ThatOneCalculator Date: Sat, 15 Jul 2023 14:28:44 -0700 Subject: [PATCH 2/5] dev78 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b38da39fa..a30d85b3d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "calckey", - "version": "14.0.0-dev77", + "version": "14.0.0-dev78", "codename": "aqua", "repository": { "type": "git", From 49b0a52f9f345aa7b81db9eec404afc1864b5ee8 Mon Sep 17 00:00:00 2001 From: ThatOneCalculator Date: Sat, 15 Jul 2023 14:50:23 -0700 Subject: [PATCH 3/5] fix: :wrench: max post length 100000 Pleroma doesn't accept >100K (ref: https://blob.cat/objects/82f33d96-534f-45ee-902a-f77c2723db8b) --- .config/example.yml | 2 +- packages/backend/src/const.ts | 7 +++++-- packages/backend/src/misc/hard-limits.ts | 3 +-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.config/example.yml b/.config/example.yml index 9716915b6..f73f4f1d7 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -121,7 +121,7 @@ redis: # ┌─────────────────────┐ #───┘ Other configuration └───────────────────────────────────── -# Maximum length of a post (default 3000, max 250000000) +# Maximum length of a post (default 3000, max 100000) #maxNoteLength: 3000 # Maximum length of an image caption (default 1500, max 8192) diff --git a/packages/backend/src/const.ts b/packages/backend/src/const.ts index 1f1e63292..49f012c5e 100644 --- a/packages/backend/src/const.ts +++ b/packages/backend/src/const.ts @@ -1,7 +1,10 @@ import config from "@/config/index.js"; -import { DB_MAX_IMAGE_COMMENT_LENGTH } from "@/misc/hard-limits.js"; +import { DB_MAX_NOTE_TEXT_LENGTH, DB_MAX_IMAGE_COMMENT_LENGTH } from "@/misc/hard-limits.js"; -export const MAX_NOTE_TEXT_LENGTH = config.maxNoteLength ?? 3000; +export const MAX_NOTE_TEXT_LENGTH = Math.min( + config.maxNoteLength ?? 3000, + DB_MAX_NOTE_TEXT_LENGTH, +); export const MAX_CAPTION_TEXT_LENGTH = Math.min( config.maxCaptionLength ?? 1500, DB_MAX_IMAGE_COMMENT_LENGTH, diff --git a/packages/backend/src/misc/hard-limits.ts b/packages/backend/src/misc/hard-limits.ts index f1a58a425..62671a706 100644 --- a/packages/backend/src/misc/hard-limits.ts +++ b/packages/backend/src/misc/hard-limits.ts @@ -3,9 +3,8 @@ /** * Maximum note text length that can be stored in DB. * Surrogate pairs count as one - * DEPRECARTED: use const/MAX_NOTE_TEXT_LENGTH instead */ -// export const DB_MAX_NOTE_TEXT_LENGTH = 8192; +export const DB_MAX_NOTE_TEXT_LENGTH = 100000; /** * Maximum image description length that can be stored in DB. From bd57b95a97098c72a1cb24891f1a3d0ea6086dfc Mon Sep 17 00:00:00 2001 From: ThatOneCalculator Date: Sat, 15 Jul 2023 15:32:39 -0700 Subject: [PATCH 4/5] docs: :bulb: going past DB_MAX_NOTE_TEXT_LENGTH --- packages/backend/src/misc/hard-limits.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/backend/src/misc/hard-limits.ts b/packages/backend/src/misc/hard-limits.ts index 62671a706..5ce3e0ac9 100644 --- a/packages/backend/src/misc/hard-limits.ts +++ b/packages/backend/src/misc/hard-limits.ts @@ -3,6 +3,11 @@ /** * Maximum note text length that can be stored in DB. * Surrogate pairs count as one + * + * NOTE: this can hypothetically be pushed further + * (up to 250000000), but will likely cause truncations + * and incompatibilities with other servers, + * as well as potential performance issues. */ export const DB_MAX_NOTE_TEXT_LENGTH = 100000; From cae6ba0edbf8a11f7a6e22bec23d213f27d1682b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=82=B3=E3=83=AB=E3=82=BB=E3=83=83=E3=83=88=E5=A7=AB=40?= =?UTF-8?q?=E3=81=8C=E3=82=93=E3=81=B0=E3=82=89=E3=81=AA=E3=81=84?= Date: Sat, 15 Jul 2023 22:56:09 +0000 Subject: [PATCH 5/5] =?UTF-8?q?feat:=20=E2=9C=A8=20Add=20media=20to=20Mast?= =?UTF-8?q?odon=20and=20Calckey=20post=20imports=20(#10496)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What does this PR do? Adding files fields in the export notes option, and corresponding import notes Current the mastodon import does not import any attachments, this pr will use the "upload from url" feature to include medias if its a valid URL. There are many way to convert the outbox.json file, can be simple as upload media_attachments to any web hosting and do string replace on the json file. I also create a tool that upload the tar.gz file with auto convert and host the media as simplify the process at https://tempfile.moegirl.live Detail example can be found at https://fedi.moegirl.live/notes/9h76gtqnp2gwl5dz https://r2temp.moegirl.live/2023/7/15/15356683-050f-423a-b331-c9a05561f52a/shana-settings-_-meng-zhai-le-yuan-xyou-yu-ou-xiang-de-luo-ke-ke-wu-yan-moe-otaku-elysian-x-gloomily-idol-s-rococo-luncheon----mozilla-firefox-private-browsing-2023-07-15-18-36-37.mp4 Co-authored-by: CGsama Reviewed-on: https://codeberg.org/calckey/calckey/pulls/10496 Co-authored-by: コルセット姫@がんばらない Co-committed-by: コルセット姫@がんばらない --- .../src/queue/processors/db/export-notes.ts | 9 +++++---- .../processors/db/import-calckey-post.ts | 19 +++++++++++++++++- .../queue/processors/db/import-masto-post.ts | 20 ++++++++++++++++++- 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/queue/processors/db/export-notes.ts b/packages/backend/src/queue/processors/db/export-notes.ts index de8fac05b..bf53f8360 100644 --- a/packages/backend/src/queue/processors/db/export-notes.ts +++ b/packages/backend/src/queue/processors/db/export-notes.ts @@ -4,7 +4,7 @@ import * as fs from "node:fs"; import { queueLogger } from "../../logger.js"; import { addFile } from "@/services/drive/add-file.js"; import { format as dateFormat } from "date-fns"; -import { Users, Notes, Polls } from "@/models/index.js"; +import { Users, Notes, Polls, DriveFiles } from "@/models/index.js"; import { MoreThan } from "typeorm"; import type { Note } from "@/models/entities/note.js"; import type { Poll } from "@/models/entities/poll.js"; @@ -75,7 +75,7 @@ export async function exportNotes( if (note.hasPoll) { poll = await Polls.findOneByOrFail({ noteId: note.id }); } - const content = JSON.stringify(serialize(note, poll)); + const content = JSON.stringify(await serialize(note, poll)); const isFirst = exportedNotesCount === 0; await write(isFirst ? content : ",\n" + content); exportedNotesCount++; @@ -112,15 +112,16 @@ export async function exportNotes( done(); } -function serialize( +async function serialize( note: Note, poll: Poll | null = null, -): Record { +): Promise> { return { id: note.id, text: note.text, createdAt: note.createdAt, fileIds: note.fileIds, + files: await DriveFiles.packMany(note.fileIds), replyId: note.replyId, renoteId: note.renoteId, poll: poll, diff --git a/packages/backend/src/queue/processors/db/import-calckey-post.ts b/packages/backend/src/queue/processors/db/import-calckey-post.ts index 28e794aa0..945c50d63 100644 --- a/packages/backend/src/queue/processors/db/import-calckey-post.ts +++ b/packages/backend/src/queue/processors/db/import-calckey-post.ts @@ -3,6 +3,8 @@ import create from "@/services/note/create.js"; import { Users } from "@/models/index.js"; import type { DbUserImportMastoPostJobData } from "@/queue/types.js"; import { queueLogger } from "../../logger.js"; +import { uploadFromUrl } from "@/services/drive/upload-from-url.js"; +import type { DriveFile } from "@/models/entities/drive-file.js"; import type Bull from "bull"; const logger = queueLogger.createSubLogger("import-calckey-post"); @@ -29,10 +31,25 @@ export async function importCkPost( done(); return; } + const urls = (post.files || []) + .map((x: any) => x.url) + .filter((x: String) => x.startsWith("http")); + const files: DriveFile[] = []; + for (const url of urls) { + try { + const file = await uploadFromUrl({ + url: url, + user: user, + }); + files.push(file); + } catch (e) { + logger.error(`Skipped adding file to drive: ${url}`); + } + } const { text, cw, localOnly, createdAt } = Post.parse(post); const note = await create(user, { createdAt: createdAt, - files: undefined, + files: files.length == 0 ? undefined : files, poll: undefined, text: text || undefined, reply: null, diff --git a/packages/backend/src/queue/processors/db/import-masto-post.ts b/packages/backend/src/queue/processors/db/import-masto-post.ts index efa4adf3f..05166b085 100644 --- a/packages/backend/src/queue/processors/db/import-masto-post.ts +++ b/packages/backend/src/queue/processors/db/import-masto-post.ts @@ -6,6 +6,8 @@ import type Bull from "bull"; import { htmlToMfm } from "@/remote/activitypub/misc/html-to-mfm.js"; import { resolveNote } from "@/remote/activitypub/models/note.js"; import { Note } from "@/models/entities/note.js"; +import { uploadFromUrl } from "@/services/drive/upload-from-url.js"; +import type { DriveFile } from "@/models/entities/drive-file.js"; const logger = queueLogger.createSubLogger("import-masto-post"); @@ -43,9 +45,25 @@ export async function importMastoPost( throw e; } job.progress(80); + const urls = post.object.attachment + .map((x: any) => x.url) + .filter((x: String) => x.startsWith("http")); + const files: DriveFile[] = []; + for (const url of urls) { + try { + const file = await uploadFromUrl({ + url: url, + user: user, + }); + files.push(file); + } catch (e) { + logger.error(`Skipped adding file to drive: ${url}`); + } + } + const note = await create(user, { createdAt: new Date(post.object.published), - files: undefined, + files: files.length == 0 ? undefined : files, poll: undefined, text: text || undefined, reply,