feat: Add media to Mastodon and Calckey post imports (#10496)

### What does this PR do?

Adding files fields in the export notes option, and corresponding import notes

Current the mastodon import does not import any attachments, this pr will use the "upload from url" feature to include medias if its a valid URL.

There are many way to convert the outbox.json file, can be simple as upload media_attachments to any web hosting and do string replace on the json file.

I also create a tool that upload the tar.gz file with auto convert and host the media as simplify the process at https://tempfile.moegirl.live

Detail example can be found at https://fedi.moegirl.live/notes/9h76gtqnp2gwl5dz

https://r2temp.moegirl.live/2023/7/15/15356683-050f-423a-b331-c9a05561f52a/shana-settings-_-meng-zhai-le-yuan-xyou-yu-ou-xiang-de-luo-ke-ke-wu-yan-moe-otaku-elysian-x-gloomily-idol-s-rococo-luncheon----mozilla-firefox-private-browsing-2023-07-15-18-36-37.mp4

Co-authored-by: CGsama <CGsama@outlook.com>
Reviewed-on: https://codeberg.org/calckey/calckey/pulls/10496
Co-authored-by: コルセット姫@がんばらない <cgsama@noreply.codeberg.org>
Co-committed-by: コルセット姫@がんばらない <cgsama@noreply.codeberg.org>
This commit is contained in:
コルセット姫@がんばらない 2023-07-15 22:56:09 +00:00 committed by Kainoa Kanter
parent bd57b95a97
commit cae6ba0edb
3 changed files with 42 additions and 6 deletions

View file

@ -4,7 +4,7 @@ import * as fs from "node:fs";
import { queueLogger } from "../../logger.js"; import { queueLogger } from "../../logger.js";
import { addFile } from "@/services/drive/add-file.js"; import { addFile } from "@/services/drive/add-file.js";
import { format as dateFormat } from "date-fns"; import { format as dateFormat } from "date-fns";
import { Users, Notes, Polls } from "@/models/index.js"; import { Users, Notes, Polls, DriveFiles } from "@/models/index.js";
import { MoreThan } from "typeorm"; import { MoreThan } from "typeorm";
import type { Note } from "@/models/entities/note.js"; import type { Note } from "@/models/entities/note.js";
import type { Poll } from "@/models/entities/poll.js"; import type { Poll } from "@/models/entities/poll.js";
@ -75,7 +75,7 @@ export async function exportNotes(
if (note.hasPoll) { if (note.hasPoll) {
poll = await Polls.findOneByOrFail({ noteId: note.id }); poll = await Polls.findOneByOrFail({ noteId: note.id });
} }
const content = JSON.stringify(serialize(note, poll)); const content = JSON.stringify(await serialize(note, poll));
const isFirst = exportedNotesCount === 0; const isFirst = exportedNotesCount === 0;
await write(isFirst ? content : ",\n" + content); await write(isFirst ? content : ",\n" + content);
exportedNotesCount++; exportedNotesCount++;
@ -112,15 +112,16 @@ export async function exportNotes(
done(); done();
} }
function serialize( async function serialize(
note: Note, note: Note,
poll: Poll | null = null, poll: Poll | null = null,
): Record<string, unknown> { ): Promise<Record<string, unknown>> {
return { return {
id: note.id, id: note.id,
text: note.text, text: note.text,
createdAt: note.createdAt, createdAt: note.createdAt,
fileIds: note.fileIds, fileIds: note.fileIds,
files: await DriveFiles.packMany(note.fileIds),
replyId: note.replyId, replyId: note.replyId,
renoteId: note.renoteId, renoteId: note.renoteId,
poll: poll, poll: poll,

View file

@ -3,6 +3,8 @@ import create from "@/services/note/create.js";
import { Users } from "@/models/index.js"; import { Users } from "@/models/index.js";
import type { DbUserImportMastoPostJobData } from "@/queue/types.js"; import type { DbUserImportMastoPostJobData } from "@/queue/types.js";
import { queueLogger } from "../../logger.js"; import { queueLogger } from "../../logger.js";
import { uploadFromUrl } from "@/services/drive/upload-from-url.js";
import type { DriveFile } from "@/models/entities/drive-file.js";
import type Bull from "bull"; import type Bull from "bull";
const logger = queueLogger.createSubLogger("import-calckey-post"); const logger = queueLogger.createSubLogger("import-calckey-post");
@ -29,10 +31,25 @@ export async function importCkPost(
done(); done();
return; return;
} }
const urls = (post.files || [])
.map((x: any) => x.url)
.filter((x: String) => x.startsWith("http"));
const files: DriveFile[] = [];
for (const url of urls) {
try {
const file = await uploadFromUrl({
url: url,
user: user,
});
files.push(file);
} catch (e) {
logger.error(`Skipped adding file to drive: ${url}`);
}
}
const { text, cw, localOnly, createdAt } = Post.parse(post); const { text, cw, localOnly, createdAt } = Post.parse(post);
const note = await create(user, { const note = await create(user, {
createdAt: createdAt, createdAt: createdAt,
files: undefined, files: files.length == 0 ? undefined : files,
poll: undefined, poll: undefined,
text: text || undefined, text: text || undefined,
reply: null, reply: null,

View file

@ -6,6 +6,8 @@ import type Bull from "bull";
import { htmlToMfm } from "@/remote/activitypub/misc/html-to-mfm.js"; import { htmlToMfm } from "@/remote/activitypub/misc/html-to-mfm.js";
import { resolveNote } from "@/remote/activitypub/models/note.js"; import { resolveNote } from "@/remote/activitypub/models/note.js";
import { Note } from "@/models/entities/note.js"; import { Note } from "@/models/entities/note.js";
import { uploadFromUrl } from "@/services/drive/upload-from-url.js";
import type { DriveFile } from "@/models/entities/drive-file.js";
const logger = queueLogger.createSubLogger("import-masto-post"); const logger = queueLogger.createSubLogger("import-masto-post");
@ -43,9 +45,25 @@ export async function importMastoPost(
throw e; throw e;
} }
job.progress(80); job.progress(80);
const urls = post.object.attachment
.map((x: any) => x.url)
.filter((x: String) => x.startsWith("http"));
const files: DriveFile[] = [];
for (const url of urls) {
try {
const file = await uploadFromUrl({
url: url,
user: user,
});
files.push(file);
} catch (e) {
logger.error(`Skipped adding file to drive: ${url}`);
}
}
const note = await create(user, { const note = await create(user, {
createdAt: new Date(post.object.published), createdAt: new Date(post.object.published),
files: undefined, files: files.length == 0 ? undefined : files,
poll: undefined, poll: undefined,
text: text || undefined, text: text || undefined,
reply, reply,