cae6ba0edb
### What does this PR do? Adding files fields in the export notes option, and corresponding import notes Current the mastodon import does not import any attachments, this pr will use the "upload from url" feature to include medias if its a valid URL. There are many way to convert the outbox.json file, can be simple as upload media_attachments to any web hosting and do string replace on the json file. I also create a tool that upload the tar.gz file with auto convert and host the media as simplify the process at https://tempfile.moegirl.live Detail example can be found at https://fedi.moegirl.live/notes/9h76gtqnp2gwl5dz https://r2temp.moegirl.live/2023/7/15/15356683-050f-423a-b331-c9a05561f52a/shana-settings-_-meng-zhai-le-yuan-xyou-yu-ou-xiang-de-luo-ke-ke-wu-yan-moe-otaku-elysian-x-gloomily-idol-s-rococo-luncheon----mozilla-firefox-private-browsing-2023-07-15-18-36-37.mp4 Co-authored-by: CGsama <CGsama@outlook.com> Reviewed-on: https://codeberg.org/calckey/calckey/pulls/10496 Co-authored-by: コルセット姫@がんばらない <cgsama@noreply.codeberg.org> Co-committed-by: コルセット姫@がんばらない <cgsama@noreply.codeberg.org>
134 lines
2.9 KiB
TypeScript
134 lines
2.9 KiB
TypeScript
import type Bull from "bull";
|
|
import * as fs from "node:fs";
|
|
|
|
import { queueLogger } from "../../logger.js";
|
|
import { addFile } from "@/services/drive/add-file.js";
|
|
import { format as dateFormat } from "date-fns";
|
|
import { Users, Notes, Polls, DriveFiles } from "@/models/index.js";
|
|
import { MoreThan } from "typeorm";
|
|
import type { Note } from "@/models/entities/note.js";
|
|
import type { Poll } from "@/models/entities/poll.js";
|
|
import type { DbUserJobData } from "@/queue/types.js";
|
|
import { createTemp } from "@/misc/create-temp.js";
|
|
|
|
const logger = queueLogger.createSubLogger("export-notes");
|
|
|
|
export async function exportNotes(
|
|
job: Bull.Job<DbUserJobData>,
|
|
done: any,
|
|
): Promise<void> {
|
|
logger.info(`Exporting notes of ${job.data.user.id} ...`);
|
|
|
|
const user = await Users.findOneBy({ id: job.data.user.id });
|
|
if (user == null) {
|
|
done();
|
|
return;
|
|
}
|
|
|
|
// Create temp file
|
|
const [path, cleanup] = await createTemp();
|
|
|
|
logger.info(`Temp file is ${path}`);
|
|
|
|
try {
|
|
const stream = fs.createWriteStream(path, { flags: "a" });
|
|
|
|
const write = (text: string): Promise<void> => {
|
|
return new Promise<void>((res, rej) => {
|
|
stream.write(text, (err) => {
|
|
if (err) {
|
|
logger.error(err);
|
|
rej(err);
|
|
} else {
|
|
res();
|
|
}
|
|
});
|
|
});
|
|
};
|
|
|
|
await write("[");
|
|
|
|
let exportedNotesCount = 0;
|
|
let cursor: Note["id"] | null = null;
|
|
|
|
while (true) {
|
|
const notes = (await Notes.find({
|
|
where: {
|
|
userId: user.id,
|
|
...(cursor ? { id: MoreThan(cursor) } : {}),
|
|
},
|
|
take: 100,
|
|
order: {
|
|
id: 1,
|
|
},
|
|
})) as Note[];
|
|
|
|
if (notes.length === 0) {
|
|
job.progress(100);
|
|
break;
|
|
}
|
|
|
|
cursor = notes[notes.length - 1].id;
|
|
|
|
for (const note of notes) {
|
|
let poll: Poll | undefined;
|
|
if (note.hasPoll) {
|
|
poll = await Polls.findOneByOrFail({ noteId: note.id });
|
|
}
|
|
const content = JSON.stringify(await serialize(note, poll));
|
|
const isFirst = exportedNotesCount === 0;
|
|
await write(isFirst ? content : ",\n" + content);
|
|
exportedNotesCount++;
|
|
}
|
|
|
|
const total = await Notes.countBy({
|
|
userId: user.id,
|
|
});
|
|
|
|
job.progress(exportedNotesCount / total);
|
|
}
|
|
|
|
await write("]");
|
|
|
|
stream.end();
|
|
logger.succ(`Exported to: ${path}`);
|
|
|
|
const fileName = `notes-${dateFormat(
|
|
new Date(),
|
|
"yyyy-MM-dd-HH-mm-ss",
|
|
)}.json`;
|
|
const driveFile = await addFile({
|
|
user,
|
|
path,
|
|
name: fileName,
|
|
force: true,
|
|
});
|
|
|
|
logger.succ(`Exported to: ${driveFile.id}`);
|
|
} finally {
|
|
cleanup();
|
|
}
|
|
|
|
done();
|
|
}
|
|
|
|
async function serialize(
|
|
note: Note,
|
|
poll: Poll | null = null,
|
|
): Promise<Record<string, unknown>> {
|
|
return {
|
|
id: note.id,
|
|
text: note.text,
|
|
createdAt: note.createdAt,
|
|
fileIds: note.fileIds,
|
|
files: await DriveFiles.packMany(note.fileIds),
|
|
replyId: note.replyId,
|
|
renoteId: note.renoteId,
|
|
poll: poll,
|
|
cw: note.cw,
|
|
visibility: note.visibility,
|
|
visibleUserIds: note.visibleUserIds,
|
|
localOnly: note.localOnly,
|
|
};
|
|
}
|