diff --git a/packages/backend/src/models/repositories/user-profile.ts b/packages/backend/src/models/repositories/user-profile.ts index 2b8c9fd93..6e68ee018 100644 --- a/packages/backend/src/models/repositories/user-profile.ts +++ b/packages/backend/src/models/repositories/user-profile.ts @@ -6,13 +6,13 @@ import { resolveMentionToUserAndProfile } from "@/remote/resolve-user.js"; import { IMentionedRemoteUsers } from "@/models/entities/note.js"; import { unique } from "@/prelude/array.js"; import config from "@/config/index.js"; -import { Semaphore } from "async-mutex"; +import { Mutex, Semaphore } from "async-mutex"; const queue = new Semaphore(5); export const UserProfileRepository = db.getRepository(UserProfile).extend({ // We must never await this without promiseEarlyReturn, otherwise giant webring-style profile mention trees will cause the queue to stop working - async updateMentions(id: UserProfile["userId"]){ + async updateMentions(id: UserProfile["userId"], limiter: RecursionLimiter = new RecursionLimiter(20)){ const profile = await this.findOneBy({ userId: id }); if (!profile) return; const tokens: mfm.MfmNode[] = []; @@ -24,16 +24,16 @@ export const UserProfileRepository = db.getRepository(UserProfile).extend({ return queue.runExclusive(async () => { const partial = { - mentions: await populateMentions(tokens, profile.userHost) + mentions: await populateMentions(tokens, profile.userHost, limiter) }; return UserProfileRepository.update(profile.userId, partial); }); }, }); -async function populateMentions(tokens: mfm.MfmNode[], objectHost: string | null): Promise { +async function populateMentions(tokens: mfm.MfmNode[], objectHost: string | null, limiter: RecursionLimiter): Promise { const mentions = extractMentions(tokens); - const resolved = await Promise.all(mentions.map(m => resolveMentionToUserAndProfile(m.username, m.host, objectHost))); + const resolved = await Promise.all(mentions.map(m => resolveMentionToUserAndProfile(m.username, m.host, objectHost, limiter))); const remote = resolved.filter(p => p && p.data.host !== config.domain && (p.data.host !== null || objectHost !== null)) .map(p => p!); const res = remote.map(m => { @@ -47,3 +47,17 @@ async function populateMentions(tokens: mfm.MfmNode[], objectHost: string | null return unique(res); } + +export class RecursionLimiter { + private counter; + private mutex = new Mutex(); + constructor(count: number = 20) { + this.counter = count; + } + + public shouldContinue(): Promise { + return this.mutex.runExclusive(() => { + return this.counter-- > 0; + }); + } +} \ No newline at end of file diff --git a/packages/backend/src/remote/activitypub/models/mention.ts b/packages/backend/src/remote/activitypub/models/mention.ts index b888fa21a..da19f0aa8 100644 --- a/packages/backend/src/remote/activitypub/models/mention.ts +++ b/packages/backend/src/remote/activitypub/models/mention.ts @@ -6,9 +6,11 @@ import type { IObject, IApMention } from "../type.js"; import { isMention } from "../type.js"; import Resolver from "../resolver.js"; import { resolvePerson } from "./person.js"; +import { RecursionLimiter } from "@/models/repositories/user-profile.js"; export async function extractApMentions( tags: IObject | IObject[] | null | undefined, + limiter: RecursionLimiter = new RecursionLimiter(20) ) { const hrefs = unique( extractApMentionObjects(tags).map((x) => x.href as string), @@ -20,7 +22,7 @@ export async function extractApMentions( const mentionedUsers = ( await Promise.all( hrefs.map((x) => - limit(() => resolvePerson(x, resolver).catch(() => null)), + limit(() => resolvePerson(x, resolver, limiter).catch(() => null)), ), ) ).filter((x): x is CacheableUser => x != null); diff --git a/packages/backend/src/remote/activitypub/models/note.ts b/packages/backend/src/remote/activitypub/models/note.ts index bd2e1b73c..243947b32 100644 --- a/packages/backend/src/remote/activitypub/models/note.ts +++ b/packages/backend/src/remote/activitypub/models/note.ts @@ -53,6 +53,7 @@ import { DB_MAX_IMAGE_COMMENT_LENGTH } from "@/misc/hard-limits.js"; import { truncate } from "@/misc/truncate.js"; import { type Size, getEmojiSize } from "@/misc/emoji-meta.js"; import { fetchMeta } from "@/misc/fetch-meta.js"; +import { RecursionLimiter } from "@/models/repositories/user-profile.js"; const logger = apLogger; @@ -108,6 +109,7 @@ export async function createNote( value: string | IObject, resolver?: Resolver, silent = false, + limiter: RecursionLimiter = new RecursionLimiter(20) ): Promise { if (resolver == null) resolver = new Resolver(); @@ -163,6 +165,7 @@ export async function createNote( const actor = (await resolvePerson( getOneApId(note.attributedTo), resolver, + limiter )) as CacheableRemoteUser; // Skip if author is suspended. @@ -188,8 +191,8 @@ export async function createNote( let isTalk = note._misskey_talk && visibility === "specified"; - const apMentions = await extractApMentions(note.tag); - const apHashtags = await extractApHashtags(note.tag); + const apMentions = await extractApMentions(note.tag, limiter); + const apHashtags = extractApHashtags(note.tag); // Attachments // TODO: attachmentは必ずしもImageではない @@ -216,7 +219,7 @@ export async function createNote( // Reply const reply: Note | null = note.inReplyTo - ? await resolveNote(note.inReplyTo, resolver) + ? await resolveNote(note.inReplyTo, resolver, limiter) .then((x) => { if (x == null) { logger.warn("Specified inReplyTo, but nout found"); @@ -262,7 +265,7 @@ export async function createNote( if (typeof uri !== "string" || !uri.match(/^https?:/)) return { status: "permerror" }; try { - const res = await resolveNote(uri); + const res = await resolveNote(uri, undefined, limiter); if (res) { return { status: "ok", @@ -403,6 +406,7 @@ export async function createNote( export async function resolveNote( value: string | IObject, resolver?: Resolver, + limiter: RecursionLimiter = new RecursionLimiter(20) ): Promise { const uri = typeof value === "string" ? value : value.id; if (uri == null) throw new Error("missing uri"); @@ -437,7 +441,7 @@ export async function resolveNote( // Fetch from remote server and register // If the attached `Note` Object is specified here instead of the uri, the note will be generated without going through the server fetch. // Since the attached Note Object may be disguised, always specify the uri and fetch it from the server. - return await createNote(uri, resolver, true); + return await createNote(uri, resolver, true, limiter); } finally { unlock(); } diff --git a/packages/backend/src/remote/activitypub/models/person.ts b/packages/backend/src/remote/activitypub/models/person.ts index fdbc58c71..e474e1f54 100644 --- a/packages/backend/src/remote/activitypub/models/person.ts +++ b/packages/backend/src/remote/activitypub/models/person.ts @@ -53,6 +53,7 @@ import { getSubjectHostFromRemoteUser, getSubjectHostFromAcctParts } from "@/remote/resolve-user.js" +import { RecursionLimiter } from "@/models/repositories/user-profile.js"; const logger = apLogger; @@ -170,7 +171,7 @@ export async function createPerson( uri: string, resolver?: Resolver, subjectHost?: string, - skipMentions: boolean = false + limiter: RecursionLimiter = new RecursionLimiter(20) ): Promise { if (typeof uri !== "string") throw new Error("uri is not string"); @@ -400,7 +401,7 @@ export async function createPerson( updateUsertags(user!, tags); // Mentions update - if (!skipMentions) UserProfiles.updateMentions(user!.id); + if (await limiter.shouldContinue()) UserProfiles.updateMentions(user!.id, limiter); //#region Fetch avatar and header image const [avatar, banner] = await Promise.all( @@ -436,7 +437,7 @@ export async function createPerson( }); //#endregion - await updateFeatured(user!.id, resolver).catch((err) => logger.error(err)); + await updateFeatured(user!.id, resolver, limiter).catch((err) => logger.error(err)); return user!; } @@ -646,6 +647,7 @@ export async function updatePerson( export async function resolvePerson( uri: string, resolver?: Resolver, + limiter: RecursionLimiter = new RecursionLimiter(20) ): Promise { if (typeof uri !== "string") throw new Error("uri is not string"); @@ -659,7 +661,7 @@ export async function resolvePerson( // Fetched from remote server and registered if (resolver == null) resolver = new Resolver(); - return await createPerson(uri, resolver); + return await createPerson(uri, resolver, undefined, limiter); } const services: { @@ -720,7 +722,7 @@ export async function analyzeAttachments( return { fields, services }; } -export async function updateFeatured(userId: User["id"], resolver?: Resolver) { +export async function updateFeatured(userId: User["id"], resolver?: Resolver, limiter: RecursionLimiter = new RecursionLimiter(20)) { const user = await Users.findOneByOrFail({ id: userId }); if (!Users.isRemoteUser(user)) return; if (!user.featured) return; @@ -742,14 +744,14 @@ export async function updateFeatured(userId: User["id"], resolver?: Resolver) { toArray(unresolvedItems).map((x) => resolver.resolve(x)), ); - // Resolve and regist Notes + // Resolve and register Notes resolver.reset(); const limit = promiseLimit(2); const featuredNotes = await Promise.all( items .filter((item) => getApType(item) === "Note") // TODO: Maybe it doesn't have to be a Note. .slice(0, 5) - .map((item) => limit(() => resolveNote(item, resolver))), + .map((item) => limit(() => resolveNote(item, resolver, limiter))), ); await db.transaction(async (transactionalEntityManager) => { diff --git a/packages/backend/src/remote/resolve-user.ts b/packages/backend/src/remote/resolve-user.ts index 1778ba3c8..c2c70ef6d 100644 --- a/packages/backend/src/remote/resolve-user.ts +++ b/packages/backend/src/remote/resolve-user.ts @@ -11,6 +11,7 @@ import { remoteLogger } from "./logger.js"; import { Cache } from "@/misc/cache.js"; import { IMentionedRemoteUsers } from "@/models/entities/note.js"; import { UserProfile } from "@/models/entities/user-profile.js"; +import { RecursionLimiter } from "@/models/repositories/user-profile.js"; const logger = remoteLogger.createSubLogger("resolve-user"); const uriHostCache = new Cache("resolveUserUriHost", 60 * 60 * 24); @@ -31,7 +32,7 @@ export async function resolveUser( host: string | null, refresh: boolean = true, awaitRefresh: boolean = true, - skipMentionsOnCreate: boolean = false + limiter: RecursionLimiter = new RecursionLimiter(20) ): Promise { const usernameLower = username.toLowerCase(); @@ -105,14 +106,14 @@ export async function resolveUser( // Otherwise create and return new user else { logger.succ(`return new remote user: ${chalk.magenta(finalAcctLower)}`); - return await createPerson(fingerRes.self.href, undefined, subjectHost, skipMentionsOnCreate); + return await createPerson(fingerRes.self.href, undefined, subjectHost, limiter); } } } // Not a split domain setup, so we can simply create and return the new user logger.succ(`return new remote user: ${chalk.magenta(finalAcctLower)}`); - return await createPerson(fingerRes.self.href, undefined, subjectHost, skipMentionsOnCreate); + return await createPerson(fingerRes.self.href, undefined, subjectHost, limiter); } // If user information is out of date, return it by starting over from WebFinger @@ -188,17 +189,17 @@ export async function resolveUser( } else if (refresh && !awaitRefresh && (user.lastFetchedAt == null || Date.now() - user.lastFetchedAt.getTime() > 1000 * 60 * 60 * 24)) { // Run the refresh in the background // noinspection ES6MissingAwait - resolveUser(username, host, true, true, skipMentionsOnCreate); + resolveUser(username, host, true, true, limiter); } logger.info(`return existing remote user: ${acctLower}`); return user; } -export async function resolveMentionToUserAndProfile(username: string, host: string | null, objectHost: string | null) { +export async function resolveMentionToUserAndProfile(username: string, host: string | null, objectHost: string | null, limiter: RecursionLimiter) { return profileMentionCache.fetch(`${username}@${host ?? objectHost}`, async () => { try { - const user = await resolveUser(username, host ?? objectHost, false, false, true); + const user = await resolveUser(username, host ?? objectHost, false, false, limiter); const profile = await UserProfiles.findOneBy({ userId: user.id }); const data = { username, host: host ?? objectHost };