diff --git a/megalodon/package.json b/megalodon/package.json index 4752224..5e9d809 100644 --- a/megalodon/package.json +++ b/megalodon/package.json @@ -68,7 +68,8 @@ "socks-proxy-agent": "^7.0.0", "typescript": "4.9.4", "uuid": "^9.0.0", - "ws": "8.12.0" + "ws": "8.12.0", + "async-lock": "1.4.0" }, "devDependencies": { "@types/core-js": "^2.5.0", @@ -79,6 +80,7 @@ "@types/uuid": "^9.0.0", "@typescript-eslint/eslint-plugin": "^5.49.0", "@typescript-eslint/parser": "^5.49.0", + "@types/async-lock": "1.4.0", "eslint": "^8.32.0", "eslint-config-prettier": "^8.6.0", "eslint-config-standard": "^16.0.3", diff --git a/megalodon/src/misskey.ts b/megalodon/src/misskey.ts index 8ee9a20..adb2b38 100644 --- a/megalodon/src/misskey.ts +++ b/megalodon/src/misskey.ts @@ -1,4 +1,5 @@ import FormData from 'form-data' +import AsyncLock from 'async-lock'; import MisskeyAPI from './misskey/api_client' import { DEFAULT_UA } from './default' @@ -9,6 +10,11 @@ import Entity from './entity' import { MegalodonInterface, WebSocketInterface, NoImplementedError, ArgumentError, UnexpectedError } from './megalodon' import MegalodonEntity from "@/entity"; +type AccountCache = { + locks: AsyncLock, + accounts: Entity.Account[] +} + export default class Misskey implements MegalodonInterface { public client: MisskeyAPI.Interface public converter: MisskeyAPI.Converter @@ -314,6 +320,8 @@ export default class Misskey implements MegalodonInterface { only_media?: boolean } ): Promise>> { + const accountCache = this.getFreshAccountCache(); + if (options?.pinned) { return this.client .post('/api/users/show', { @@ -323,7 +331,7 @@ export default class Misskey implements MegalodonInterface { if (res.data.pinnedNotes) { return { ...res, - data: await Promise.all(res.data.pinnedNotes.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl)))) + data: await Promise.all(res.data.pinnedNotes.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl), accountCache))) } } return {...res, data: []} @@ -375,7 +383,7 @@ export default class Misskey implements MegalodonInterface { }) } return this.client.post>('/api/users/notes', params).then(async res => { - const statuses: Array = await Promise.all(res.data.map(note => this.noteWithMentions(note, this.baseUrlToHost(this.baseUrl)))) + const statuses: Array = await Promise.all(res.data.map(note => this.noteWithMentions(note, this.baseUrlToHost(this.baseUrl), accountCache))) return Object.assign(res, { data: statuses }) @@ -390,6 +398,8 @@ export default class Misskey implements MegalodonInterface { since_id?: string } ): Promise>> { + const accountCache = this.getFreshAccountCache(); + let params = { userId: id }; @@ -412,7 +422,7 @@ export default class Misskey implements MegalodonInterface { } return this.client.post>('/api/users/reactions', params).then(async res => { return Object.assign(res, { - data: await Promise.all(res.data.map(fav => this.noteWithMentions(fav.note, this.baseUrlToHost(this.baseUrl)))) + data: await Promise.all(res.data.map(fav => this.noteWithMentions(fav.note, this.baseUrlToHost(this.baseUrl), accountCache))) }) }) } @@ -700,6 +710,8 @@ export default class Misskey implements MegalodonInterface { since_id?: string min_id?: string }): Promise>> { + const accountCache = this.getFreshAccountCache(); + let params = {} if (options) { if (options.limit) { @@ -720,7 +732,7 @@ export default class Misskey implements MegalodonInterface { } return this.client.post>('/api/i/favorites', params).then(async res => { return Object.assign(res, { - data: await Promise.all(res.data.map(s => this.noteWithMentions(s.note, this.baseUrlToHost(this.baseUrl)))) + data: await Promise.all(res.data.map(s => this.noteWithMentions(s.note, this.baseUrlToHost(this.baseUrl), accountCache))) }) }) } @@ -1159,7 +1171,7 @@ export default class Misskey implements MegalodonInterface { .post('/api/notes/create', params) .then(async res => ({ ...res, - data: await this.noteWithMentions(res.data.createdNote, this.baseUrlToHost(this.baseUrl)) + data: await this.noteWithMentions(res.data.createdNote, this.baseUrlToHost(this.baseUrl), this.getFreshAccountCache()) })) } @@ -1171,23 +1183,31 @@ export default class Misskey implements MegalodonInterface { .post('/api/notes/show', { noteId: id }) - .then(async res => ({ ...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl))})); + .then(async res => ({ ...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl), this.getFreshAccountCache())})); } - public async noteWithMentions(n: MisskeyAPI.Entity.Note, host: string): Promise { + private getFreshAccountCache() :AccountCache { + return { + locks: new AsyncLock(), + accounts: [] + } + } + + public async noteWithMentions(n: MisskeyAPI.Entity.Note, host: string, cache: AccountCache): Promise { const status = await this.converter.note(n, host); - return status.mentions.length === 0 ? this.addMentionsToStatus(status) : status; + return status.mentions.length === 0 ? this.addMentionsToStatus(status, cache) : status; } - public async addMentionsToStatus(status: Entity.Status) : Promise { - status.mentions = (await this.getMentions(status.plain_content!)).filter(p => p != null); + public async addMentionsToStatus(status: Entity.Status, cache: AccountCache) : Promise { + status.mentions = (await this.getMentions(status.plain_content!, cache)).filter(p => p != null); for (const m of status.mentions.filter((value, index, array) => array.indexOf(value) === index)) { status.content = status.content.replace(`@${m.acct}`, `@${m.acct}`); } return status; } - public async getMentions(text: string): Promise { + public async getMentions(text: string, cache: AccountCache): Promise { + console.log(`cache length: ${cache.accounts.length}`); const mentions :Entity.Mention[] = []; const mentionMatch = text.matchAll(/(?<=^|\s)@(?.*?)(?:@(?.*?)|)(?=\s|$)/g); @@ -1197,22 +1217,46 @@ export default class Misskey implements MegalodonInterface { if (m.groups == null) continue; - const account = await this.getAccountByName(m.groups.user, m.groups?.host ?? null); + const account = await this.getAccountByNameCached(m.groups.user, m.groups.host, cache); - if (!account) + if (account == null) continue; mentions.push({ - id: account.data.id, - url: account.data.url, - username: account.data.username, - acct: account.data.acct + id: account.id, + url: account.url, + username: account.username, + acct: account.acct }); } return mentions; } + public async getAccountByNameCached(user: string, host: string | null, cache: AccountCache): Promise { + const acctToFind = host == null ? user : `${user}@${host}`; + + return await cache.locks.acquire(acctToFind, async () => { + const cacheHit = cache.accounts.find(p => p.acct === acctToFind); + const account = cacheHit ?? (await this.getAccountByName(user, host ?? null)).data; + + if (!account) { + console.log(`cache fail: ${acctToFind}`); + return null; + } + + if (cacheHit == null) { + console.log(`cache miss: ${acctToFind}`); + cache.accounts.push(account); + } + else { + console.log(`cache hit: ${acctToFind}`); + } + + return account; + }) + } + public async editStatus( _id: string, _options: { @@ -1281,12 +1325,13 @@ export default class Misskey implements MegalodonInterface { return this.client.post>('/api/notes/children', params).then(async res => { console.log(JSON.stringify(res, null, 2)); + const accountCache = this.getFreshAccountCache(); const conversation = await this.client.post>('/api/notes/conversation', params); - const parents = await Promise.all(conversation.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl)))); + const parents = await Promise.all(conversation.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl), accountCache))); const context: Entity.Context = { ancestors: parents.reverse(), - descendants: this.dfs(await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl))))) + descendants: this.dfs(await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl), accountCache)))) } return { ...res, @@ -1399,7 +1444,7 @@ export default class Misskey implements MegalodonInterface { }) .then(async res => ({ ...res, - data: await this.noteWithMentions(res.data.createdNote, this.baseUrlToHost(this.baseUrl)) + data: await this.noteWithMentions(res.data.createdNote, this.baseUrlToHost(this.baseUrl), this.getFreshAccountCache()) })) } @@ -1414,7 +1459,7 @@ export default class Misskey implements MegalodonInterface { .post('/api/notes/show', { noteId: id }) - .then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl))})) + .then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl), this.getFreshAccountCache())})) } /** @@ -1428,7 +1473,7 @@ export default class Misskey implements MegalodonInterface { .post('/api/notes/show', { noteId: id }) - .then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl))})) + .then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl), this.getFreshAccountCache())})) } /** @@ -1442,7 +1487,7 @@ export default class Misskey implements MegalodonInterface { .post('/api/notes/show', { noteId: id }) - .then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl))})) + .then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl), this.getFreshAccountCache())})) } public async muteStatus(_id: string): Promise> { @@ -1470,7 +1515,7 @@ export default class Misskey implements MegalodonInterface { .post('/api/notes/show', { noteId: id }) - .then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl))})) + .then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl), this.getFreshAccountCache())})) } /** @@ -1484,7 +1529,7 @@ export default class Misskey implements MegalodonInterface { .post('/api/notes/show', { noteId: id }) - .then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl))})) + .then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl), this.getFreshAccountCache())})) } // ====================================== @@ -1570,7 +1615,7 @@ export default class Misskey implements MegalodonInterface { noteId: status_id }) .then(async res => { - const note = await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl)) + const note = await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl), this.getFreshAccountCache()) return {...res, data: note.poll} }) if (!res.data) { @@ -1631,6 +1676,8 @@ export default class Misskey implements MegalodonInterface { since_id?: string min_id?: string }): Promise>> { + const accountCache = this.getFreshAccountCache(); + let params = {} if (options) { if (options.only_media !== undefined) { @@ -1673,7 +1720,7 @@ export default class Misskey implements MegalodonInterface { .post>('/api/notes/global-timeline', params) .then(async res => ({ ...res, - data: await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl)))) + data: await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl), accountCache))) })) } @@ -1687,6 +1734,8 @@ export default class Misskey implements MegalodonInterface { since_id?: string min_id?: string }): Promise>> { + const accountCache = this.getFreshAccountCache(); + let params = {} if (options) { if (options.only_media !== undefined) { @@ -1729,7 +1778,7 @@ export default class Misskey implements MegalodonInterface { .post>('/api/notes/local-timeline', params) .then(async res => ({ ...res, - data: await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl)))) + data: await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl), accountCache))) })) } @@ -1747,6 +1796,8 @@ export default class Misskey implements MegalodonInterface { min_id?: string } ): Promise>> { + const accountCache = this.getFreshAccountCache(); + let params = { tag: hashtag } @@ -1791,7 +1842,7 @@ export default class Misskey implements MegalodonInterface { .post>('/api/notes/search-by-tag', params) .then(async res => ({ ...res, - data: await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl)))) + data: await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl), accountCache))) })) } @@ -1805,6 +1856,8 @@ export default class Misskey implements MegalodonInterface { since_id?: string min_id?: string }): Promise>> { + const accountCache = this.getFreshAccountCache(); + let params = { withFiles: false } @@ -1844,7 +1897,7 @@ export default class Misskey implements MegalodonInterface { .post>('/api/notes/timeline', params) .then(async res => ({ ...res, - data: await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl)))) + data: await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl), accountCache))) })) } @@ -1860,6 +1913,8 @@ export default class Misskey implements MegalodonInterface { min_id?: string } ): Promise>> { + const accountCache = this.getFreshAccountCache(); + let params = { listId: list_id, withFiles: false @@ -1898,7 +1953,7 @@ export default class Misskey implements MegalodonInterface { } return this.client .post>('/api/notes/user-list-timeline', params) - .then(async res => ({ ...res, data: await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl)))) })) + .then(async res => ({ ...res, data: await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl), accountCache))) })) } // ====================================== @@ -1951,6 +2006,7 @@ export default class Misskey implements MegalodonInterface { return this.client .post>('/api/notes/mentions', params) .then(res => ({ ...res, data: res.data.map(n => this.converter.noteToConversation(n, this.baseUrlToHost(this.baseUrl))) })) + // FIXME: ^ this should also parse mentions } public async deleteConversation(_id: string): Promise> { @@ -2225,6 +2281,8 @@ export default class Misskey implements MegalodonInterface { exclude_unreviewed?: boolean } ): Promise> { + const accountCache = this.getFreshAccountCache(); + switch (type) { case 'accounts': { let params = { @@ -2291,7 +2349,7 @@ export default class Misskey implements MegalodonInterface { ...res, data: { accounts: [], - statuses: await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl)))), + statuses: await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl), accountCache))), hashtags: [] } })) @@ -2429,7 +2487,7 @@ export default class Misskey implements MegalodonInterface { .post('/api/notes/show', { noteId: id }) - .then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl))})) + .then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl), this.getFreshAccountCache())})) } /** @@ -2443,7 +2501,7 @@ export default class Misskey implements MegalodonInterface { .post('/api/notes/show', { noteId: id }) - .then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl))})) + .then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl), this.getFreshAccountCache())})) } public async getEmojiReactions(id: string): Promise>> {