zotan/megalodon
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Improve mention account lookup performance

Browse source
This commit is contained in:
Laura Hausmann 2023-07-05 17:59:48 +02:00
parent b033328ee4
commit 9462e8b20d
Signed by: zotan
GPG key ID: D044E84C5BE01605
2 changed files with 95 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
megalodon/package.json
View file

@ -68,7 +68,8 @@
"socks-proxy-agent": "^7.0.0",
"typescript": "4.9.4",
"uuid": "^9.0.0",
"ws": "8.12.0"
"ws": "8.12.0",
"async-lock": "1.4.0"
},
"devDependencies": {
"@types/core-js": "^2.5.0",
@ -79,6 +80,7 @@
"@types/uuid": "^9.0.0",
"@typescript-eslint/eslint-plugin": "^5.49.0",
"@typescript-eslint/parser": "^5.49.0",
"@types/async-lock": "1.4.0",
"eslint": "^8.32.0",
"eslint-config-prettier": "^8.6.0",
"eslint-config-standard": "^16.0.3",

126
megalodon/src/misskey.ts
View file

@ -1,4 +1,5 @@
import FormData from 'form-data'
import AsyncLock from 'async-lock';
import MisskeyAPI from './misskey/api_client'
import { DEFAULT_UA } from './default'
@ -9,6 +10,11 @@ import Entity from './entity'
import { MegalodonInterface, WebSocketInterface, NoImplementedError, ArgumentError, UnexpectedError } from './megalodon'
import MegalodonEntity from "@/entity";
type AccountCache = {
locks: AsyncLock,
accounts: Entity.Account[]
}
export default class Misskey implements MegalodonInterface {
public client: MisskeyAPI.Interface
public converter: MisskeyAPI.Converter
@ -314,6 +320,8 @@ export default class Misskey implements MegalodonInterface {
only_media?: boolean
}
): Promise<Response<Array<Entity.Status>>> {
const accountCache = this.getFreshAccountCache();
if (options?.pinned) {
return this.client
.post<MisskeyAPI.Entity.UserDetail>('/api/users/show', {
@ -323,7 +331,7 @@ export default class Misskey implements MegalodonInterface {
if (res.data.pinnedNotes) {
return {
...res,
data: await Promise.all(res.data.pinnedNotes.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl))))
data: await Promise.all(res.data.pinnedNotes.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl), accountCache)))
}
}
return {...res, data: []}
@ -375,7 +383,7 @@ export default class Misskey implements MegalodonInterface {
})
}
return this.client.post<Array<MisskeyAPI.Entity.Note>>('/api/users/notes', params).then(async res => {
const statuses: Array<Entity.Status> = await Promise.all(res.data.map(note => this.noteWithMentions(note, this.baseUrlToHost(this.baseUrl))))
const statuses: Array<Entity.Status> = await Promise.all(res.data.map(note => this.noteWithMentions(note, this.baseUrlToHost(this.baseUrl), accountCache)))
return Object.assign(res, {
data: statuses
})
@ -390,6 +398,8 @@ export default class Misskey implements MegalodonInterface {
since_id?: string
}
): Promise<Response<Array<Entity.Status>>> {
const accountCache = this.getFreshAccountCache();
let params = {
userId: id
};
@ -412,7 +422,7 @@ export default class Misskey implements MegalodonInterface {
}
return this.client.post<Array<MisskeyAPI.Entity.Favorite>>('/api/users/reactions', params).then(async res => {
return Object.assign(res, {
data: await Promise.all(res.data.map(fav => this.noteWithMentions(fav.note, this.baseUrlToHost(this.baseUrl))))
data: await Promise.all(res.data.map(fav => this.noteWithMentions(fav.note, this.baseUrlToHost(this.baseUrl), accountCache)))
})
})
}
@ -700,6 +710,8 @@ export default class Misskey implements MegalodonInterface {
since_id?: string
min_id?: string
}): Promise<Response<Array<Entity.Status>>> {
const accountCache = this.getFreshAccountCache();
let params = {}
if (options) {
if (options.limit) {
@ -720,7 +732,7 @@ export default class Misskey implements MegalodonInterface {
}
return this.client.post<Array<MisskeyAPI.Entity.Favorite>>('/api/i/favorites', params).then(async res => {
return Object.assign(res, {
data: await Promise.all(res.data.map(s => this.noteWithMentions(s.note, this.baseUrlToHost(this.baseUrl))))
data: await Promise.all(res.data.map(s => this.noteWithMentions(s.note, this.baseUrlToHost(this.baseUrl), accountCache)))
})
})
}
@ -1159,7 +1171,7 @@ export default class Misskey implements MegalodonInterface {
.post<MisskeyAPI.Entity.CreatedNote>('/api/notes/create', params)
.then(async res => ({
...res,
data: await this.noteWithMentions(res.data.createdNote, this.baseUrlToHost(this.baseUrl))
data: await this.noteWithMentions(res.data.createdNote, this.baseUrlToHost(this.baseUrl), this.getFreshAccountCache())
}))
}
@ -1171,23 +1183,31 @@ export default class Misskey implements MegalodonInterface {
.post<MisskeyAPI.Entity.Note>('/api/notes/show', {
noteId: id
})
.then(async res => ({ ...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl))}));
.then(async res => ({ ...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl), this.getFreshAccountCache())}));
}
public async noteWithMentions(n: MisskeyAPI.Entity.Note, host: string): Promise<MegalodonEntity.Status> {
private getFreshAccountCache() :AccountCache {
return {
locks: new AsyncLock(),
accounts: []
}
}
public async noteWithMentions(n: MisskeyAPI.Entity.Note, host: string, cache: AccountCache): Promise<MegalodonEntity.Status> {
const status = await this.converter.note(n, host);
return status.mentions.length === 0 ? this.addMentionsToStatus(status) : status;
return status.mentions.length === 0 ? this.addMentionsToStatus(status, cache) : status;
}
public async addMentionsToStatus(status: Entity.Status) : Promise<Entity.Status> {
status.mentions = (await this.getMentions(status.plain_content!)).filter(p => p != null);
public async addMentionsToStatus(status: Entity.Status, cache: AccountCache) : Promise<Entity.Status> {
status.mentions = (await this.getMentions(status.plain_content!, cache)).filter(p => p != null);
for (const m of status.mentions.filter((value, index, array) => array.indexOf(value) === index)) {
status.content = status.content.replace(`@${m.acct}`, `<a href="${m.url}" class="u-url mention" rel="nofollow noopener noreferrer" target="_blank">@${m.acct}</a>`);
}
return status;
}
public async getMentions(text: string): Promise<Entity.Mention[]> {
public async getMentions(text: string, cache: AccountCache): Promise<Entity.Mention[]> {
console.log(`cache length: ${cache.accounts.length}`);
const mentions :Entity.Mention[] = [];
const mentionMatch = text.matchAll(/(?<=^|\s)@(?<user>.*?)(?:@(?<host>.*?)|)(?=\s|$)/g);
@ -1197,22 +1217,46 @@ export default class Misskey implements MegalodonInterface {
if (m.groups == null)
continue;
const account = await this.getAccountByName(m.groups.user, m.groups?.host ?? null);
const account = await this.getAccountByNameCached(m.groups.user, m.groups.host, cache);
if (!account)
if (account == null)
continue;
mentions.push({
id: account.data.id,
url: account.data.url,
username: account.data.username,
acct: account.data.acct
id: account.id,
url: account.url,
username: account.username,
acct: account.acct
});
}
return mentions;
}
public async getAccountByNameCached(user: string, host: string | null, cache: AccountCache): Promise<Entity.Account | undefined | null> {
const acctToFind = host == null ? user : `${user}@${host}`;
return await cache.locks.acquire(acctToFind, async () => {
const cacheHit = cache.accounts.find(p => p.acct === acctToFind);
const account = cacheHit ?? (await this.getAccountByName(user, host ?? null)).data;
if (!account) {
console.log(`cache fail: ${acctToFind}`);
return null;
}
if (cacheHit == null) {
console.log(`cache miss: ${acctToFind}`);
cache.accounts.push(account);
}
else {
console.log(`cache hit: ${acctToFind}`);
}
return account;
})
}
public async editStatus(
_id: string,
_options: {
@ -1281,12 +1325,13 @@ export default class Misskey implements MegalodonInterface {
return this.client.post<Array<MisskeyAPI.Entity.Note>>('/api/notes/children', params).then(async res => {
console.log(JSON.stringify(res, null, 2));
const accountCache = this.getFreshAccountCache();
const conversation = await this.client.post<Array<MisskeyAPI.Entity.Note>>('/api/notes/conversation', params);
const parents = await Promise.all(conversation.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl))));
const parents = await Promise.all(conversation.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl), accountCache)));
const context: Entity.Context = {
ancestors: parents.reverse(),
descendants: this.dfs(await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl)))))
descendants: this.dfs(await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl), accountCache))))
}
return {
...res,
@ -1399,7 +1444,7 @@ export default class Misskey implements MegalodonInterface {
})
.then(async res => ({
...res,
data: await this.noteWithMentions(res.data.createdNote, this.baseUrlToHost(this.baseUrl))
data: await this.noteWithMentions(res.data.createdNote, this.baseUrlToHost(this.baseUrl), this.getFreshAccountCache())
}))
}
@ -1414,7 +1459,7 @@ export default class Misskey implements MegalodonInterface {
.post<MisskeyAPI.Entity.Note>('/api/notes/show', {
noteId: id
})
.then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl))}))
.then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl), this.getFreshAccountCache())}))
}
/**
@ -1428,7 +1473,7 @@ export default class Misskey implements MegalodonInterface {
.post<MisskeyAPI.Entity.Note>('/api/notes/show', {
noteId: id
})
.then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl))}))
.then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl), this.getFreshAccountCache())}))
}
/**
@ -1442,7 +1487,7 @@ export default class Misskey implements MegalodonInterface {
.post<MisskeyAPI.Entity.Note>('/api/notes/show', {
noteId: id
})
.then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl))}))
.then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl), this.getFreshAccountCache())}))
}
public async muteStatus(_id: string): Promise<Response<Entity.Status>> {
@ -1470,7 +1515,7 @@ export default class Misskey implements MegalodonInterface {
.post<MisskeyAPI.Entity.Note>('/api/notes/show', {
noteId: id
})
.then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl))}))
.then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl), this.getFreshAccountCache())}))
}
/**
@ -1484,7 +1529,7 @@ export default class Misskey implements MegalodonInterface {
.post<MisskeyAPI.Entity.Note>('/api/notes/show', {
noteId: id
})
.then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl))}))
.then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl), this.getFreshAccountCache())}))
}
// ======================================
@ -1570,7 +1615,7 @@ export default class Misskey implements MegalodonInterface {
noteId: status_id
})
.then(async res => {
const note = await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl))
const note = await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl), this.getFreshAccountCache())
return {...res, data: note.poll}
})
if (!res.data) {
@ -1631,6 +1676,8 @@ export default class Misskey implements MegalodonInterface {
since_id?: string
min_id?: string
}): Promise<Response<Array<Entity.Status>>> {
const accountCache = this.getFreshAccountCache();
let params = {}
if (options) {
if (options.only_media !== undefined) {
@ -1673,7 +1720,7 @@ export default class Misskey implements MegalodonInterface {
.post<Array<MisskeyAPI.Entity.Note>>('/api/notes/global-timeline', params)
.then(async res => ({
...res,
data: await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl))))
data: await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl), accountCache)))
}))
}
@ -1687,6 +1734,8 @@ export default class Misskey implements MegalodonInterface {
since_id?: string
min_id?: string
}): Promise<Response<Array<Entity.Status>>> {
const accountCache = this.getFreshAccountCache();
let params = {}
if (options) {
if (options.only_media !== undefined) {
@ -1729,7 +1778,7 @@ export default class Misskey implements MegalodonInterface {
.post<Array<MisskeyAPI.Entity.Note>>('/api/notes/local-timeline', params)
.then(async res => ({
...res,
data: await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl))))
data: await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl), accountCache)))
}))
}
@ -1747,6 +1796,8 @@ export default class Misskey implements MegalodonInterface {
min_id?: string
}
): Promise<Response<Array<Entity.Status>>> {
const accountCache = this.getFreshAccountCache();
let params = {
tag: hashtag
}
@ -1791,7 +1842,7 @@ export default class Misskey implements MegalodonInterface {
.post<Array<MisskeyAPI.Entity.Note>>('/api/notes/search-by-tag', params)
.then(async res => ({
...res,
data: await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl))))
data: await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl), accountCache)))
}))
}
@ -1805,6 +1856,8 @@ export default class Misskey implements MegalodonInterface {
since_id?: string
min_id?: string
}): Promise<Response<Array<Entity.Status>>> {
const accountCache = this.getFreshAccountCache();
let params = {
withFiles: false
}
@ -1844,7 +1897,7 @@ export default class Misskey implements MegalodonInterface {
.post<Array<MisskeyAPI.Entity.Note>>('/api/notes/timeline', params)
.then(async res => ({
...res,
data: await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl))))
data: await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl), accountCache)))
}))
}
@ -1860,6 +1913,8 @@ export default class Misskey implements MegalodonInterface {
min_id?: string
}
): Promise<Response<Array<Entity.Status>>> {
const accountCache = this.getFreshAccountCache();
let params = {
listId: list_id,
withFiles: false
@ -1898,7 +1953,7 @@ export default class Misskey implements MegalodonInterface {
}
return this.client
.post<Array<MisskeyAPI.Entity.Note>>('/api/notes/user-list-timeline', params)
.then(async res => ({ ...res, data: await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl)))) }))
.then(async res => ({ ...res, data: await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl), accountCache))) }))
}
// ======================================
@ -1951,6 +2006,7 @@ export default class Misskey implements MegalodonInterface {
return this.client
.post<Array<MisskeyAPI.Entity.Note>>('/api/notes/mentions', params)
.then(res => ({ ...res, data: res.data.map(n => this.converter.noteToConversation(n, this.baseUrlToHost(this.baseUrl))) }))
// FIXME: ^ this should also parse mentions
}
public async deleteConversation(_id: string): Promise<Response<{}>> {
@ -2225,6 +2281,8 @@ export default class Misskey implements MegalodonInterface {
exclude_unreviewed?: boolean
}
): Promise<Response<Entity.Results>> {
const accountCache = this.getFreshAccountCache();
switch (type) {
case 'accounts': {
let params = {
@ -2291,7 +2349,7 @@ export default class Misskey implements MegalodonInterface {
...res,
data: {
accounts: [],
statuses: await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl)))),
statuses: await Promise.all(res.data.map(n => this.noteWithMentions(n, this.baseUrlToHost(this.baseUrl), accountCache))),
hashtags: []
}
}))
@ -2429,7 +2487,7 @@ export default class Misskey implements MegalodonInterface {
.post<MisskeyAPI.Entity.Note>('/api/notes/show', {
noteId: id
})
.then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl))}))
.then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl), this.getFreshAccountCache())}))
}
/**
@ -2443,7 +2501,7 @@ export default class Misskey implements MegalodonInterface {
.post<MisskeyAPI.Entity.Note>('/api/notes/show', {
noteId: id
})
.then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl))}))
.then(async res => ({...res, data: await this.noteWithMentions(res.data, this.baseUrlToHost(this.baseUrl), this.getFreshAccountCache())}))
}
public async getEmojiReactions(id: string): Promise<Response<Array<Entity.Reaction>>> {