e2net/iceshrimp
3
0
Fork 0
forked from mirrors/iceshrimp
Code Pull requests Activity

[mastodon-client] Cache account/user data per api call

Browse source
This commit is contained in:
Laura Hausmann 2023-09-18 22:19:33 +02:00
parent 941f44dc71
commit e90b679864
Signed by: zotan
GPG key ID: D044E84C5BE01605
9 changed files with 103 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.pnp.cjs generated
View file

@ -14960,6 +14960,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) {
["@tensorflow/tfjs-core", "npm:4.9.0"],\
["@tensorflow/tfjs-node", "npm:3.21.1"],\
["@types/adm-zip", "npm:0.5.0"],\
["@types/async-lock", "npm:1.4.0"],\
["@types/bcryptjs", "npm:2.4.2"],\
["@types/cbor", "npm:6.0.0"],\
["@types/escape-regexp", "npm:0.0.1"],\
@ -15006,6 +15007,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) {
["ajv", "npm:8.12.0"],\
["archiver", "npm:5.3.1"],\
["argon2", "npm:0.30.3"],\
["async-lock", "npm:1.4.0"],\
["autolinker", "npm:4.0.0"],\
["autwh", "npm:0.1.0"],\
["aws-sdk", "npm:2.1413.0"],\

2
packages/backend/package.json
View file

@ -44,6 +44,7 @@
"ajv": "8.12.0",
"archiver": "5.3.1",
"argon2": "^0.30.3",
"async-lock": "1.4.0",
"autolinker": "4.0.0",
"autwh": "0.1.0",
"aws-sdk": "2.1413.0",
@ -146,6 +147,7 @@
"@swc/cli": "^0.1.62",
"@swc/core": "^1.3.68",
"@types/adm-zip": "^0.5.0",
"@types/async-lock": "1.4.0",
"@types/bcryptjs": "2.4.2",
"@types/cbor": "6.0.0",
"@types/escape-regexp": "0.0.1",

19
packages/backend/src/server/api/mastodon/converters/note.ts
View file

@ -16,10 +16,11 @@ import { PollConverter } from "@/server/api/mastodon/converters/poll.js";
import { populatePoll } from "@/models/repositories/note.js";
import { FileConverter } from "@/server/api/mastodon/converters/file.js";
import { awaitAll } from "@/prelude/await-all.js";
import { AccountCache, UserHelpers } from "@/server/api/mastodon/helpers/user.js";
export class NoteConverter {
public static async encode(note: Note, user: ILocalUser | null): Promise<MastodonEntity.Status> {
const noteUser = note.user ?? getUser(note.userId);
public static async encode(note: Note, user: ILocalUser | null, cache: AccountCache = UserHelpers.getFreshAccountCache()): Promise<MastodonEntity.Status> {
const noteUser = note.user ?? UserHelpers.getUserCached(note.userId, cache);
if (!await Notes.isVisibleForMe(note, user?.id ?? null))
throw new Error('Cannot encode note not visible for user');
@ -72,22 +73,20 @@ export class NoteConverter {
const files = DriveFiles.packMany(note.fileIds);
const mentions = Promise.all(note.mentions.map(p =>
getUser(p)
UserHelpers.getUserCached(p, cache)
.then(u => MentionConverter.encode(u, JSON.parse(note.mentionedRemoteUsers)))
.catch(() => null)))
.then(p => p.filter(m => m)) as Promise<MastodonEntity.Mention[]>;
// FIXME use await-all
// noinspection ES6MissingAwait
return await awaitAll({
id: note.id,
uri: note.uri ? note.uri : `https://${config.host}/notes/${note.id}`,
url: note.uri ? note.uri : `https://${config.host}/notes/${note.id}`,
account: Promise.resolve(noteUser).then(p => UserConverter.encode(p)),
account: Promise.resolve(noteUser).then(p => UserConverter.encode(p, cache)),
in_reply_to_id: note.replyId,
in_reply_to_account_id: Promise.resolve(reply).then(reply => reply?.userId ?? null),
reblog: note.renote ? this.encode(note.renote, user) : null,
reblog: note.renote ? this.encode(note.renote, user, cache) : null,
content: note.text ? toHtml(mfm.parse(note.text), JSON.parse(note.mentionedRemoteUsers)) ?? escapeMFM(note.text) : "",
text: note.text ? note.text : null,
created_at: note.createdAt.toISOString(),
@ -116,12 +115,12 @@ export class NoteConverter {
// Use emojis list to provide URLs for emoji reactions.
reactions: [], //FIXME: this.mapReactions(n.emojis, n.reactions, n.myReaction),
bookmarked: isBookmarked,
quote: note.renote && note.text ? this.encode(note.renote, user) : null,
quote: note.renote && note.text ? this.encode(note.renote, user, cache) : null,
});
}
public static async encodeMany(notes: Note[], user: ILocalUser | null): Promise<MastodonEntity.Status[]> {
const encoded = notes.map(n => this.encode(n, user));
public static async encodeMany(notes: Note[], user: ILocalUser | null, cache: AccountCache = UserHelpers.getFreshAccountCache()): Promise<MastodonEntity.Status[]> {
const encoded = notes.map(n => this.encode(n, user, cache));
return Promise.all(encoded);
}
}

83
packages/backend/src/server/api/mastodon/converters/user.ts
View file

@ -7,6 +7,7 @@ import { toHtml } from "@/mfm/to-html.js";
import { escapeMFM } from "@/server/api/mastodon/converters/mfm.js";
import mfm from "mfm-js";
import { awaitAll } from "@/prelude/await-all.js";
import { AccountCache, UserHelpers } from "@/server/api/mastodon/helpers/user.js";
type Field = {
name: string;
@ -15,44 +16,52 @@ type Field = {
};
export class UserConverter {
public static async encode(u: User): Promise<MastodonEntity.Account> {
let acct = u.username;
let acctUrl = `https://${u.host || config.host}/@${u.username}`;
if (u.host) {
acct = `${u.username}@${u.host}`;
acctUrl = `https://${u.host}/@${u.username}`;
}
const profile = UserProfiles.findOneBy({userId: u.id});
const bio = profile.then(profile => toHtml(mfm.parse(profile?.description ?? "")) ?? escapeMFM(profile?.description ?? ""));
const avatar = u.avatarId
? (DriveFiles.findOneBy({ id: u.avatarId }))
.then(p => p?.url ?? Users.getIdenticonUrl(u.id))
: Users.getIdenticonUrl(u.id);
const banner = u.bannerId
? (DriveFiles.findOneBy({ id: u.bannerId }))
.then(p => p?.url ?? `${config.url}/static-assets/transparent.png`)
: `${config.url}/static-assets/transparent.png`;
public static async encode(u: User, cache: AccountCache = UserHelpers.getFreshAccountCache()): Promise<MastodonEntity.Account> {
return cache.locks.acquire(u.id, async () => {
const cacheHit = cache.accounts.find(p => p.id == u.id);
if (cacheHit) return cacheHit;
return awaitAll({
id: u.id,
username: u.username,
acct: acct,
display_name: u.name || u.username,
locked: u.isLocked,
created_at: new Date().toISOString(),
followers_count: u.followersCount,
following_count: u.followingCount,
statuses_count: u.notesCount,
note: bio,
url: u.uri ?? acctUrl,
avatar: avatar,
avatar_static: avatar,
header: banner,
header_static: banner,
emojis: populateEmojis(u.emojis, u.host).then(emoji => emoji.map((e) => EmojiConverter.encode(e))),
moved: null, //FIXME
fields: profile.then(profile => profile?.fields.map(p => this.encodeField(p)) ?? []),
bot: u.isBot
let acct = u.username;
let acctUrl = `https://${u.host || config.host}/@${u.username}`;
if (u.host) {
acct = `${u.username}@${u.host}`;
acctUrl = `https://${u.host}/@${u.username}`;
}
const profile = UserProfiles.findOneBy({userId: u.id});
const bio = profile.then(profile => toHtml(mfm.parse(profile?.description ?? "")) ?? escapeMFM(profile?.description ?? ""));
const avatar = u.avatarId
? (DriveFiles.findOneBy({ id: u.avatarId }))
.then(p => p?.url ?? Users.getIdenticonUrl(u.id))
: Users.getIdenticonUrl(u.id);
const banner = u.bannerId
? (DriveFiles.findOneBy({ id: u.bannerId }))
.then(p => p?.url ?? `${config.url}/static-assets/transparent.png`)
: `${config.url}/static-assets/transparent.png`;
return awaitAll({
id: u.id,
username: u.username,
acct: acct,
display_name: u.name || u.username,
locked: u.isLocked,
created_at: new Date().toISOString(),
followers_count: u.followersCount,
following_count: u.followingCount,
statuses_count: u.notesCount,
note: bio,
url: u.uri ?? acctUrl,
avatar: avatar,
avatar_static: avatar,
header: banner,
header_static: banner,
emojis: populateEmojis(u.emojis, u.host).then(emoji => emoji.map((e) => EmojiConverter.encode(e))),
moved: null, //FIXME
fields: profile.then(profile => profile?.fields.map(p => this.encodeField(p)) ?? []),
bot: u.isBot
}).then(p => {
cache.accounts.push(p);
return p;
});
});
}

5
packages/backend/src/server/api/mastodon/endpoints/account.ts
View file

@ -161,10 +161,11 @@ export function apiAccountMastodon(router: Router): void {
}
const userId = convertId(ctx.params.id, IdType.IceshrimpId);
const query = await getUser(userId);
const cache = UserHelpers.getFreshAccountCache();
const query = await UserHelpers.getUserCached(userId, cache);
const args = normalizeUrlQuery(convertTimelinesArgsId(argsToBools(limitToInt(ctx.query))));
const tl = await UserHelpers.getUserStatuses(query, user, args.max_id, args.since_id, args.min_id, args.limit, args.only_media, args.exclude_replies, args.exclude_reblogs, args.pinned, args.tagged)
.then(n => NoteConverter.encodeMany(n, user));
.then(n => NoteConverter.encodeMany(n, user, cache));
ctx.body = tl.map(s => convertStatus(s));
} catch (e: any) {

6
packages/backend/src/server/api/mastodon/endpoints/status.ts
View file

@ -11,6 +11,7 @@ import { getNote } from "@/server/api/common/getters.js";
import authenticate from "@/server/api/authenticate.js";
import { NoteHelpers } from "@/server/api/mastodon/helpers/note.js";
import { Note } from "@/models/entities/note.js";
import { UserHelpers } from "@/server/api/mastodon/helpers/user.js";
function normalizeQuery(data: any) {
const str = querystring.stringify(data);
@ -197,6 +198,7 @@ export function apiStatusMastodon(router: Router): void {
const user = auth[0] ?? null;
const id = convertId(ctx.params.id, IdType.IceshrimpId);
const cache = UserHelpers.getFreshAccountCache();
const note = await getNote(id, user ?? null).then(n => n).catch(() => null);
if (!note) {
if (!note) {
@ -206,10 +208,10 @@ export function apiStatusMastodon(router: Router): void {
}
const ancestors = await NoteHelpers.getNoteAncestors(note, user, user ? 4096 : 60)
.then(n => NoteConverter.encodeMany(n, user))
.then(n => NoteConverter.encodeMany(n, user, cache))
.then(n => n.map(s => convertStatus(s)));
const descendants = await NoteHelpers.getNoteDescendants(note, user, user ? 4096 : 40, user ? 4096 : 20)
.then(n => NoteConverter.encodeMany(n, user))
.then(n => NoteConverter.encodeMany(n, user, cache))
.then(n => n.map(s => convertStatus(s)));
ctx.body = {

7
packages/backend/src/server/api/mastodon/endpoints/timeline.ts
View file

@ -12,6 +12,7 @@ import authenticate from "@/server/api/authenticate.js";
import { TimelineHelpers } from "@/server/api/mastodon/helpers/timeline.js";
import { NoteHelpers } from "@/server/api/mastodon/helpers/note.js";
import { NoteConverter } from "@/server/api/mastodon/converters/note.js";
import { UserHelpers } from "@/server/api/mastodon/helpers/user.js";
export function limitToInt(q: ParsedUrlQuery) {
let object: any = q;
@ -82,8 +83,9 @@ export function apiTimelineMastodon(router: Router): void {
}
const args = normalizeUrlQuery(convertTimelinesArgsId(argsToBools(limitToInt(ctx.query))));
const cache = UserHelpers.getFreshAccountCache();
const tl = await TimelineHelpers.getPublicTimeline(user, args.max_id, args.since_id, args.min_id, args.limit, args.only_media, args.local, args.remote)
.then(n => NoteConverter.encodeMany(n, user));
.then(n => NoteConverter.encodeMany(n, user, cache));
ctx.body = tl.map(s => convertStatus(s));
} catch (e: any) {
@ -124,8 +126,9 @@ export function apiTimelineMastodon(router: Router): void {
}
const args = normalizeUrlQuery(convertTimelinesArgsId(limitToInt(ctx.query)));
const cache = UserHelpers.getFreshAccountCache();
const tl = await TimelineHelpers.getHomeTimeline(user, args.max_id, args.since_id, args.min_id, args.limit)
.then(n => NoteConverter.encodeMany(n, user));
.then(n => NoteConverter.encodeMany(n, user, cache));
ctx.body = tl.map(s => convertStatus(s));
} catch (e: any) {

40
packages/backend/src/server/api/mastodon/helpers/user.ts
View file

@ -1,20 +1,21 @@
import { Note } from "@/models/entities/note.js";
import { User } from "@/models/entities/user.js";
import { ILocalUser } from "@/models/entities/user.js";
import { Followings, Notes } from "@/models/index.js";
import { ILocalUser, User } from "@/models/entities/user.js";
import { Notes } from "@/models/index.js";
import { makePaginationQuery } from "@/server/api/common/make-pagination-query.js";
import { Brackets, SelectQueryBuilder } from "typeorm";
import { generateChannelQuery } from "@/server/api/common/generate-channel-query.js";
import { generateRepliesQuery } from "@/server/api/common/generate-replies-query.js";
import { generateVisibilityQuery } from "@/server/api/common/generate-visibility-query.js";
import { generateMutedUserQuery } from "@/server/api/common/generate-muted-user-query.js";
import { generateMutedNoteQuery } from "@/server/api/common/generate-muted-note-query.js";
import { generateBlockedUserQuery } from "@/server/api/common/generate-block-query.js";
import { generateMutedUserRenotesQueryForNotes } from "@/server/api/common/generated-muted-renote-query.js";
import { fetchMeta } from "@/misc/fetch-meta.js";
import { ApiError } from "@/server/api/error.js";
import { meta } from "@/server/api/endpoints/notes/global-timeline.js";
import { NoteHelpers } from "@/server/api/mastodon/helpers/note.js";
import Entity from "megalodon/src/entity.js";
import AsyncLock from "async-lock";
import { getUser } from "@/server/api/common/getters.js";
export type AccountCache = {
locks: AsyncLock;
accounts: Entity.Account[];
users: User[];
};
export class UserHelpers {
public static async getUserStatuses(user: User, localUser: ILocalUser | null, maxId: string | undefined, sinceId: string | undefined, minId: string | undefined, limit: number = 20, onlyMedia: boolean = false, excludeReplies: boolean = false, excludeReblogs: boolean = false, pinned: boolean = false, tagged: string | undefined): Promise<Note[]> {
@ -67,4 +68,23 @@ export class UserHelpers {
return NoteHelpers.execQuery(query, limit);
}
public static async getUserCached(id: string, cache: AccountCache = UserHelpers.getFreshAccountCache()): Promise<User> {
return cache.locks.acquire(id, async () => {
const cacheHit = cache.users.find(p => p.id == id);
if (cacheHit) return cacheHit;
return getUser(id).then(p => {
cache.users.push(p);
return p;
});
});
}
public static getFreshAccountCache(): AccountCache {
return {
locks: new AsyncLock(),
accounts: [],
users: [],
};
}
}

2
yarn.lock
View file

@ -5757,6 +5757,7 @@ __metadata:
"@tensorflow/tfjs-core": ^4.2.0
"@tensorflow/tfjs-node": 3.21.1
"@types/adm-zip": ^0.5.0
"@types/async-lock": 1.4.0
"@types/bcryptjs": 2.4.2
"@types/cbor": 6.0.0
"@types/escape-regexp": 0.0.1
@ -5803,6 +5804,7 @@ __metadata:
ajv: 8.12.0
archiver: 5.3.1
argon2: ^0.30.3
async-lock: 1.4.0
autolinker: 4.0.0
autwh: 0.1.0
aws-sdk: 2.1413.0