From a88d5814137fffd7ed44b54f338dc1f1348be1b3 Mon Sep 17 00:00:00 2001 From: Laura Hausmann Date: Fri, 17 Nov 2023 20:57:45 +0100 Subject: [PATCH] [backend] Implement filters for postgres FTS --- .../server/api/common/generate-fts-query.ts | 167 ++++++++++++++++++ .../src/server/api/endpoints/notes/search.ts | 12 +- .../src/server/api/mastodon/helpers/search.ts | 9 +- 3 files changed, 178 insertions(+), 10 deletions(-) create mode 100644 packages/backend/src/server/api/common/generate-fts-query.ts diff --git a/packages/backend/src/server/api/common/generate-fts-query.ts b/packages/backend/src/server/api/common/generate-fts-query.ts new file mode 100644 index 000000000..de86cf9f3 --- /dev/null +++ b/packages/backend/src/server/api/common/generate-fts-query.ts @@ -0,0 +1,167 @@ +import { Brackets, IsNull, Not, SelectQueryBuilder } from "typeorm"; +import { sqlLikeEscape } from "@/misc/sql-like-escape.js"; +import { Followings, Users } from "@/models/index.js"; +import { FILE_TYPE_BROWSERSAFE } from "@/const.js"; + +const filters = { + "from": fromFilter, + "-from": fromFilterInverse, + "mention": mentionFilter, + "-mention": mentionFilterInverse, + "reply": replyFilter, + "-reply": replyFilterInverse, + "replyto": replyFilter, + "-replyto": replyFilterInverse, + "to": replyFilter, + "-to": replyFilterInverse, + "before": beforeFilter, + "until": beforeFilter, + "after": afterFilter, + "since": afterFilter, + "domain": domainFilter, + "host": domainFilter, + "filter": miscFilter, + "-filter": miscFilterInverse, + "has": attachmentFilter, +} as Record, search: string) => any> + +//TODO: (phrase OR phrase2) should be treated as an OR part of the query +//TODO: "phrase with multiple words" should be treated as one term +//TODO: editing the query should be possible, clicking search again resets it (it should be a twitter-like top of the page kind of deal) + +export function generateFtsQuery(query: SelectQueryBuilder, q: string): void { + const components = q.split(" "); + const terms: string[] = []; + + for (const component of components) { + const split = component.split(":"); + if (split.length > 1 && filters[split[0]] !== undefined) + filters[split[0]](query, split.slice(1).join(":")); + else terms.push(component); + } + + for (const term of terms) { + if (term.startsWith('-')) query.andWhere("note.text NOT ILIKE :q", { q: `%${sqlLikeEscape(term.substring(1))}%` }); + else query.andWhere("note.text ILIKE :q", { q: `%${sqlLikeEscape(term)}%` }); + } +} + +function fromFilter(query: SelectQueryBuilder, filter: string) { + const userQuery = generateUserSubquery(filter); + query.andWhere(`note.userId = (${userQuery.getQuery()})`); + query.setParameters(userQuery.getParameters()); +} + +function fromFilterInverse(query: SelectQueryBuilder, filter: string) { + const userQuery = generateUserSubquery(filter); + query.andWhere(`note.userId <> (${userQuery.getQuery()})`); + query.setParameters(userQuery.getParameters()); +} + +function mentionFilter(query: SelectQueryBuilder, filter: string) { + const userQuery = generateUserSubquery(filter); + query.andWhere(`note.mentions @> array[(${userQuery.getQuery()})]`); + query.setParameters(userQuery.getParameters()); +} + +function mentionFilterInverse(query: SelectQueryBuilder, filter: string) { + const userQuery = generateUserSubquery(filter); + query.andWhere(`NOT (note.mentions @> array[(${userQuery.getQuery()})])`); + query.setParameters(userQuery.getParameters()); +} + +function replyFilter(query: SelectQueryBuilder, filter: string) { + const userQuery = generateUserSubquery(filter); + query.andWhere(`note.replyUserId = (${userQuery.getQuery()})`); + query.setParameters(userQuery.getParameters()); +} + +function replyFilterInverse(query: SelectQueryBuilder, filter: string) { + const userQuery = generateUserSubquery(filter); + query.andWhere(`note.replyUserId <> (${userQuery.getQuery()})`); + query.setParameters(userQuery.getParameters()); +} + +function beforeFilter(query: SelectQueryBuilder, filter: string) { + query.andWhere('note.createdAt < :before', { before: filter }); +} + +function afterFilter(query: SelectQueryBuilder, filter: string) { + query.andWhere('note.createdAt > :after', { after: filter }); +} + +function domainFilter(query: SelectQueryBuilder, filter: string) { + query.andWhere('note.userHost = :domain', { domain: filter }); +} + +function miscFilter(query: SelectQueryBuilder, filter: string) { + let subQuery: SelectQueryBuilder | null = null; + if (filter === 'followers') { + subQuery = Followings.createQueryBuilder('following') + .select('following.followerId') + .where('following.followeeId = :meId') + } else if (filter === 'following') { + subQuery = Followings.createQueryBuilder('following') + .select('following.followeeId') + .where('following.followerId = :meId') + } else if (filter === 'replies') { + query.andWhere('note.replyId IS NOT NULL'); + } else if (filter === 'boosts') { + query.andWhere('note.renoteId IS NOT NULL'); + } + + if (subQuery !== null) query.andWhere(`note.userId IN (${subQuery.getQuery()})`); +} + +function miscFilterInverse(query: SelectQueryBuilder, filter: string) { + let subQuery: SelectQueryBuilder | null = null; + if (filter === 'followers') { + subQuery = Followings.createQueryBuilder('following') + .select('following.followerId') + .where('following.followeeId = :meId') + } else if (filter === 'following') { + subQuery = Followings.createQueryBuilder('following') + .select('following.followeeId') + .where('following.followerId = :meId') + } else if (filter === 'replies') { + query.andWhere('note.replyId IS NULL'); + } else if (filter === 'boosts' || filter === 'renotes') { + query.andWhere('note.renoteId IS NULL'); + } + + if (subQuery !== null) query.andWhere(`note.userId NOT IN (${subQuery.getQuery()})`); +} + +function attachmentFilter(query: SelectQueryBuilder, filter: string) { + switch(filter) { + case 'image': + case 'video': + case 'audio': + query.andWhere(`note.attachedFileTypes && array[:...types]::varchar[]`, { types: FILE_TYPE_BROWSERSAFE.filter(t => t.startsWith(`${filter}/`)) }); + break; + case 'file': + query.andWhere(`note.attachedFileTypes <> '{}'`); + query.andWhere(`NOT (note.attachedFileTypes && array[:...types]::varchar[])`, { types: FILE_TYPE_BROWSERSAFE }); + break; + default: + break; + } +} + +function generateUserSubquery(filter: string) { + if (filter.startsWith('@')) filter = filter.substring(1); + const split = filter.split('@'); + const id = Buffer.from(filter).toString('hex'); + + const query = Users.createQueryBuilder('user') + .select('user.id') + .where(`user.usernameLower = :user_${id}`) + .andWhere(`user.host ${split[1] !== undefined ? `= :host_${id}` : 'IS NULL'}`); + + query.setParameter(`user_${id}`, split[0].toLowerCase()); + + if (split[1] !== undefined) + query.setParameter(`host_${id}`, split[1].toLowerCase()); + + return query; +} diff --git a/packages/backend/src/server/api/endpoints/notes/search.ts b/packages/backend/src/server/api/endpoints/notes/search.ts index f7da1a837..fd211c254 100644 --- a/packages/backend/src/server/api/endpoints/notes/search.ts +++ b/packages/backend/src/server/api/endpoints/notes/search.ts @@ -6,6 +6,7 @@ import { generateVisibilityQuery } from "../../common/generate-visibility-query. import { generateMutedUserQuery } from "../../common/generate-muted-user-query.js"; import { generateBlockedUserQuery } from "../../common/generate-block-query.js"; import { sqlLikeEscape } from "@/misc/sql-like-escape.js"; +import { generateFtsQuery } from "@/server/api/common/generate-fts-query.js"; export const meta = { tags: ["notes"], @@ -73,7 +74,6 @@ export default define(meta, paramDef, async (ps, me) => { } query - .andWhere("note.text ILIKE :q", { q: `%${sqlLikeEscape(ps.query)}%` }) .innerJoinAndSelect("note.user", "user") .leftJoinAndSelect("user.avatar", "avatar") .leftJoinAndSelect("user.banner", "banner") @@ -86,11 +86,11 @@ export default define(meta, paramDef, async (ps, me) => { .leftJoinAndSelect("renoteUser.avatar", "renoteUserAvatar") .leftJoinAndSelect("renoteUser.banner", "renoteUserBanner"); + generateFtsQuery(query, ps.query); generateVisibilityQuery(query, me); - if (me) generateMutedUserQuery(query, me); - if (me) generateBlockedUserQuery(query, me); + generateMutedUserQuery(query, me); + generateBlockedUserQuery(query, me); + query.setParameter("meId", me.id); - const notes: Note[] = await query.take(ps.limit).getMany(); - - return await Notes.packMany(notes, me); + return await query.take(ps.limit).getMany().then(notes => Notes.packMany(notes, me)); }); diff --git a/packages/backend/src/server/api/mastodon/helpers/search.ts b/packages/backend/src/server/api/mastodon/helpers/search.ts index 128e61daa..32c5bbb2e 100644 --- a/packages/backend/src/server/api/mastodon/helpers/search.ts +++ b/packages/backend/src/server/api/mastodon/helpers/search.ts @@ -18,6 +18,7 @@ import { resolveUser } from "@/remote/resolve-user.js"; import { createNote } from "@/remote/activitypub/models/note.js"; import config from "@/config/index.js"; import { logger, MastoContext } from "@/server/api/mastodon/index.js"; +import { generateFtsQuery } from "@/server/api/common/generate-fts-query.js"; export class SearchHelpers { public static async search(q: string | undefined, type: string | undefined, resolve: boolean = false, following: boolean = false, accountId: string | undefined, excludeUnreviewed: boolean = false, maxId: string | undefined, minId: string | undefined, limit: number = 20, offset: number | undefined, ctx: MastoContext): Promise { @@ -163,11 +164,9 @@ export class SearchHelpers { ) } - query - .andWhere("note.text ILIKE :q", { q: `%${sqlLikeEscape(q)}%` }) - .leftJoinAndSelect("note.renote", "renote"); - + query.leftJoinAndSelect("note.renote", "renote"); + generateFtsQuery(query, q); generateVisibilityQuery(query, user); if (!accountId) { @@ -175,6 +174,8 @@ export class SearchHelpers { generateBlockedUserQuery(query, user); } + query.setParameter("meId", user); + return query.skip(offset ?? 0).take(limit).getMany().then(p => minId ? p.reverse() : p); }