From 236d72685dfec013135fc1450b890e33ec377de1 Mon Sep 17 00:00:00 2001 From: syuilo Date: Wed, 10 Apr 2019 00:59:41 +0900 Subject: [PATCH] More puny --- src/models/repositories/drive-file.ts | 3 ++- src/queue/processors/inbox.ts | 26 ++++++++++++------- src/remote/activitypub/models/note.ts | 7 +++-- src/remote/activitypub/models/person.ts | 7 +++-- src/remote/activitypub/request.ts | 4 +-- src/remote/resolve-user.ts | 19 +++++--------- src/server/api/common/get-host-lower.ts | 6 ----- src/server/api/endpoints/admin/emoji/list.ts | 3 ++- .../admin/federation/update-instance.ts | 5 ++-- .../api/endpoints/federation/show-instance.ts | 3 ++- src/server/api/endpoints/users/followers.ts | 3 ++- src/server/api/endpoints/users/following.ts | 3 ++- src/server/api/index.ts | 5 +--- src/services/chart/charts/classes/instance.ts | 13 +++++----- 14 files changed, 52 insertions(+), 55 deletions(-) delete mode 100644 src/server/api/common/get-host-lower.ts diff --git a/src/models/repositories/drive-file.ts b/src/models/repositories/drive-file.ts index fe0ca72bf..817677fa3 100644 --- a/src/models/repositories/drive-file.ts +++ b/src/models/repositories/drive-file.ts @@ -3,6 +3,7 @@ import { DriveFile } from '../entities/drive-file'; import { Users, DriveFolders } from '..'; import rap from '@prezzemolo/rap'; import { User } from '../entities/user'; +import { toPuny } from '../../misc/convert-host'; @EntityRepository(DriveFile) export class DriveFileRepository extends Repository { @@ -39,7 +40,7 @@ export class DriveFileRepository extends Repository { public async clacDriveUsageOfHost(host: string): Promise { const { sum } = await this .createQueryBuilder('file') - .where('file.userHost = :host', { host: host }) + .where('file.userHost = :host', { host: toPuny(host) }) .select('SUM(file.size)', 'sum') .getRawOne(); diff --git a/src/queue/processors/inbox.ts b/src/queue/processors/inbox.ts index 16badabcf..481bcbb1c 100644 --- a/src/queue/processors/inbox.ts +++ b/src/queue/processors/inbox.ts @@ -4,7 +4,6 @@ import parseAcct from '../../misc/acct/parse'; import { IRemoteUser } from '../../models/entities/user'; import perform from '../../remote/activitypub/perform'; import { resolvePerson, updatePerson } from '../../remote/activitypub/models/person'; -import { toUnicode } from 'punycode'; import { URL } from 'url'; import { publishApLogStream } from '../../services/stream'; import Logger from '../../services/logger'; @@ -13,6 +12,7 @@ import { Instances, Users, UserPublickeys } from '../../models'; import { instanceChart } from '../../services/chart'; import { UserPublickey } from '../../models/entities/user-publickey'; import fetchMeta from '../../misc/fetch-meta'; +import { toPuny } from '../../misc/convert-host'; const logger = new Logger('inbox'); @@ -33,7 +33,10 @@ export default async (job: Bull.Job): Promise => { let key: UserPublickey; if (keyIdLower.startsWith('acct:')) { - const { username, host } = parseAcct(keyIdLower.slice('acct:'.length)); + const acct = parseAcct(keyIdLower.slice('acct:'.length)); + const host = toPuny(acct.host); + const username = toPuny(acct.username); + if (host === null) { logger.warn(`request was made by local user: @${username}`); return; @@ -50,19 +53,22 @@ export default async (job: Bull.Job): Promise => { // ブロックしてたら中断 // TODO: いちいちデータベースにアクセスするのはコスト高そうなのでどっかにキャッシュしておく const meta = await fetchMeta(); - if (meta.blockedHosts.includes(host.toLowerCase())) { + if (meta.blockedHosts.includes(host)) { logger.info(`Blocked request: ${host}`); return; } - user = await Users.findOne({ usernameLower: username, host: host.toLowerCase() }) as IRemoteUser; + user = await Users.findOne({ + usernameLower: username.toLowerCase(), + host: host + }) as IRemoteUser; key = await UserPublickeys.findOne({ userId: user.id }); } else { // アクティビティ内のホストの検証 - const host = toUnicode(new URL(signature.keyId).hostname.toLowerCase()); + const host = toPuny(new URL(signature.keyId).hostname); try { ValidateActivity(activity, host); } catch (e) { @@ -73,7 +79,7 @@ export default async (job: Bull.Job): Promise => { // ブロックしてたら中断 // TODO: いちいちデータベースにアクセスするのはコスト高そうなのでどっかにキャッシュしておく const meta = await fetchMeta(); - if (meta.blockedHosts.includes(host.toLowerCase())) { + if (meta.blockedHosts.includes(host)) { logger.info(`Blocked request: ${host}`); return; } @@ -145,7 +151,7 @@ export default async (job: Bull.Job): Promise => { function ValidateActivity(activity: any, host: string) { // id (if exists) if (typeof activity.id === 'string') { - const uriHost = toUnicode(new URL(activity.id).hostname.toLowerCase()); + const uriHost = toPuny(new URL(activity.id).hostname); if (host !== uriHost) { const diag = activity.signature ? '. Has LD-Signature. Forwarded?' : ''; throw new Error(`activity.id(${activity.id}) has different host(${host})${diag}`); @@ -154,7 +160,7 @@ function ValidateActivity(activity: any, host: string) { // actor (if exists) if (typeof activity.actor === 'string') { - const uriHost = toUnicode(new URL(activity.actor).hostname.toLowerCase()); + const uriHost = toPuny(new URL(activity.actor).hostname); if (host !== uriHost) throw new Error('activity.actor has different host'); } @@ -162,13 +168,13 @@ function ValidateActivity(activity: any, host: string) { if (activity.type === 'Create' && activity.object) { // object.id (if exists) if (typeof activity.object.id === 'string') { - const uriHost = toUnicode(new URL(activity.object.id).hostname.toLowerCase()); + const uriHost = toPuny(new URL(activity.object.id).hostname); if (host !== uriHost) throw new Error('activity.object.id has different host'); } // object.attributedTo (if exists) if (typeof activity.object.attributedTo === 'string') { - const uriHost = toUnicode(new URL(activity.object.attributedTo).hostname.toLowerCase()); + const uriHost = toPuny(new URL(activity.object.attributedTo).hostname); if (host !== uriHost) throw new Error('activity.object.attributedTo has different host'); } } diff --git a/src/remote/activitypub/models/note.ts b/src/remote/activitypub/models/note.ts index cd587c51c..07991d4eb 100644 --- a/src/remote/activitypub/models/note.ts +++ b/src/remote/activitypub/models/note.ts @@ -8,14 +8,13 @@ import { resolveImage } from './image'; import { IRemoteUser, User } from '../../../models/entities/user'; import { fromHtml } from '../../../mfm/fromHtml'; import { ITag, extractHashtags } from './tag'; -import { toUnicode } from 'punycode'; import { unique, concat, difference } from '../../../prelude/array'; import { extractPollFromQuestion } from './question'; import vote from '../../../services/note/polls/vote'; import { apLogger } from '../logger'; import { DriveFile } from '../../../models/entities/drive-file'; import { deliverQuestionUpdate } from '../../../services/note/polls/update'; -import { extractDbHost } from '../../../misc/convert-host'; +import { extractDbHost, toPuny } from '../../../misc/convert-host'; import { Notes, Emojis, Polls } from '../../../models'; import { Note } from '../../../models/entities/note'; import { IObject, INote } from '../type'; @@ -246,8 +245,8 @@ export async function resolveNote(value: string | IObject, resolver?: Resolver): return await createNote(uri, resolver); } -export async function extractEmojis(tags: ITag[], host_: string) { - const host = toUnicode(host_.toLowerCase()); +export async function extractEmojis(tags: ITag[], host: string) { + host = toPuny(host); if (!tags) return []; diff --git a/src/remote/activitypub/models/person.ts b/src/remote/activitypub/models/person.ts index 2362455e0..715bae40e 100644 --- a/src/remote/activitypub/models/person.ts +++ b/src/remote/activitypub/models/person.ts @@ -1,5 +1,4 @@ import * as promiseLimit from 'promise-limit'; -import { toUnicode } from 'punycode'; import config from '../../../config'; import Resolver from '../resolver'; @@ -33,7 +32,7 @@ const logger = apLogger; * @param uri Fetch target URI */ function validatePerson(x: any, uri: string) { - const expectHost = toUnicode(new URL(uri).hostname.toLowerCase()); + const expectHost = toPuny(new URL(uri).hostname); if (x == null) { return new Error('invalid person: object is null'); @@ -63,7 +62,7 @@ function validatePerson(x: any, uri: string) { return new Error('invalid person: id is not a string'); } - const idHost = toUnicode(new URL(x.id).hostname.toLowerCase()); + const idHost = toPuny(new URL(x.id).hostname); if (idHost !== expectHost) { return new Error('invalid person: id has different host'); } @@ -72,7 +71,7 @@ function validatePerson(x: any, uri: string) { return new Error('invalid person: publicKey.id is not a string'); } - const publicKeyIdHost = toUnicode(new URL(x.publicKey.id).hostname.toLowerCase()); + const publicKeyIdHost = toPuny(new URL(x.publicKey.id).hostname); if (publicKeyIdHost !== expectHost) { return new Error('invalid person: publicKey.id has different host'); } diff --git a/src/remote/activitypub/request.ts b/src/remote/activitypub/request.ts index c50d05e2a..8aca5e810 100644 --- a/src/remote/activitypub/request.ts +++ b/src/remote/activitypub/request.ts @@ -4,7 +4,6 @@ import { URL } from 'url'; import * as crypto from 'crypto'; import { lookup, IRunOptions } from 'lookup-dns-cache'; import * as promiseAny from 'promise-any'; -import { toUnicode } from 'punycode'; import config from '../../config'; import { ILocalUser } from '../../models/entities/user'; @@ -12,6 +11,7 @@ import { publishApLogStream } from '../../services/stream'; import { apLogger } from './logger'; import { UserKeypairs } from '../../models'; import fetchMeta from '../../misc/fetch-meta'; +import { toPuny } from '../../misc/convert-host'; export const logger = apLogger.createSubLogger('deliver'); @@ -25,7 +25,7 @@ export default async (user: ILocalUser, url: string, object: any) => { // ブロックしてたら中断 // TODO: いちいちデータベースにアクセスするのはコスト高そうなのでどっかにキャッシュしておく const meta = await fetchMeta(); - if (meta.blockedHosts.includes(toUnicode(host))) return; + if (meta.blockedHosts.includes(toPuny(host))) return; const data = JSON.stringify(object); diff --git a/src/remote/resolve-user.ts b/src/remote/resolve-user.ts index a10d3c2d8..e6a11bc0d 100644 --- a/src/remote/resolve-user.ts +++ b/src/remote/resolve-user.ts @@ -1,4 +1,3 @@ -import { toUnicode, toASCII } from 'punycode'; import webFinger from './webfinger'; import config from '../config'; import { createPerson, updatePerson } from './activitypub/models/person'; @@ -7,31 +6,27 @@ import { remoteLogger } from './logger'; import chalk from 'chalk'; import { User, IRemoteUser } from '../models/entities/user'; import { Users } from '../models'; +import { toPuny } from '../misc/convert-host'; const logger = remoteLogger.createSubLogger('resolve-user'); -export async function resolveUser(username: string, _host: string, option?: any, resync = false): Promise { +export async function resolveUser(username: string, host: string, option?: any, resync = false): Promise { const usernameLower = username.toLowerCase(); + host = toPuny(host); - if (_host == null) { + if (host == null) { logger.info(`return local user: ${usernameLower}`); return await Users.findOne({ usernameLower, host: null }); } - const configHostAscii = toASCII(config.host).toLowerCase(); - const configHost = toUnicode(configHostAscii); - - const hostAscii = toASCII(_host).toLowerCase(); - const host = toUnicode(hostAscii); - - if (configHost == host) { + if (config.host == host) { logger.info(`return local user: ${usernameLower}`); return await Users.findOne({ usernameLower, host: null }); } const user = await Users.findOne({ usernameLower, host }, option); - const acctLower = `${usernameLower}@${hostAscii}`; + const acctLower = `${usernameLower}@${host}`; if (user == null) { const self = await resolveSelf(acctLower); @@ -51,7 +46,7 @@ export async function resolveUser(username: string, _host: string, option?: any, // validate uri const uri = new URL(self.href); - if (uri.hostname !== hostAscii) { + if (uri.hostname !== host) { throw new Error(`Invalied uri`); } diff --git a/src/server/api/common/get-host-lower.ts b/src/server/api/common/get-host-lower.ts deleted file mode 100644 index 26ddf6c6d..000000000 --- a/src/server/api/common/get-host-lower.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { toUnicode } from 'punycode'; - -export default (host: string) => { - if (host == null) return null; - return toUnicode(host).toLowerCase(); -}; diff --git a/src/server/api/endpoints/admin/emoji/list.ts b/src/server/api/endpoints/admin/emoji/list.ts index 07174723b..26385d4e2 100644 --- a/src/server/api/endpoints/admin/emoji/list.ts +++ b/src/server/api/endpoints/admin/emoji/list.ts @@ -1,6 +1,7 @@ import $ from 'cafy'; import define from '../../../define'; import { Emojis } from '../../../../../models'; +import { toPuny } from '../../../../../misc/convert-host'; export const meta = { desc: { @@ -22,7 +23,7 @@ export const meta = { export default define(meta, async (ps) => { const emojis = await Emojis.find({ - host: ps.host + host: toPuny(ps.host) }); return emojis.map(e => ({ diff --git a/src/server/api/endpoints/admin/federation/update-instance.ts b/src/server/api/endpoints/admin/federation/update-instance.ts index d1abe95a5..90ab7a3ec 100644 --- a/src/server/api/endpoints/admin/federation/update-instance.ts +++ b/src/server/api/endpoints/admin/federation/update-instance.ts @@ -1,6 +1,7 @@ import $ from 'cafy'; import define from '../../../define'; import { Instances } from '../../../../../models'; +import { toPuny } from '../../../../../misc/convert-host'; export const meta = { tags: ['admin'], @@ -20,13 +21,13 @@ export const meta = { }; export default define(meta, async (ps, me) => { - const instance = await Instances.findOne({ host: ps.host }); + const instance = await Instances.findOne({ host: toPuny(ps.host) }); if (instance == null) { throw new Error('instance not found'); } - Instances.update({ host: ps.host }, { + Instances.update({ host: toPuny(ps.host) }, { isMarkedAsClosed: ps.isClosed }); }); diff --git a/src/server/api/endpoints/federation/show-instance.ts b/src/server/api/endpoints/federation/show-instance.ts index 875afa05b..9afcf28a7 100644 --- a/src/server/api/endpoints/federation/show-instance.ts +++ b/src/server/api/endpoints/federation/show-instance.ts @@ -1,6 +1,7 @@ import $ from 'cafy'; import define from '../../define'; import { Instances } from '../../../../models'; +import { toPuny } from '../../../../misc/convert-host'; export const meta = { tags: ['federation'], @@ -16,7 +17,7 @@ export const meta = { export default define(meta, async (ps, me) => { const instance = await Instances - .findOne({ host: ps.host }); + .findOne({ host: toPuny(ps.host) }); return instance; }); diff --git a/src/server/api/endpoints/users/followers.ts b/src/server/api/endpoints/users/followers.ts index 51b007dda..64d63e2d0 100644 --- a/src/server/api/endpoints/users/followers.ts +++ b/src/server/api/endpoints/users/followers.ts @@ -4,6 +4,7 @@ import define from '../../define'; import { ApiError } from '../../error'; import { Users, Followings } from '../../../../models'; import { makePaginationQuery } from '../../common/make-pagination-query'; +import { toPuny } from '../../../../misc/convert-host'; export const meta = { desc: { @@ -65,7 +66,7 @@ export const meta = { export default define(meta, async (ps, me) => { const user = await Users.findOne(ps.userId != null ? { id: ps.userId } - : { usernameLower: ps.username.toLowerCase(), host: ps.host }); + : { usernameLower: ps.username.toLowerCase(), host: toPuny(ps.host) }); if (user == null) { throw new ApiError(meta.errors.noSuchUser); diff --git a/src/server/api/endpoints/users/following.ts b/src/server/api/endpoints/users/following.ts index 46550f0f7..0e2800168 100644 --- a/src/server/api/endpoints/users/following.ts +++ b/src/server/api/endpoints/users/following.ts @@ -4,6 +4,7 @@ import define from '../../define'; import { ApiError } from '../../error'; import { Users, Followings } from '../../../../models'; import { makePaginationQuery } from '../../common/make-pagination-query'; +import { toPuny } from '../../../../misc/convert-host'; export const meta = { desc: { @@ -65,7 +66,7 @@ export const meta = { export default define(meta, async (ps, me) => { const user = await Users.findOne(ps.userId != null ? { id: ps.userId } - : { usernameLower: ps.username.toLowerCase(), host: ps.host }); + : { usernameLower: ps.username.toLowerCase(), host: toPuny(ps.host) }); if (user == null) { throw new ApiError(meta.errors.noSuchUser); diff --git a/src/server/api/index.ts b/src/server/api/index.ts index 7858efd92..8c2b97775 100644 --- a/src/server/api/index.ts +++ b/src/server/api/index.ts @@ -15,7 +15,6 @@ import signin from './private/signin'; import discord from './service/discord'; import github from './service/github'; import twitter from './service/twitter'; -import { toASCII } from 'punycode'; import { Instances } from '../../models'; // Init app @@ -71,9 +70,7 @@ router.get('/v1/instance/peers', async ctx => { select: ['host'] }); - const punyCodes = instances.map(instance => toASCII(instance.host)); - - ctx.body = punyCodes; + ctx.body = instances.map(instance => instance.host); }); // Return 404 for unknown API diff --git a/src/services/chart/charts/classes/instance.ts b/src/services/chart/charts/classes/instance.ts index 55db53457..f3d341f38 100644 --- a/src/services/chart/charts/classes/instance.ts +++ b/src/services/chart/charts/classes/instance.ts @@ -5,6 +5,7 @@ import { DriveFiles, Followings, Users, Notes } from '../../../../models'; import { DriveFile } from '../../../../models/entities/drive-file'; import { name, schema } from '../schemas/instance'; import { Note } from '../../../../models/entities/note'; +import { toPuny } from '../../../../misc/convert-host'; type InstanceLog = SchemaType; @@ -79,7 +80,7 @@ export default class InstanceChart extends Chart { requests: { received: 1 } - }, host); + }, toPuny(host)); } @autobind @@ -94,7 +95,7 @@ export default class InstanceChart extends Chart { await this.inc({ requests: update - }, host); + }, toPuny(host)); } @autobind @@ -104,7 +105,7 @@ export default class InstanceChart extends Chart { total: 1, inc: 1 } - }, host); + }, toPuny(host)); } @autobind @@ -126,7 +127,7 @@ export default class InstanceChart extends Chart { dec: isAdditional ? 0 : 1, diffs: diffs } - }, host); + }, toPuny(host)); } @autobind @@ -137,7 +138,7 @@ export default class InstanceChart extends Chart { inc: isAdditional ? 1 : 0, dec: isAdditional ? 0 : 1, } - }, host); + }, toPuny(host)); } @autobind @@ -148,7 +149,7 @@ export default class InstanceChart extends Chart { inc: isAdditional ? 1 : 0, dec: isAdditional ? 0 : 1, } - }, host); + }, toPuny(host)); } @autobind