From eccc90c843f63b2dc08d8fbf80e4f54a601e477d Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 2 Jul 2022 15:12:11 +0900 Subject: [PATCH] feat: Log user ips (#8872) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * wip * store ip and headers * Update admin-file.vue * require admin for view ip/headers * IP (recent) 消した * admin必須 * opt in * clean ips periodically * respect logging setting in drive/files/create --- locales/ja-JP.yml | 1 + .../migration/1655918165614-user-ip.js | 17 +++++++ .../migration/1656122560740-file-ip.js | 13 +++++ .../backend/migration/1656328812281-ip-2.js | 13 +++++ packages/backend/src/db/postgre.ts | 4 +- .../backend/src/models/entities/drive-file.ts | 13 ++++- packages/backend/src/models/entities/meta.ts | 7 ++- .../backend/src/models/entities/user-ip.ts | 24 +++++++++ packages/backend/src/models/index.ts | 2 + packages/backend/src/queue/index.ts | 18 ++++--- .../src/queue/processors/system/clean.ts | 18 +++++++ .../src/queue/processors/system/index.ts | 2 + .../backend/src/server/api/api-handler.ts | 34 +++++++++++++ packages/backend/src/server/api/call.ts | 2 +- packages/backend/src/server/api/define.ts | 34 +++++++------ packages/backend/src/server/api/endpoints.ts | 2 + .../api/endpoints/admin/drive/show-file.ts | 7 ++- .../api/endpoints/admin/get-user-ips.ts | 31 +++++++++++ .../src/server/api/endpoints/admin/meta.ts | 8 ++- .../server/api/endpoints/admin/update-meta.ts | 7 ++- .../api/endpoints/drive/files/create.ts | 21 ++++++-- .../endpoints/drive/files/upload-from-url.ts | 6 +-- .../backend/src/services/drive/add-file.ts | 51 +++++++++++-------- .../src/services/drive/upload-from-url.ts | 10 ++-- packages/client/src/account.ts | 1 + packages/client/src/pages/admin-file.vue | 26 ++++++++-- packages/client/src/pages/admin/security.vue | 15 ++++++ packages/client/src/pages/user-info.vue | 47 +++++++++++++++-- packages/client/src/scripts/upload.ts | 10 ++-- 29 files changed, 371 insertions(+), 73 deletions(-) create mode 100644 packages/backend/migration/1655918165614-user-ip.js create mode 100644 packages/backend/migration/1656122560740-file-ip.js create mode 100644 packages/backend/migration/1656328812281-ip-2.js create mode 100644 packages/backend/src/models/entities/user-ip.ts create mode 100644 packages/backend/src/queue/processors/system/clean.ts create mode 100644 packages/backend/src/server/api/endpoints/admin/get-user-ips.ts diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index b97b64dc5..ec726c821 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -854,6 +854,7 @@ noEmailServerWarning: "メールサーバーの設定がされていません。 thereIsUnresolvedAbuseReportWarning: "未対応の通報があります。" recommended: "推奨" check: "チェック" +requireAdminForView: "閲覧するには管理者アカウントでログインしている必要があります。" isSystemAccount: "システムにより自動で作成・管理されているアカウントです。" typeToConfirm: "この操作を行うには {x} と入力してください" deleteAccount: "アカウント削除" diff --git a/packages/backend/migration/1655918165614-user-ip.js b/packages/backend/migration/1655918165614-user-ip.js new file mode 100644 index 000000000..2294fbaf1 --- /dev/null +++ b/packages/backend/migration/1655918165614-user-ip.js @@ -0,0 +1,17 @@ +export class userIp1655918165614 { + name = 'userIp1655918165614' + + async up(queryRunner) { + await queryRunner.query(`CREATE TABLE "user_ip" ("id" SERIAL NOT NULL, "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL, "userId" character varying(32) NOT NULL, "ip" character varying(128) NOT NULL, CONSTRAINT "PK_2c44ddfbf7c0464d028dcef325e" PRIMARY KEY ("id"))`); + await queryRunner.query(`CREATE INDEX "IDX_7f7f1c66f48e9a8e18a33bc515" ON "user_ip" ("userId") `); + await queryRunner.query(`CREATE UNIQUE INDEX "IDX_361b500e06721013c124b7b6c5" ON "user_ip" ("userId", "ip") `); + await queryRunner.query(`ALTER TABLE "user_ip" ADD CONSTRAINT "FK_7f7f1c66f48e9a8e18a33bc5150" FOREIGN KEY ("userId") REFERENCES "user"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "user_ip" DROP CONSTRAINT "FK_7f7f1c66f48e9a8e18a33bc5150"`); + await queryRunner.query(`DROP INDEX "public"."IDX_361b500e06721013c124b7b6c5"`); + await queryRunner.query(`DROP INDEX "public"."IDX_7f7f1c66f48e9a8e18a33bc515"`); + await queryRunner.query(`DROP TABLE "user_ip"`); + } +} diff --git a/packages/backend/migration/1656122560740-file-ip.js b/packages/backend/migration/1656122560740-file-ip.js new file mode 100644 index 000000000..b59e7a911 --- /dev/null +++ b/packages/backend/migration/1656122560740-file-ip.js @@ -0,0 +1,13 @@ +export class fileIp1656122560740 { + name = 'fileIp1656122560740' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "drive_file" ADD "requestHeaders" jsonb DEFAULT '{}'`); + await queryRunner.query(`ALTER TABLE "drive_file" ADD "requestIp" character varying(128)`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "drive_file" DROP COLUMN "requestIp"`); + await queryRunner.query(`ALTER TABLE "drive_file" DROP COLUMN "requestHeaders"`); + } +} diff --git a/packages/backend/migration/1656328812281-ip-2.js b/packages/backend/migration/1656328812281-ip-2.js new file mode 100644 index 000000000..b0ee1ebfc --- /dev/null +++ b/packages/backend/migration/1656328812281-ip-2.js @@ -0,0 +1,13 @@ +export class ip21656328812281 { + name = 'ip21656328812281' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "user_ip" DROP CONSTRAINT "FK_7f7f1c66f48e9a8e18a33bc5150"`); + await queryRunner.query(`ALTER TABLE "meta" ADD "enableIpLogging" boolean NOT NULL DEFAULT false`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "enableIpLogging"`); + await queryRunner.query(`ALTER TABLE "user_ip" ADD CONSTRAINT "FK_7f7f1c66f48e9a8e18a33bc5150" FOREIGN KEY ("userId") REFERENCES "user"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`); + } +} diff --git a/packages/backend/src/db/postgre.ts b/packages/backend/src/db/postgre.ts index 904bbb8b7..94d55e431 100644 --- a/packages/backend/src/db/postgre.ts +++ b/packages/backend/src/db/postgre.ts @@ -68,9 +68,10 @@ import { RegistryItem } from '@/models/entities/registry-item.js'; import { Ad } from '@/models/entities/ad.js'; import { PasswordResetRequest } from '@/models/entities/password-reset-request.js'; import { UserPending } from '@/models/entities/user-pending.js'; +import { Webhook } from '@/models/entities/webhook.js'; +import { UserIp } from '@/models/entities/user-ip.js'; import { entities as charts } from '@/services/chart/entities.js'; -import { Webhook } from '@/models/entities/webhook.js'; import { envOption } from '../env.js'; import { dbLogger } from './logger.js'; import { redisClient } from './redis.js'; @@ -173,6 +174,7 @@ export const entities = [ PasswordResetRequest, UserPending, Webhook, + UserIp, ...charts, ]; diff --git a/packages/backend/src/models/entities/drive-file.ts b/packages/backend/src/models/entities/drive-file.ts index a636d1d51..32387290d 100644 --- a/packages/backend/src/models/entities/drive-file.ts +++ b/packages/backend/src/models/entities/drive-file.ts @@ -1,7 +1,7 @@ import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne } from 'typeorm'; +import { id } from '../id.js'; import { User } from './user.js'; import { DriveFolder } from './drive-folder.js'; -import { id } from '../id.js'; @Entity() @Index(['userId', 'folderId', 'id']) @@ -165,4 +165,15 @@ export class DriveFile { comment: 'Whether the DriveFile is direct link to remote server.', }) public isLink: boolean; + + @Column('jsonb', { + default: {}, + nullable: true, + }) + public requestHeaders: Record | null; + + @Column('varchar', { + length: 128, nullable: true, + }) + public requestIp: string | null; } diff --git a/packages/backend/src/models/entities/meta.ts b/packages/backend/src/models/entities/meta.ts index 80b5228bc..2be43bdd4 100644 --- a/packages/backend/src/models/entities/meta.ts +++ b/packages/backend/src/models/entities/meta.ts @@ -1,6 +1,6 @@ import { Entity, Column, PrimaryColumn, ManyToOne, JoinColumn } from 'typeorm'; -import { User } from './user.js'; import { id } from '../id.js'; +import { User } from './user.js'; import { Clip } from './clip.js'; @Entity() @@ -427,4 +427,9 @@ export class Meta { default: true, }) public objectStorageS3ForcePathStyle: boolean; + + @Column('boolean', { + default: false, + }) + public enableIpLogging: boolean; } diff --git a/packages/backend/src/models/entities/user-ip.ts b/packages/backend/src/models/entities/user-ip.ts new file mode 100644 index 000000000..543e9e728 --- /dev/null +++ b/packages/backend/src/models/entities/user-ip.ts @@ -0,0 +1,24 @@ +import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne, PrimaryGeneratedColumn } from 'typeorm'; +import { id } from '../id.js'; +import { Note } from './note.js'; +import { User } from './user.js'; + +@Entity() +@Index(['userId', 'ip'], { unique: true }) +export class UserIp { + @PrimaryGeneratedColumn() + public id: string; + + @Column('timestamp with time zone', { + }) + public createdAt: Date; + + @Index() + @Column(id()) + public userId: User['id']; + + @Column('varchar', { + length: 128, + }) + public ip: string; +} diff --git a/packages/backend/src/models/index.ts b/packages/backend/src/models/index.ts index 814b37d44..3f7326931 100644 --- a/packages/backend/src/models/index.ts +++ b/packages/backend/src/models/index.ts @@ -65,6 +65,7 @@ import { PasswordResetRequest } from './entities/password-reset-request.js'; import { UserPending } from './entities/user-pending.js'; import { InstanceRepository } from './repositories/instance.js'; import { Webhook } from './entities/webhook.js'; +import { UserIp } from './entities/user-ip.js'; export const Announcements = db.getRepository(Announcement); export const AnnouncementReads = db.getRepository(AnnouncementRead); @@ -90,6 +91,7 @@ export const UserGroups = (UserGroupRepository); export const UserGroupJoinings = db.getRepository(UserGroupJoining); export const UserGroupInvitations = (UserGroupInvitationRepository); export const UserNotePinings = db.getRepository(UserNotePining); +export const UserIps = db.getRepository(UserIp); export const UsedUsernames = db.getRepository(UsedUsername); export const Followings = (FollowingRepository); export const FollowRequests = (FollowRequestRepository); diff --git a/packages/backend/src/queue/index.ts b/packages/backend/src/queue/index.ts index c5fd7de1c..ebb3a77ca 100644 --- a/packages/backend/src/queue/index.ts +++ b/packages/backend/src/queue/index.ts @@ -2,6 +2,9 @@ import httpSignature from '@peertube/http-signature'; import { v4 as uuid } from 'uuid'; import config from '@/config/index.js'; +import { DriveFile } from '@/models/entities/drive-file.js'; +import { IActivity } from '@/remote/activitypub/type.js'; +import { Webhook, webhookEventTypes } from '@/models/entities/webhook.js'; import { envOption } from '../env.js'; import processDeliver from './processors/deliver.js'; @@ -12,18 +15,15 @@ import processSystemQueue from './processors/system/index.js'; import processWebhookDeliver from './processors/webhook-deliver.js'; import { endedPollNotification } from './processors/ended-poll-notification.js'; import { queueLogger } from './logger.js'; -import { DriveFile } from '@/models/entities/drive-file.js'; import { getJobInfo } from './get-job-info.js'; import { systemQueue, dbQueue, deliverQueue, inboxQueue, objectStorageQueue, endedPollNotificationQueue, webhookDeliverQueue } from './queues.js'; import { ThinUser } from './types.js'; -import { IActivity } from '@/remote/activitypub/type.js'; -import { Webhook, webhookEventTypes } from '@/models/entities/webhook.js'; function renderError(e: Error): any { return { - stack: e?.stack, - message: e?.message, - name: e?.name, + stack: e.stack, + message: e.message, + name: e.name, }; } @@ -314,6 +314,12 @@ export default function() { removeOnComplete: true, }); + systemQueue.add('clean', { + }, { + repeat: { cron: '0 0 * * *' }, + removeOnComplete: true, + }); + systemQueue.add('checkExpiredMutings', { }, { repeat: { cron: '*/5 * * * *' }, diff --git a/packages/backend/src/queue/processors/system/clean.ts b/packages/backend/src/queue/processors/system/clean.ts new file mode 100644 index 000000000..c4f978d7c --- /dev/null +++ b/packages/backend/src/queue/processors/system/clean.ts @@ -0,0 +1,18 @@ +import Bull from 'bull'; +import { LessThan } from 'typeorm'; +import { UserIps } from '@/models/index.js'; + +import { queueLogger } from '../../logger.js'; + +const logger = queueLogger.createSubLogger('clean'); + +export async function clean(job: Bull.Job>, done: any): Promise { + logger.info('Cleaning...'); + + UserIps.delete({ + createdAt: LessThan(new Date(Date.now() - (1000 * 60 * 60 * 24 * 90))), + }); + + logger.succ('Cleaned.'); + done(); +} diff --git a/packages/backend/src/queue/processors/system/index.ts b/packages/backend/src/queue/processors/system/index.ts index f90f6efaf..9527d40b0 100644 --- a/packages/backend/src/queue/processors/system/index.ts +++ b/packages/backend/src/queue/processors/system/index.ts @@ -3,12 +3,14 @@ import { tickCharts } from './tick-charts.js'; import { resyncCharts } from './resync-charts.js'; import { cleanCharts } from './clean-charts.js'; import { checkExpiredMutings } from './check-expired-mutings.js'; +import { clean } from './clean.js'; const jobs = { tickCharts, resyncCharts, cleanCharts, checkExpiredMutings, + clean, } as Record> | Bull.ProcessPromiseFunction>>; export default function(dbQueue: Bull.Queue>) { diff --git a/packages/backend/src/server/api/api-handler.ts b/packages/backend/src/server/api/api-handler.ts index c22c868c8..34ff970b4 100644 --- a/packages/backend/src/server/api/api-handler.ts +++ b/packages/backend/src/server/api/api-handler.ts @@ -1,10 +1,19 @@ import Koa from 'koa'; +import { User } from '@/models/entities/user.js'; +import { UserIps } from '@/models/index.js'; +import { fetchMeta } from '@/misc/fetch-meta.js'; import { IEndpoint } from './endpoints.js'; import authenticate, { AuthenticationError } from './authenticate.js'; import call from './call.js'; import { ApiError } from './error.js'; +const userIpHistories = new Map>(); + +setInterval(() => { + userIpHistories.clear(); +}, 1000 * 60 * 60); + export default (endpoint: IEndpoint, ctx: Koa.Context) => new Promise((res) => { const body = ctx.is('multipart/form-data') ? (ctx.request as any).body @@ -44,6 +53,31 @@ export default (endpoint: IEndpoint, ctx: Koa.Context) => new Promise((res }).catch((e: ApiError) => { reply(e.httpStatusCode ? e.httpStatusCode : e.kind === 'client' ? 400 : 500, e); }); + + // Log IP + if (user) { + fetchMeta().then(meta => { + if (!meta.enableIpLogging) return; + const ip = ctx.ip; + const ips = userIpHistories.get(user.id); + if (ips == null || !ips.has(ip)) { + if (ips == null) { + userIpHistories.set(user.id, new Set([ip])); + } else { + ips.add(ip); + } + + try { + UserIps.insert({ + createdAt: new Date(), + userId: user.id, + ip: ip, + }); + } catch { + } + } + }); + } }).catch(e => { if (e instanceof AuthenticationError) { reply(403, new ApiError({ diff --git a/packages/backend/src/server/api/call.ts b/packages/backend/src/server/api/call.ts index 75bbc9f90..aa130459a 100644 --- a/packages/backend/src/server/api/call.ts +++ b/packages/backend/src/server/api/call.ts @@ -116,7 +116,7 @@ export default async (endpoint: string, user: CacheableLocalUser | null | undefi // API invoking const before = performance.now(); - return await ep.exec(data, user, token, ctx?.file).catch((e: Error) => { + return await ep.exec(data, user, token, ctx?.file, ctx?.ip, ctx?.headers).catch((e: Error) => { if (e instanceof ApiError) { throw e; } else { diff --git a/packages/backend/src/server/api/define.ts b/packages/backend/src/server/api/define.ts index 47dcb44ea..c1b56b8a8 100644 --- a/packages/backend/src/server/api/define.ts +++ b/packages/backend/src/server/api/define.ts @@ -1,16 +1,16 @@ import * as fs from 'node:fs'; import Ajv from 'ajv'; import { CacheableLocalUser, ILocalUser } from '@/models/entities/user.js'; -import { IEndpointMeta } from './endpoints.js'; -import { ApiError } from './error.js'; import { Schema, SchemaType } from '@/misc/schema.js'; import { AccessToken } from '@/models/entities/access-token.js'; +import { IEndpointMeta } from './endpoints.js'; +import { ApiError } from './error.js'; export type Response = Record | void; // TODO: paramsの型をT['params']のスキーマ定義から推論する type executor = - (params: SchemaType, user: T['requireCredential'] extends true ? CacheableLocalUser : CacheableLocalUser | null, token: AccessToken | null, file?: any, cleanup?: () => any) => + (params: SchemaType, user: T['requireCredential'] extends true ? CacheableLocalUser : CacheableLocalUser | null, token: AccessToken | null, file?: any, cleanup?: () => any, ip?: string | null, headers?: Record | null) => Promise>>; const ajv = new Ajv({ @@ -20,23 +20,27 @@ const ajv = new Ajv({ ajv.addFormat('misskey:id', /^[a-zA-Z0-9]+$/); export default function (meta: T, paramDef: Ps, cb: executor) - : (params: any, user: T['requireCredential'] extends true ? CacheableLocalUser : CacheableLocalUser | null, token: AccessToken | null, file?: any) => Promise { + : (params: any, user: T['requireCredential'] extends true ? CacheableLocalUser : CacheableLocalUser | null, token: AccessToken | null, file?: any, ip?: string | null, headers?: Record | null) => Promise { const validate = ajv.compile(paramDef); - return (params: any, user: T['requireCredential'] extends true ? CacheableLocalUser : CacheableLocalUser | null, token: AccessToken | null, file?: any) => { - function cleanup() { - fs.unlink(file.path, () => {}); - } + return (params: any, user: T['requireCredential'] extends true ? CacheableLocalUser : CacheableLocalUser | null, token: AccessToken | null, file?: any, ip?: string | null, headers?: Record | null) => { + let cleanup: undefined | (() => void) = undefined; - if (meta.requireFile && file == null) return Promise.reject(new ApiError({ - message: 'File required.', - code: 'FILE_REQUIRED', - id: '4267801e-70d1-416a-b011-4ee502885d8b', - })); + if (meta.requireFile) { + cleanup = () => { + fs.unlink(file.path, () => {}); + }; + + if (file == null) return Promise.reject(new ApiError({ + message: 'File required.', + code: 'FILE_REQUIRED', + id: '4267801e-70d1-416a-b011-4ee502885d8b', + })); + } const valid = validate(params); if (!valid) { - if (file) cleanup(); + if (file) cleanup!(); const errors = validate.errors!; const err = new ApiError({ @@ -50,6 +54,6 @@ export default function (meta: T, pa return Promise.reject(err); } - return cb(params as SchemaType, user, token, file, cleanup); + return cb(params as SchemaType, user, token, file, cleanup, ip, headers); }; } diff --git a/packages/backend/src/server/api/endpoints.ts b/packages/backend/src/server/api/endpoints.ts index 1a3fc199d..f01967754 100644 --- a/packages/backend/src/server/api/endpoints.ts +++ b/packages/backend/src/server/api/endpoints.ts @@ -35,6 +35,7 @@ import * as ep___admin_federation_removeAllFollowing from './endpoints/admin/fed import * as ep___admin_federation_updateInstance from './endpoints/admin/federation/update-instance.js'; import * as ep___admin_getIndexStats from './endpoints/admin/get-index-stats.js'; import * as ep___admin_getTableStats from './endpoints/admin/get-table-stats.js'; +import * as ep___admin_getUserIps from './endpoints/admin/get-user-ips.js'; import * as ep___admin_invite from './endpoints/admin/invite.js'; import * as ep___admin_moderators_add from './endpoints/admin/moderators/add.js'; import * as ep___admin_moderators_remove from './endpoints/admin/moderators/remove.js'; @@ -348,6 +349,7 @@ const eps = [ ['admin/federation/update-instance', ep___admin_federation_updateInstance], ['admin/get-index-stats', ep___admin_getIndexStats], ['admin/get-table-stats', ep___admin_getTableStats], + ['admin/get-user-ips', ep___admin_getUserIps], ['admin/invite', ep___admin_invite], ['admin/moderators/add', ep___admin_moderators_add], ['admin/moderators/remove', ep___admin_moderators_remove], diff --git a/packages/backend/src/server/api/endpoints/admin/drive/show-file.ts b/packages/backend/src/server/api/endpoints/admin/drive/show-file.ts index 039df74f1..e9117a23c 100644 --- a/packages/backend/src/server/api/endpoints/admin/drive/show-file.ts +++ b/packages/backend/src/server/api/endpoints/admin/drive/show-file.ts @@ -1,6 +1,6 @@ +import { DriveFiles } from '@/models/index.js'; import define from '../../../define.js'; import { ApiError } from '../../../error.js'; -import { DriveFiles } from '@/models/index.js'; export const meta = { tags: ['admin'], @@ -184,5 +184,10 @@ export default define(meta, paramDef, async (ps, me) => { throw new ApiError(meta.errors.noSuchFile); } + if (!me.isAdmin) { + delete file.requestIp; + delete file.requestHeaders; + } + return file; }); diff --git a/packages/backend/src/server/api/endpoints/admin/get-user-ips.ts b/packages/backend/src/server/api/endpoints/admin/get-user-ips.ts new file mode 100644 index 000000000..e8b9cb3b0 --- /dev/null +++ b/packages/backend/src/server/api/endpoints/admin/get-user-ips.ts @@ -0,0 +1,31 @@ +import { UserIps } from '@/models/index.js'; +import define from '../../define.js'; + +export const meta = { + tags: ['admin'], + + requireCredential: true, + requireAdmin: true, +} as const; + +export const paramDef = { + type: 'object', + properties: { + userId: { type: 'string', format: 'misskey:id' }, + }, + required: ['userId'], +} as const; + +// eslint-disable-next-line import/no-default-export +export default define(meta, paramDef, async (ps, me) => { + const ips = await UserIps.find({ + where: { userId: ps.userId }, + order: { createdAt: 'DESC' }, + take: 30, + }); + + return ips.map(x => ({ + ip: x.ip, + createdAt: x.createdAt.toISOString(), + })); +}); diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts index 8d50486ef..8b7162895 100644 --- a/packages/backend/src/server/api/endpoints/admin/meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/meta.ts @@ -1,7 +1,7 @@ import config from '@/config/index.js'; -import define from '../../define.js'; import { fetchMeta } from '@/misc/fetch-meta.js'; import { MAX_NOTE_TEXT_LENGTH } from '@/const.js'; +import define from '../../define.js'; export const meta = { tags: ['meta'], @@ -304,6 +304,10 @@ export const meta = { type: 'boolean', optional: true, nullable: false, }, + enableIpLogging: { + type: 'boolean', + optional: true, nullable: false, + }, }, }, } as const; @@ -360,7 +364,6 @@ export default define(meta, paramDef, async (ps, me) => { pinnedPages: instance.pinnedPages, pinnedClipId: instance.pinnedClipId, cacheRemoteFiles: instance.cacheRemoteFiles, - useStarForReactionFallback: instance.useStarForReactionFallback, pinnedUsers: instance.pinnedUsers, hiddenTags: instance.hiddenTags, @@ -397,5 +400,6 @@ export default define(meta, paramDef, async (ps, me) => { objectStorageS3ForcePathStyle: instance.objectStorageS3ForcePathStyle, deeplAuthKey: instance.deeplAuthKey, deeplIsPro: instance.deeplIsPro, + enableIpLogging: instance.enableIpLogging, }; }); diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts index 09e43301b..4dc4726a2 100644 --- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts @@ -1,8 +1,8 @@ -import define from '../../define.js'; import { Meta } from '@/models/entities/meta.js'; import { insertModerationLog } from '@/services/insert-moderation-log.js'; import { DB_MAX_NOTE_TEXT_LENGTH } from '@/misc/hard-limits.js'; import { db } from '@/db/postgre.js'; +import define from '../../define.js'; export const meta = { tags: ['admin'], @@ -96,6 +96,7 @@ export const paramDef = { objectStorageUseProxy: { type: 'boolean' }, objectStorageSetPublicRead: { type: 'boolean' }, objectStorageS3ForcePathStyle: { type: 'boolean' }, + enableIpLogging: { type: 'boolean' }, }, required: [], } as const; @@ -396,6 +397,10 @@ export default define(meta, paramDef, async (ps, me) => { set.deeplIsPro = ps.deeplIsPro; } + if (ps.enableIpLogging !== undefined) { + set.enableIpLogging = ps.enableIpLogging; + } + await db.transaction(async transactionalEntityManager => { const metas = await transactionalEntityManager.find(Meta, { order: { diff --git a/packages/backend/src/server/api/endpoints/drive/files/create.ts b/packages/backend/src/server/api/endpoints/drive/files/create.ts index 7397fd9ce..3a76a5d98 100644 --- a/packages/backend/src/server/api/endpoints/drive/files/create.ts +++ b/packages/backend/src/server/api/endpoints/drive/files/create.ts @@ -1,10 +1,11 @@ import ms from 'ms'; import { addFile } from '@/services/drive/add-file.js'; +import { DriveFiles } from '@/models/index.js'; +import { DB_MAX_IMAGE_COMMENT_LENGTH } from '@/misc/hard-limits.js'; +import { fetchMeta } from '@/misc/fetch-meta.js'; import define from '../../../define.js'; import { apiLogger } from '../../../logger.js'; import { ApiError } from '../../../error.js'; -import { DriveFiles } from '@/models/index.js'; -import { DB_MAX_IMAGE_COMMENT_LENGTH } from '@/misc/hard-limits.js'; export const meta = { tags: ['drive'], @@ -50,7 +51,7 @@ export const paramDef = { } as const; // eslint-disable-next-line import/no-default-export -export default define(meta, paramDef, async (ps, user, _, file, cleanup) => { +export default define(meta, paramDef, async (ps, user, _, file, cleanup, ip, headers) => { // Get 'name' parameter let name = ps.name || file.originalname; if (name !== undefined && name !== null) { @@ -66,9 +67,21 @@ export default define(meta, paramDef, async (ps, user, _, file, cleanup) => { name = null; } + const meta = await fetchMeta(); + try { // Create file - const driveFile = await addFile({ user, path: file.path, name, comment: ps.comment, folderId: ps.folderId, force: ps.force, sensitive: ps.isSensitive }); + const driveFile = await addFile({ + user, + path: file.path, + name, + comment: ps.comment, + folderId: ps.folderId, + force: ps.force, + sensitive: ps.isSensitive, + requestIp: meta.enableIpLogging ? ip : null, + requestHeaders: meta.enableIpLogging ? headers : null, + }); return await DriveFiles.pack(driveFile, { self: true }); } catch (e) { if (e instanceof Error || typeof e === 'string') { diff --git a/packages/backend/src/server/api/endpoints/drive/files/upload-from-url.ts b/packages/backend/src/server/api/endpoints/drive/files/upload-from-url.ts index 53f2298f2..eb8071c3c 100644 --- a/packages/backend/src/server/api/endpoints/drive/files/upload-from-url.ts +++ b/packages/backend/src/server/api/endpoints/drive/files/upload-from-url.ts @@ -1,9 +1,9 @@ import ms from 'ms'; import { uploadFromUrl } from '@/services/drive/upload-from-url.js'; -import define from '../../../define.js'; import { DriveFiles } from '@/models/index.js'; import { publishMainStream } from '@/services/stream.js'; import { DB_MAX_IMAGE_COMMENT_LENGTH } from '@/misc/hard-limits.js'; +import define from '../../../define.js'; export const meta = { tags: ['drive'], @@ -34,8 +34,8 @@ export const paramDef = { } as const; // eslint-disable-next-line import/no-default-export -export default define(meta, paramDef, async (ps, user) => { - uploadFromUrl({ url: ps.url, user, folderId: ps.folderId, sensitive: ps.isSensitive, force: ps.force, comment: ps.comment }).then(file => { +export default define(meta, paramDef, async (ps, user, _1, _2, _3, ip, headers) => { + uploadFromUrl({ url: ps.url, user, folderId: ps.folderId, sensitive: ps.isSensitive, force: ps.force, comment: ps.comment, requestIp: ip, requestHeaders: headers }).then(file => { DriveFiles.pack(file, { self: true }).then(packedFile => { publishMainStream(user.id, 'urlUploadFinished', { marker: ps.marker, diff --git a/packages/backend/src/services/drive/add-file.ts b/packages/backend/src/services/drive/add-file.ts index cfbcb60dd..a25413187 100644 --- a/packages/backend/src/services/drive/add-file.ts +++ b/packages/backend/src/services/drive/add-file.ts @@ -2,26 +2,26 @@ import * as fs from 'node:fs'; import { v4 as uuid } from 'uuid'; +import S3 from 'aws-sdk/clients/s3.js'; +import sharp from 'sharp'; +import { IsNull } from 'typeorm'; import { publishMainStream, publishDriveStream } from '@/services/stream.js'; -import { deleteFile } from './delete-file.js'; import { fetchMeta } from '@/misc/fetch-meta.js'; -import { GenerateVideoThumbnail } from './generate-video-thumbnail.js'; -import { driveLogger } from './logger.js'; -import { IImage, convertSharpToJpeg, convertSharpToWebp, convertSharpToPng } from './image-processor.js'; import { contentDisposition } from '@/misc/content-disposition.js'; import { getFileInfo } from '@/misc/get-file-info.js'; import { DriveFiles, DriveFolders, Users, Instances, UserProfiles } from '@/models/index.js'; -import { InternalStorage } from './internal-storage.js'; import { DriveFile } from '@/models/entities/drive-file.js'; import { IRemoteUser, User } from '@/models/entities/user.js'; import { driveChart, perUserDriveChart, instanceChart } from '@/services/chart/index.js'; import { genId } from '@/misc/gen-id.js'; import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js'; -import S3 from 'aws-sdk/clients/s3.js'; -import { getS3 } from './s3.js'; -import sharp from 'sharp'; import { FILE_TYPE_BROWSERSAFE } from '@/const.js'; -import { IsNull } from 'typeorm'; +import { getS3 } from './s3.js'; +import { InternalStorage } from './internal-storage.js'; +import { IImage, convertSharpToJpeg, convertSharpToWebp, convertSharpToPng } from './image-processor.js'; +import { driveLogger } from './logger.js'; +import { GenerateVideoThumbnail } from './generate-video-thumbnail.js'; +import { deleteFile } from './delete-file.js'; const logger = driveLogger.createSubLogger('register', 'yellow'); @@ -171,7 +171,7 @@ export async function generateAlts(path: string, type: string, generateWeb: bool } if (!['image/jpeg', 'image/png', 'image/webp', 'image/svg+xml'].includes(type)) { - logger.debug(`web image and thumbnail not created (not an required file)`); + logger.debug('web image and thumbnail not created (not an required file)'); return { webpublic: null, thumbnail: null, @@ -212,7 +212,7 @@ export async function generateAlts(path: string, type: string, generateWeb: bool let webpublic: IImage | null = null; if (generateWeb && !satisfyWebpublic) { - logger.info(`creating web image`); + logger.info('creating web image'); try { if (['image/jpeg', 'image/webp'].includes(type)) { @@ -222,14 +222,14 @@ export async function generateAlts(path: string, type: string, generateWeb: bool } else if (['image/svg+xml'].includes(type)) { webpublic = await convertSharpToPng(img, 2048, 2048); } else { - logger.debug(`web image not created (not an required image)`); + logger.debug('web image not created (not an required image)'); } } catch (err) { - logger.warn(`web image not created (an error occured)`, err as Error); + logger.warn('web image not created (an error occured)', err as Error); } } else { - if (satisfyWebpublic) logger.info(`web image not created (original satisfies webpublic)`); - else logger.info(`web image not created (from remote)`); + if (satisfyWebpublic) logger.info('web image not created (original satisfies webpublic)'); + else logger.info('web image not created (from remote)'); } // #endregion webpublic @@ -240,10 +240,10 @@ export async function generateAlts(path: string, type: string, generateWeb: bool if (['image/jpeg', 'image/webp', 'image/png', 'image/svg+xml'].includes(type)) { thumbnail = await convertSharpToWebp(img, 498, 280); } else { - logger.debug(`thumbnail not created (not an required file)`); + logger.debug('thumbnail not created (not an required file)'); } } catch (err) { - logger.warn(`thumbnail not created (an error occured)`, err as Error); + logger.warn('thumbnail not created (an error occured)', err as Error); } // #endregion thumbnail @@ -276,7 +276,7 @@ async function upload(key: string, stream: fs.ReadStream | Buffer, type: string, const s3 = getS3(meta); const upload = s3.upload(params, { - partSize: s3.endpoint?.hostname === 'storage.googleapis.com' ? 500 * 1024 * 1024 : 8 * 1024 * 1024, + partSize: s3.endpoint.hostname === 'storage.googleapis.com' ? 500 * 1024 * 1024 : 8 * 1024 * 1024, }); const result = await upload.promise(); @@ -326,6 +326,9 @@ type AddFileArgs = { uri?: string | null; /** Mark file as sensitive */ sensitive?: boolean | null; + + requestIp?: string | null; + requestHeaders?: Record | null; }; /** @@ -342,7 +345,9 @@ export async function addFile({ isLink = false, url = null, uri = null, - sensitive = null + sensitive = null, + requestIp = null, + requestHeaders = null, }: AddFileArgs): Promise { const info = await getFileInfo(path); logger.info(`${JSON.stringify(info)}`); @@ -427,11 +432,13 @@ export async function addFile({ file.properties = properties; file.blurhash = info.blurhash || null; file.isLink = isLink; + file.requestIp = requestIp; + file.requestHeaders = requestHeaders; file.isSensitive = user ? Users.isLocalUser(user) && profile!.alwaysMarkNsfw ? true : - (sensitive !== null && sensitive !== undefined) - ? sensitive - : false + (sensitive !== null && sensitive !== undefined) + ? sensitive + : false : false; if (url !== null) { diff --git a/packages/backend/src/services/drive/upload-from-url.ts b/packages/backend/src/services/drive/upload-from-url.ts index 001fc49ee..3c5e1aa5c 100644 --- a/packages/backend/src/services/drive/upload-from-url.ts +++ b/packages/backend/src/services/drive/upload-from-url.ts @@ -1,12 +1,12 @@ import { URL } from 'node:url'; -import { addFile } from './add-file.js'; import { User } from '@/models/entities/user.js'; -import { driveLogger } from './logger.js'; import { createTemp } from '@/misc/create-temp.js'; import { downloadUrl } from '@/misc/download-url.js'; import { DriveFolder } from '@/models/entities/drive-folder.js'; import { DriveFile } from '@/models/entities/drive-file.js'; import { DriveFiles } from '@/models/index.js'; +import { driveLogger } from './logger.js'; +import { addFile } from './add-file.js'; const logger = driveLogger.createSubLogger('downloader'); @@ -19,6 +19,8 @@ type Args = { force?: boolean; isLink?: boolean; comment?: string | null; + requestIp?: string | null; + requestHeaders?: Record | null; }; export async function uploadFromUrl({ @@ -30,6 +32,8 @@ export async function uploadFromUrl({ force = false, isLink = false, comment = null, + requestIp = null, + requestHeaders = null, }: Args): Promise { let name = new URL(url).pathname.split('/').pop() || null; if (name == null || !DriveFiles.validateFileName(name)) { @@ -49,7 +53,7 @@ export async function uploadFromUrl({ // write content at URL to temp file await downloadUrl(url, path); - const driveFile = await addFile({ user, path, name, comment, folderId, force, isLink, url, uri, sensitive }); + const driveFile = await addFile({ user, path, name, comment, folderId, force, isLink, url, uri, sensitive, requestIp, requestHeaders }); logger.succ(`Got: ${driveFile.id}`); return driveFile!; } catch (e) { diff --git a/packages/client/src/account.ts b/packages/client/src/account.ts index eb2ba0a1e..38f2ee4b3 100644 --- a/packages/client/src/account.ts +++ b/packages/client/src/account.ts @@ -17,6 +17,7 @@ const accountData = localStorage.getItem('account'); export const $i = accountData ? reactive(JSON.parse(accountData) as Account) : null; export const iAmModerator = $i != null && ($i.isAdmin || $i.isModerator); +export const iAmAdmin = $i != null && $i.isAdmin; export async function signout() { waiting(); diff --git a/packages/client/src/pages/admin-file.vue b/packages/client/src/pages/admin-file.vue index 7bfbed35f..f96a41a7e 100644 --- a/packages/client/src/pages/admin-file.vue +++ b/packages/client/src/pages/admin-file.vue @@ -1,7 +1,7 @@