From ef354e94f20ace67b94faa2859c458a436cdd3f7 Mon Sep 17 00:00:00 2001 From: Kagami Sascha Rosylight Date: Sun, 25 Jun 2023 04:04:33 +0200 Subject: refactor(backend): replace rndstr with secureRndstr (#11044) * refactor(backend): replace rndstr with secureRndstr * Update pnpm-lock.yaml * .js --- packages/backend/test/unit/activitypub.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'packages/backend/test/unit/activitypub.ts') diff --git a/packages/backend/test/unit/activitypub.ts b/packages/backend/test/unit/activitypub.ts index 146998937e..7cd740a2fa 100644 --- a/packages/backend/test/unit/activitypub.ts +++ b/packages/backend/test/unit/activitypub.ts @@ -1,7 +1,6 @@ process.env.NODE_ENV = 'test'; import * as assert from 'assert'; -import rndstr from 'rndstr'; import { Test } from '@nestjs/testing'; import { jest } from '@jest/globals'; @@ -13,13 +12,14 @@ import { CoreModule } from '@/core/CoreModule.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { LoggerService } from '@/core/LoggerService.js'; import type { IActor } from '@/core/activitypub/type.js'; -import { MockResolver } from '../misc/mock-resolver.js'; import { Note } from '@/models/index.js'; +import { secureRndstr } from '@/misc/secure-rndstr.js'; +import { MockResolver } from '../misc/mock-resolver.js'; const host = 'https://host1.test'; function createRandomActor(): IActor & { id: string } { - const preferredUsername = `${rndstr('A-Z', 4)}${rndstr('a-z', 4)}`; + const preferredUsername = secureRndstr(8); const actorId = `${host}/users/${preferredUsername.toLowerCase()}`; return { @@ -61,7 +61,7 @@ describe('ActivityPub', () => { const post = { '@context': 'https://www.w3.org/ns/activitystreams', - id: `${host}/users/${rndstr('0-9a-z', 8)}`, + id: `${host}/users/${secureRndstr(8)}`, type: 'Note', attributedTo: actor.id, to: 'https://www.w3.org/ns/activitystreams#Public', @@ -94,7 +94,7 @@ describe('ActivityPub', () => { test('Truncate long name', async () => { const actor = { ...createRandomActor(), - name: rndstr('0-9a-z', 129), + name: secureRndstr(129), }; resolver._register(actor.id, actor); -- cgit v1.2.3-freya From 5059d4d7e1fc98c5a86ebe73997e21b8c9bda0f7 Mon Sep 17 00:00:00 2001 From: Kagami Sascha Rosylight Date: Sun, 9 Jul 2023 01:59:44 +0200 Subject: refactor(backend): skip fetching notes when the data is same-origin (#11200) * refactor(backend): skip fetching notes when the data is same-origin * Update CHANGELOG.md * sentFrom --- CHANGELOG.md | 1 + .../src/core/activitypub/models/ApNoteService.ts | 10 +- .../src/core/activitypub/models/ApPersonService.ts | 11 +- packages/backend/test/misc/mock-resolver.ts | 19 +++- packages/backend/test/unit/activitypub.ts | 113 +++++++++++++++++++-- 5 files changed, 131 insertions(+), 23 deletions(-) (limited to 'packages/backend/test/unit/activitypub.ts') diff --git a/CHANGELOG.md b/CHANGELOG.md index 35162f6b0f..c28d0b9bfb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ - nsfwjs のモデルロードを排他することで、重複ロードによってメモリ使用量が増加しないように - 連合の配送ジョブのパフォーマンスを向上（ロック機構の見直し、Redisキャッシュの活用） - 全体的なDBクエリのパフォーマンスを向上 +- featuredノートのsignedGet回数を減らしました ## 13.13.2 diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index 35865a819d..d3359ef900 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -177,7 +177,7 @@ export class ApNoteService { // リプライ const reply: Note | null = note.inReplyTo - ? await this.resolveNote(note.inReplyTo, resolver) + ? await this.resolveNote(note.inReplyTo, { resolver }) .then(x => { if (x == null) { this.logger.warn('Specified inReplyTo, but not found'); @@ -293,9 +293,8 @@ export class ApNoteService { * リモートサーバーからフェッチしてMisskeyに登録しそれを返します。 */ @bindThis - public async resolveNote(value: string | IObject, resolver?: Resolver): Promise { - const uri = typeof value === 'string' ? value : value.id; - if (uri == null) throw new Error('missing uri'); + public async resolveNote(value: string | IObject, options: { sentFrom?: URL, resolver?: Resolver } = {}): Promise { + const uri = getApId(value); // ブロックしていたら中断 const meta = await this.metaService.fetch(); @@ -318,7 +317,8 @@ export class ApNoteService { // リモートサーバーからフェッチしてきて登録 // ここでuriの代わりに添付されてきたNote Objectが指定されていると、サーバーフェッチを経ずにノートが生成されるが // 添付されてきたNote Objectは偽装されている可能性があるため、常にuriを指定してサーバーフェッチを行う。 - return await this.createNote(uri, resolver, true); + const createFrom = options.sentFrom?.origin === new URL(uri).origin ? value : uri; + return await this.createNote(createFrom, options.resolver, true); } finally { unlock(); } diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index e8b65b3d42..e89ee4632c 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -260,7 +260,7 @@ export class ApPersonService implements OnModuleInit { // Create user let user: RemoteUser | null = null; try { - // Start transaction + // Start transaction await this.db.transaction(async transactionalEntityManager => { user = await transactionalEntityManager.save(new User({ id: this.idService.genId(), @@ -306,9 +306,9 @@ export class ApPersonService implements OnModuleInit { } }); } catch (e) { - // duplicate key error + // duplicate key error if (isDuplicateKeyValueError(e)) { - // /users/@a => /users/:id のように入力がaliasなときにエラーになることがあるのを対応 + // /users/@a => /users/:id のように入力がaliasなときにエラーになることがあるのを対応 const u = await this.usersRepository.findOneBy({ uri: person.id }); if (u == null) throw new Error('already registered'); @@ -604,7 +604,10 @@ export class ApPersonService implements OnModuleInit { const featuredNotes = await Promise.all(items .filter(item => getApType(item) === 'Note') // TODO: Noteでなくてもいいかも .slice(0, 5) - .map(item => limit(() => this.apNoteService.resolveNote(item, _resolver)))); + .map(item => limit(() => this.apNoteService.resolveNote(item, { + resolver: _resolver, + sentFrom: new URL(user.uri), + })))); await this.db.transaction(async transactionalEntityManager => { await transactionalEntityManager.delete(UserNotePining, { userId: user.id }); diff --git a/packages/backend/test/misc/mock-resolver.ts b/packages/backend/test/misc/mock-resolver.ts index a7bcd859ae..9dbe77a7c4 100644 --- a/packages/backend/test/misc/mock-resolver.ts +++ b/packages/backend/test/misc/mock-resolver.ts @@ -18,7 +18,8 @@ type MockResponse = { }; export class MockResolver extends Resolver { - private _rs = new Map(); + #responseMap = new Map(); + #remoteGetTrials: string[] = []; constructor(loggerService: LoggerService) { super( @@ -38,18 +39,28 @@ export class MockResolver extends Resolver { ); } - public async _register(uri: string, content: string | Record, type = 'application/activity+json') { - this._rs.set(uri, { + public register(uri: string, content: string | Record, type = 'application/activity+json'): void { + this.#responseMap.set(uri, { type, content: typeof content === 'string' ? content : JSON.stringify(content), }); } + public clear(): void { + this.#responseMap.clear(); + this.#remoteGetTrials.length = 0; + } + + public remoteGetTrials(): string[] { + return this.#remoteGetTrials; + } + @bindThis public async resolve(value: string | IObject): Promise { if (typeof value !== 'string') return value; - const r = this._rs.get(value); + this.#remoteGetTrials.push(value); + const r = this.#responseMap.get(value); if (!r) { throw new Error('Not registed for mock'); diff --git a/packages/backend/test/unit/activitypub.ts b/packages/backend/test/unit/activitypub.ts index 7cd740a2fa..02b900da9b 100644 --- a/packages/backend/test/unit/activitypub.ts +++ b/packages/backend/test/unit/activitypub.ts @@ -11,16 +11,19 @@ import { GlobalModule } from '@/GlobalModule.js'; import { CoreModule } from '@/core/CoreModule.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { LoggerService } from '@/core/LoggerService.js'; -import type { IActor } from '@/core/activitypub/type.js'; +import type { IActor, ICollection, IPost } from '@/core/activitypub/type.js'; import { Note } from '@/models/index.js'; import { secureRndstr } from '@/misc/secure-rndstr.js'; import { MockResolver } from '../misc/mock-resolver.js'; const host = 'https://host1.test'; -function createRandomActor(): IActor & { id: string } { +type NonTransientIActor = IActor & { id: string }; +type NonTransientIPost = IPost & { id: string }; + +function createRandomActor({ actorHost = host } = {}): NonTransientIActor { const preferredUsername = secureRndstr(8); - const actorId = `${host}/users/${preferredUsername.toLowerCase()}`; + const actorId = `${actorHost}/users/${preferredUsername.toLowerCase()}`; return { '@context': 'https://www.w3.org/ns/activitystreams', @@ -32,13 +35,41 @@ function createRandomActor(): IActor & { id: string } { }; } +function createRandomNote(actor: NonTransientIActor): NonTransientIPost { + const id = secureRndstr(8); + const noteId = `${new URL(actor.id).origin}/notes/${id}`; + + return { + id: noteId, + type: 'Note', + attributedTo: actor.id, + content: 'test test foo', + }; +} + +function createRandomNotes(actor: NonTransientIActor, length: number): NonTransientIPost[] { + return new Array(length).fill(null).map(() => createRandomNote(actor)); +} + +function createRandomFeaturedCollection(actor: NonTransientIActor, length: number): ICollection { + const items = createRandomNotes(actor, length); + + return { + '@context': 'https://www.w3.org/ns/activitystreams', + type: 'Collection', + id: actor.outbox as string, + totalItems: items.length, + items, + }; +} + describe('ActivityPub', () => { let noteService: ApNoteService; let personService: ApPersonService; let rendererService: ApRendererService; let resolver: MockResolver; - beforeEach(async () => { + beforeAll(async () => { const app = await Test.createTestingModule({ imports: [GlobalModule, CoreModule], }).compile(); @@ -53,7 +84,11 @@ describe('ActivityPub', () => { // Prevent ApPersonService from fetching instance, as it causes Jest import-after-test error const federatedInstanceService = app.get(FederatedInstanceService); - jest.spyOn(federatedInstanceService, 'fetch').mockImplementation(() => new Promise(() => {})); + jest.spyOn(federatedInstanceService, 'fetch').mockImplementation(() => new Promise(() => { })); + }); + + beforeEach(() => { + resolver.clear(); }); describe('Parse minimum object', () => { @@ -69,7 +104,7 @@ describe('ActivityPub', () => { }; test('Minimum Actor', async () => { - resolver._register(actor.id, actor); + resolver.register(actor.id, actor); const user = await personService.createPerson(actor.id, resolver); @@ -79,8 +114,8 @@ describe('ActivityPub', () => { }); test('Minimum Note', async () => { - resolver._register(actor.id, actor); - resolver._register(post.id, post); + resolver.register(actor.id, actor); + resolver.register(post.id, post); const note = await noteService.createNote(post.id, resolver, true); @@ -97,7 +132,7 @@ describe('ActivityPub', () => { name: secureRndstr(129), }; - resolver._register(actor.id, actor); + resolver.register(actor.id, actor); const user = await personService.createPerson(actor.id, resolver); @@ -110,7 +145,7 @@ describe('ActivityPub', () => { name: '', }; - resolver._register(actor.id, actor); + resolver.register(actor.id, actor); const user = await personService.createPerson(actor.id, resolver); @@ -126,4 +161,62 @@ describe('ActivityPub', () => { } as Note); }); }); + + describe('Featured', () => { + test('Fetch featured notes from IActor', async () => { + const actor = createRandomActor(); + actor.featured = `${actor.id}/collections/featured`; + + const featured = createRandomFeaturedCollection(actor, 5); + + resolver.register(actor.id, actor); + resolver.register(actor.featured, featured); + + await personService.createPerson(actor.id, resolver); + + // All notes in `featured` are same-origin, no need to fetch notes again + assert.deepStrictEqual(resolver.remoteGetTrials(), [actor.id, actor.featured]); + + // Created notes without resolving anything + for (const item of featured.items as IPost[]) { + const note = await noteService.fetchNote(item); + assert.ok(note); + assert.strictEqual(note.text, 'test test foo'); + assert.strictEqual(note.uri, item.id); + } + }); + + test('Fetch featured notes from IActor pointing to another remote server', async () => { + const actor1 = createRandomActor(); + actor1.featured = `${actor1.id}/collections/featured`; + const actor2 = createRandomActor({ actorHost: 'https://host2.test' }); + + const actor2Note = createRandomNote(actor2); + const featured = createRandomFeaturedCollection(actor1, 0); + (featured.items as IPost[]).push({ + ...actor2Note, + content: 'test test bar', // fraud! + }); + + resolver.register(actor1.id, actor1); + resolver.register(actor1.featured, featured); + resolver.register(actor2.id, actor2); + resolver.register(actor2Note.id, actor2Note); + + await personService.createPerson(actor1.id, resolver); + + // actor2Note is from a different server and needs to be fetched again + assert.deepStrictEqual( + resolver.remoteGetTrials(), + [actor1.id, actor1.featured, actor2Note.id, actor2.id], + ); + + const note = await noteService.fetchNote(actor2Note.id); + assert.ok(note); + + // Reflects the original content instead of the fraud + assert.strictEqual(note.text, 'test test foo'); + assert.strictEqual(note.uri, actor2Note.id); + }); + }); }); -- cgit v1.2.3-freya From d5f30ecb86289f2791b774f0620ea474a0ccb7cf Mon Sep 17 00:00:00 2001 From: Kagami Sascha Rosylight Date: Sat, 15 Jul 2023 13:12:20 +0200 Subject: feat(backend): allow disabling cache for sensitive files (#11245) * feat(backend): allow disabling cache for sensitive files * Update CHANGELOG.md * fix storybook * Update locales/ja-JP.yml --------- Co-authored-by: syuilo --- CHANGELOG.md | 1 + locales/index.d.ts | 2 + locales/ja-JP.yml | 2 + .../backend/migration/1689102832143-nsfw-cache.js | 11 ++ .../src/core/activitypub/models/ApImageService.ts | 13 ++- packages/backend/src/models/entities/Meta.ts | 6 +- .../backend/src/server/api/endpoints/admin/meta.ts | 6 +- .../src/server/api/endpoints/admin/update-meta.ts | 5 + packages/backend/src/server/api/endpoints/meta.ts | 7 +- packages/backend/test/unit/activitypub.ts | 129 ++++++++++++++++++++- packages/frontend/src/pages/admin/settings.vue | 15 ++- packages/misskey-js/etc/misskey-js.api.md | 7 +- packages/misskey-js/src/api.types.ts | 1 + packages/misskey-js/src/entities.ts | 1 + 14 files changed, 190 insertions(+), 16 deletions(-) create mode 100644 packages/backend/migration/1689102832143-nsfw-cache.js (limited to 'packages/backend/test/unit/activitypub.ts') diff --git a/CHANGELOG.md b/CHANGELOG.md index ed7168ace4..9023222878 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,7 @@ - nsfwjs のモデルロードを排他することで、重複ロードによってメモリ使用量が増加しないように - 連合の配送ジョブのパフォーマンスを向上（ロック機構の見直し、Redisキャッシュの活用） - featuredノートのsignedGet回数を減らしました +- リモートサーバーからのNSFW映像のキャッシュだけを無効化できるオプションを追加 - MeilisearchにIndexするノートの範囲を設定できるように - Fix: Remove Meilisearch index when notes are deleted - Fix: 非英語環境でのPostgreSQLのエラーハンドリングを修正 diff --git a/locales/index.d.ts b/locales/index.d.ts index e3ad4ed003..082cde078e 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -159,6 +159,8 @@ export interface Locale { "settingGuide": string; "cacheRemoteFiles": string; "cacheRemoteFilesDescription": string; + "cacheRemoteSensitiveFiles": string; + "cacheRemoteSensitiveFilesDescription": string; "flagAsBot": string; "flagAsBotDescription": string; "flagAsCat": string; diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index c66b42284d..ceff2a7cff 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -156,6 +156,8 @@ addEmoji: "絵文字を追加" settingGuide: "おすすめ設定" cacheRemoteFiles: "リモートのファイルをキャッシュする" cacheRemoteFilesDescription: "この設定を無効にすると、リモートファイルをキャッシュせず直リンクするようになります。サーバーのストレージを節約できますが、サムネイルが生成されないので通信量が増加します。" +cacheRemoteSensitiveFiles: "リモートのセンシティブなファイルをキャッシュする" +cacheRemoteSensitiveFilesDescription: "この設定を無効にすると、リモートのセンシティブなファイルはキャッシュせず直リンクするようになります。" flagAsBot: "Botとして設定" flagAsBotDescription: "このアカウントがプログラムによって運用される場合は、このフラグをオンにします。オンにすると、反応の連鎖を防ぐためのフラグとして他の開発者に役立ったり、Misskeyのシステム上での扱いがBotに合ったものになります。" flagAsCat: "にゃああああああああああああああ！！！！！！！！！！！！" diff --git a/packages/backend/migration/1689102832143-nsfw-cache.js b/packages/backend/migration/1689102832143-nsfw-cache.js new file mode 100644 index 0000000000..cdce0dae09 --- /dev/null +++ b/packages/backend/migration/1689102832143-nsfw-cache.js @@ -0,0 +1,11 @@ +export class NsfwCache1689102832143 { + name = 'NsfwCache1689102832143' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" ADD "cacheRemoteSensitiveFiles" boolean NOT NULL DEFAULT true`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "cacheRemoteSensitiveFiles"`); + } +} diff --git a/packages/backend/src/core/activitypub/models/ApImageService.ts b/packages/backend/src/core/activitypub/models/ApImageService.ts index 0da312241f..1f2984894c 100644 --- a/packages/backend/src/core/activitypub/models/ApImageService.ts +++ b/packages/backend/src/core/activitypub/models/ApImageService.ts @@ -1,7 +1,6 @@ import { Inject, Injectable } from '@nestjs/common'; import { DI } from '@/di-symbols.js'; import type { DriveFilesRepository } from '@/models/index.js'; -import type { Config } from '@/config.js'; import type { RemoteUser } from '@/models/entities/User.js'; import type { DriveFile } from '@/models/entities/DriveFile.js'; import { MetaService } from '@/core/MetaService.js'; @@ -20,9 +19,6 @@ export class ApImageService { private logger: Logger; constructor( - @Inject(DI.config) - private config: Config, - @Inject(DI.driveFilesRepository) private driveFilesRepository: DriveFilesRepository, @@ -47,7 +43,7 @@ export class ApImageService { const image = await this.apResolverService.createResolver().resolve(value); if (image.url == null) { - throw new Error('invalid image: url not privided'); + throw new Error('invalid image: url not provided'); } if (typeof image.url !== 'string') { @@ -62,12 +58,17 @@ export class ApImageService { const instance = await this.metaService.fetch(); + // Cache if remote file cache is on AND either + // 1. remote sensitive file is also on + // 2. or the image is not sensitive + const shouldBeCached = instance.cacheRemoteFiles && (instance.cacheRemoteSensitiveFiles || !image.sensitive); + const file = await this.driveService.uploadFromUrl({ url: image.url, user: actor, uri: image.url, sensitive: image.sensitive, - isLink: !instance.cacheRemoteFiles, + isLink: !shouldBeCached, comment: truncate(image.name ?? undefined, DB_MAX_IMAGE_COMMENT_LENGTH), }); if (!file.isLink || file.url === image.url) return file; diff --git a/packages/backend/src/models/entities/Meta.ts b/packages/backend/src/models/entities/Meta.ts index a251c0b31c..7bb1b67712 100644 --- a/packages/backend/src/models/entities/Meta.ts +++ b/packages/backend/src/models/entities/Meta.ts @@ -1,7 +1,6 @@ import { Entity, Column, PrimaryColumn, ManyToOne, JoinColumn } from 'typeorm'; import { id } from '../id.js'; import { User } from './User.js'; -import type { Clip } from './Clip.js'; @Entity() export class Meta { @@ -126,6 +125,11 @@ export class Meta { }) public cacheRemoteFiles: boolean; + @Column('boolean', { + default: true, + }) + public cacheRemoteSensitiveFiles: boolean; + @Column({ ...id(), nullable: true, diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts index 28aec7a090..084bdb598b 100644 --- a/packages/backend/src/server/api/endpoints/admin/meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/meta.ts @@ -1,5 +1,4 @@ import { Inject, Injectable } from '@nestjs/common'; -import { MAX_NOTE_TEXT_LENGTH } from '@/const.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { MetaService } from '@/core/MetaService.js'; import type { Config } from '@/config.js'; @@ -20,6 +19,10 @@ export const meta = { type: 'boolean', optional: false, nullable: false, }, + cacheRemoteSensitiveFiles: { + type: 'boolean', + optional: false, nullable: false, + }, emailRequiredForSignup: { type: 'boolean', optional: false, nullable: false, @@ -332,6 +335,7 @@ export default class extends Endpoint { enableServiceWorker: instance.enableServiceWorker, translatorAvailable: instance.deeplAuthKey != null, cacheRemoteFiles: instance.cacheRemoteFiles, + cacheRemoteSensitiveFiles: instance.cacheRemoteSensitiveFiles, pinnedUsers: instance.pinnedUsers, hiddenTags: instance.hiddenTags, blockedHosts: instance.blockedHosts, diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts index 0b20b058fd..144360a921 100644 --- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts @@ -43,6 +43,7 @@ export const paramDef = { defaultLightTheme: { type: 'string', nullable: true }, defaultDarkTheme: { type: 'string', nullable: true }, cacheRemoteFiles: { type: 'boolean' }, + cacheRemoteSensitiveFiles: { type: 'boolean' }, emailRequiredForSignup: { type: 'boolean' }, enableHcaptcha: { type: 'boolean' }, hcaptchaSiteKey: { type: 'string', nullable: true }, @@ -193,6 +194,10 @@ export default class extends Endpoint { set.cacheRemoteFiles = ps.cacheRemoteFiles; } + if (ps.cacheRemoteSensitiveFiles !== undefined) { + set.cacheRemoteSensitiveFiles = ps.cacheRemoteSensitiveFiles; + } + if (ps.emailRequiredForSignup !== undefined) { set.emailRequiredForSignup = ps.emailRequiredForSignup; } diff --git a/packages/backend/src/server/api/endpoints/meta.ts b/packages/backend/src/server/api/endpoints/meta.ts index 915a1e54f8..3d0146e315 100644 --- a/packages/backend/src/server/api/endpoints/meta.ts +++ b/packages/backend/src/server/api/endpoints/meta.ts @@ -83,6 +83,10 @@ export const meta = { type: 'boolean', optional: false, nullable: false, }, + cacheRemoteSensitiveFiles: { + type: 'boolean', + optional: false, nullable: false, + }, emailRequiredForSignup: { type: 'boolean', optional: false, nullable: false, @@ -272,7 +276,7 @@ export default class extends Endpoint { .orWhere('ads.dayOfWeek = 0'); })) .getMany(); - + const response: any = { maintainerName: instance.maintainerName, maintainerEmail: instance.maintainerEmail, @@ -329,6 +333,7 @@ export default class extends Endpoint { ...(ps.detail ? { cacheRemoteFiles: instance.cacheRemoteFiles, + cacheRemoteSensitiveFiles: instance.cacheRemoteSensitiveFiles, requireSetup: (await this.usersRepository.countBy({ host: IsNull(), })) === 0, diff --git a/packages/backend/test/unit/activitypub.ts b/packages/backend/test/unit/activitypub.ts index 02b900da9b..78b916c112 100644 --- a/packages/backend/test/unit/activitypub.ts +++ b/packages/backend/test/unit/activitypub.ts @@ -4,6 +4,7 @@ import * as assert from 'assert'; import { Test } from '@nestjs/testing'; import { jest } from '@jest/globals'; +import { ApImageService } from '@/core/activitypub/models/ApImageService.js'; import { ApNoteService } from '@/core/activitypub/models/ApNoteService.js'; import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; @@ -11,9 +12,12 @@ import { GlobalModule } from '@/GlobalModule.js'; import { CoreModule } from '@/core/CoreModule.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { LoggerService } from '@/core/LoggerService.js'; -import type { IActor, ICollection, IPost } from '@/core/activitypub/type.js'; -import { Note } from '@/models/index.js'; +import type { IActor, IApDocument, ICollection, IPost } from '@/core/activitypub/type.js'; +import { Meta, Note } from '@/models/index.js'; import { secureRndstr } from '@/misc/secure-rndstr.js'; +import { DownloadService } from '@/core/DownloadService.js'; +import { MetaService } from '@/core/MetaService.js'; +import type { RemoteUser } from '@/models/entities/User.js'; import { MockResolver } from '../misc/mock-resolver.js'; const host = 'https://host1.test'; @@ -63,16 +67,47 @@ function createRandomFeaturedCollection(actor: NonTransientIActor, length: numbe }; } +async function createRandomRemoteUser( + resolver: MockResolver, + personService: ApPersonService, +): Promise { + const actor = createRandomActor(); + resolver.register(actor.id, actor); + + return await personService.createPerson(actor.id, resolver); +} + describe('ActivityPub', () => { + let imageService: ApImageService; let noteService: ApNoteService; let personService: ApPersonService; let rendererService: ApRendererService; let resolver: MockResolver; + const metaInitial = { + cacheRemoteFiles: true, + cacheRemoteSensitiveFiles: true, + blockedHosts: [] as string[], + sensitiveWords: [] as string[], + } as Meta; + let meta = metaInitial; + beforeAll(async () => { const app = await Test.createTestingModule({ imports: [GlobalModule, CoreModule], - }).compile(); + }) + .overrideProvider(DownloadService).useValue({ + async downloadUrl(): Promise<{ filename: string }> { + return { + filename: 'dummy.tmp', + }; + }, + }) + .overrideProvider(MetaService).useValue({ + async fetch(): Promise { + return meta; + }, + }).compile(); await app.init(); app.enableShutdownHooks(); @@ -80,6 +115,7 @@ describe('ActivityPub', () => { noteService = app.get(ApNoteService); personService = app.get(ApPersonService); rendererService = app.get(ApRendererService); + imageService = app.get(ApImageService); resolver = new MockResolver(await app.resolve(LoggerService)); // Prevent ApPersonService from fetching instance, as it causes Jest import-after-test error @@ -219,4 +255,91 @@ describe('ActivityPub', () => { assert.strictEqual(note.uri, actor2Note.id); }); }); + + describe('Images', () => { + test('Create images', async () => { + const imageObject: IApDocument = { + type: 'Document', + mediaType: 'image/png', + url: 'http://host1.test/foo.png', + name: '', + }; + const driveFile = await imageService.createImage( + await createRandomRemoteUser(resolver, personService), + imageObject, + ); + assert.ok(!driveFile.isLink); + + const sensitiveImageObject: IApDocument = { + type: 'Document', + mediaType: 'image/png', + url: 'http://host1.test/bar.png', + name: '', + sensitive: true, + }; + const sensitiveDriveFile = await imageService.createImage( + await createRandomRemoteUser(resolver, personService), + sensitiveImageObject, + ); + assert.ok(!sensitiveDriveFile.isLink); + }); + + test('cacheRemoteFiles=false disables caching', async () => { + meta = { ...metaInitial, cacheRemoteFiles: false }; + + const imageObject: IApDocument = { + type: 'Document', + mediaType: 'image/png', + url: 'http://host1.test/foo.png', + name: '', + }; + const driveFile = await imageService.createImage( + await createRandomRemoteUser(resolver, personService), + imageObject, + ); + assert.ok(driveFile.isLink); + + const sensitiveImageObject: IApDocument = { + type: 'Document', + mediaType: 'image/png', + url: 'http://host1.test/bar.png', + name: '', + sensitive: true, + }; + const sensitiveDriveFile = await imageService.createImage( + await createRandomRemoteUser(resolver, personService), + sensitiveImageObject, + ); + assert.ok(sensitiveDriveFile.isLink); + }); + + test('cacheRemoteSensitiveFiles=false only affects sensitive files', async () => { + meta = { ...metaInitial, cacheRemoteSensitiveFiles: false }; + + const imageObject: IApDocument = { + type: 'Document', + mediaType: 'image/png', + url: 'http://host1.test/foo.png', + name: '', + }; + const driveFile = await imageService.createImage( + await createRandomRemoteUser(resolver, personService), + imageObject, + ); + assert.ok(!driveFile.isLink); + + const sensitiveImageObject: IApDocument = { + type: 'Document', + mediaType: 'image/png', + url: 'http://host1.test/bar.png', + name: '', + sensitive: true, + }; + const sensitiveDriveFile = await imageService.createImage( + await createRandomRemoteUser(resolver, personService), + sensitiveImageObject, + ); + assert.ok(sensitiveDriveFile.isLink); + }); + }); }); diff --git a/packages/frontend/src/pages/admin/settings.vue b/packages/frontend/src/pages/admin/settings.vue index 4c2fe46f28..bd57c06181 100644 --- a/packages/frontend/src/pages/admin/settings.vue +++ b/packages/frontend/src/pages/admin/settings.vue @@ -37,6 +37,13 @@ + + @@ -104,7 +111,6 @@ import { fetchInstance } from '@/instance'; import { i18n } from '@/i18n'; import { definePageMetadata } from '@/scripts/page-metadata'; import MkButton from '@/components/MkButton.vue'; -import MkColorInput from '@/components/MkColorInput.vue'; let name: string | null = $ref(null); let description: string | null = $ref(null); @@ -112,13 +118,14 @@ let maintainerName: string | null = $ref(null); let maintainerEmail: string | null = $ref(null); let pinnedUsers: string = $ref(''); let cacheRemoteFiles: boolean = $ref(false); +let cacheRemoteSensitiveFiles: boolean = $ref(false); let enableServiceWorker: boolean = $ref(false); let swPublicKey: any = $ref(null); let swPrivateKey: any = $ref(null); let deeplAuthKey: string = $ref(''); let deeplIsPro: boolean = $ref(false); -async function init() { +async function init(): Promise { const meta = await os.api('admin/meta'); name = meta.name; description = meta.description; @@ -126,6 +133,7 @@ async function init() { maintainerEmail = meta.maintainerEmail; pinnedUsers = meta.pinnedUsers.join('\n'); cacheRemoteFiles = meta.cacheRemoteFiles; + cacheRemoteSensitiveFiles = meta.cacheRemoteSensitiveFiles; enableServiceWorker = meta.enableServiceWorker; swPublicKey = meta.swPublickey; swPrivateKey = meta.swPrivateKey; @@ -133,7 +141,7 @@ async function init() { deeplIsPro = meta.deeplIsPro; } -function save() { +function save(): void { os.apiWithDialog('admin/update-meta', { name, description, @@ -141,6 +149,7 @@ function save() { maintainerEmail, pinnedUsers: pinnedUsers.split('\n'), cacheRemoteFiles, + cacheRemoteSensitiveFiles, enableServiceWorker, swPublicKey, swPrivateKey, diff --git a/packages/misskey-js/etc/misskey-js.api.md b/packages/misskey-js/etc/misskey-js.api.md index 851b21abc5..65d6e2e0b1 100644 --- a/packages/misskey-js/etc/misskey-js.api.md +++ b/packages/misskey-js/etc/misskey-js.api.md @@ -271,6 +271,7 @@ type DetailedInstanceMetadata = LiteInstanceMetadata & { pinnedPages: string[]; pinnedClipId: string | null; cacheRemoteFiles: boolean; + cacheRemoteSensitiveFiles: boolean; requireSetup: boolean; proxyAccountName: string | null; features: Record; @@ -327,6 +328,10 @@ export type Endpoints = { req: TODO; res: TODO; }; + 'admin/meta': { + req: TODO; + res: TODO; + }; 'admin/reset-password': { req: TODO; res: TODO; @@ -2805,7 +2810,7 @@ type UserSorting = '+follower' | '-follower' | '+createdAt' | '-createdAt' | '+u // // src/api.types.ts:16:32 - (ae-forgotten-export) The symbol "TODO" needs to be exported by the entry point index.d.ts // src/api.types.ts:18:25 - (ae-forgotten-export) The symbol "NoParams" needs to be exported by the entry point index.d.ts -// src/api.types.ts:628:18 - (ae-forgotten-export) The symbol "ShowUserReq" needs to be exported by the entry point index.d.ts +// src/api.types.ts:629:18 - (ae-forgotten-export) The symbol "ShowUserReq" needs to be exported by the entry point index.d.ts // src/streaming.types.ts:33:4 - (ae-forgotten-export) The symbol "FIXME" needs to be exported by the entry point index.d.ts // (No @packageDocumentation comment for this package) diff --git a/packages/misskey-js/src/api.types.ts b/packages/misskey-js/src/api.types.ts index ed12be4a06..343977f4be 100644 --- a/packages/misskey-js/src/api.types.ts +++ b/packages/misskey-js/src/api.types.ts @@ -20,6 +20,7 @@ export type Endpoints = { 'admin/get-table-stats': { req: TODO; res: TODO; }; 'admin/invite': { req: TODO; res: TODO; }; 'admin/logs': { req: TODO; res: TODO; }; + 'admin/meta': { req: TODO; res: TODO; }; 'admin/reset-password': { req: TODO; res: TODO; }; 'admin/resolve-abuse-user-report': { req: TODO; res: TODO; }; 'admin/resync-chart': { req: TODO; res: TODO; }; diff --git a/packages/misskey-js/src/entities.ts b/packages/misskey-js/src/entities.ts index 9d8fd6a49b..ea85bedf63 100644 --- a/packages/misskey-js/src/entities.ts +++ b/packages/misskey-js/src/entities.ts @@ -338,6 +338,7 @@ export type DetailedInstanceMetadata = LiteInstanceMetadata & { pinnedPages: string[]; pinnedClipId: string | null; cacheRemoteFiles: boolean; + cacheRemoteSensitiveFiles: boolean; requireSetup: boolean; proxyAccountName: string | null; features: Record; -- cgit v1.2.3-freya