From 3c949f0b8113e88f474b7e27b1c0abcfe0664081 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 11 May 2025 03:34:47 -0400 Subject: overhaul trending polls * Split into local, global, and completed sections * Don't require credential, but check for local/global timeline perms * Fix rate limit * Return polls where the current user has already voted * Return non-public polls if the user has permission to view them * Apply user/instance blocks * Fetch polls + notes + users in a single step to speed up pack --- .../api/endpoints/notes/polls/recommendation.ts | 96 +++++++++++++++++----- 1 file changed, 76 insertions(+), 20 deletions(-) (limited to 'packages/backend/src') diff --git a/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts b/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts index 33a9c281b3..8fed8ae590 100644 --- a/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts +++ b/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts @@ -9,13 +9,13 @@ import type { NotesRepository, MutingsRepository, PollsRepository, PollVotesRepo import { Endpoint } from '@/server/api/endpoint-base.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { DI } from '@/di-symbols.js'; +import { QueryService } from '@/core/QueryService.js'; +import { RoleService } from '@/core/RoleService.js'; +import { ApiError } from '@/server/api/error.js'; export const meta = { tags: ['notes'], - requireCredential: true, - kind: 'read:account', - res: { type: 'array', optional: false, nullable: false, @@ -26,10 +26,24 @@ export const meta = { }, }, - // 2 calls per second + errors: { + ltlDisabled: { + message: 'Local timeline has been disabled.', + code: 'LTL_DISABLED', + id: '45a6eb02-7695-4393-b023-dd3be9aaaefd', + }, + gtlDisabled: { + message: 'Global timeline has been disabled.', + code: 'GTL_DISABLED', + id: '0332fc13-6ab2-4427-ae80-a9fadffd1a6b', + }, + }, + + // Up to 10 calls, then 2 per second limit: { - duration: 1000, - max: 2, + type: 'bucket', + size: 10, + dripRate: 500, }, } as const; @@ -39,6 +53,8 @@ export const paramDef = { limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 }, offset: { type: 'integer', default: 0 }, excludeChannels: { type: 'boolean', default: false }, + local: { type: 'boolean', nullable: true, default: null }, + expired: { type: 'boolean', default: false }, }, required: [], } as const; @@ -59,18 +75,54 @@ export default class extends Endpoint { // eslint- private mutingsRepository: MutingsRepository, private noteEntityService: NoteEntityService, + private readonly queryService: QueryService, + private readonly roleService: RoleService, ) { super(meta, paramDef, async (ps, me) => { const query = this.pollsRepository.createQueryBuilder('poll') - .where('poll.userHost IS NULL') - .andWhere('poll.userId != :meId', { meId: me.id }) - .andWhere('poll.noteVisibility = \'public\'') - .andWhere(new Brackets(qb => { + .innerJoinAndSelect('poll.note', 'note') + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('renote.user', 'renoteUser') + .leftJoinAndSelect('reply.user', 'replyUser') + ; + + if (me) { + query.andWhere('poll.userId != :meId', { meId: me.id }); + } + + if (ps.expired) { + query.andWhere('poll.expiresAt IS NOT NULL'); + query.andWhere('poll.expiresAt < :expiresMax', { + expiresMax: new Date(), + }); + query.andWhere('poll.expiresAt >= :expiresMin', { + expiresMin: new Date(Date.now() - (1000 * 60 * 60 * 24 * 7)), + }); + } else { + query.andWhere(new Brackets(qb => { qb .where('poll.expiresAt IS NULL') .orWhere('poll.expiresAt > :now', { now: new Date() }); })); + } + const policies = await this.roleService.getUserPolicies(me?.id ?? null); + if (ps.local != null) { + if (ps.local) { + if (!policies.ltlAvailable) throw new ApiError(meta.errors.ltlDisabled); + query.andWhere('poll.userHost IS NULL'); + } else { + if (!policies.gtlAvailable) throw new ApiError(meta.errors.gtlDisabled); + query.andWhere('poll.userHost IS NOT NULL'); + } + } else { + if (!policies.ltlAvailable) throw new ApiError(meta.errors.ltlDisabled); + if (!policies.gtlAvailable) throw new ApiError(meta.errors.gtlDisabled); + } + + /* //#region exclude arleady voted polls const votedQuery = this.pollVotesRepository.createQueryBuilder('vote') .select('vote.noteId') @@ -81,16 +133,15 @@ export default class extends Endpoint { // eslint- query.setParameters(votedQuery.getParameters()); //#endregion - - //#region mute - const mutingQuery = this.mutingsRepository.createQueryBuilder('muting') - .select('muting.muteeId') - .where('muting.muterId = :muterId', { muterId: me.id }); - - query - .andWhere(`poll.userId NOT IN (${ mutingQuery.getQuery() })`); - - query.setParameters(mutingQuery.getParameters()); + */ + + //#region block/mute/vis + this.queryService.generateVisibilityQuery(query, me); + this.queryService.generateBlockedHostQueryForNote(query); + if (me) { + this.queryService.generateBlockedUserQueryForNotes(query, me); + this.queryService.generateMutedUserQueryForNotes(query, me); + } //#endregion //#region exclude channels @@ -107,6 +158,7 @@ export default class extends Endpoint { // eslint- if (polls.length === 0) return []; + /* const notes = await this.notesRepository.find({ where: { id: In(polls.map(poll => poll.noteId)), @@ -115,6 +167,10 @@ export default class extends Endpoint { // eslint- id: 'DESC', }, }); + */ + + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const notes = polls.map(poll => poll.note!); return await this.noteEntityService.packMany(notes, me, { detail: true, -- cgit v1.2.3-freya From f01dc57ec9f57d925880b7b86ed99bc3e1a92628 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 11 May 2025 05:28:13 -0400 Subject: only show polls from explorable users --- packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts | 1 + 1 file changed, 1 insertion(+) (limited to 'packages/backend/src') diff --git a/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts b/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts index 8fed8ae590..5dd6db54f9 100644 --- a/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts +++ b/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts @@ -86,6 +86,7 @@ export default class extends Endpoint { // eslint- .leftJoinAndSelect('note.reply', 'reply') .leftJoinAndSelect('renote.user', 'renoteUser') .leftJoinAndSelect('reply.user', 'replyUser') + .andWhere('user.isExplorable = TRUE') ; if (me) { -- cgit v1.2.3-freya From 63950fea3161985e8c1aa13c301893e9073168a2 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 12 May 2025 13:47:12 -0400 Subject: catch polls that are expiring exactly in the current instant --- packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'packages/backend/src') diff --git a/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts b/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts index 5dd6db54f9..9840f07832 100644 --- a/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts +++ b/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts @@ -95,7 +95,7 @@ export default class extends Endpoint { // eslint- if (ps.expired) { query.andWhere('poll.expiresAt IS NOT NULL'); - query.andWhere('poll.expiresAt < :expiresMax', { + query.andWhere('poll.expiresAt <= :expiresMax', { expiresMax: new Date(), }); query.andWhere('poll.expiresAt >= :expiresMin', { -- cgit v1.2.3-freya From 31e00a09067716463656785ddd6d7b87172a88e0 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 12 May 2025 13:56:12 -0400 Subject: fix policy check for unspecified "local" property in polls/recommendation endpoint --- packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts | 1 - 1 file changed, 1 deletion(-) (limited to 'packages/backend/src') diff --git a/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts b/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts index 9840f07832..6f96821a63 100644 --- a/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts +++ b/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts @@ -119,7 +119,6 @@ export default class extends Endpoint { // eslint- query.andWhere('poll.userHost IS NOT NULL'); } } else { - if (!policies.ltlAvailable) throw new ApiError(meta.errors.ltlDisabled); if (!policies.gtlAvailable) throw new ApiError(meta.errors.gtlDisabled); } -- cgit v1.2.3-freya From 4ed1ea8f89e367c3320fb396b09a31a9955fa033 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 25 May 2025 00:52:41 -0400 Subject: increase chart rate limits to max=200 and rate=5/s --- packages/backend/src/server/api/endpoints/charts/active-users.ts | 6 +++--- packages/backend/src/server/api/endpoints/charts/ap-request.ts | 6 +++--- packages/backend/src/server/api/endpoints/charts/drive.ts | 6 +++--- packages/backend/src/server/api/endpoints/charts/federation.ts | 6 +++--- packages/backend/src/server/api/endpoints/charts/instance.ts | 6 +++--- packages/backend/src/server/api/endpoints/charts/notes.ts | 6 +++--- packages/backend/src/server/api/endpoints/charts/user/drive.ts | 6 +++--- packages/backend/src/server/api/endpoints/charts/user/following.ts | 6 +++--- packages/backend/src/server/api/endpoints/charts/user/notes.ts | 6 +++--- packages/backend/src/server/api/endpoints/charts/user/pv.ts | 6 +++--- packages/backend/src/server/api/endpoints/charts/user/reactions.ts | 6 +++--- packages/backend/src/server/api/endpoints/charts/users.ts | 6 +++--- 12 files changed, 36 insertions(+), 36 deletions(-) (limited to 'packages/backend/src') diff --git a/packages/backend/src/server/api/endpoints/charts/active-users.ts b/packages/backend/src/server/api/endpoints/charts/active-users.ts index dcdcf46d0b..9f5064fe83 100644 --- a/packages/backend/src/server/api/endpoints/charts/active-users.ts +++ b/packages/backend/src/server/api/endpoints/charts/active-users.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/ap-request.ts b/packages/backend/src/server/api/endpoints/charts/ap-request.ts index 28c64229e7..68dc87546e 100644 --- a/packages/backend/src/server/api/endpoints/charts/ap-request.ts +++ b/packages/backend/src/server/api/endpoints/charts/ap-request.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/drive.ts b/packages/backend/src/server/api/endpoints/charts/drive.ts index 69ff3c5d7a..c0bfb00608 100644 --- a/packages/backend/src/server/api/endpoints/charts/drive.ts +++ b/packages/backend/src/server/api/endpoints/charts/drive.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/federation.ts b/packages/backend/src/server/api/endpoints/charts/federation.ts index bd870cc3d9..bd15700670 100644 --- a/packages/backend/src/server/api/endpoints/charts/federation.ts +++ b/packages/backend/src/server/api/endpoints/charts/federation.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/instance.ts b/packages/backend/src/server/api/endpoints/charts/instance.ts index 765bf024ee..e1053d05d8 100644 --- a/packages/backend/src/server/api/endpoints/charts/instance.ts +++ b/packages/backend/src/server/api/endpoints/charts/instance.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/notes.ts b/packages/backend/src/server/api/endpoints/charts/notes.ts index ecac436311..4550e2f17e 100644 --- a/packages/backend/src/server/api/endpoints/charts/notes.ts +++ b/packages/backend/src/server/api/endpoints/charts/notes.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/user/drive.ts b/packages/backend/src/server/api/endpoints/charts/user/drive.ts index 98ec40ade2..9475a8ab0a 100644 --- a/packages/backend/src/server/api/endpoints/charts/user/drive.ts +++ b/packages/backend/src/server/api/endpoints/charts/user/drive.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/user/following.ts b/packages/backend/src/server/api/endpoints/charts/user/following.ts index cb3dd36bab..20d0ecb25d 100644 --- a/packages/backend/src/server/api/endpoints/charts/user/following.ts +++ b/packages/backend/src/server/api/endpoints/charts/user/following.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/user/notes.ts b/packages/backend/src/server/api/endpoints/charts/user/notes.ts index 0742a21210..1d24dc2b77 100644 --- a/packages/backend/src/server/api/endpoints/charts/user/notes.ts +++ b/packages/backend/src/server/api/endpoints/charts/user/notes.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/user/pv.ts b/packages/backend/src/server/api/endpoints/charts/user/pv.ts index a220381b00..e0026d5ff3 100644 --- a/packages/backend/src/server/api/endpoints/charts/user/pv.ts +++ b/packages/backend/src/server/api/endpoints/charts/user/pv.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/user/reactions.ts b/packages/backend/src/server/api/endpoints/charts/user/reactions.ts index 3bb33622c2..c15056466f 100644 --- a/packages/backend/src/server/api/endpoints/charts/user/reactions.ts +++ b/packages/backend/src/server/api/endpoints/charts/user/reactions.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/users.ts b/packages/backend/src/server/api/endpoints/charts/users.ts index b5452517ab..0f96fae202 100644 --- a/packages/backend/src/server/api/endpoints/charts/users.ts +++ b/packages/backend/src/server/api/endpoints/charts/users.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; -- cgit v1.2.3-freya From b506dd564b25066b921dccc294010cbd510c53a3 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 26 May 2025 11:15:59 -0400 Subject: support fetching anonymous AP objects --- packages/backend/src/core/HttpRequestService.ts | 8 +++-- .../src/core/activitypub/ApRequestService.ts | 13 ++++++--- .../src/core/activitypub/ApResolverService.ts | 34 ++++++++++++++-------- 3 files changed, 37 insertions(+), 18 deletions(-) (limited to 'packages/backend/src') diff --git a/packages/backend/src/core/HttpRequestService.ts b/packages/backend/src/core/HttpRequestService.ts index 5c271b81e3..a0f2607ddc 100644 --- a/packages/backend/src/core/HttpRequestService.ts +++ b/packages/backend/src/core/HttpRequestService.ts @@ -235,7 +235,7 @@ export class HttpRequestService { } @bindThis - public async getActivityJson(url: string, isLocalAddressAllowed = false): Promise { + public async getActivityJson(url: string, isLocalAddressAllowed = false, allowAnonymous = false): Promise { this.apUtilityService.assertApUrl(url); const res = await this.send(url, { @@ -255,7 +255,11 @@ export class HttpRequestService { // Make sure the object ID matches the final URL (which is where it actually exists). // The caller (ApResolverService) will verify the ID against the original / entry URL, which ensures that all three match. - this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url); + if (allowAnonymous && activity.id == null) { + activity.id = res.url; + } else { + this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url); + } return activity as IObjectWithId; } diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index b665b51700..4c7cac2169 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -184,10 +184,11 @@ export class ApRequestService { * Get AP object with http-signature * @param user http-signature user * @param url URL to fetch - * @param followAlternate + * @param allowAnonymous If a fetched object lacks an ID, then it will be auto-generated from the final URL. (default: false) + * @param followAlternate Whether to resolve HTML responses to their referenced canonical AP endpoint. (default: true) */ @bindThis - public async signedGet(url: string, user: { id: MiUser['id'] }, followAlternate?: boolean): Promise { + public async signedGet(url: string, user: { id: MiUser['id'] }, allowAnonymous = false, followAlternate?: boolean): Promise { this.apUtilityService.assertApUrl(url); const _followAlternate = followAlternate ?? true; @@ -258,7 +259,7 @@ export class ApRequestService { if (alternate) { const href = alternate.getAttribute('href'); if (href && this.apUtilityService.haveSameAuthority(url, href)) { - return await this.signedGet(href, user, false); + return await this.signedGet(href, user, allowAnonymous, false); } } } catch { @@ -275,7 +276,11 @@ export class ApRequestService { // Make sure the object ID matches the final URL (which is where it actually exists). // The caller (ApResolverService) will verify the ID against the original / entry URL, which ensures that all three match. - this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url); + if (allowAnonymous && activity.id == null) { + activity.id = res.url; + } else { + this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url); + } return activity as IObjectWithId; } diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index 5e58f848c0..a7c5125928 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -63,10 +63,12 @@ export class Resolver { return this.recursionLimit; } + public async resolveCollection(value: string, allowAnonymous?: boolean): Promise; + public async resolveCollection(value: string | IObject, allowAnonymous?: boolean): Promise; @bindThis - public async resolveCollection(value: string | IObject): Promise { + public async resolveCollection(value: string | IObject, allowAnonymous?: boolean): Promise { const collection = typeof value === 'string' - ? await this.resolve(value) + ? await this.resolve(value, allowAnonymous) : value; if (isCollectionOrOrderedCollection(collection)) { @@ -103,25 +105,33 @@ export class Resolver { return await this.resolve(id); } - public async resolve(value: string | [string]): Promise; - public async resolve(value: string | IObject | [string | IObject]): Promise; + public async resolve(value: string | [string], allowAnonymous?: boolean): Promise; + public async resolve(value: string | IObject | [string | IObject], allowAnonymous?: boolean): Promise; + /** + * Resolves a URL or object to an AP object. + * Tuples are expanded to their first element before anything else, and non-string inputs are returned as-is. + * Otherwise, the string URL is fetched and validated to represent a valid ActivityPub object. + * @param value The input value to resolve + * @param allowAnonymous Determines what to do if a response object lacks an ID field. If false (default), then an exception is thrown. If true, then the ID is populated from the final response URL. + */ @bindThis - public async resolve(value: string | IObject | [string | IObject]): Promise { + public async resolve(value: string | IObject | [string | IObject], allowAnonymous = false): Promise { value = fromTuple(value); + // TODO try and remove this eventually, as it's a major security foot-gun if (typeof value !== 'string') { return value; } const host = this.utilityService.extractDbHost(value); if (this.config.activityLogging.enabled && !this.utilityService.isSelfHost(host)) { - return await this._resolveLogged(value, host); + return await this._resolveLogged(value, host, allowAnonymous); } else { - return await this._resolve(value, host); + return await this._resolve(value, host, allowAnonymous); } } - private async _resolveLogged(requestUri: string, host: string): Promise { + private async _resolveLogged(requestUri: string, host: string, allowAnonymous: boolean): Promise { const startTime = process.hrtime.bigint(); const log = await this.apLogService.createFetchLog({ @@ -130,7 +140,7 @@ export class Resolver { }); try { - const result = await this._resolve(requestUri, host, log); + const result = await this._resolve(requestUri, host, allowAnonymous, log); log.accepted = true; log.result = 'ok'; @@ -150,7 +160,7 @@ export class Resolver { } } - private async _resolve(value: string, host: string, log?: SkApFetchLog): Promise { + private async _resolve(value: string, host: string, allowAnonymous: boolean, log?: SkApFetchLog): Promise { if (value.includes('#')) { // URLs with fragment parts cannot be resolved correctly because // the fragment part does not get transmitted over HTTP(S). @@ -181,8 +191,8 @@ export class Resolver { } const object = (this.user - ? await this.apRequestService.signedGet(value, this.user) - : await this.httpRequestService.getActivityJson(value)); + ? await this.apRequestService.signedGet(value, this.user, allowAnonymous) + : await this.httpRequestService.getActivityJson(value, false, allowAnonymous)); if (log) { const { object: objectOnly, context, contextHash } = extractObjectContext(object); -- cgit v1.2.3-freya From 5f0bb5dcd7ced04035e409f7768c6bfccb950683 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 26 May 2025 11:16:48 -0400 Subject: implement resolver.resolveCollectionItems --- .../src/core/activitypub/ApResolverService.ts | 64 +++++++++++++++++++++- packages/backend/src/core/activitypub/type.ts | 46 ++++++++-------- packages/misskey-js/src/autogen/types.ts | 2 + 3 files changed, 86 insertions(+), 26 deletions(-) (limited to 'packages/backend/src') diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index a7c5125928..74050f456b 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -19,11 +19,12 @@ import { ApLogService, calculateDurationSince, extractObjectContext } from '@/co import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js'; import { SystemAccountService } from '@/core/SystemAccountService.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; -import { getApId, getNullableApId, IObjectWithId, isCollectionOrOrderedCollection } from './type.js'; +import { toArray } from '@/misc/prelude/array.js'; +import { AnyCollection, getApId, getNullableApId, IObjectWithId, isCollection, isCollectionOrOrderedCollection, isCollectionPage, isOrderedCollection, isOrderedCollectionPage } from './type.js'; import { ApDbResolverService } from './ApDbResolverService.js'; import { ApRendererService } from './ApRendererService.js'; import { ApRequestService } from './ApRequestService.js'; -import type { IObject, ICollection, IOrderedCollection, ApObject } from './type.js'; +import type { IObject, ApObject } from './type.js'; export class Resolver { private history: Set; @@ -78,6 +79,65 @@ export class Resolver { } } + @bindThis + public async resolveCollectionItems(value: string | IObject, limit?: number, allowAnonymousItems?: boolean): Promise { + const items: IObjectWithId[] = []; + + const collection = await this.resolveCollection(value); + await this.resolveCollectionItemsTo(collection, limit, allowAnonymousItems, collection.id, items); + + return items; + } + + private async resolveCollectionItemsTo(current: AnyCollection | null, limit: number | undefined, allowAnonymousItems: boolean | undefined, sourceUri: string | undefined, destination: IObjectWithId[]): Promise { + // This is pulled up to avoid code duplication below + const iterate = async(items: ApObject): Promise => { + for (const item of toArray(items)) { + // Stop when we reach the fetch limit + if (this.history.size > this.recursionLimit) break; + + // Stop when we reach the item limit + if (limit != null && limit < 1) break; + + // Use secureResolve whenever possible, to avoid re-fetching items that were included inline. + const resolved = (sourceUri && !allowAnonymousItems) + ? await this.secureResolve(item, sourceUri) + : await this.resolve(getApId(item), allowAnonymousItems); + destination.push(resolved); + + // Decrement the outer variable directly, because the code below checks it too + if (limit != null) limit--; + } + }; + + while (current != null) { + // Iterate all items in the current page + if (current.items) { + await iterate(current.items); + } + if (current.orderedItems) { + await iterate(current.orderedItems); + } + + if (this.history.size >= this.recursionLimit) { + // Stop when we reach the fetch limit + current = null; + } else if (limit != null && limit < 1) { + // Stop when we reach the item limit + current = null; + } else if (isCollection(current) || isOrderedCollection(current)) { + // Continue to first page + current = current.first ? await this.resolveCollection(current.first, true) : null; + } else if (isCollectionPage(current) || isOrderedCollectionPage(current)) { + // Continue to next page + current = current.next ? await this.resolveCollection(current.next, true) : null; + } else { + // Stop in all other conditions + current = null; + } + } + } + /** * Securely resolves an AP object or URL that has been sent from another instance. * An input object is trusted if and only if its ID matches the authority of sentFromUri. diff --git a/packages/backend/src/core/activitypub/type.ts b/packages/backend/src/core/activitypub/type.ts index 281733d484..e33dec18d7 100644 --- a/packages/backend/src/core/activitypub/type.ts +++ b/packages/backend/src/core/activitypub/type.ts @@ -125,48 +125,46 @@ export interface IActivity extends IObject { }; } -export interface ICollection extends IObject { - type: 'Collection'; - totalItems: number; +export interface CollectionBase extends IObject { + totalItems?: number; first?: IObject | string; last?: IObject | string; current?: IObject | string; + partOf?: IObject | string; + next?: IObject | string; + prev?: IObject | string; items?: ApObject; + orderedItems?: ApObject; } -export interface IOrderedCollection extends IObject { +export interface ICollection extends CollectionBase { + type: 'Collection'; + totalItems: number; + items?: ApObject; + orderedItems?: undefined; +} + +export interface IOrderedCollection extends CollectionBase { type: 'OrderedCollection'; totalItems: number; - first?: IObject | string; - last?: IObject | string; - current?: IObject | string; + items?: undefined; orderedItems?: ApObject; } -export interface ICollectionPage extends IObject { +export interface ICollectionPage extends CollectionBase { type: 'CollectionPage'; - totalItems: number; - first?: IObject | string; - last?: IObject | string; - current?: IObject | string; - partOf?: IObject | string; - next?: IObject | string; - prev?: IObject | string; items?: ApObject; + orderedItems?: undefined; } -export interface IOrderedCollectionPage extends IObject { +export interface IOrderedCollectionPage extends CollectionBase { type: 'OrderedCollectionPage'; - totalItems: number; - first?: IObject | string; - last?: IObject | string; - current?: IObject | string; - partOf?: IObject | string; - next?: IObject | string; - prev?: IObject | string; + items?: undefined; orderedItems?: ApObject; } +export type AnyCollection = ICollection | IOrderedCollection | ICollectionPage | IOrderedCollectionPage; + export const validPost = ['Note', 'Question', 'Article', 'Audio', 'Document', 'Image', 'Page', 'Video', 'Event']; export const isPost = (object: IObject): object is IPost => { @@ -269,7 +267,7 @@ export const isCollectionPage = (object: IObject): object is ICollectionPage => export const isOrderedCollectionPage = (object: IObject): object is IOrderedCollectionPage => getApType(object) === 'OrderedCollectionPage'; -export const isCollectionOrOrderedCollection = (object: IObject): object is ICollection | IOrderedCollection => +export const isCollectionOrOrderedCollection = (object: IObject): object is AnyCollection => isCollection(object) || isOrderedCollection(object) || isCollectionPage(object) || isOrderedCollectionPage(object); export interface IApPropertyValue extends IObject { diff --git a/packages/misskey-js/src/autogen/types.ts b/packages/misskey-js/src/autogen/types.ts index 55302960dc..678e980892 100644 --- a/packages/misskey-js/src/autogen/types.ts +++ b/packages/misskey-js/src/autogen/types.ts @@ -12918,6 +12918,8 @@ export type operations = { content: { 'application/json': { uri: string; + expandCollectionItems?: boolean; + allowAnonymous?: boolean; }; }; }; -- cgit v1.2.3-freya From bdccb203ea01c897a6818498d54681dd137f63aa Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 26 May 2025 11:17:01 -0400 Subject: resolve collection items in ApInboxService --- packages/backend/src/core/activitypub/ApInboxService.ts | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) (limited to 'packages/backend/src') diff --git a/packages/backend/src/core/activitypub/ApInboxService.ts b/packages/backend/src/core/activitypub/ApInboxService.ts index b8526a972c..2b0da52332 100644 --- a/packages/backend/src/core/activitypub/ApInboxService.ts +++ b/packages/backend/src/core/activitypub/ApInboxService.ts @@ -36,7 +36,7 @@ import InstanceChart from '@/core/chart/charts/instance.js'; import FederationChart from '@/core/chart/charts/federation.js'; import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js'; import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js'; -import { getApHrefNullable, getApId, getApIds, getApType, getNullableApId, isAccept, isActor, isAdd, isAnnounce, isApObject, isBlock, isCollection, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isDislike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost, isActivity, IObjectWithId } from './type.js'; +import { getApHrefNullable, getApId, getApIds, getApType, getNullableApId, isAccept, isActor, isAdd, isAnnounce, isApObject, isBlock, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isDislike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost, isActivity, IObjectWithId } from './type.js'; import { ApNoteService } from './models/ApNoteService.js'; import { ApLoggerService } from './ApLoggerService.js'; import { ApDbResolverService } from './ApDbResolverService.js'; @@ -106,22 +106,16 @@ export class ApInboxService { let result = undefined as string | void; if (isCollectionOrOrderedCollection(activity)) { const results = [] as [string, string | void][]; - // eslint-disable-next-line no-param-reassign resolver ??= this.apResolverService.createResolver(); - const items = toArray(isCollection(activity) ? activity.items : activity.orderedItems); - if (items.length >= resolver.getRecursionLimit()) { - throw new Error(`skipping activity: collection would surpass recursion limit: ${this.utilityService.extractDbHost(actor.uri)}`); - } - - for (const item of items) { - const act = await resolver.resolve(item); - if (act.id == null || this.utilityService.extractDbHost(act.id) !== this.utilityService.extractDbHost(actor.uri)) { + const items = await resolver.resolveCollectionItems(activity); + for (const act of items) { + if (this.utilityService.extractDbHost(act.id) !== this.utilityService.extractDbHost(actor.uri)) { this.logger.debug('skipping activity: activity id is null or mismatching'); continue; } try { - results.push([getApId(item), await this.performOneActivity(actor, act, resolver)]); + results.push([getApId(act), await this.performOneActivity(actor, act, resolver)]); } catch (err) { if (err instanceof Error || typeof err === 'string') { this.logger.error(err); -- cgit v1.2.3-freya From e689c047644e4083bafe8e2a79ff9a677fa96e1b Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 26 May 2025 11:17:20 -0400 Subject: add options expandCollectionItems and allowAnonymous to ap/get endpoint --- packages/backend/src/server/api/endpoints/ap/get.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'packages/backend/src') diff --git a/packages/backend/src/server/api/endpoints/ap/get.ts b/packages/backend/src/server/api/endpoints/ap/get.ts index 14286bc23e..3fe5c60a44 100644 --- a/packages/backend/src/server/api/endpoints/ap/get.ts +++ b/packages/backend/src/server/api/endpoints/ap/get.ts @@ -7,6 +7,7 @@ import { Injectable } from '@nestjs/common'; import ms from 'ms'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { ApResolverService } from '@/core/activitypub/ApResolverService.js'; +import { isCollectionOrOrderedCollection, isOrderedCollection, isOrderedCollectionPage } from '@/core/activitypub/type.js'; export const meta = { tags: ['federation'], @@ -33,6 +34,8 @@ export const paramDef = { type: 'object', properties: { uri: { type: 'string' }, + expandCollectionItems: { type: 'boolean' }, + allowAnonymous: { type: 'boolean' }, }, required: ['uri'], } as const; @@ -44,7 +47,18 @@ export default class extends Endpoint { // eslint- ) { super(meta, paramDef, async (ps, me) => { const resolver = this.apResolverService.createResolver(); - const object = await resolver.resolve(ps.uri); + const object = await resolver.resolve(ps.uri, ps.allowAnonymous ?? false); + + if (ps.expandCollectionItems && isCollectionOrOrderedCollection(object)) { + const items = await resolver.resolveCollectionItems(object, undefined, ps.allowAnonymous ?? false); + + if (isOrderedCollection(object) || isOrderedCollectionPage(object)) { + object.orderedItems = items; + } else { + object.items = items; + } + } + return object; }); } -- cgit v1.2.3-freya From 02787f75ef194474d23c94cf5ac8a34a5f9a19d0 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 26 May 2025 11:22:18 -0400 Subject: add JSDocs to resolveCollectionItems --- .../backend/src/core/activitypub/ApResolverService.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'packages/backend/src') diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index 74050f456b..8312294a1f 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -79,12 +79,21 @@ export class Resolver { } } + /** + * Recursively resolves items from a collection. + * Stops when reaching the resolution limit or an optional item limit - whichever is lower. + * This method supports Collection, OrderedCollection, and individual pages of either type. + * Malformed collections (mixing Ordered and un-Ordered types) are also supported. + * @param collection Collection to resolve from - can be a URL or object of any supported collection type. + * @param limit Maximum number of items to resolve. If null or undefined (default), then items will be resolved until reaching the recursion limit. + * @param allowAnonymousItems If true, collection items can be anonymous (lack an ID). If false (default), then an error is thrown when reaching an item without ID. + */ @bindThis - public async resolveCollectionItems(value: string | IObject, limit?: number, allowAnonymousItems?: boolean): Promise { + public async resolveCollectionItems(collection: string | IObject, limit?: number | null, allowAnonymousItems?: boolean): Promise { const items: IObjectWithId[] = []; - const collection = await this.resolveCollection(value); - await this.resolveCollectionItemsTo(collection, limit, allowAnonymousItems, collection.id, items); + const collectionObj = await this.resolveCollection(collection); + await this.resolveCollectionItemsTo(collectionObj, limit ?? undefined, allowAnonymousItems, collectionObj.id, items); return items; } -- cgit v1.2.3-freya From facedd364667e09f280cdb903d86748ba7d22629 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 26 May 2025 11:33:53 -0400 Subject: allow anonymous objects in secureResolve --- .../backend/src/core/activitypub/ApResolverService.ts | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) (limited to 'packages/backend/src') diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index 8312294a1f..1b4fed4461 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -151,16 +151,26 @@ export class Resolver { * Securely resolves an AP object or URL that has been sent from another instance. * An input object is trusted if and only if its ID matches the authority of sentFromUri. * In all other cases, the object is re-fetched from remote by input string or object ID. + * @param input The input object or URL to resolve + * @param sentFromUri The URL where this object originated. This MUST be accurate - all security checks depend on this value! + * @param allowAnonymous If true, anonymous objects are allowed and will have their ID set to sentFromUri. If false (default) then anonymous objects will be rejected with an error. */ @bindThis - public async secureResolve(input: ApObject, sentFromUri: string): Promise { + public async secureResolve(input: ApObject, sentFromUri: string, allowAnonymous?: boolean): Promise { // Unpack arrays to get the value element. const value = fromTuple(input); if (value == null) { throw new IdentifiableError('20058164-9de1-4573-8715-425753a21c1d', 'Cannot resolve null input'); } - // This will throw if the input has no ID, which is good because we can't verify an anonymous object anyway. + // If anonymous input is allowed, then any object is automatically valid if we set the ID. + // We can short-circuit here and avoid un-necessary checks. + if (allowAnonymous && typeof(value) === 'object' && value.id == null) { + value.id = sentFromUri; + return value as IObjectWithId; + } + + // This ensures the input has a string ID, protecting against type confusion and rejecting anonymous objects. const id = getApId(value); // Check if we can use the provided object as-is. @@ -171,7 +181,7 @@ export class Resolver { } // If the checks didn't pass, then we must fetch the object and use that. - return await this.resolve(id); + return await this.resolve(id, allowAnonymous); } public async resolve(value: string | [string], allowAnonymous?: boolean): Promise; -- cgit v1.2.3-freya From 1ab5ceb65aab5170a556e1880ea07b6eccc5d710 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 26 May 2025 11:34:46 -0400 Subject: fix ID checks in resolveCollectionItems --- packages/backend/src/core/activitypub/ApResolverService.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'packages/backend/src') diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index 1b4fed4461..6b95d9c93b 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -93,12 +93,12 @@ export class Resolver { const items: IObjectWithId[] = []; const collectionObj = await this.resolveCollection(collection); - await this.resolveCollectionItemsTo(collectionObj, limit ?? undefined, allowAnonymousItems, collectionObj.id, items); + await this.resolveCollectionItemsTo(collectionObj, limit ?? undefined, allowAnonymousItems, items); return items; } - private async resolveCollectionItemsTo(current: AnyCollection | null, limit: number | undefined, allowAnonymousItems: boolean | undefined, sourceUri: string | undefined, destination: IObjectWithId[]): Promise { + private async resolveCollectionItemsTo(current: AnyCollection | null, limit: number | undefined, allowAnonymousItems: boolean | undefined, destination: IObjectWithId[]): Promise { // This is pulled up to avoid code duplication below const iterate = async(items: ApObject): Promise => { for (const item of toArray(items)) { @@ -109,8 +109,8 @@ export class Resolver { if (limit != null && limit < 1) break; // Use secureResolve whenever possible, to avoid re-fetching items that were included inline. - const resolved = (sourceUri && !allowAnonymousItems) - ? await this.secureResolve(item, sourceUri) + const resolved = current?.id + ? await this.secureResolve(item, current.id, allowAnonymousItems) : await this.resolve(getApId(item), allowAnonymousItems); destination.push(resolved); -- cgit v1.2.3-freya From 3da3ce9a40085f55da7dc9a911d1c03796ec0681 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 26 May 2025 11:43:05 -0400 Subject: pass limit from ap/get to resolveCollectionItems --- packages/backend/src/server/api/endpoints/ap/get.ts | 3 ++- packages/misskey-js/src/autogen/types.ts | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'packages/backend/src') diff --git a/packages/backend/src/server/api/endpoints/ap/get.ts b/packages/backend/src/server/api/endpoints/ap/get.ts index 3fe5c60a44..06dd37a140 100644 --- a/packages/backend/src/server/api/endpoints/ap/get.ts +++ b/packages/backend/src/server/api/endpoints/ap/get.ts @@ -35,6 +35,7 @@ export const paramDef = { properties: { uri: { type: 'string' }, expandCollectionItems: { type: 'boolean' }, + expandCollectionLimit: { type: 'integer', nullable: true }, allowAnonymous: { type: 'boolean' }, }, required: ['uri'], @@ -50,7 +51,7 @@ export default class extends Endpoint { // eslint- const object = await resolver.resolve(ps.uri, ps.allowAnonymous ?? false); if (ps.expandCollectionItems && isCollectionOrOrderedCollection(object)) { - const items = await resolver.resolveCollectionItems(object, undefined, ps.allowAnonymous ?? false); + const items = await resolver.resolveCollectionItems(object, ps.expandCollectionLimit, ps.allowAnonymous ?? false); if (isOrderedCollection(object) || isOrderedCollectionPage(object)) { object.orderedItems = items; diff --git a/packages/misskey-js/src/autogen/types.ts b/packages/misskey-js/src/autogen/types.ts index 678e980892..5e5f4f5db5 100644 --- a/packages/misskey-js/src/autogen/types.ts +++ b/packages/misskey-js/src/autogen/types.ts @@ -12919,6 +12919,7 @@ export type operations = { 'application/json': { uri: string; expandCollectionItems?: boolean; + expandCollectionLimit?: number | null; allowAnonymous?: boolean; }; }; -- cgit v1.2.3-freya From a3f9ff68fa2e1b08d9a32560f72288dd550a5d99 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 26 May 2025 13:08:05 -0400 Subject: resolve collection items in parallel --- .../src/core/activitypub/ApResolverService.ts | 82 ++++++++++++---------- 1 file changed, 44 insertions(+), 38 deletions(-) (limited to 'packages/backend/src') diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index 6b95d9c93b..cf370c730c 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -5,6 +5,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { IsNull, Not } from 'typeorm'; +import promiseLimit from 'promise-limit'; import type { MiLocalUser, MiRemoteUser } from '@/models/User.js'; import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository, FollowRequestsRepository, MiMeta, SkApFetchLog } from '@/models/_.js'; import type { Config } from '@/config.js'; @@ -64,13 +65,16 @@ export class Resolver { return this.recursionLimit; } - public async resolveCollection(value: string, allowAnonymous?: boolean): Promise; - public async resolveCollection(value: string | IObject, allowAnonymous?: boolean): Promise; + public async resolveCollection(value: string | IObjectWithId, allowAnonymous?: boolean, sentFromUri?: string): Promise; + public async resolveCollection(value: string | IObject, allowAnonymous: boolean | undefined, sentFromUri: string): Promise; + public async resolveCollection(value: string | IObject, allowAnonymous?: boolean, sentFromUri?: string): Promise; @bindThis - public async resolveCollection(value: string | IObject, allowAnonymous?: boolean): Promise { + public async resolveCollection(value: string | IObject, allowAnonymous?: boolean, sentFromUri?: string): Promise { const collection = typeof value === 'string' - ? await this.resolve(value, allowAnonymous) - : value; + ? sentFromUri + ? await this.secureResolve(value, sentFromUri, allowAnonymous) + : await this.resolve(value, allowAnonymous) + : value; // TODO try and remove this eventually, as it's a major security foot-gun if (isCollectionOrOrderedCollection(collection)) { return collection; @@ -87,66 +91,67 @@ export class Resolver { * @param collection Collection to resolve from - can be a URL or object of any supported collection type. * @param limit Maximum number of items to resolve. If null or undefined (default), then items will be resolved until reaching the recursion limit. * @param allowAnonymousItems If true, collection items can be anonymous (lack an ID). If false (default), then an error is thrown when reaching an item without ID. + * @param concurrency Maximum number of items to resolve at once. (default: 4) */ @bindThis - public async resolveCollectionItems(collection: string | IObject, limit?: number | null, allowAnonymousItems?: boolean): Promise { - const items: IObjectWithId[] = []; + public async resolveCollectionItems(collection: string | IObjectWithId, limit?: number | null, allowAnonymousItems?: boolean, concurrency = 4): Promise { + const resolvedItems: IObjectWithId[] = []; - const collectionObj = await this.resolveCollection(collection); - await this.resolveCollectionItemsTo(collectionObj, limit ?? undefined, allowAnonymousItems, items); - - return items; - } - - private async resolveCollectionItemsTo(current: AnyCollection | null, limit: number | undefined, allowAnonymousItems: boolean | undefined, destination: IObjectWithId[]): Promise { // This is pulled up to avoid code duplication below - const iterate = async(items: ApObject): Promise => { - for (const item of toArray(items)) { - // Stop when we reach the fetch limit - if (this.history.size > this.recursionLimit) break; - - // Stop when we reach the item limit - if (limit != null && limit < 1) break; - - // Use secureResolve whenever possible, to avoid re-fetching items that were included inline. - const resolved = current?.id - ? await this.secureResolve(item, current.id, allowAnonymousItems) - : await this.resolve(getApId(item), allowAnonymousItems); - destination.push(resolved); - - // Decrement the outer variable directly, because the code below checks it too - if (limit != null) limit--; - } + const iterate = async(items: ApObject, current: AnyCollection & IObjectWithId) => { + const sentFrom = current.id; + const itemArr = toArray(items); + const itemLimit = limit ?? Number.MAX_SAFE_INTEGER; + const allowAnonymous = allowAnonymousItems ?? false; + await this.resolveItemArray(itemArr, sentFrom, itemLimit, concurrency, allowAnonymous, resolvedItems); }; - while (current != null) { + let current: (AnyCollection & IObjectWithId) | null = await this.resolveCollection(collection); + do { // Iterate all items in the current page if (current.items) { - await iterate(current.items); + await iterate(current.items, current); } if (current.orderedItems) { - await iterate(current.orderedItems); + await iterate(current.orderedItems, current); } if (this.history.size >= this.recursionLimit) { // Stop when we reach the fetch limit current = null; - } else if (limit != null && limit < 1) { + } else if (limit != null && resolvedItems.length >= limit) { // Stop when we reach the item limit current = null; } else if (isCollection(current) || isOrderedCollection(current)) { // Continue to first page - current = current.first ? await this.resolveCollection(current.first, true) : null; + current = current.first ? await this.resolveCollection(current.first, true, current.id) : null; } else if (isCollectionPage(current) || isOrderedCollectionPage(current)) { // Continue to next page - current = current.next ? await this.resolveCollection(current.next, true) : null; + current = current.next ? await this.resolveCollection(current.next, true, current.id) : null; } else { // Stop in all other conditions current = null; } - } + } while (current != null); + + return resolvedItems; } + private async resolveItemArray(source: (string | IObject)[], sentFrom: string, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObjectWithId[]): Promise { + const recursionLimit = this.recursionLimit - this.history.size; + const batchLimit = Math.min(source.length, recursionLimit, itemLimit); + + const limiter = promiseLimit(concurrency); + const batch = await Promise.all(source + .slice(0, batchLimit) + .map(item => limiter(async () => { + // Use secureResolve to avoid re-fetching items that were included inline. + return await this.secureResolve(item, sentFrom, allowAnonymousItems); + }))); + + destination.push(...batch); + }; + /** * Securely resolves an AP object or URL that has been sent from another instance. * An input object is trusted if and only if its ID matches the authority of sentFromUri. @@ -185,6 +190,7 @@ export class Resolver { } public async resolve(value: string | [string], allowAnonymous?: boolean): Promise; + public async resolve(value: string | IObjectWithId | [string | IObjectWithId], allowAnonymous?: boolean): Promise; public async resolve(value: string | IObject | [string | IObject], allowAnonymous?: boolean): Promise; /** * Resolves a URL or object to an AP object. -- cgit v1.2.3-freya From ca32c231d0aaab9dacaa88a0b0564d626827e797 Mon Sep 17 00:00:00 2001 From: piuvas Date: Mon, 26 May 2025 22:44:29 -0300 Subject: initial link attributions work. --- locales/index.d.ts | 16 ++++++ .../1748096357260-AddAttributionDomains.js | 19 ++++++++ .../src/core/activitypub/ApRendererService.ts | 1 + .../backend/src/core/activitypub/misc/contexts.ts | 4 ++ .../src/core/activitypub/models/ApPersonService.ts | 2 + packages/backend/src/core/activitypub/type.ts | 1 + .../backend/src/core/entities/UserEntityService.ts | 1 + packages/backend/src/models/User.ts | 6 +++ packages/backend/src/models/json-schema/user.ts | 7 +++ .../backend/src/server/api/endpoints/i/update.ts | 6 ++- .../backend/src/server/web/UrlPreviewService.ts | 50 ++++++++++++++++++- packages/frontend/src/components/MkUrlPreview.vue | 46 ++++++++++++++++- .../pages/settings/attribution-domains-setting.vue | 57 ++++++++++++++++++++++ packages/frontend/src/pages/settings/profile.vue | 5 ++ sharkey-locales/en-US.yml | 6 +++ sharkey-locales/pt-PT.yml | 4 ++ 16 files changed, 228 insertions(+), 3 deletions(-) create mode 100644 packages/backend/migration/1748096357260-AddAttributionDomains.js create mode 100644 packages/frontend/src/pages/settings/attribution-domains-setting.vue (limited to 'packages/backend/src') diff --git a/locales/index.d.ts b/locales/index.d.ts index 69c63cc714..9b71db33ac 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -13129,6 +13129,22 @@ export interface Locale extends ILocale { * Timeout in milliseconds for translation API requests. */ "translationTimeoutCaption": string; + /** + * Attribution Domains + */ + "attributionDomains": string; + /** + * A list of domains whose content can be attributed to you on link previews, separated by new-line. Any subdomain will also be valid. + */ + "attributionDomainsDescription": string; + /** + * Webpages with `` can then be attributed to you. + */ + "attributionDomainsTutorial": ParameterizedString<"user" | "host">; + /** + * Written by + */ + "writtenBy": string; } declare const locales: { [lang: string]: Locale; diff --git a/packages/backend/migration/1748096357260-AddAttributionDomains.js b/packages/backend/migration/1748096357260-AddAttributionDomains.js new file mode 100644 index 0000000000..90c0d0c655 --- /dev/null +++ b/packages/backend/migration/1748096357260-AddAttributionDomains.js @@ -0,0 +1,19 @@ +/** + * @typedef {import('typeorm').MigrationInterface} MigrationInterface + */ + +/** + * @class + * @implements {MigrationInterface} + */ +module.exports = class AddAttributionDomains1748096357260 { + name = 'AddAttributionDomains1748096357260' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "user" ADD "attributionDomains" text array NOT NULL DEFAULT '{}'`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "user" DROP COLUMN "attributionDomains"`); + } +} diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index f41eeba39f..46a78687f3 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -613,6 +613,7 @@ export class ApRendererService { enableRss: user.enableRss, speakAsCat: user.speakAsCat, attachment: attachment.length ? attachment : undefined, + attributionDomains: user.attributionDomains, }; if (user.movedToUri) { diff --git a/packages/backend/src/core/activitypub/misc/contexts.ts b/packages/backend/src/core/activitypub/misc/contexts.ts index 5c0b8ffcbb..cedd1d8dd5 100644 --- a/packages/backend/src/core/activitypub/misc/contexts.ts +++ b/packages/backend/src/core/activitypub/misc/contexts.ts @@ -546,6 +546,10 @@ const extension_context_definition = { featured: 'toot:featured', discoverable: 'toot:discoverable', indexable: 'toot:indexable', + attributionDomains: { + '@id': 'toot:attributionDomains', + '@type': '@id', + }, // schema schema: 'http://schema.org#', PropertyValue: 'schema:PropertyValue', diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index 4b685f7e1b..744b339254 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -433,6 +433,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { makeNotesFollowersOnlyBefore: (person as any).makeNotesFollowersOnlyBefore ?? null, makeNotesHiddenBefore: (person as any).makeNotesHiddenBefore ?? null, emojis, + attributionDomains: (person as any).attributionDomains, })) as MiRemoteUser; let _description: string | null = null; @@ -616,6 +617,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { // We use "!== false" to handle incorrect types, missing / null values, and "default to true" logic. hideOnlineStatus: person.hideOnlineStatus !== false, isExplorable: person.discoverable !== false, + attributionDomains: person.attributionDomains, ...(await this.resolveAvatarAndBanner(exist, person.icon, person.image, person.backgroundUrl).catch(() => ({}))), } as Partial & Pick; diff --git a/packages/backend/src/core/activitypub/type.ts b/packages/backend/src/core/activitypub/type.ts index 281733d484..0122697f2a 100644 --- a/packages/backend/src/core/activitypub/type.ts +++ b/packages/backend/src/core/activitypub/type.ts @@ -255,6 +255,7 @@ export interface IActor extends IObject { enableRss?: boolean; listenbrainz?: string; backgroundUrl?: string; + attributionDomains?: string[]; } export const isCollection = (object: IObject): object is ICollection => diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index 56506a5fa4..f66a36336d 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -603,6 +603,7 @@ export class UserEntityService implements OnModuleInit { enableRss: user.enableRss, mandatoryCW: user.mandatoryCW, rejectQuotes: user.rejectQuotes, + attributionDomains: user.attributionDomains, isSilenced: user.isSilenced || this.roleService.getUserPolicies(user.id).then(r => !r.canPublicNote), speakAsCat: user.speakAsCat ?? false, approved: user.approved, diff --git a/packages/backend/src/models/User.ts b/packages/backend/src/models/User.ts index 46f8e84a94..d5f572a879 100644 --- a/packages/backend/src/models/User.ts +++ b/packages/backend/src/models/User.ts @@ -378,6 +378,12 @@ export class MiUser { }) public allowUnsignedFetch: UserUnsignedFetchOption; + @Column('varchar', { + name: 'attributionDomains', + length: 128, array: true, default: '{}', + }) + public attributionDomains: string[]; + constructor(data: Partial) { if (data == null) return; diff --git a/packages/backend/src/models/json-schema/user.ts b/packages/backend/src/models/json-schema/user.ts index 964a179244..9bfb57fc66 100644 --- a/packages/backend/src/models/json-schema/user.ts +++ b/packages/backend/src/models/json-schema/user.ts @@ -236,6 +236,13 @@ export const packedUserLiteSchema = { }, }, }, + attributionDomains: { + type: 'object', + nullable: false, optional: false, + additionalProperties: { + type: 'string', + }, + }, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/i/update.ts b/packages/backend/src/server/api/endpoints/i/update.ts index f35e395841..dad605f151 100644 --- a/packages/backend/src/server/api/endpoints/i/update.ts +++ b/packages/backend/src/server/api/endpoints/i/update.ts @@ -263,6 +263,9 @@ export const paramDef = { enum: userUnsignedFetchOptions, nullable: false, }, + attributionDomains: { type: 'array', items: { + type: 'string', + } }, }, } as const; @@ -373,6 +376,7 @@ export default class extends Endpoint { // eslint- } if (ps.mutedInstances !== undefined) profileUpdates.mutedInstances = ps.mutedInstances; if (ps.notificationRecieveConfig !== undefined) profileUpdates.notificationRecieveConfig = ps.notificationRecieveConfig; + if (ps.attributionDomains !== undefined) updates.attributionDomains = ps.attributionDomains; if (typeof ps.isLocked === 'boolean') updates.isLocked = ps.isLocked; if (typeof ps.isExplorable === 'boolean') updates.isExplorable = ps.isExplorable; if (typeof ps.hideOnlineStatus === 'boolean') updates.hideOnlineStatus = ps.hideOnlineStatus; @@ -663,7 +667,7 @@ export default class extends Endpoint { // eslint- // these two methods need to be kept in sync with // `ApRendererService.renderPerson` private userNeedsPublishing(oldUser: MiLocalUser, newUser: Partial): boolean { - const basicFields: (keyof MiUser)[] = ['avatarId', 'bannerId', 'backgroundId', 'isBot', 'username', 'name', 'isLocked', 'isExplorable', 'isCat', 'noindex', 'speakAsCat', 'movedToUri', 'alsoKnownAs', 'hideOnlineStatus', 'enableRss', 'requireSigninToViewContents', 'makeNotesFollowersOnlyBefore', 'makeNotesHiddenBefore']; + const basicFields: (keyof MiUser)[] = ['avatarId', 'bannerId', 'backgroundId', 'isBot', 'username', 'name', 'isLocked', 'isExplorable', 'isCat', 'noindex', 'speakAsCat', 'movedToUri', 'alsoKnownAs', 'hideOnlineStatus', 'enableRss', 'requireSigninToViewContents', 'makeNotesFollowersOnlyBefore', 'makeNotesHiddenBefore', 'attributionDomains']; for (const field of basicFields) { if ((field in newUser) && oldUser[field] !== newUser[field]) { return true; diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index 203bc908a8..d5d62c67c0 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -19,7 +19,8 @@ import { MiMeta } from '@/models/Meta.js'; import { RedisKVCache } from '@/misc/cache.js'; import { UtilityService } from '@/core/UtilityService.js'; import { ApDbResolverService } from '@/core/activitypub/ApDbResolverService.js'; -import type { MiAccessToken, NotesRepository } from '@/models/_.js'; +import type { MiAccessToken, NotesRepository, UsersRepository } from '@/models/_.js'; +import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js'; import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js'; import { ApRequestService } from '@/core/activitypub/ApRequestService.js'; import { SystemAccountService } from '@/core/SystemAccountService.js'; @@ -34,6 +35,12 @@ import type { FastifyRequest, FastifyReply } from 'fastify'; export type LocalSummalyResult = SummalyResult & { haveNoteLocally?: boolean; + linkAttribution?: { + name: string, + username: string, + avatarUrl: string, + avatarBlurhash: string, + } }; // Increment this to invalidate cached previews after a major change. @@ -77,11 +84,15 @@ export class UrlPreviewService { @Inject(DI.notesRepository) private readonly notesRepository: NotesRepository, + @Inject(DI.usersRepository) + private readonly usersRepository: UsersRepository, + private httpRequestService: HttpRequestService, private loggerService: LoggerService, private readonly utilityService: UtilityService, private readonly apUtilityService: ApUtilityService, private readonly apDbResolverService: ApDbResolverService, + private readonly remoteUserResolveService: RemoteUserResolveService, private readonly apRequestService: ApRequestService, private readonly systemAccountService: SystemAccountService, private readonly apNoteService: ApNoteService, @@ -206,6 +217,8 @@ export class UrlPreviewService { } } + await this.validateLinkAttribution(summary); + // Await this to avoid hammering redis when a bunch of URLs are fetched at once await this.previewCache.set(cacheKey, summary); @@ -426,6 +439,41 @@ export class UrlPreviewService { } } + private async validateLinkAttribution(summary: LocalSummalyResult) { + if (!summary.fediverseCreator) return; + + const url = URL.parse(summary.url); + if (!url) return; + let fediverseCreator = summary.fediverseCreator; + // expecting either '@username@host' or 'username@host' + if (fediverseCreator.startsWith('@')) { + fediverseCreator = fediverseCreator.substring(1); + } + + // + const array = fediverseCreator.split('@'); + const username = array[0].toLowerCase(); + let host: string | null = array[1]; + if (host.toLowerCase() === this.config.host) { + host = null; + } + try { + const user = await this.remoteUserResolveService.resolveUser(username, host) + + const attributionDomains = user.attributionDomains; + if (attributionDomains.some(x => `.${url.host.toLowerCase()}`.endsWith(`.${x}`))) { + summary.linkAttribution = { + name: user.name ?? user.username, + username: fediverseCreator, + avatarUrl: user.avatarUrl ?? '', + avatarBlurhash: user.avatarBlurhash ?? '', + } + } + } catch { + console.warn('user not found: ' + fediverseCreator) + } + } + // Adapted from ApiCallService private async checkFetchPermissions(auth: AuthArray, reply: FastifyReply): Promise { const [user, app] = auth; diff --git a/packages/frontend/src/components/MkUrlPreview.vue b/packages/frontend/src/components/MkUrlPreview.vue index a14c2ecef9..029d2212eb 100644 --- a/packages/frontend/src/components/MkUrlPreview.vue +++ b/packages/frontend/src/components/MkUrlPreview.vue @@ -65,6 +65,13 @@ SPDX-License-Identifier: AGPL-3.0-only +