support Mastodon v4 "link header" pagination

4 files changed, 143 insertions, 76 deletions

diff --git a/packages/backend/src/server/api/mastodon/endpoints/account.ts b/packages/backend/src/server/api/mastodon/endpoints/account.ts
index 6f6999f2e1..f669b71efb 100644
--- a/packages/backend/src/server/api/mastodon/endpoints/account.ts
+++ b/packages/backend/src/server/api/mastodon/endpoints/account.ts
@@ -9,6 +9,7 @@ import { MastodonClientService } from '@/server/api/mastodon/MastodonClientServi
 import { DriveService } from '@/core/DriveService.js';
 import { DI } from '@/di-symbols.js';
 import type { AccessTokensRepository, UserProfilesRepository } from '@/models/_.js';
+import { attachMinMaxPagination } from '@/server/api/mastodon/pagination.js';
 import { MastoConverters, convertRelationship, convertFeaturedTag, convertList } from '../converters.js';
 import type multer from 'fastify-multer';
 import type { FastifyInstance } from 'fastify';
@@ -173,14 +174,15 @@ export class ApiAccountMastodon {
 			reply.send(account);
 		});
 
-		fastify.get<ApiAccountMastodonRoute & { Params: { id?: string } }>('/v1/accounts/:id/statuses', async (_request, reply) => {
-			if (!_request.params.id) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required parameter "id"' });
+		fastify.get<ApiAccountMastodonRoute & { Params: { id?: string } }>('/v1/accounts/:id/statuses', async (request, reply) => {
+			if (!request.params.id) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required parameter "id"' });
 
-			const { client, me } = await this.clientService.getAuthClient(_request);
-			const args = parseTimelineArgs(_request.query);
-			const data = await client.getAccountStatuses(_request.params.id, args);
+			const { client, me } = await this.clientService.getAuthClient(request);
+			const args = parseTimelineArgs(request.query);
+			const data = await client.getAccountStatuses(request.params.id, args);
 			const response = await Promise.all(data.data.map(async (status) => await this.mastoConverters.convertStatus(status, me)));
 
+			attachMinMaxPagination(request, reply, response);
 			reply.send(response);
 		});
 
@@ -194,29 +196,31 @@ export class ApiAccountMastodon {
 			reply.send(response);
 		});
 
-		fastify.get<ApiAccountMastodonRoute & { Params: { id?: string } }>('/v1/accounts/:id/followers', async (_request, reply) => {
-			if (!_request.params.id) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required parameter "id"' });
+		fastify.get<ApiAccountMastodonRoute & { Params: { id?: string } }>('/v1/accounts/:id/followers', async (request, reply) => {
+			if (!request.params.id) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required parameter "id"' });
 
-			const client = this.clientService.getClient(_request);
+			const client = this.clientService.getClient(request);
 			const data = await client.getAccountFollowers(
-				_request.params.id,
-				parseTimelineArgs(_request.query),
+				request.params.id,
+				parseTimelineArgs(request.query),
 			);
 			const response = await Promise.all(data.data.map(async (account) => await this.mastoConverters.convertAccount(account)));
 
+			attachMinMaxPagination(request, reply, response);
 			reply.send(response);
 		});
 
-		fastify.get<ApiAccountMastodonRoute & { Params: { id?: string } }>('/v1/accounts/:id/following', async (_request, reply) => {
-			if (!_request.params.id) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required parameter "id"' });
+		fastify.get<ApiAccountMastodonRoute & { Params: { id?: string } }>('/v1/accounts/:id/following', async (request, reply) => {
+			if (!request.params.id) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required parameter "id"' });
 
-			const client = this.clientService.getClient(_request);
+			const client = this.clientService.getClient(request);
 			const data = await client.getAccountFollowing(
-				_request.params.id,
-				parseTimelineArgs(_request.query),
+				request.params.id,
+				parseTimelineArgs(request.query),
 			);
 			const response = await Promise.all(data.data.map(async (account) => await this.mastoConverters.convertAccount(account)));
 
+			attachMinMaxPagination(request, reply, response);
 			reply.send(response);
 		});
 
@@ -236,7 +240,7 @@ export class ApiAccountMastodon {
 			const client = this.clientService.getClient(_request);
 			const data = await client.followAccount(_request.params.id);
 			const acct = convertRelationship(data.data);
-			acct.following = true;
+			acct.following = true; // TODO this is wrong, follow may not have processed immediately
 
 			reply.send(acct);
 		});
diff --git a/packages/backend/src/server/api/mastodon/endpoints/notifications.ts b/packages/backend/src/server/api/mastodon/endpoints/notifications.ts
index 3b2833bf86..6acb9edd6b 100644
--- a/packages/backend/src/server/api/mastodon/endpoints/notifications.ts
+++ b/packages/backend/src/server/api/mastodon/endpoints/notifications.ts
@@ -6,6 +6,7 @@
 import { Injectable } from '@nestjs/common';
 import { parseTimelineArgs, TimelineArgs } from '@/server/api/mastodon/argsUtils.js';
 import { MastoConverters } from '@/server/api/mastodon/converters.js';
+import { attachMinMaxPagination } from '@/server/api/mastodon/pagination.js';
 import { MastodonClientService } from '../MastodonClientService.js';
 import type { FastifyInstance } from 'fastify';
 import type multer from 'fastify-multer';
@@ -25,9 +26,9 @@ export class ApiNotificationsMastodon {
 	) {}
 
 	public register(fastify: FastifyInstance, upload: ReturnType<typeof multer>): void {
-		fastify.get<ApiNotifyMastodonRoute>('/v1/notifications', async (_request, reply) => {
-			const { client, me } = await this.clientService.getAuthClient(_request);
-			const data = await client.getNotifications(parseTimelineArgs(_request.query));
+		fastify.get<ApiNotifyMastodonRoute>('/v1/notifications', async (request, reply) => {
+			const { client, me } = await this.clientService.getAuthClient(request);
+			const data = await client.getNotifications(parseTimelineArgs(request.query));
 			const response = await Promise.all(data.data.map(async n => {
 				const converted = await this.mastoConverters.convertNotification(n, me);
 				if (converted.type === 'reaction') {
@@ -36,6 +37,7 @@ export class ApiNotificationsMastodon {
 				return converted;
 			}));
 
+			attachMinMaxPagination(request, reply, response);
 			reply.send(response);
 		});
 
diff --git a/packages/backend/src/server/api/mastodon/endpoints/search.ts b/packages/backend/src/server/api/mastodon/endpoints/search.ts
index 34d82096ba..997a585077 100644
--- a/packages/backend/src/server/api/mastodon/endpoints/search.ts
+++ b/packages/backend/src/server/api/mastodon/endpoints/search.ts
@@ -5,16 +5,18 @@
 
 import { Injectable } from '@nestjs/common';
 import { MastodonClientService } from '@/server/api/mastodon/MastodonClientService.js';
+import { attachMinMaxPagination, attachOffsetPagination } from '@/server/api/mastodon/pagination.js';
 import { MastoConverters } from '../converters.js';
-import { parseTimelineArgs, TimelineArgs } from '../argsUtils.js';
+import { parseTimelineArgs, TimelineArgs, toBoolean, toInt } from '../argsUtils.js';
 import Account = Entity.Account;
 import Status = Entity.Status;
 import type { FastifyInstance } from 'fastify';
 
 interface ApiSearchMastodonRoute {
 	Querystring: TimelineArgs & {
-		type?: 'accounts' | 'hashtags' | 'statuses';
+		type?: string;
 		q?: string;
+		resolve?: string;
 	}
 }
 
@@ -26,66 +28,116 @@ export class ApiSearchMastodon {
 	) {}
 
 	public register(fastify: FastifyInstance): void {
-		fastify.get<ApiSearchMastodonRoute>('/v1/search', async (_request, reply) => {
-			if (!_request.query.q) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required property "q"' });
-			if (!_request.query.type) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required property "type"' });
+		fastify.get<ApiSearchMastodonRoute>('/v1/search', async (request, reply) => {
+			if (!request.query.q) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required property "q"' });
+			if (!request.query.type) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required property "type"' });
 
-			const query = parseTimelineArgs(_request.query);
-			const client = this.clientService.getClient(_request);
-			const data = await client.search(_request.query.q, { type: _request.query.type, ...query });
+			const type = request.query.type;
+			if (type !== 'hashtags' && type !== 'statuses' && type !== 'accounts') {
+				return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Invalid type' });
+			}
 
-			reply.send(data.data);
+			const { client, me } = await this.clientService.getAuthClient(request);
+
+			if (toBoolean(request.query.resolve) && !me) {
+				return reply.code(401).send({ error: 'The access token is invalid', error_description: 'Authentication is required to use the "resolve" property' });
+			}
+			if (toInt(request.query.offset) && !me) {
+				return reply.code(401).send({ error: 'The access token is invalid', error_description: 'Authentication is required to use the "offset" property' });
+			}
+
+			// TODO implement resolve
+
+			const query = parseTimelineArgs(request.query);
+			const { data } = await client.search(request.query.q, { type, ...query });
+			const response = {
+				...data,
+				accounts: await Promise.all(data.accounts.map((account: Account) => this.mastoConverters.convertAccount(account))),
+				statuses: await Promise.all(data.statuses.map((status: Status) => this.mastoConverters.convertStatus(status, me))),
+			};
+
+			if (type === 'hashtags') {
+				attachOffsetPagination(request, reply, response.hashtags);
+			} else {
+				attachMinMaxPagination(request, reply, response[type]);
+			}
+
+			reply.send(response);
 		});
 
-		fastify.get<ApiSearchMastodonRoute>('/v2/search', async (_request, reply) => {
-			if (!_request.query.q) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required property "q"' });
+		fastify.get<ApiSearchMastodonRoute>('/v2/search', async (request, reply) => {
+			if (!request.query.q) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required property "q"' });
+
+			const type = request.query.type;
+			if (type !== undefined && type !== 'hashtags' && type !== 'statuses' && type !== 'accounts') {
+				return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Invalid type' });
+			}
+
+			const { client, me } = await this.clientService.getAuthClient(request);
+
+			if (toBoolean(request.query.resolve) && !me) {
+				return reply.code(401).send({ error: 'The access token is invalid', error_description: 'Authentication is required to use the "resolve" property' });
+			}
+			if (toInt(request.query.offset) && !me) {
+				return reply.code(401).send({ error: 'The access token is invalid', error_description: 'Authentication is required to use the "offset" property' });
+			}
 
-			const query = parseTimelineArgs(_request.query);
-			const type = _request.query.type;
-			const { client, me } = await this.clientService.getAuthClient(_request);
-			const acct = !type || type === 'accounts' ? await client.search(_request.query.q, { type: 'accounts', ...query }) : null;
-			const stat = !type || type === 'statuses' ? await client.search(_request.query.q, { type: 'statuses', ...query }) : null;
-			const tags = !type || type === 'hashtags' ? await client.search(_request.query.q, { type: 'hashtags', ...query }) : null;
+			// TODO implement resolve
+
+			const query = parseTimelineArgs(request.query);
+			const acct = !type || type === 'accounts' ? await client.search(request.query.q, { type: 'accounts', ...query }) : null;
+			const stat = !type || type === 'statuses' ? await client.search(request.query.q, { type: 'statuses', ...query }) : null;
+			const tags = !type || type === 'hashtags' ? await client.search(request.query.q, { type: 'hashtags', ...query }) : null;
 			const response = {
-				accounts: await Promise.all(acct?.data.accounts.map(async (account: Account) => await this.mastoConverters.convertAccount(account)) ?? []),
-				statuses: await Promise.all(stat?.data.statuses.map(async (status: Status) => await this.mastoConverters.convertStatus(status, me)) ?? []),
+				accounts: await Promise.all(acct?.data.accounts.map((account: Account) => this.mastoConverters.convertAccount(account)) ?? []),
+				statuses: await Promise.all(stat?.data.statuses.map((status: Status) => this.mastoConverters.convertStatus(status, me)) ?? []),
 				hashtags: tags?.data.hashtags ?? [],
 			};
 
+			// Pagination hack, based on "best guess" expected behavior.
+			// Mastodon doesn't document this part at all!
+			const longestResult = [response.statuses, response.hashtags]
+				.reduce((longest: unknown[], current: unknown[]) => current.length > longest.length ? current : longest, response.accounts);
+
+			// Ignore min/max pagination because how TF would that work with multiple result sets??
+			// Offset pagination is the only possible option
+			attachOffsetPagination(request, reply, longestResult);
+
 			reply.send(response);
 		});
 
-		fastify.get<ApiSearchMastodonRoute>('/v1/trends/statuses', async (_request, reply) => {
-			const baseUrl = this.clientService.getBaseUrl(_request);
+		fastify.get<ApiSearchMastodonRoute>('/v1/trends/statuses', async (request, reply) => {
+			const baseUrl = this.clientService.getBaseUrl(request);
 			const res = await fetch(`${baseUrl}/api/notes/featured`,
 				{
 					method: 'POST',
 					headers: {
-						..._request.headers as HeadersInit,
+						...request.headers as HeadersInit,
 						'Accept': 'application/json',
 						'Content-Type': 'application/json',
 					},
 					body: '{}',
 				});
 			const data = await res.json() as Status[];
-			const me = await this.clientService.getAuth(_request);
+			const me = await this.clientService.getAuth(request);
 			const response = await Promise.all(data.map(status => this.mastoConverters.convertStatus(status, me)));
 
+			attachMinMaxPagination(request, reply, response);
 			reply.send(response);
 		});
 
-		fastify.get<ApiSearchMastodonRoute>('/v2/suggestions', async (_request, reply) => {
-			const baseUrl = this.clientService.getBaseUrl(_request);
+		fastify.get<ApiSearchMastodonRoute>('/v2/suggestions', async (request, reply) => {
+			const baseUrl = this.clientService.getBaseUrl(request);
 			const res = await fetch(`${baseUrl}/api/users`,
 				{
 					method: 'POST',
 					headers: {
-						..._request.headers as HeadersInit,
+						...request.headers as HeadersInit,
 						'Accept': 'application/json',
 						'Content-Type': 'application/json',
 					},
 					body: JSON.stringify({
-						limit: parseTimelineArgs(_request.query).limit ?? 20,
+						limit: parseTimelineArgs(request.query).limit ?? 20,
 						origin: 'local',
 						sort: '+follower',
 						state: 'alive',
@@ -99,6 +151,7 @@ export class ApiSearchMastodon {
 				};
 			}));
 
+			attachOffsetPagination(request, reply, response);
 			reply.send(response);
 		});
 	}
diff --git a/packages/backend/src/server/api/mastodon/endpoints/timeline.ts b/packages/backend/src/server/api/mastodon/endpoints/timeline.ts
index 975aa9d04b..a333e77c3e 100644
--- a/packages/backend/src/server/api/mastodon/endpoints/timeline.ts
+++ b/packages/backend/src/server/api/mastodon/endpoints/timeline.ts
@@ -5,6 +5,7 @@
 
 import { Injectable } from '@nestjs/common';
 import { MastodonClientService } from '@/server/api/mastodon/MastodonClientService.js';
+import { attachMinMaxPagination } from '@/server/api/mastodon/pagination.js';
 import { convertList, MastoConverters } from '../converters.js';
 import { parseTimelineArgs, TimelineArgs, toBoolean } from '../argsUtils.js';
 import type { Entity } from 'megalodon';
@@ -18,55 +19,60 @@ export class ApiTimelineMastodon {
 	) {}
 
 	public register(fastify: FastifyInstance): void {
-		fastify.get<{ Querystring: TimelineArgs }>('/v1/timelines/public', async (_request, reply) => {
-			const { client, me } = await this.clientService.getAuthClient(_request);
-			const query = parseTimelineArgs(_request.query);
-			const data = toBoolean(_request.query.local)
+		fastify.get<{ Querystring: TimelineArgs }>('/v1/timelines/public', async (request, reply) => {
+			const { client, me } = await this.clientService.getAuthClient(request);
+			const query = parseTimelineArgs(request.query);
+			const data = toBoolean(request.query.local)
 				? await client.getLocalTimeline(query)
 				: await client.getPublicTimeline(query);
 			const response = await Promise.all(data.data.map((status: Entity.Status) => this.mastoConverters.convertStatus(status, me)));
 
+			attachMinMaxPagination(request, reply, response);
 			reply.send(response);
 		});
 
-		fastify.get<{ Querystring: TimelineArgs }>('/v1/timelines/home', async (_request, reply) => {
-			const { client, me } = await this.clientService.getAuthClient(_request);
-			const query = parseTimelineArgs(_request.query);
+		fastify.get<{ Querystring: TimelineArgs }>('/v1/timelines/home', async (request, reply) => {
+			const { client, me } = await this.clientService.getAuthClient(request);
+			const query = parseTimelineArgs(request.query);
 			const data = await client.getHomeTimeline(query);
 			const response = await Promise.all(data.data.map((status: Entity.Status) => this.mastoConverters.convertStatus(status, me)));
 
+			attachMinMaxPagination(request, reply, response);
 			reply.send(response);
 		});
 
-		fastify.get<{ Params: { hashtag?: string }, Querystring: TimelineArgs }>('/v1/timelines/tag/:hashtag', async (_request, reply) => {
-			if (!_request.params.hashtag) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required parameter "hashtag"' });
+		fastify.get<{ Params: { hashtag?: string }, Querystring: TimelineArgs }>('/v1/timelines/tag/:hashtag', async (request, reply) => {
+			if (!request.params.hashtag) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required parameter "hashtag"' });
 
-			const { client, me } = await this.clientService.getAuthClient(_request);
-			const query = parseTimelineArgs(_request.query);
-			const data = await client.getTagTimeline(_request.params.hashtag, query);
+			const { client, me } = await this.clientService.getAuthClient(request);
+			const query = parseTimelineArgs(request.query);
+			const data = await client.getTagTimeline(request.params.hashtag, query);
 			const response = await Promise.all(data.data.map((status: Entity.Status) => this.mastoConverters.convertStatus(status, me)));
 
+			attachMinMaxPagination(request, reply, response);
 			reply.send(response);
 		});
 
-		fastify.get<{ Params: { id?: string }, Querystring: TimelineArgs }>('/v1/timelines/list/:id', async (_request, reply) => {
-			if (!_request.params.id) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required parameter "id"' });
+		fastify.get<{ Params: { id?: string }, Querystring: TimelineArgs }>('/v1/timelines/list/:id', async (request, reply) => {
+			if (!request.params.id) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required parameter "id"' });
 
-			const { client, me } = await this.clientService.getAuthClient(_request);
-			const query = parseTimelineArgs(_request.query);
-			const data = await client.getListTimeline(_request.params.id, query);
+			const { client, me } = await this.clientService.getAuthClient(request);
+			const query = parseTimelineArgs(request.query);
+			const data = await client.getListTimeline(request.params.id, query);
 			const response = await Promise.all(data.data.map(async (status: Entity.Status) => await this.mastoConverters.convertStatus(status, me)));
 
+			attachMinMaxPagination(request, reply, response);
 			reply.send(response);
 		});
 
-		fastify.get<{ Querystring: TimelineArgs }>('/v1/conversations', async (_request, reply) => {
-			const { client, me } = await this.clientService.getAuthClient(_request);
-			const query = parseTimelineArgs(_request.query);
+		fastify.get<{ Querystring: TimelineArgs }>('/v1/conversations', async (request, reply) => {
+			const { client, me } = await this.clientService.getAuthClient(request);
+			const query = parseTimelineArgs(request.query);
 			const data = await client.getConversationTimeline(query);
-			const conversations = await Promise.all(data.data.map((conversation: Entity.Conversation) => this.mastoConverters.convertConversation(conversation, me)));
+			const response = await Promise.all(data.data.map((conversation: Entity.Conversation) => this.mastoConverters.convertConversation(conversation, me)));
 
-			reply.send(conversations);
+			attachMinMaxPagination(request, reply, response);
+			reply.send(response);
 		});
 
 		fastify.get<{ Params: { id?: string } }>('/v1/lists/:id', async (_request, reply) => {
@@ -79,22 +85,24 @@ export class ApiTimelineMastodon {
 			reply.send(response);
 		});
 
-		fastify.get('/v1/lists', async (_request, reply) => {
-			const client = this.clientService.getClient(_request);
+		fastify.get('/v1/lists', async (request, reply) => {
+			const client = this.clientService.getClient(request);
 			const data = await client.getLists();
 			const response = data.data.map((list: Entity.List) => convertList(list));
 
+			attachMinMaxPagination(request, reply, response);
 			reply.send(response);
 		});
 
-		fastify.get<{ Params: { id?: string }, Querystring: { limit?: number, max_id?: string, since_id?: string } }>('/v1/lists/:id/accounts', async (_request, reply) => {
-			if (!_request.params.id) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required parameter "id"' });
+		fastify.get<{ Params: { id?: string }, Querystring: TimelineArgs }>('/v1/lists/:id/accounts', async (request, reply) => {
+			if (!request.params.id) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required parameter "id"' });
 
-			const client = this.clientService.getClient(_request);
-			const data = await client.getAccountsInList(_request.params.id, _request.query);
-			const accounts = await Promise.all(data.data.map((account: Entity.Account) => this.mastoConverters.convertAccount(account)));
+			const client = this.clientService.getClient(request);
+			const data = await client.getAccountsInList(request.params.id, parseTimelineArgs(request.query));
+			const response = await Promise.all(data.data.map((account: Entity.Account) => this.mastoConverters.convertAccount(account)));
 
-			reply.send(accounts);
+			attachMinMaxPagination(request, reply, response);
+			reply.send(response);
 		});
 
 		fastify.post<{ Params: { id?: string }, Querystring: { accounts_id?: string[] } }>('/v1/lists/:id/accounts', async (_request, reply) => {