From d20542c495dd3342b23ef9f1c759c2f4f2dce63e Mon Sep 17 00:00:00 2001 From: かっこかり <67428053+kakkokari-gtyih@users.noreply.github.com> Date: Fri, 23 Feb 2024 10:47:17 +0900 Subject: enhance: `meta`をSSR HTMLに埋め込む (#13436) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * enhance: `meta`をSSR HTMLに埋め込む * HTML Metaの有効時間を指定 * １時間 * MetaEntityService * JSONをPackするように * :v: --------- Co-authored-by: syuilo <4439005+syuilo@users.noreply.github.com> --- packages/backend/src/misc/json-schema.ts | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'packages/backend/src/misc') diff --git a/packages/backend/src/misc/json-schema.ts b/packages/backend/src/misc/json-schema.ts index de38f145b2..8449e5ff07 100644 --- a/packages/backend/src/misc/json-schema.ts +++ b/packages/backend/src/misc/json-schema.ts @@ -50,6 +50,11 @@ import { } from '@/models/json-schema/role.js'; import { packedAdSchema } from '@/models/json-schema/ad.js'; import { packedReversiGameLiteSchema, packedReversiGameDetailedSchema } from '@/models/json-schema/reversi-game.js'; +import { + packedMetaLiteSchema, + packedMetaDetailedOnlySchema, + packedMetaDetailedSchema, +} from '@/models/json-schema/meta.js'; export const refs = { UserLite: packedUserLiteSchema, @@ -99,6 +104,9 @@ export const refs = { RolePolicies: packedRolePoliciesSchema, ReversiGameLite: packedReversiGameLiteSchema, ReversiGameDetailed: packedReversiGameDetailedSchema, + MetaLite: packedMetaLiteSchema, + MetaDetailedOnly: packedMetaDetailedOnlySchema, + MetaDetailed: packedMetaDetailedSchema, }; export type Packed = SchemaType; -- cgit v1.2.3-freya From 64953fadc92abf3fd83186733282bcf4f39bdb49 Mon Sep 17 00:00:00 2001 From: okayurisotto <47853651+okayurisotto@users.noreply.github.com> Date: Fri, 23 Feb 2024 14:12:57 +0900 Subject: refactor(backend): `Array.prototype.filter`での非null確認では`isNotNull`関数を使うように (#13442) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * `Array.prototype.filter`での非null確認では`isNotNull`関数を使うように * `{}` -> `NonNullable` --- packages/backend/src/core/NoteCreateService.ts | 3 ++- packages/backend/src/core/activitypub/ApAudienceService.ts | 3 ++- packages/backend/src/core/activitypub/ApInboxService.ts | 3 ++- packages/backend/src/core/activitypub/ApRendererService.ts | 2 +- packages/backend/src/core/activitypub/models/ApMentionService.ts | 3 ++- packages/backend/src/core/activitypub/models/ApNoteService.ts | 3 ++- packages/backend/src/core/activitypub/models/ApPersonService.ts | 3 ++- packages/backend/src/core/activitypub/models/ApQuestionService.ts | 3 ++- packages/backend/src/core/activitypub/models/tag.ts | 3 ++- packages/backend/src/core/entities/DriveFileEntityService.ts | 2 +- packages/backend/src/core/entities/PageEntityService.ts | 3 ++- packages/backend/src/core/entities/UserEntityService.ts | 3 ++- packages/backend/src/misc/is-not-null.ts | 4 +--- packages/backend/src/server/api/endpoints/gallery/posts/create.ts | 3 ++- packages/backend/src/server/api/endpoints/gallery/posts/update.ts | 3 ++- packages/backend/src/server/api/endpoints/pinned-users.ts | 3 ++- 16 files changed, 29 insertions(+), 18 deletions(-) (limited to 'packages/backend/src/misc') diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 2a5fd2e1a6..b412d5db11 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -59,6 +59,7 @@ import { UtilityService } from '@/core/UtilityService.js'; import { UserBlockingService } from '@/core/UserBlockingService.js'; import { isReply } from '@/misc/is-reply.js'; import { trackPromise } from '@/misc/promise-tracker.js'; +import { isNotNull } from '@/misc/is-not-null.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; type NotificationType = 'reply' | 'renote' | 'quote' | 'mention'; @@ -816,7 +817,7 @@ export class NoteCreateService implements OnApplicationShutdown { const mentions = extractMentions(tokens); let mentionedUsers = (await Promise.all(mentions.map(m => this.remoteUserResolveService.resolveUser(m.username, m.host ?? user.host).catch(() => null), - ))).filter(x => x != null) as MiUser[]; + ))).filter(isNotNull); // Drop duplicate users mentionedUsers = mentionedUsers.filter((u, i, self) => diff --git a/packages/backend/src/core/activitypub/ApAudienceService.ts b/packages/backend/src/core/activitypub/ApAudienceService.ts index d47be79441..0fccc7b950 100644 --- a/packages/backend/src/core/activitypub/ApAudienceService.ts +++ b/packages/backend/src/core/activitypub/ApAudienceService.ts @@ -8,6 +8,7 @@ import promiseLimit from 'promise-limit'; import type { MiRemoteUser, MiUser } from '@/models/User.js'; import { concat, unique } from '@/misc/prelude/array.js'; import { bindThis } from '@/decorators.js'; +import { isNotNull } from '@/misc/is-not-null.js'; import { getApIds } from './type.js'; import { ApPersonService } from './models/ApPersonService.js'; import type { ApObject } from './type.js'; @@ -40,7 +41,7 @@ export class ApAudienceService { const limit = promiseLimit(2); const mentionedUsers = (await Promise.all( others.map(id => limit(() => this.apPersonService.resolvePerson(id, resolver).catch(() => null))), - )).filter((x): x is MiUser => x != null); + )).filter(isNotNull); if (toGroups.public.length > 0) { return { diff --git a/packages/backend/src/core/activitypub/ApInboxService.ts b/packages/backend/src/core/activitypub/ApInboxService.ts index 1cc54b6ff6..8d9cd74a25 100644 --- a/packages/backend/src/core/activitypub/ApInboxService.ts +++ b/packages/backend/src/core/activitypub/ApInboxService.ts @@ -27,6 +27,7 @@ import { QueueService } from '@/core/QueueService.js'; import type { UsersRepository, NotesRepository, FollowingsRepository, AbuseUserReportsRepository, FollowRequestsRepository } from '@/models/_.js'; import { bindThis } from '@/decorators.js'; import type { MiRemoteUser } from '@/models/User.js'; +import { isNotNull } from '@/misc/is-not-null.js'; import { getApHrefNullable, getApId, getApIds, getApType, isAccept, isActor, isAdd, isAnnounce, isBlock, isCollection, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost } from './type.js'; import { ApNoteService } from './models/ApNoteService.js'; import { ApLoggerService } from './ApLoggerService.js'; @@ -521,7 +522,7 @@ export class ApInboxService { const userIds = uris .filter(uri => uri.startsWith(this.config.url + '/users/')) .map(uri => uri.split('/').at(-1)) - .filter((userId): userId is string => userId !== undefined); + .filter(isNotNull); const users = await this.usersRepository.findBy({ id: In(userIds), }); diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index 494622909a..d7fb977a99 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -315,7 +315,7 @@ export class ApRendererService { const getPromisedFiles = async (ids: string[]): Promise => { if (ids.length === 0) return []; const items = await this.driveFilesRepository.findBy({ id: In(ids) }); - return ids.map(id => items.find(item => item.id === id)).filter((item): item is MiDriveFile => item != null); + return ids.map(id => items.find(item => item.id === id)).filter(isNotNull); }; let inReplyTo; diff --git a/packages/backend/src/core/activitypub/models/ApMentionService.ts b/packages/backend/src/core/activitypub/models/ApMentionService.ts index 73eea1edf0..0ced7e88af 100644 --- a/packages/backend/src/core/activitypub/models/ApMentionService.ts +++ b/packages/backend/src/core/activitypub/models/ApMentionService.ts @@ -8,6 +8,7 @@ import promiseLimit from 'promise-limit'; import type { MiUser } from '@/models/_.js'; import { toArray, unique } from '@/misc/prelude/array.js'; import { bindThis } from '@/decorators.js'; +import { isNotNull } from '@/misc/is-not-null.js'; import { isMention } from '../type.js'; import { Resolver } from '../ApResolverService.js'; import { ApPersonService } from './ApPersonService.js'; @@ -27,7 +28,7 @@ export class ApMentionService { const limit = promiseLimit(2); const mentionedUsers = (await Promise.all( hrefs.map(x => limit(() => this.apPersonService.resolvePerson(x, resolver).catch(() => null))), - )).filter((x): x is MiUser => x != null); + )).filter(isNotNull); return mentionedUsers; } diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index 8da9407216..e201b88173 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -37,6 +37,7 @@ import { ApQuestionService } from './ApQuestionService.js'; import { ApImageService } from './ApImageService.js'; import type { Resolver } from '../ApResolverService.js'; import type { IObject, IPost } from '../type.js'; +import { isNotNull } from '@/misc/is-not-null.js'; @Injectable() export class ApNoteService { @@ -221,7 +222,7 @@ export class ApNoteService { } }; - const uris = unique([note._misskey_quote, note.quoteUrl].filter((x): x is string => typeof x === 'string')); + const uris = unique([note._misskey_quote, note.quoteUrl].filter(isNotNull)); const results = await Promise.all(uris.map(tryResolveNote)); quote = results.filter((x): x is { status: 'ok', res: MiNote } => x.status === 'ok').map(x => x.res).at(0); diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index e80cd34a56..744b1ea683 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -38,6 +38,7 @@ import { MetaService } from '@/core/MetaService.js'; import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; import type { AccountMoveService } from '@/core/AccountMoveService.js'; import { checkHttps } from '@/misc/check-https.js'; +import { isNotNull } from '@/misc/is-not-null.js'; import { getApId, getApType, getOneApHrefNullable, isActor, isCollection, isCollectionOrOrderedCollection, isPropertyValue } from '../type.js'; import { extractApHashtags } from './tag.js'; import type { OnModuleInit } from '@nestjs/common'; @@ -636,7 +637,7 @@ export class ApPersonService implements OnModuleInit { // とりあえずidを別の時間で生成して順番を維持 let td = 0; - for (const note of featuredNotes.filter((note): note is MiNote => note != null)) { + for (const note of featuredNotes.filter(isNotNull)) { td -= 1000; transactionalEntityManager.insert(MiUserNotePining, { id: this.idService.gen(Date.now() + td), diff --git a/packages/backend/src/core/activitypub/models/ApQuestionService.ts b/packages/backend/src/core/activitypub/models/ApQuestionService.ts index e78b3a3599..d1936cfe1d 100644 --- a/packages/backend/src/core/activitypub/models/ApQuestionService.ts +++ b/packages/backend/src/core/activitypub/models/ApQuestionService.ts @@ -10,6 +10,7 @@ import type { Config } from '@/config.js'; import type { IPoll } from '@/models/Poll.js'; import type Logger from '@/logger.js'; import { bindThis } from '@/decorators.js'; +import { isNotNull } from '@/misc/is-not-null.js'; import { isQuestion } from '../type.js'; import { ApLoggerService } from '../ApLoggerService.js'; import { ApResolverService } from '../ApResolverService.js'; @@ -51,7 +52,7 @@ export class ApQuestionService { const choices = question[multiple ? 'anyOf' : 'oneOf'] ?.map((x) => x.name) - .filter((x): x is string => typeof x === 'string') + .filter(isNotNull) ?? []; const votes = question[multiple ? 'anyOf' : 'oneOf']?.map((x) => x.replies?.totalItems ?? x._misskey_votes ?? 0); diff --git a/packages/backend/src/core/activitypub/models/tag.ts b/packages/backend/src/core/activitypub/models/tag.ts index ced101b764..e7ceec3262 100644 --- a/packages/backend/src/core/activitypub/models/tag.ts +++ b/packages/backend/src/core/activitypub/models/tag.ts @@ -4,6 +4,7 @@ */ import { toArray } from '@/misc/prelude/array.js'; +import { isNotNull } from '@/misc/is-not-null.js'; import { isHashtag } from '../type.js'; import type { IObject, IApHashtag } from '../type.js'; @@ -15,7 +16,7 @@ export function extractApHashtags(tags: IObject | IObject[] | null | undefined): return hashtags.map(tag => { const m = tag.name.match(/^#(.+)/); return m ? m[1] : null; - }).filter((x): x is string => x != null); + }).filter(isNotNull); } export function extractApHashtagObjects(tags: IObject | IObject[] | null | undefined): IApHashtag[] { diff --git a/packages/backend/src/core/entities/DriveFileEntityService.ts b/packages/backend/src/core/entities/DriveFileEntityService.ts index 50f1c49b48..8affe2b3bf 100644 --- a/packages/backend/src/core/entities/DriveFileEntityService.ts +++ b/packages/backend/src/core/entities/DriveFileEntityService.ts @@ -259,7 +259,7 @@ export class DriveFileEntityService { options?: PackOptions, ): Promise[]> { const items = await Promise.all(files.map(f => this.packNullable(f, options))); - return items.filter((x): x is Packed<'DriveFile'> => x != null); + return items.filter(isNotNull); } @bindThis diff --git a/packages/backend/src/core/entities/PageEntityService.ts b/packages/backend/src/core/entities/PageEntityService.ts index fe7b137bd2..65c69a49a7 100644 --- a/packages/backend/src/core/entities/PageEntityService.ts +++ b/packages/backend/src/core/entities/PageEntityService.ts @@ -14,6 +14,7 @@ import type { MiPage } from '@/models/Page.js'; import type { MiDriveFile } from '@/models/DriveFile.js'; import { bindThis } from '@/decorators.js'; import { IdService } from '@/core/IdService.js'; +import { isNotNull } from '@/misc/is-not-null.js'; import { UserEntityService } from './UserEntityService.js'; import { DriveFileEntityService } from './DriveFileEntityService.js'; @@ -102,7 +103,7 @@ export class PageEntityService { script: page.script, eyeCatchingImageId: page.eyeCatchingImageId, eyeCatchingImage: page.eyeCatchingImageId ? await this.driveFileEntityService.pack(page.eyeCatchingImageId) : null, - attachedFiles: this.driveFileEntityService.packMany((await Promise.all(attachedFiles)).filter((x): x is MiDriveFile => x != null)), + attachedFiles: this.driveFileEntityService.packMany((await Promise.all(attachedFiles)).filter(isNotNull)), likedCount: page.likedCount, isLiked: meId ? await this.pageLikesRepository.exists({ where: { pageId: page.id, userId: meId } }) : undefined, }); diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index 53df32f210..14761357a5 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -25,6 +25,7 @@ import { IdService } from '@/core/IdService.js'; import type { AnnouncementService } from '@/core/AnnouncementService.js'; import type { CustomEmojiService } from '@/core/CustomEmojiService.js'; import { AvatarDecorationService } from '@/core/AvatarDecorationService.js'; +import { isNotNull } from '@/misc/is-not-null.js'; import type { OnModuleInit } from '@nestjs/common'; import type { NoteEntityService } from './NoteEntityService.js'; import type { DriveFileEntityService } from './DriveFileEntityService.js'; @@ -384,7 +385,7 @@ export class UserEntityService implements OnModuleInit { movedTo: user.movedToUri ? this.apPersonService.resolvePerson(user.movedToUri).then(user => user.id).catch(() => null) : null, alsoKnownAs: user.alsoKnownAs ? Promise.all(user.alsoKnownAs.map(uri => this.apPersonService.fetchPerson(uri).then(user => user?.id).catch(() => null))) - .then(xs => xs.length === 0 ? null : xs.filter(x => x != null) as string[]) + .then(xs => xs.length === 0 ? null : xs.filter(isNotNull)) : null, createdAt: this.idService.parse(user.id).date.toISOString(), updatedAt: user.updatedAt ? user.updatedAt.toISOString() : null, diff --git a/packages/backend/src/misc/is-not-null.ts b/packages/backend/src/misc/is-not-null.ts index 584a09d35a..8d9dc8bb39 100644 --- a/packages/backend/src/misc/is-not-null.ts +++ b/packages/backend/src/misc/is-not-null.ts @@ -3,8 +3,6 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -// we are using {} as "any non-nullish value" as expected -// eslint-disable-next-line @typescript-eslint/ban-types -export function isNotNull(input: T | undefined | null): input is T { +export function isNotNull>(input: T | undefined | null): input is T { return input != null; } diff --git a/packages/backend/src/server/api/endpoints/gallery/posts/create.ts b/packages/backend/src/server/api/endpoints/gallery/posts/create.ts index 784ae5088f..b07cdf1ed9 100644 --- a/packages/backend/src/server/api/endpoints/gallery/posts/create.ts +++ b/packages/backend/src/server/api/endpoints/gallery/posts/create.ts @@ -12,6 +12,7 @@ import type { MiDriveFile } from '@/models/DriveFile.js'; import { IdService } from '@/core/IdService.js'; import { GalleryPostEntityService } from '@/core/entities/GalleryPostEntityService.js'; import { DI } from '@/di-symbols.js'; +import { isNotNull } from '@/misc/is-not-null.js'; export const meta = { tags: ['gallery'], @@ -69,7 +70,7 @@ export default class extends Endpoint { // eslint- id: fileId, userId: me.id, }), - ))).filter((file): file is MiDriveFile => file != null); + ))).filter(isNotNull); if (files.length === 0) { throw new Error(); diff --git a/packages/backend/src/server/api/endpoints/gallery/posts/update.ts b/packages/backend/src/server/api/endpoints/gallery/posts/update.ts index 8872b261dd..8bd83ff5ba 100644 --- a/packages/backend/src/server/api/endpoints/gallery/posts/update.ts +++ b/packages/backend/src/server/api/endpoints/gallery/posts/update.ts @@ -10,6 +10,7 @@ import type { DriveFilesRepository, GalleryPostsRepository } from '@/models/_.js import type { MiDriveFile } from '@/models/DriveFile.js'; import { GalleryPostEntityService } from '@/core/entities/GalleryPostEntityService.js'; import { DI } from '@/di-symbols.js'; +import { isNotNull } from '@/misc/is-not-null.js'; export const meta = { tags: ['gallery'], @@ -67,7 +68,7 @@ export default class extends Endpoint { // eslint- id: fileId, userId: me.id, }), - ))).filter((file): file is MiDriveFile => file != null); + ))).filter(isNotNull); if (files.length === 0) { throw new Error(); diff --git a/packages/backend/src/server/api/endpoints/pinned-users.ts b/packages/backend/src/server/api/endpoints/pinned-users.ts index 1f4509764f..784766bcb5 100644 --- a/packages/backend/src/server/api/endpoints/pinned-users.ts +++ b/packages/backend/src/server/api/endpoints/pinned-users.ts @@ -12,6 +12,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; import { MetaService } from '@/core/MetaService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { DI } from '@/di-symbols.js'; +import { isNotNull } from '@/misc/is-not-null.js'; export const meta = { tags: ['users'], @@ -52,7 +53,7 @@ export default class extends Endpoint { // eslint- host: acct.host ?? IsNull(), }))); - return await this.userEntityService.packMany(users.filter(x => x !== null) as MiUser[], me, { schema: 'UserDetailed' }); + return await this.userEntityService.packMany(users.filter(isNotNull), me, { schema: 'UserDetailed' }); }); } } -- cgit v1.2.3-freya From 2c6f25b710b4f8095458fe88ddd56e6c6a41d006 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sun, 25 Feb 2024 12:36:10 +0900 Subject: fix: 古いキャッシュを使うのを修正 (#13453) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/core/AccountMoveService.ts | 4 +--- packages/backend/src/core/CacheService.ts | 4 +++- packages/backend/src/core/GlobalEventService.ts | 1 + packages/backend/src/core/UserFollowingService.ts | 27 ++++++++-------------- packages/backend/src/misc/cache.ts | 8 +++++++ .../processors/RelationshipProcessorService.ts | 2 +- .../src/server/api/endpoints/following/create.ts | 2 +- .../backend/src/server/api/endpoints/i/update.ts | 6 ++--- 8 files changed, 27 insertions(+), 27 deletions(-) (limited to 'packages/backend/src/misc') diff --git a/packages/backend/src/core/AccountMoveService.ts b/packages/backend/src/core/AccountMoveService.ts index b7796a5183..5bd885df40 100644 --- a/packages/backend/src/core/AccountMoveService.ts +++ b/packages/backend/src/core/AccountMoveService.ts @@ -20,7 +20,6 @@ import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; import { ApDeliverManagerService } from '@/core/activitypub/ApDeliverManagerService.js'; import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; -import { CacheService } from '@/core/CacheService.js'; import { ProxyAccountService } from '@/core/ProxyAccountService.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { MetaService } from '@/core/MetaService.js'; @@ -60,7 +59,6 @@ export class AccountMoveService { private instanceChart: InstanceChart, private metaService: MetaService, private relayService: RelayService, - private cacheService: CacheService, private queueService: QueueService, ) { } @@ -84,7 +82,7 @@ export class AccountMoveService { Object.assign(src, update); // Update cache - this.cacheService.uriPersonCache.set(srcUri, src); + this.globalEventService.publishInternalEvent('localUserUpdated', src); const srcPerson = await this.apRendererService.renderPerson(src); const updateAct = this.apRendererService.addContext(this.apRendererService.renderUpdate(srcPerson, src)); diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index 0fc47bf8ec..d008e7ec52 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -129,10 +129,12 @@ export class CacheService implements OnApplicationShutdown { switch (type) { case 'userChangeSuspendedState': case 'userChangeDeletedState': - case 'remoteUserUpdated': { + case 'remoteUserUpdated': + case 'localUserUpdated': { const user = await this.usersRepository.findOneBy({ id: body.id }); if (user == null) { this.userByIdCache.delete(body.id); + this.localUserByIdCache.delete(body.id); for (const [k, v] of this.uriPersonCache.cache.entries()) { if (v.value?.id === body.id) { this.uriPersonCache.delete(k); diff --git a/packages/backend/src/core/GlobalEventService.ts b/packages/backend/src/core/GlobalEventService.ts index a127a6df3b..7c1b34da05 100644 --- a/packages/backend/src/core/GlobalEventService.ts +++ b/packages/backend/src/core/GlobalEventService.ts @@ -212,6 +212,7 @@ export interface InternalEventTypes { userChangeDeletedState: { id: MiUser['id']; isDeleted: MiUser['isDeleted']; }; userTokenRegenerated: { id: MiUser['id']; oldToken: string; newToken: string; }; remoteUserUpdated: { id: MiUser['id']; }; + localUserUpdated: { id: MiUser['id']; }; follow: { followerId: MiUser['id']; followeeId: MiUser['id']; }; unfollow: { followerId: MiUser['id']; followeeId: MiUser['id']; }; blockingCreated: { blockerId: MiUser['id']; blockeeId: MiUser['id']; }; diff --git a/packages/backend/src/core/UserFollowingService.ts b/packages/backend/src/core/UserFollowingService.ts index d87cbacdcb..0a492c06e4 100644 --- a/packages/backend/src/core/UserFollowingService.ts +++ b/packages/backend/src/core/UserFollowingService.ts @@ -101,33 +101,24 @@ export class UserFollowingService implements OnModuleInit { this.queueService.deliver(followee, content, follower.inbox, false); } - /** - * ThinUserでなくともユーザーの情報が最新でない場合はこちらを使うべき - */ @bindThis - public async followByThinUser( + public async follow( _follower: ThinUser, _followee: ThinUser, - options: Parameters[2] = {}, - ) { - const [follower, followee] = await Promise.all([ - this.usersRepository.findOneByOrFail({ id: _follower.id }), - this.usersRepository.findOneByOrFail({ id: _followee.id }), - ]) as [MiLocalUser | MiRemoteUser, MiLocalUser | MiRemoteUser]; - - await this.follow(follower, followee, options); - } - - @bindThis - public async follow( - follower: MiLocalUser | MiRemoteUser, - followee: MiLocalUser | MiRemoteUser, { requestId, silent = false, withReplies }: { requestId?: string, silent?: boolean, withReplies?: boolean, } = {}, ): Promise { + /** + * 必ず最新のユーザー情報を取得する + */ + const [follower, followee] = await Promise.all([ + this.usersRepository.findOneByOrFail({ id: _follower.id }), + this.usersRepository.findOneByOrFail({ id: _followee.id }), + ]) as [MiLocalUser | MiRemoteUser, MiLocalUser | MiRemoteUser]; + if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isRemoteUser(followee)) { // What? throw new Error('Remote user cannot follow remote user.'); diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index 7f4d1521b5..bba64a06ef 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -187,6 +187,10 @@ export class RedisSingleCache { // TODO: メモリ節約のためあまり参照されないキーを定期的に削除できるようにする？ export class MemoryKVCache { + /** + * データを持つマップ + * @deprecated これを直接操作するべきではない + */ public cache: Map; private lifetime: number; private gcIntervalHandle: NodeJS.Timeout; @@ -201,6 +205,10 @@ export class MemoryKVCache { } @bindThis + /** + * Mapにキャッシュをセットします + * @deprecated これを直接呼び出すべきではない。InternalEventなどで変更を全てのプロセス/マシンに通知するべき + */ public set(key: string, value: T): void { this.cache.set(key, { date: Date.now(), diff --git a/packages/backend/src/queue/processors/RelationshipProcessorService.ts b/packages/backend/src/queue/processors/RelationshipProcessorService.ts index 53dbb42169..408b02fb38 100644 --- a/packages/backend/src/queue/processors/RelationshipProcessorService.ts +++ b/packages/backend/src/queue/processors/RelationshipProcessorService.ts @@ -35,7 +35,7 @@ export class RelationshipProcessorService { @bindThis public async processFollow(job: Bull.Job): Promise { this.logger.info(`${job.data.from.id} is trying to follow ${job.data.to.id} ${job.data.withReplies ? "with replies" : "without replies"}`); - await this.userFollowingService.followByThinUser(job.data.from, job.data.to, { + await this.userFollowingService.follow(job.data.from, job.data.to, { requestId: job.data.requestId, silent: job.data.silent, withReplies: job.data.withReplies, diff --git a/packages/backend/src/server/api/endpoints/following/create.ts b/packages/backend/src/server/api/endpoints/following/create.ts index 042d7f119d..db320e7129 100644 --- a/packages/backend/src/server/api/endpoints/following/create.ts +++ b/packages/backend/src/server/api/endpoints/following/create.ts @@ -71,7 +71,7 @@ export const paramDef = { type: 'object', properties: { userId: { type: 'string', format: 'misskey:id' }, - withReplies: { type: 'boolean' } + withReplies: { type: 'boolean' }, }, required: ['userId'], } as const; diff --git a/packages/backend/src/server/api/endpoints/i/update.ts b/packages/backend/src/server/api/endpoints/i/update.ts index bf6c53d8eb..84a1931a3d 100644 --- a/packages/backend/src/server/api/endpoints/i/update.ts +++ b/packages/backend/src/server/api/endpoints/i/update.ts @@ -456,9 +456,9 @@ export default class extends Endpoint { // eslint- this.hashtagService.updateUsertags(user, tags); //#endregion - if (Object.keys(updates).length > 0) await this.usersRepository.update(user.id, updates); - if (Object.keys(updates).includes('alsoKnownAs')) { - this.cacheService.uriPersonCache.set(this.userEntityService.genLocalUserUri(user.id), { ...user, ...updates }); + if (Object.keys(updates).length > 0) { + await this.usersRepository.update(user.id, updates); + this.globalEventService.publishInternalEvent('localUserUpdated', { id: user.id }); } await this.userProfilesRepository.update(user.id, { -- cgit v1.2.3-freya From f906ad6ca7044e4c509a5fe01f398f841a44027a Mon Sep 17 00:00:00 2001 From: zawa-ch Date: Tue, 27 Feb 2024 18:45:46 +0900 Subject: Enhance: コンディショナルロールの条件に「マニュアルロールへのアサイン」を追加 (#13463) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * コンディショナルロールの条件に「マニュアルロールへのアサイン」を追加 * コメント修正 --- CHANGELOG.md | 1 + locales/index.d.ts | 4 ++++ locales/ja-JP.yml | 1 + packages/backend/src/core/RoleService.ts | 19 ++++++++------- packages/backend/src/misc/json-schema.ts | 2 ++ packages/backend/src/models/Role.ts | 6 +++++ packages/backend/src/models/json-schema/role.ts | 20 ++++++++++++++++ packages/backend/test/unit/RoleService.ts | 28 ++++++++++++++++++++++ .../src/pages/admin/RolesEditorFormula.vue | 9 +++++++ packages/misskey-js/etc/misskey-js.api.md | 4 ++++ packages/misskey-js/src/autogen/models.ts | 1 + packages/misskey-js/src/autogen/types.ts | 11 ++++++++- 12 files changed, 97 insertions(+), 9 deletions(-) (limited to 'packages/backend/src/misc') diff --git a/CHANGELOG.md b/CHANGELOG.md index ebbe22f2f6..513338e667 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ ### General - Enhance: サーバーごとにモデレーションノートを残せるように +- Enhance: コンディショナルロールの条件に「マニュアルロールへのアサイン」を追加 ### Client - Enhance: ノート作成画面のファイル添付メニューの区切り線の位置を調整 diff --git a/locales/index.d.ts b/locales/index.d.ts index 1a2565b06a..7d5f8ce732 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -6528,6 +6528,10 @@ export interface Locale extends ILocale { "avatarDecorationLimit": string; }; "_condition": { + /** + * マニュアルロールにアサイン済み + */ + "roleAssignedTo": string; /** * ローカルユーザー */ diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 61c61b8f96..1bb56738c6 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1687,6 +1687,7 @@ _role: canUseTranslator: "翻訳機能の利用" avatarDecorationLimit: "アイコンデコレーションの最大取付個数" _condition: + roleAssignedTo: "マニュアルロールにアサイン済み" isLocal: "ローカルユーザー" isRemote: "リモートユーザー" createdLessThan: "アカウント作成から～以内" diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index c5baaf3fff..8312489a78 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -200,17 +200,20 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { } @bindThis - private evalCond(user: MiUser, value: RoleCondFormulaValue): boolean { + private evalCond(user: MiUser, roles: MiRole[], value: RoleCondFormulaValue): boolean { try { switch (value.type) { case 'and': { - return value.values.every(v => this.evalCond(user, v)); + return value.values.every(v => this.evalCond(user, roles, v)); } case 'or': { - return value.values.some(v => this.evalCond(user, v)); + return value.values.some(v => this.evalCond(user, roles, v)); } case 'not': { - return !this.evalCond(user, value.value); + return !this.evalCond(user, roles, value.value); + } + case 'roleAssignedTo': { + return roles.some(r => r.id === value.roleId); } case 'isLocal': { return this.userEntityService.isLocalUser(user); @@ -272,7 +275,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { const assigns = await this.getUserAssigns(userId); const assignedRoles = roles.filter(r => assigns.map(x => x.roleId).includes(r.id)); const user = roles.some(r => r.target === 'conditional') ? await this.cacheService.findUserById(userId) : null; - const matchedCondRoles = roles.filter(r => r.target === 'conditional' && this.evalCond(user!, r.condFormula)); + const matchedCondRoles = roles.filter(r => r.target === 'conditional' && this.evalCond(user!, assignedRoles, r.condFormula)); return [...assignedRoles, ...matchedCondRoles]; } @@ -285,13 +288,13 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { let assigns = await this.roleAssignmentByUserIdCache.fetch(userId, () => this.roleAssignmentsRepository.findBy({ userId })); // 期限切れのロールを除外 assigns = assigns.filter(a => a.expiresAt == null || (a.expiresAt.getTime() > now)); - const assignedRoleIds = assigns.map(x => x.roleId); const roles = await this.rolesCache.fetch(() => this.rolesRepository.findBy({})); - const assignedBadgeRoles = roles.filter(r => r.asBadge && assignedRoleIds.includes(r.id)); + const assignedRoles = roles.filter(r => assigns.map(x => x.roleId).includes(r.id)); + const assignedBadgeRoles = assignedRoles.filter(r => r.asBadge); const badgeCondRoles = roles.filter(r => r.asBadge && (r.target === 'conditional')); if (badgeCondRoles.length > 0) { const user = roles.some(r => r.target === 'conditional') ? await this.cacheService.findUserById(userId) : null; - const matchedBadgeCondRoles = badgeCondRoles.filter(r => this.evalCond(user!, r.condFormula)); + const matchedBadgeCondRoles = badgeCondRoles.filter(r => this.evalCond(user!, assignedRoles, r.condFormula)); return [...assignedBadgeRoles, ...matchedBadgeCondRoles]; } else { return assignedBadgeRoles; diff --git a/packages/backend/src/misc/json-schema.ts b/packages/backend/src/misc/json-schema.ts index 8449e5ff07..46b0bb2fab 100644 --- a/packages/backend/src/misc/json-schema.ts +++ b/packages/backend/src/misc/json-schema.ts @@ -44,6 +44,7 @@ import { packedRoleCondFormulaLogicsSchema, packedRoleCondFormulaValueNot, packedRoleCondFormulaValueIsLocalOrRemoteSchema, + packedRoleCondFormulaValueAssignedRoleSchema, packedRoleCondFormulaValueCreatedSchema, packedRoleCondFormulaFollowersOrFollowingOrNotesSchema, packedRoleCondFormulaValueSchema, @@ -96,6 +97,7 @@ export const refs = { RoleCondFormulaLogics: packedRoleCondFormulaLogicsSchema, RoleCondFormulaValueNot: packedRoleCondFormulaValueNot, RoleCondFormulaValueIsLocalOrRemote: packedRoleCondFormulaValueIsLocalOrRemoteSchema, + RoleCondFormulaValueAssignedRole: packedRoleCondFormulaValueAssignedRoleSchema, RoleCondFormulaValueCreated: packedRoleCondFormulaValueCreatedSchema, RoleCondFormulaFollowersOrFollowingOrNotes: packedRoleCondFormulaFollowersOrFollowingOrNotesSchema, RoleCondFormulaValue: packedRoleCondFormulaValueSchema, diff --git a/packages/backend/src/models/Role.ts b/packages/backend/src/models/Role.ts index fa05ea8637..058abe3118 100644 --- a/packages/backend/src/models/Role.ts +++ b/packages/backend/src/models/Role.ts @@ -29,6 +29,11 @@ type CondFormulaValueIsRemote = { type: 'isRemote'; }; +type CondFormulaValueRoleAssignedTo = { + type: 'roleAssignedTo'; + roleId: string; +}; + type CondFormulaValueCreatedLessThan = { type: 'createdLessThan'; sec: number; @@ -75,6 +80,7 @@ export type RoleCondFormulaValue = { id: string } & ( CondFormulaValueNot | CondFormulaValueIsLocal | CondFormulaValueIsRemote | + CondFormulaValueRoleAssignedTo | CondFormulaValueCreatedLessThan | CondFormulaValueCreatedMoreThan | CondFormulaValueFollowersLessThanOrEq | diff --git a/packages/backend/src/models/json-schema/role.ts b/packages/backend/src/models/json-schema/role.ts index ef6b279bee..9f2b5b17ed 100644 --- a/packages/backend/src/models/json-schema/role.ts +++ b/packages/backend/src/models/json-schema/role.ts @@ -57,6 +57,23 @@ export const packedRoleCondFormulaValueIsLocalOrRemoteSchema = { }, } as const; +export const packedRoleCondFormulaValueAssignedRoleSchema = { + type: 'object', + properties: { + type: { + type: 'string', + nullable: false, optional: false, + enum: ['roleAssignedTo'], + }, + roleId: { + type: 'string', + nullable: false, optional: false, + format: 'id', + example: 'xxxxxxxxxx', + }, + }, +} as const; + export const packedRoleCondFormulaValueCreatedSchema = { type: 'object', properties: { @@ -115,6 +132,9 @@ export const packedRoleCondFormulaValueSchema = { { ref: 'RoleCondFormulaValueIsLocalOrRemote', }, + { + ref: 'RoleCondFormulaValueAssignedRole', + }, { ref: 'RoleCondFormulaValueCreated', }, diff --git a/packages/backend/test/unit/RoleService.ts b/packages/backend/test/unit/RoleService.ts index 5222745b7f..fe5ad31597 100644 --- a/packages/backend/test/unit/RoleService.ts +++ b/packages/backend/test/unit/RoleService.ts @@ -251,6 +251,34 @@ describe('RoleService', () => { expect(user2Policies.canManageCustomEmojis).toBe(true); }); + test('コンディショナルロール: マニュアルロールにアサイン済み', async () => { + const [user1, user2, role1] = await Promise.all([ + createUser(), + createUser(), + createRole({ + name: 'manual role', + }), + ]); + const role2 = await createRole({ + name: 'conditional role', + target: 'conditional', + condFormula: { + // idはバックエンドのロジックに必要ない？ + id: 'bdc612bd-9d54-4675-ae83-0499c82ea670', + type: 'roleAssignedTo', + roleId: role1.id, + }, + }); + await roleService.assign(user2.id, role1.id); + + const [u1role, u2role] = await Promise.all([ + roleService.getUserRoles(user1.id), + roleService.getUserRoles(user2.id), + ]); + expect(u1role.some(r => r.id === role2.id)).toBe(false); + expect(u2role.some(r => r.id === role2.id)).toBe(true); + }); + test('expired role', async () => { const user = await createUser(); const role = await createRole({ diff --git a/packages/frontend/src/pages/admin/RolesEditorFormula.vue b/packages/frontend/src/pages/admin/RolesEditorFormula.vue index f4a8f44955..2f5b4c47d8 100644 --- a/packages/frontend/src/pages/admin/RolesEditorFormula.vue +++ b/packages/frontend/src/pages/admin/RolesEditorFormula.vue @@ -9,6 +9,7 @@ SPDX-License-Identifier: AGPL-3.0-only + @@ -51,6 +52,10 @@ SPDX-License-Identifier: AGPL-3.0-only + + + + @@ -62,6 +67,7 @@ import MkSelect from '@/components/MkSelect.vue'; import MkButton from '@/components/MkButton.vue'; import { i18n } from '@/i18n.js'; import { deepClone } from '@/scripts/clone.js'; +import { rolesCache } from '@/cache.js'; const Sortable = defineAsyncComponent(() => import('vuedraggable').then(x => x.default)); @@ -77,6 +83,8 @@ const props = defineProps<{ const v = ref(deepClone(props.modelValue)); +const roles = await rolesCache.fetch(); + watch(() => props.modelValue, () => { if (JSON.stringify(props.modelValue) === JSON.stringify(v.value)) return; v.value = deepClone(props.modelValue); @@ -92,6 +100,7 @@ const type = computed({ if (t === 'and') v.value.values = []; if (t === 'or') v.value.values = []; if (t === 'not') v.value.value = { id: uuid(), type: 'isRemote' }; + if (t === 'roleAssignedTo') v.value.roleId = ''; if (t === 'createdLessThan') v.value.sec = 86400; if (t === 'createdMoreThan') v.value.sec = 86400; if (t === 'followersLessThanOrEq') v.value.value = 10; diff --git a/packages/misskey-js/etc/misskey-js.api.md b/packages/misskey-js/etc/misskey-js.api.md index b5e7ec7548..0e990ffd5a 100644 --- a/packages/misskey-js/etc/misskey-js.api.md +++ b/packages/misskey-js/etc/misskey-js.api.md @@ -1712,6 +1712,7 @@ declare namespace entities { RoleCondFormulaLogics, RoleCondFormulaValueNot, RoleCondFormulaValueIsLocalOrRemote, + RoleCondFormulaValueAssignedRole, RoleCondFormulaValueCreated, RoleCondFormulaFollowersOrFollowingOrNotes, RoleCondFormulaValue, @@ -2731,6 +2732,9 @@ type RoleCondFormulaLogics = components['schemas']['RoleCondFormulaLogics']; // @public (undocumented) type RoleCondFormulaValue = components['schemas']['RoleCondFormulaValue']; +// @public (undocumented) +type RoleCondFormulaValueAssignedRole = components['schemas']['RoleCondFormulaValueAssignedRole']; + // @public (undocumented) type RoleCondFormulaValueCreated = components['schemas']['RoleCondFormulaValueCreated']; diff --git a/packages/misskey-js/src/autogen/models.ts b/packages/misskey-js/src/autogen/models.ts index ab49f9478a..6f61458600 100644 --- a/packages/misskey-js/src/autogen/models.ts +++ b/packages/misskey-js/src/autogen/models.ts @@ -38,6 +38,7 @@ export type Signin = components['schemas']['Signin']; export type RoleCondFormulaLogics = components['schemas']['RoleCondFormulaLogics']; export type RoleCondFormulaValueNot = components['schemas']['RoleCondFormulaValueNot']; export type RoleCondFormulaValueIsLocalOrRemote = components['schemas']['RoleCondFormulaValueIsLocalOrRemote']; +export type RoleCondFormulaValueAssignedRole = components['schemas']['RoleCondFormulaValueAssignedRole']; export type RoleCondFormulaValueCreated = components['schemas']['RoleCondFormulaValueCreated']; export type RoleCondFormulaFollowersOrFollowingOrNotes = components['schemas']['RoleCondFormulaFollowersOrFollowingOrNotes']; export type RoleCondFormulaValue = components['schemas']['RoleCondFormulaValue']; diff --git a/packages/misskey-js/src/autogen/types.ts b/packages/misskey-js/src/autogen/types.ts index 733670d704..8d700fb828 100644 --- a/packages/misskey-js/src/autogen/types.ts +++ b/packages/misskey-js/src/autogen/types.ts @@ -4573,6 +4573,15 @@ export type components = { /** @enum {string} */ type: 'isLocal' | 'isRemote'; }; + RoleCondFormulaValueAssignedRole: { + /** @enum {string} */ + type: 'roleAssignedTo'; + /** + * Format: id + * @example xxxxxxxxxx + */ + roleId: string; + }; RoleCondFormulaValueCreated: { id: string; /** @enum {string} */ @@ -4585,7 +4594,7 @@ export type components = { type: 'followersLessThanOrEq' | 'followersMoreThanOrEq' | 'followingLessThanOrEq' | 'followingMoreThanOrEq' | 'notesLessThanOrEq' | 'notesMoreThanOrEq'; value: number; }; - RoleCondFormulaValue: components['schemas']['RoleCondFormulaLogics'] | components['schemas']['RoleCondFormulaValueNot'] | components['schemas']['RoleCondFormulaValueIsLocalOrRemote'] | components['schemas']['RoleCondFormulaValueCreated'] | components['schemas']['RoleCondFormulaFollowersOrFollowingOrNotes']; + RoleCondFormulaValue: components['schemas']['RoleCondFormulaLogics'] | components['schemas']['RoleCondFormulaValueNot'] | components['schemas']['RoleCondFormulaValueIsLocalOrRemote'] | components['schemas']['RoleCondFormulaValueAssignedRole'] | components['schemas']['RoleCondFormulaValueCreated'] | components['schemas']['RoleCondFormulaFollowersOrFollowingOrNotes']; RoleLite: { /** * Format: id -- cgit v1.2.3-freya From b7d9d1620161a728e34ab20d8ea99160b3eb4196 Mon Sep 17 00:00:00 2001 From: okayurisotto <47853651+okayurisotto@users.noreply.github.com> Date: Wed, 28 Feb 2024 15:34:58 +0900 Subject: refactor(backend): ノートのエクスポート処理でStreams APIを使うように (#13465) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(backend): ノートのエクスポート処理でStreams APIを使うように * fixup! refactor(backend): ノートのエクスポート処理でStreams APIを使うように `await`忘れにより、ジョブがすぐに完了したことになり削除されてしまっていた。 それによって、`NoteStream`内での`updateProgress`メソッドの呼び出しで、`Missing key for job`のエラーが発生することがあった。 --------- Co-authored-by: syuilo <4439005+syuilo@users.noreply.github.com> --- packages/backend/src/misc/FileWriterStream.ts | 31 ++++ packages/backend/src/misc/JsonArrayStream.ts | 30 ++++ .../processors/ExportNotesProcessorService.ts | 164 +++++++++++---------- 3 files changed, 146 insertions(+), 79 deletions(-) create mode 100644 packages/backend/src/misc/FileWriterStream.ts create mode 100644 packages/backend/src/misc/JsonArrayStream.ts (limited to 'packages/backend/src/misc') diff --git a/packages/backend/src/misc/FileWriterStream.ts b/packages/backend/src/misc/FileWriterStream.ts new file mode 100644 index 0000000000..828851df0e --- /dev/null +++ b/packages/backend/src/misc/FileWriterStream.ts @@ -0,0 +1,31 @@ +import * as fs from 'node:fs/promises'; +import type { PathLike } from 'node:fs'; + +/** + * `fs.createWriteStream()`相当のことを行う`WritableStream` (Web標準) + */ +export class FileWriterStream extends WritableStream { + constructor(path: PathLike) { + let file: fs.FileHandle | null = null; + + super({ + start: async () => { + file = await fs.open(path, 'a'); + }, + write: async (chunk, controller) => { + if (file === null) { + controller.error(); + throw new Error(); + } + + await file.write(chunk); + }, + close: async () => { + await file?.close(); + }, + abort: async () => { + await file?.close(); + }, + }); + } +} diff --git a/packages/backend/src/misc/JsonArrayStream.ts b/packages/backend/src/misc/JsonArrayStream.ts new file mode 100644 index 0000000000..ad35bb3a79 --- /dev/null +++ b/packages/backend/src/misc/JsonArrayStream.ts @@ -0,0 +1,30 @@ +import { TransformStream } from 'node:stream/web'; + +/** + * ストリームに流れてきた各データについて`JSON.stringify()`した上で、それらを一つの配列にまとめる + */ +export class JsonArrayStream extends TransformStream { + constructor() { + /** 最初の要素かどうかを変数に記録 */ + let isFirst = true; + + super({ + start(controller) { + controller.enqueue('['); + }, + flush(controller) { + controller.enqueue(']'); + }, + transform(chunk, controller) { + if (isFirst) { + isFirst = false; + } else { + // 妥当なJSON配列にするためには最初以外の要素の前に`,`を挿入しなければならない + controller.enqueue(',\n'); + } + + controller.enqueue(JSON.stringify(chunk)); + }, + }); + } +} diff --git a/packages/backend/src/queue/processors/ExportNotesProcessorService.ts b/packages/backend/src/queue/processors/ExportNotesProcessorService.ts index f2ae0ce4b4..c7611012d7 100644 --- a/packages/backend/src/queue/processors/ExportNotesProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportNotesProcessorService.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import * as fs from 'node:fs'; +import { ReadableStream, TextEncoderStream } from 'node:stream/web'; import { Inject, Injectable } from '@nestjs/common'; import { MoreThan } from 'typeorm'; import { format as dateFormat } from 'date-fns'; @@ -18,10 +18,82 @@ import { bindThis } from '@/decorators.js'; import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; import { Packed } from '@/misc/json-schema.js'; import { IdService } from '@/core/IdService.js'; +import { JsonArrayStream } from '@/misc/JsonArrayStream.js'; +import { FileWriterStream } from '@/misc/FileWriterStream.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; import type { DbJobDataWithUser } from '../types.js'; +class NoteStream extends ReadableStream> { + constructor( + job: Bull.Job, + notesRepository: NotesRepository, + pollsRepository: PollsRepository, + driveFileEntityService: DriveFileEntityService, + idService: IdService, + userId: string, + ) { + let exportedNotesCount = 0; + let cursor: MiNote['id'] | null = null; + + const serialize = ( + note: MiNote, + poll: MiPoll | null, + files: Packed<'DriveFile'>[], + ): Record => { + return { + id: note.id, + text: note.text, + createdAt: idService.parse(note.id).date.toISOString(), + fileIds: note.fileIds, + files: files, + replyId: note.replyId, + renoteId: note.renoteId, + poll: poll, + cw: note.cw, + visibility: note.visibility, + visibleUserIds: note.visibleUserIds, + localOnly: note.localOnly, + reactionAcceptance: note.reactionAcceptance, + }; + }; + + super({ + async pull(controller): Promise { + const notes = await notesRepository.find({ + where: { + userId, + ...(cursor !== null ? { id: MoreThan(cursor) } : {}), + }, + take: 100, // 100件ずつ取得 + order: { id: 1 }, + }); + + if (notes.length === 0) { + job.updateProgress(100); + controller.close(); + } + + cursor = notes.at(-1)?.id ?? null; + + for (const note of notes) { + const poll = note.hasPoll + ? await pollsRepository.findOneByOrFail({ noteId: note.id }) // N+1 + : null; + const files = await driveFileEntityService.packManyByIds(note.fileIds); // N+1 + const content = serialize(note, poll, files); + + controller.enqueue(content); + exportedNotesCount++; + } + + const total = await notesRepository.countBy({ userId }); + job.updateProgress(exportedNotesCount / total); + }, + }); + } +} + @Injectable() export class ExportNotesProcessorService { private logger: Logger; @@ -59,67 +131,19 @@ export class ExportNotesProcessorService { this.logger.info(`Temp file is ${path}`); try { - const stream = fs.createWriteStream(path, { flags: 'a' }); - - const write = (text: string): Promise => { - return new Promise((res, rej) => { - stream.write(text, err => { - if (err) { - this.logger.error(err); - rej(err); - } else { - res(); - } - }); - }); - }; - - await write('['); - - let exportedNotesCount = 0; - let cursor: MiNote['id'] | null = null; - - while (true) { - const notes = await this.notesRepository.find({ - where: { - userId: user.id, - ...(cursor ? { id: MoreThan(cursor) } : {}), - }, - take: 100, - order: { - id: 1, - }, - }) as MiNote[]; - - if (notes.length === 0) { - job.updateProgress(100); - break; - } - - cursor = notes.at(-1)?.id ?? null; + // メモリが足りなくならないようにストリームで処理する + await new NoteStream( + job, + this.notesRepository, + this.pollsRepository, + this.driveFileEntityService, + this.idService, + user.id, + ) + .pipeThrough(new JsonArrayStream()) + .pipeThrough(new TextEncoderStream()) + .pipeTo(new FileWriterStream(path)); - for (const note of notes) { - let poll: MiPoll | undefined; - if (note.hasPoll) { - poll = await this.pollsRepository.findOneByOrFail({ noteId: note.id }); - } - const files = await this.driveFileEntityService.packManyByIds(note.fileIds); - const content = JSON.stringify(this.serialize(note, poll, files)); - const isFirst = exportedNotesCount === 0; - await write(isFirst ? content : ',\n' + content); - exportedNotesCount++; - } - - const total = await this.notesRepository.countBy({ - userId: user.id, - }); - - job.updateProgress(exportedNotesCount / total); - } - - await write(']'); - - stream.end(); this.logger.succ(`Exported to: ${path}`); const fileName = 'notes-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json'; @@ -130,22 +154,4 @@ export class ExportNotesProcessorService { cleanup(); } } - - private serialize(note: MiNote, poll: MiPoll | null = null, files: Packed<'DriveFile'>[]): Record { - return { - id: note.id, - text: note.text, - createdAt: this.idService.parse(note.id).date.toISOString(), - fileIds: note.fileIds, - files: files, - replyId: note.replyId, - renoteId: note.renoteId, - poll: poll, - cw: note.cw, - visibility: note.visibility, - visibleUserIds: note.visibleUserIds, - localOnly: note.localOnly, - reactionAcceptance: note.reactionAcceptance, - }; - } } -- cgit v1.2.3-freya From d1bf432e14faa7bb420b6338038195963f6b73d2 Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Fri, 1 Mar 2024 17:28:46 +0900 Subject: add missing license headers --- packages/backend/src/misc/FileWriterStream.ts | 5 +++++ packages/backend/src/misc/JsonArrayStream.ts | 5 +++++ packages/frontend/src/scripts/search-emoji.ts | 5 +++++ 3 files changed, 15 insertions(+) (limited to 'packages/backend/src/misc') diff --git a/packages/backend/src/misc/FileWriterStream.ts b/packages/backend/src/misc/FileWriterStream.ts index 828851df0e..367a8eb560 100644 --- a/packages/backend/src/misc/FileWriterStream.ts +++ b/packages/backend/src/misc/FileWriterStream.ts @@ -1,3 +1,8 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + import * as fs from 'node:fs/promises'; import type { PathLike } from 'node:fs'; diff --git a/packages/backend/src/misc/JsonArrayStream.ts b/packages/backend/src/misc/JsonArrayStream.ts index ad35bb3a79..754938989d 100644 --- a/packages/backend/src/misc/JsonArrayStream.ts +++ b/packages/backend/src/misc/JsonArrayStream.ts @@ -1,3 +1,8 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + import { TransformStream } from 'node:stream/web'; /** diff --git a/packages/frontend/src/scripts/search-emoji.ts b/packages/frontend/src/scripts/search-emoji.ts index 07f55e5531..371f69b9a7 100644 --- a/packages/frontend/src/scripts/search-emoji.ts +++ b/packages/frontend/src/scripts/search-emoji.ts @@ -1,3 +1,8 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + export type EmojiDef = { emoji: string; name: string; -- cgit v1.2.3-freya