From b75184ec8e3436200bacdcd832e3324702553d20 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sun, 18 Sep 2022 03:27:08 +0900 Subject: なんかもうめっちゃ変えた MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/queue/QueueProcessorModule.ts | 72 ++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 packages/backend/src/queue/QueueProcessorModule.ts (limited to 'packages/backend/src/queue/QueueProcessorModule.ts') diff --git a/packages/backend/src/queue/QueueProcessorModule.ts b/packages/backend/src/queue/QueueProcessorModule.ts new file mode 100644 index 0000000000..f13dd3ef19 --- /dev/null +++ b/packages/backend/src/queue/QueueProcessorModule.ts @@ -0,0 +1,72 @@ +import { Module } from '@nestjs/common'; +import { CoreModule } from '@/core/CoreModule.js'; +import { QueueLoggerService } from './QueueLoggerService.js'; +import { QueueProcessorService } from './QueueProcessorService.js'; +import { DbQueueProcessorsService } from './DbQueueProcessorsService.js'; +import { ObjectStorageQueueProcessorsService } from './ObjectStorageQueueProcessorsService.js'; +import { DeliverProcessorService } from './processors/DeliverProcessorService.js'; +import { EndedPollNotificationProcessorService } from './processors/EndedPollNotificationProcessorService.js'; +import { InboxProcessorService } from './processors/InboxProcessorService.js'; +import { WebhookDeliverProcessorService } from './processors/WebhookDeliverProcessorService.js'; +import { SystemQueueProcessorsService } from './SystemQueueProcessorsService.js'; +import { CheckExpiredMutingsProcessorService } from './processors/CheckExpiredMutingsProcessorService.js'; +import { CleanChartsProcessorService } from './processors/CleanChartsProcessorService.js'; +import { CleanProcessorService } from './processors/CleanProcessorService.js'; +import { CleanRemoteFilesProcessorService } from './processors/CleanRemoteFilesProcessorService.js'; +import { DeleteAccountProcessorService } from './processors/DeleteAccountProcessorService.js'; +import { DeleteDriveFilesProcessorService } from './processors/DeleteDriveFilesProcessorService.js'; +import { DeleteFileProcessorService } from './processors/DeleteFileProcessorService.js'; +import { ExportBlockingProcessorService } from './processors/ExportBlockingProcessorService.js'; +import { ExportCustomEmojisProcessorService } from './processors/ExportCustomEmojisProcessorService.js'; +import { ExportFollowingProcessorService } from './processors/ExportFollowingProcessorService.js'; +import { ExportMutingProcessorService } from './processors/ExportMutingProcessorService.js'; +import { ExportNotesProcessorService } from './processors/ExportNotesProcessorService.js'; +import { ExportUserListsProcessorService } from './processors/ExportUserListsProcessorService.js'; +import { ImportBlockingProcessorService } from './processors/ImportBlockingProcessorService.js'; +import { ImportCustomEmojisProcessorService } from './processors/ImportCustomEmojisProcessorService.js'; +import { ImportFollowingProcessorService } from './processors/ImportFollowingProcessorService.js'; +import { ImportMutingProcessorService } from './processors/ImportMutingProcessorService.js'; +import { ImportUserListsProcessorService } from './processors/ImportUserListsProcessorService.js'; +import { ResyncChartsProcessorService } from './processors/ResyncChartsProcessorService.js'; +import { TickChartsProcessorService } from './processors/TickChartsProcessorService.js'; + +@Module({ + imports: [ + CoreModule, + ], + providers: [ + QueueLoggerService, + TickChartsProcessorService, + ResyncChartsProcessorService, + CleanChartsProcessorService, + CheckExpiredMutingsProcessorService, + CleanProcessorService, + DeleteDriveFilesProcessorService, + ExportCustomEmojisProcessorService, + ExportNotesProcessorService, + ExportFollowingProcessorService, + ExportMutingProcessorService, + ExportBlockingProcessorService, + ExportUserListsProcessorService, + ImportFollowingProcessorService, + ImportMutingProcessorService, + ImportBlockingProcessorService, + ImportUserListsProcessorService, + ImportCustomEmojisProcessorService, + DeleteAccountProcessorService, + DeleteFileProcessorService, + CleanRemoteFilesProcessorService, + SystemQueueProcessorsService, + ObjectStorageQueueProcessorsService, + DbQueueProcessorsService, + WebhookDeliverProcessorService, + EndedPollNotificationProcessorService, + DeliverProcessorService, + InboxProcessorService, + QueueProcessorService, + ], + exports: [ + QueueProcessorService, + ], +}) +export class QueueProcessorModule {} -- cgit v1.2.3-freya From 3e81913b6a161cfc8405bda64b4a00e8e3b1fccd Mon Sep 17 00:00:00 2001 From: syuilo Date: Sun, 25 Dec 2022 09:09:46 +0900 Subject: feat: introduce retention-rate aggregation --- CHANGELOG.md | 1 + .../1671924750884-RetentionAggregation.js | 13 ++++ .../1671926422832-RetentionAggregation2.js | 15 +++++ packages/backend/src/di-symbols.ts | 1 + packages/backend/src/models/RepositoryModule.ts | 10 ++- .../src/models/entities/RetentionAggregation.ts | 35 ++++++++++ packages/backend/src/models/index.ts | 3 + packages/backend/src/postgre.ts | 2 + packages/backend/src/queue/QueueProcessorModule.ts | 2 + .../backend/src/queue/QueueProcessorService.ts | 8 ++- .../src/queue/SystemQueueProcessorsService.ts | 5 +- .../AggregateRetentionProcessorService.ts | 75 ++++++++++++++++++++++ packages/backend/src/server/api/EndpointsModule.ts | 4 ++ packages/backend/src/server/api/endpoints.ts | 2 + .../backend/src/server/api/endpoints/retention.ts | 47 ++++++++++++++ 15 files changed, 220 insertions(+), 3 deletions(-) create mode 100644 packages/backend/migration/1671924750884-RetentionAggregation.js create mode 100644 packages/backend/migration/1671926422832-RetentionAggregation2.js create mode 100644 packages/backend/src/models/entities/RetentionAggregation.ts create mode 100644 packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts create mode 100644 packages/backend/src/server/api/endpoints/retention.ts (limited to 'packages/backend/src/queue/QueueProcessorModule.ts') diff --git a/CHANGELOG.md b/CHANGELOG.md index 4079840ae8..d58ff7691f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ You should also include the user name that made the change. - Push notification of Antenna note @tamaina - AVIF support @tamaina - Add Cloudflare Turnstile CAPTCHA support @CyberRex0 +- Introduce retention-rate aggregation @syuilo - Server: improve syslog performance @syuilo - Server: improve note scoring for featured notes @CyberRex0 - Server: delete outdated notifications regularly to improve db performance @syuilo diff --git a/packages/backend/migration/1671924750884-RetentionAggregation.js b/packages/backend/migration/1671924750884-RetentionAggregation.js new file mode 100644 index 0000000000..ed81a4b5e9 --- /dev/null +++ b/packages/backend/migration/1671924750884-RetentionAggregation.js @@ -0,0 +1,13 @@ +export class RetentionAggregation1671924750884 { + name = 'RetentionAggregation1671924750884' + + async up(queryRunner) { + await queryRunner.query(`CREATE TABLE "retention_aggregation" ("id" character varying(32) NOT NULL, "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL, "userIds" character varying(32) array NOT NULL, "data" jsonb NOT NULL DEFAULT '{}', CONSTRAINT "PK_22aad3e8640b15fb3b90ee02d18" PRIMARY KEY ("id")); COMMENT ON COLUMN "retention_aggregation"."createdAt" IS 'The created date of the Note.'`); + await queryRunner.query(`CREATE INDEX "IDX_09f4e5b9e4a2f268d3e284e4b3" ON "retention_aggregation" ("createdAt") `); + } + + async down(queryRunner) { + await queryRunner.query(`DROP INDEX "public"."IDX_09f4e5b9e4a2f268d3e284e4b3"`); + await queryRunner.query(`DROP TABLE "retention_aggregation"`); + } +} diff --git a/packages/backend/migration/1671926422832-RetentionAggregation2.js b/packages/backend/migration/1671926422832-RetentionAggregation2.js new file mode 100644 index 0000000000..725429e6ef --- /dev/null +++ b/packages/backend/migration/1671926422832-RetentionAggregation2.js @@ -0,0 +1,15 @@ +export class RetentionAggregation21671926422832 { + name = 'RetentionAggregation21671926422832' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "retention_aggregation" ADD "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL`); + await queryRunner.query(`COMMENT ON COLUMN "retention_aggregation"."updatedAt" IS 'The updated date of the GalleryPost.'`); + await queryRunner.query(`ALTER TABLE "retention_aggregation" ADD "usersCount" integer NOT NULL`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "retention_aggregation" DROP COLUMN "usersCount"`); + await queryRunner.query(`COMMENT ON COLUMN "retention_aggregation"."updatedAt" IS 'The updated date of the GalleryPost.'`); + await queryRunner.query(`ALTER TABLE "retention_aggregation" DROP COLUMN "updatedAt"`); + } +} diff --git a/packages/backend/src/di-symbols.ts b/packages/backend/src/di-symbols.ts index cc775a9c8a..d2a361405f 100644 --- a/packages/backend/src/di-symbols.ts +++ b/packages/backend/src/di-symbols.ts @@ -68,5 +68,6 @@ export const DI = { webhooksRepository: Symbol('webhooksRepository'), adsRepository: Symbol('adsRepository'), passwordResetRequestsRepository: Symbol('passwordResetRequestsRepository'), + retentionAggregationsRepository: Symbol('retentionAggregationsRepository'), //#endregion }; diff --git a/packages/backend/src/models/RepositoryModule.ts b/packages/backend/src/models/RepositoryModule.ts index 9edef10e87..e22f0517ca 100644 --- a/packages/backend/src/models/RepositoryModule.ts +++ b/packages/backend/src/models/RepositoryModule.ts @@ -1,6 +1,6 @@ import { Module } from '@nestjs/common'; import { DI } from '@/di-symbols.js'; -import { User, Note, Announcement, AnnouncementRead, App, NoteFavorite, NoteThreadMuting, NoteReaction, NoteUnread, Notification, Poll, PollVote, UserProfile, UserKeypair, UserPending, AttestationChallenge, UserSecurityKey, UserPublickey, UserList, UserListJoining, UserGroup, UserGroupJoining, UserGroupInvitation, UserNotePining, UserIp, UsedUsername, Following, FollowRequest, Instance, Emoji, DriveFile, DriveFolder, Meta, Muting, Blocking, SwSubscription, Hashtag, AbuseUserReport, RegistrationTicket, AuthSession, AccessToken, Signin, MessagingMessage, Page, PageLike, GalleryPost, GalleryLike, ModerationLog, Clip, ClipNote, Antenna, AntennaNote, PromoNote, PromoRead, Relay, MutedNote, Channel, ChannelFollowing, ChannelNotePining, RegistryItem, Webhook, Ad, PasswordResetRequest } from './index.js'; +import { User, Note, Announcement, AnnouncementRead, App, NoteFavorite, NoteThreadMuting, NoteReaction, NoteUnread, Notification, Poll, PollVote, UserProfile, UserKeypair, UserPending, AttestationChallenge, UserSecurityKey, UserPublickey, UserList, UserListJoining, UserGroup, UserGroupJoining, UserGroupInvitation, UserNotePining, UserIp, UsedUsername, Following, FollowRequest, Instance, Emoji, DriveFile, DriveFolder, Meta, Muting, Blocking, SwSubscription, Hashtag, AbuseUserReport, RegistrationTicket, AuthSession, AccessToken, Signin, MessagingMessage, Page, PageLike, GalleryPost, GalleryLike, ModerationLog, Clip, ClipNote, Antenna, AntennaNote, PromoNote, PromoRead, Relay, MutedNote, Channel, ChannelFollowing, ChannelNotePining, RegistryItem, Webhook, Ad, PasswordResetRequest, RetentionAggregation } from './index.js'; import type { DataSource } from 'typeorm'; import type { Provider } from '@nestjs/common'; @@ -382,6 +382,12 @@ const $passwordResetRequestsRepository: Provider = { inject: [DI.db], }; +const $retentionAggregationsRepository: Provider = { + provide: DI.retentionAggregationsRepository, + useFactory: (db: DataSource) => db.getRepository(RetentionAggregation), + inject: [DI.db], +}; + @Module({ imports: [ ], @@ -449,6 +455,7 @@ const $passwordResetRequestsRepository: Provider = { $webhooksRepository, $adsRepository, $passwordResetRequestsRepository, + $retentionAggregationsRepository, ], exports: [ $usersRepository, @@ -514,6 +521,7 @@ const $passwordResetRequestsRepository: Provider = { $webhooksRepository, $adsRepository, $passwordResetRequestsRepository, + $retentionAggregationsRepository, ], }) export class RepositoryModule {} diff --git a/packages/backend/src/models/entities/RetentionAggregation.ts b/packages/backend/src/models/entities/RetentionAggregation.ts new file mode 100644 index 0000000000..c79b762d71 --- /dev/null +++ b/packages/backend/src/models/entities/RetentionAggregation.ts @@ -0,0 +1,35 @@ +import { Entity, PrimaryColumn, Index, Column } from 'typeorm'; +import { id } from '../id.js'; +import type { User } from './User.js'; + +@Entity() +export class RetentionAggregation { + @PrimaryColumn(id()) + public id: string; + + @Index() + @Column('timestamp with time zone', { + comment: 'The created date of the Note.', + }) + public createdAt: Date; + + @Column('timestamp with time zone', { + comment: 'The updated date of the GalleryPost.', + }) + public updatedAt: Date; + + @Column({ + ...id(), + array: true, + }) + public userIds: User['id'][]; + + @Column('integer', { + }) + public usersCount: number; + + @Column('jsonb', { + default: {}, + }) + public data: Record; +} diff --git a/packages/backend/src/models/index.ts b/packages/backend/src/models/index.ts index 7fde3fbedb..ca7a7c9e56 100644 --- a/packages/backend/src/models/index.ts +++ b/packages/backend/src/models/index.ts @@ -61,6 +61,7 @@ import { UserPublickey } from '@/models/entities/UserPublickey.js'; import { UserSecurityKey } from '@/models/entities/UserSecurityKey.js'; import { Webhook } from '@/models/entities/Webhook.js'; import { Channel } from '@/models/entities/Channel.js'; +import { RetentionAggregation } from '@/models/entities/RetentionAggregation.js'; import type { Repository } from 'typeorm'; export { @@ -127,6 +128,7 @@ export { UserSecurityKey, Webhook, Channel, + RetentionAggregation, }; export type AbuseUserReportsRepository = Repository; @@ -192,3 +194,4 @@ export type UserPublickeysRepository = Repository; export type UserSecurityKeysRepository = Repository; export type WebhooksRepository = Repository; export type ChannelsRepository = Repository; +export type RetentionAggregationsRepository = Repository; diff --git a/packages/backend/src/postgre.ts b/packages/backend/src/postgre.ts index 829edbd7cf..4b4490a0c3 100644 --- a/packages/backend/src/postgre.ts +++ b/packages/backend/src/postgre.ts @@ -69,6 +69,7 @@ import { UserPublickey } from '@/models/entities/UserPublickey.js'; import { UserSecurityKey } from '@/models/entities/UserSecurityKey.js'; import { Webhook } from '@/models/entities/Webhook.js'; import { Channel } from '@/models/entities/Channel.js'; +import { RetentionAggregation } from '@/models/entities/RetentionAggregation.js'; import { Config } from '@/config.js'; import MisskeyLogger from '@/logger.js'; @@ -182,6 +183,7 @@ export const entities = [ UserPending, Webhook, UserIp, + RetentionAggregation, ...charts, ]; diff --git a/packages/backend/src/queue/QueueProcessorModule.ts b/packages/backend/src/queue/QueueProcessorModule.ts index f13dd3ef19..620296498c 100644 --- a/packages/backend/src/queue/QueueProcessorModule.ts +++ b/packages/backend/src/queue/QueueProcessorModule.ts @@ -29,6 +29,7 @@ import { ImportMutingProcessorService } from './processors/ImportMutingProcessor import { ImportUserListsProcessorService } from './processors/ImportUserListsProcessorService.js'; import { ResyncChartsProcessorService } from './processors/ResyncChartsProcessorService.js'; import { TickChartsProcessorService } from './processors/TickChartsProcessorService.js'; +import { AggregateRetentionProcessorService } from './processors/AggregateRetentionProcessorService.js'; @Module({ imports: [ @@ -63,6 +64,7 @@ import { TickChartsProcessorService } from './processors/TickChartsProcessorServ EndedPollNotificationProcessorService, DeliverProcessorService, InboxProcessorService, + AggregateRetentionProcessorService, QueueProcessorService, ], exports: [ diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index 1d2feb5ef8..2123815c4c 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -4,6 +4,7 @@ import type { Config } from '@/config.js'; import { DI } from '@/di-symbols.js'; import type Logger from '@/logger.js'; import { QueueService } from '@/core/QueueService.js'; +import { bindThis } from '@/decorators.js'; import { getJobInfo } from './get-job-info.js'; import { SystemQueueProcessorsService } from './SystemQueueProcessorsService.js'; import { ObjectStorageQueueProcessorsService } from './ObjectStorageQueueProcessorsService.js'; @@ -13,7 +14,6 @@ import { EndedPollNotificationProcessorService } from './processors/EndedPollNot import { DeliverProcessorService } from './processors/DeliverProcessorService.js'; import { InboxProcessorService } from './processors/InboxProcessorService.js'; import { QueueLoggerService } from './QueueLoggerService.js'; -import { bindThis } from '@/decorators.js'; @Injectable() export class QueueProcessorService { @@ -133,6 +133,12 @@ export class QueueProcessorService { repeat: { cron: '0 0 * * *' }, removeOnComplete: true, }); + + this.queueService.systemQueue.add('aggregateRetention', { + }, { + repeat: { cron: '0 0 * * *' }, + removeOnComplete: true, + }); this.queueService.systemQueue.add('clean', { }, { diff --git a/packages/backend/src/queue/SystemQueueProcessorsService.ts b/packages/backend/src/queue/SystemQueueProcessorsService.ts index 1ce4152b2c..7fb0da4b10 100644 --- a/packages/backend/src/queue/SystemQueueProcessorsService.ts +++ b/packages/backend/src/queue/SystemQueueProcessorsService.ts @@ -1,13 +1,14 @@ import { Inject, Injectable } from '@nestjs/common'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; +import { bindThis } from '@/decorators.js'; import { TickChartsProcessorService } from './processors/TickChartsProcessorService.js'; import { ResyncChartsProcessorService } from './processors/ResyncChartsProcessorService.js'; import { CleanChartsProcessorService } from './processors/CleanChartsProcessorService.js'; import { CheckExpiredMutingsProcessorService } from './processors/CheckExpiredMutingsProcessorService.js'; import { CleanProcessorService } from './processors/CleanProcessorService.js'; +import { AggregateRetentionProcessorService } from './processors/AggregateRetentionProcessorService.js'; import type Bull from 'bull'; -import { bindThis } from '@/decorators.js'; @Injectable() export class SystemQueueProcessorsService { @@ -18,6 +19,7 @@ export class SystemQueueProcessorsService { private tickChartsProcessorService: TickChartsProcessorService, private resyncChartsProcessorService: ResyncChartsProcessorService, private cleanChartsProcessorService: CleanChartsProcessorService, + private aggregateRetentionProcessorService: AggregateRetentionProcessorService, private checkExpiredMutingsProcessorService: CheckExpiredMutingsProcessorService, private cleanProcessorService: CleanProcessorService, ) { @@ -28,6 +30,7 @@ export class SystemQueueProcessorsService { q.process('tickCharts', (job, done) => this.tickChartsProcessorService.process(job, done)); q.process('resyncCharts', (job, done) => this.resyncChartsProcessorService.process(job, done)); q.process('cleanCharts', (job, done) => this.cleanChartsProcessorService.process(job, done)); + q.process('aggregateRetention', (job, done) => this.aggregateRetentionProcessorService.process(job, done)); q.process('checkExpiredMutings', (job, done) => this.checkExpiredMutingsProcessorService.process(job, done)); q.process('clean', (job, done) => this.cleanProcessorService.process(job, done)); } diff --git a/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts b/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts new file mode 100644 index 0000000000..4650da76bb --- /dev/null +++ b/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts @@ -0,0 +1,75 @@ +import { Inject, Injectable } from '@nestjs/common'; +import { In, IsNull, MoreThan } from 'typeorm'; +import { DI } from '@/di-symbols.js'; +import type { Config } from '@/config.js'; +import type Logger from '@/logger.js'; +import { bindThis } from '@/decorators.js'; +import type { RetentionAggregationsRepository, UsersRepository } from '@/models/index.js'; +import { deepClone } from '@/misc/clone.js'; +import { IdService } from '@/core/IdService.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type Bull from 'bull'; + +@Injectable() +export class AggregateRetentionProcessorService { + private logger: Logger; + + constructor( + @Inject(DI.config) + private config: Config, + + @Inject(DI.usersRepository) + private usersRepository: UsersRepository, + + @Inject(DI.retentionAggregationsRepository) + private retentionAggregationsRepository: RetentionAggregationsRepository, + + private idService: IdService, + private queueLoggerService: QueueLoggerService, + ) { + this.logger = this.queueLoggerService.logger.createSubLogger('aggregate-retention'); + } + + @bindThis + public async process(job: Bull.Job>, done: () => void): Promise { + this.logger.info('Aggregating retention...'); + + const now = new Date(); + const dateKey = `${now.getFullYear()}-${now.getMonth() + 1}-${now.getDate()}`; + + // 過去(だいたい)30日分のレコードを取得 + const pastRecords = await this.retentionAggregationsRepository.findBy({ + createdAt: MoreThan(new Date(Date.now() - (1000 * 60 * 60 * 24 * 31))), + }); + + // 今日登録したユーザーを全て取得 + const targetUsers = await this.usersRepository.findBy({ + host: IsNull(), + createdAt: MoreThan(new Date(Date.now() - (1000 * 60 * 60 * 24))), + }); + const targetUserIds = targetUsers.map(u => u.id); + + await this.retentionAggregationsRepository.insert({ + id: this.idService.genId(), + createdAt: now, + updatedAt: now, + userIds: targetUserIds, + usersCount: targetUserIds.length, + }); + + for (const record of pastRecords) { + const retention = record.userIds.filter(id => targetUserIds.includes(id)).length; + + const data = deepClone(record.data); + data[dateKey] = retention; + + this.retentionAggregationsRepository.update(record.id, { + updatedAt: now, + data, + }); + } + + this.logger.succ('Retention aggregated.'); + done(); + } +} diff --git a/packages/backend/src/server/api/EndpointsModule.ts b/packages/backend/src/server/api/EndpointsModule.ts index 647f60317a..1f96647e7d 100644 --- a/packages/backend/src/server/api/EndpointsModule.ts +++ b/packages/backend/src/server/api/EndpointsModule.ts @@ -315,6 +315,7 @@ import * as ep___users_show from './endpoints/users/show.js'; import * as ep___users_stats from './endpoints/users/stats.js'; import * as ep___fetchRss from './endpoints/fetch-rss.js'; import * as ep___admin_driveCapOverride from './endpoints/admin/drive-capacity-override.js'; +import * as ep___retention from './endpoints/retention.js'; import { GetterService } from './GetterService.js'; import { ApiLoggerService } from './ApiLoggerService.js'; import type { Provider } from '@nestjs/common'; @@ -633,6 +634,7 @@ const $users_show: Provider = { provide: 'ep:users/show', useClass: ep___users_s const $users_stats: Provider = { provide: 'ep:users/stats', useClass: ep___users_stats.default }; const $admin_driveCapOverride: Provider = { provide: 'ep:admin/drive-capacity-override', useClass: ep___admin_driveCapOverride.default }; const $fetchRss: Provider = { provide: 'ep:fetch-rss', useClass: ep___fetchRss.default }; +const $retention: Provider = { provide: 'ep:retention', useClass: ep___retention.default }; @Module({ imports: [ @@ -955,6 +957,7 @@ const $fetchRss: Provider = { provide: 'ep:fetch-rss', useClass: ep___fetchRss.d $users_stats, $admin_driveCapOverride, $fetchRss, + $retention, ], exports: [ $admin_meta, @@ -1269,6 +1272,7 @@ const $fetchRss: Provider = { provide: 'ep:fetch-rss', useClass: ep___fetchRss.d $users_stats, $admin_driveCapOverride, $fetchRss, + $retention, ], }) export class EndpointsModule {} diff --git a/packages/backend/src/server/api/endpoints.ts b/packages/backend/src/server/api/endpoints.ts index 6d10cb8f35..e8dc5abfa1 100644 --- a/packages/backend/src/server/api/endpoints.ts +++ b/packages/backend/src/server/api/endpoints.ts @@ -314,6 +314,7 @@ import * as ep___users_show from './endpoints/users/show.js'; import * as ep___users_stats from './endpoints/users/stats.js'; import * as ep___fetchRss from './endpoints/fetch-rss.js'; import * as ep___admin_driveCapOverride from './endpoints/admin/drive-capacity-override.js'; +import * as ep___retention from './endpoints/retention.js'; const eps = [ ['admin/meta', ep___admin_meta], @@ -630,6 +631,7 @@ const eps = [ ['users/stats', ep___users_stats], ['admin/drive-capacity-override', ep___admin_driveCapOverride], ['fetch-rss', ep___fetchRss], + ['retention', ep___retention], ]; export interface IEndpointMeta { diff --git a/packages/backend/src/server/api/endpoints/retention.ts b/packages/backend/src/server/api/endpoints/retention.ts new file mode 100644 index 0000000000..e3c2249cdd --- /dev/null +++ b/packages/backend/src/server/api/endpoints/retention.ts @@ -0,0 +1,47 @@ +import { IsNull } from 'typeorm'; +import { Inject, Injectable } from '@nestjs/common'; +import type { RetentionAggregationsRepository } from '@/models/index.js'; +import { Endpoint } from '@/server/api/endpoint-base.js'; +import { DI } from '@/di-symbols.js'; + +export const meta = { + tags: ['users'], + + requireCredential: false, + + res: { + }, + + allowGet: true, + cacheSec: 60 * 60, +} as const; + +export const paramDef = { + type: 'object', + properties: {}, + required: [], +} as const; + +// eslint-disable-next-line import/no-default-export +@Injectable() +export default class extends Endpoint { + constructor( + @Inject(DI.retentionAggregationsRepository) + private retentionAggregationsRepository: RetentionAggregationsRepository, + ) { + super(meta, paramDef, async (ps, me) => { + const records = await this.retentionAggregationsRepository.find({ + order: { + id: 'DESC', + }, + take: 30, + }); + + return records.map(record => ({ + createdAt: record.createdAt.toISOString(), + users: record.usersCount, + data: record.data, + })); + }); + } +} -- cgit v1.2.3-freya From e414737179bff4b744f49e431d7f3620be999e46 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sun, 1 Jan 2023 16:53:10 +0900 Subject: feat: make possible to export favorited notes #9331 --- CHANGELOG.md | 1 + locales/ja-JP.yml | 1 + packages/backend/src/core/QueueService.ts | 12 +- .../backend/src/queue/DbQueueProcessorsService.ts | 5 +- packages/backend/src/queue/QueueProcessorModule.ts | 2 + .../processors/ExportFavoritesProcessorService.ts | 162 +++++++++++++++++++++ packages/backend/src/server/api/EndpointsModule.ts | 4 + packages/backend/src/server/api/endpoints.ts | 2 + .../src/server/api/endpoints/i/export-favorites.ts | 31 ++++ .../frontend/src/pages/settings/import-export.vue | 12 ++ 10 files changed, 230 insertions(+), 2 deletions(-) create mode 100644 packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts create mode 100644 packages/backend/src/server/api/endpoints/i/export-favorites.ts (limited to 'packages/backend/src/queue/QueueProcessorModule.ts') diff --git a/CHANGELOG.md b/CHANGELOG.md index 550a5fcf95..25e175fa85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ You should also include the user name that made the change. - AVIF support @tamaina - Add Cloudflare Turnstile CAPTCHA support @CyberRex0 - Introduce retention-rate aggregation @syuilo +- Make possible to export favorited notes @syuilo - Server: signToActivityPubGet is set to true by default @syuilo - Server: improve syslog performance @syuilo - Server: improve note scoring for featured notes @CyberRex0 diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 2764a85abb..a07fb5ff91 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1421,6 +1421,7 @@ _profile: _exportOrImport: allNotes: "全てのノート" + favoritedNotes: "お気に入りにしたノート" followingList: "フォロー" muteList: "ミュート" blockingList: "ブロック" diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 7956a3a8f9..4bf41e0ac1 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -5,10 +5,10 @@ import type { DriveFile } from '@/models/entities/DriveFile.js'; import type { Webhook, webhookEventTypes } from '@/models/entities/Webhook.js'; import type { Config } from '@/config.js'; import { DI } from '@/di-symbols.js'; +import { bindThis } from '@/decorators.js'; import type { DbQueue, DeliverQueue, EndedPollNotificationQueue, InboxQueue, ObjectStorageQueue, SystemQueue, WebhookDeliverQueue } from './QueueModule.js'; import type { ThinUser } from '../queue/types.js'; import type httpSignature from '@peertube/http-signature'; -import { bindThis } from '@/decorators.js'; @Injectable() export class QueueService { @@ -97,6 +97,16 @@ export class QueueService { }); } + @bindThis + public createExportFavoritesJob(user: ThinUser) { + return this.dbQueue.add('exportFavorites', { + user: user, + }, { + removeOnComplete: true, + removeOnFail: true, + }); + } + @bindThis public createExportFollowingJob(user: ThinUser, excludeMuting = false, excludeInactive = false) { return this.dbQueue.add('exportFollowing', { diff --git a/packages/backend/src/queue/DbQueueProcessorsService.ts b/packages/backend/src/queue/DbQueueProcessorsService.ts index e5568ab9bf..df337ad810 100644 --- a/packages/backend/src/queue/DbQueueProcessorsService.ts +++ b/packages/backend/src/queue/DbQueueProcessorsService.ts @@ -2,6 +2,7 @@ import { Inject, Injectable } from '@nestjs/common'; import type { DbJobData } from '@/queue/types.js'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; +import { bindThis } from '@/decorators.js'; import { DeleteDriveFilesProcessorService } from './processors/DeleteDriveFilesProcessorService.js'; import { ExportCustomEmojisProcessorService } from './processors/ExportCustomEmojisProcessorService.js'; import { ExportNotesProcessorService } from './processors/ExportNotesProcessorService.js'; @@ -15,8 +16,8 @@ import { ImportBlockingProcessorService } from './processors/ImportBlockingProce import { ImportUserListsProcessorService } from './processors/ImportUserListsProcessorService.js'; import { ImportCustomEmojisProcessorService } from './processors/ImportCustomEmojisProcessorService.js'; import { DeleteAccountProcessorService } from './processors/DeleteAccountProcessorService.js'; +import { ExportFavoritesProcessorService } from './processors/ExportFavoritesProcessorService.js'; import type Bull from 'bull'; -import { bindThis } from '@/decorators.js'; @Injectable() export class DbQueueProcessorsService { @@ -27,6 +28,7 @@ export class DbQueueProcessorsService { private deleteDriveFilesProcessorService: DeleteDriveFilesProcessorService, private exportCustomEmojisProcessorService: ExportCustomEmojisProcessorService, private exportNotesProcessorService: ExportNotesProcessorService, + private exportFavoritesProcessorService: ExportFavoritesProcessorService, private exportFollowingProcessorService: ExportFollowingProcessorService, private exportMutingProcessorService: ExportMutingProcessorService, private exportBlockingProcessorService: ExportBlockingProcessorService, @@ -45,6 +47,7 @@ export class DbQueueProcessorsService { q.process('deleteDriveFiles', (job, done) => this.deleteDriveFilesProcessorService.process(job, done)); q.process('exportCustomEmojis', (job, done) => this.exportCustomEmojisProcessorService.process(job, done)); q.process('exportNotes', (job, done) => this.exportNotesProcessorService.process(job, done)); + q.process('exportFavorites', (job, done) => this.exportFavoritesProcessorService.process(job, done)); q.process('exportFollowing', (job, done) => this.exportFollowingProcessorService.process(job, done)); q.process('exportMuting', (job, done) => this.exportMutingProcessorService.process(job, done)); q.process('exportBlocking', (job, done) => this.exportBlockingProcessorService.process(job, done)); diff --git a/packages/backend/src/queue/QueueProcessorModule.ts b/packages/backend/src/queue/QueueProcessorModule.ts index 620296498c..034e9cc5a5 100644 --- a/packages/backend/src/queue/QueueProcessorModule.ts +++ b/packages/backend/src/queue/QueueProcessorModule.ts @@ -30,6 +30,7 @@ import { ImportUserListsProcessorService } from './processors/ImportUserListsPro import { ResyncChartsProcessorService } from './processors/ResyncChartsProcessorService.js'; import { TickChartsProcessorService } from './processors/TickChartsProcessorService.js'; import { AggregateRetentionProcessorService } from './processors/AggregateRetentionProcessorService.js'; +import { ExportFavoritesProcessorService } from './processors/ExportFavoritesProcessorService.js'; @Module({ imports: [ @@ -45,6 +46,7 @@ import { AggregateRetentionProcessorService } from './processors/AggregateRetent DeleteDriveFilesProcessorService, ExportCustomEmojisProcessorService, ExportNotesProcessorService, + ExportFavoritesProcessorService, ExportFollowingProcessorService, ExportMutingProcessorService, ExportBlockingProcessorService, diff --git a/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts b/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts new file mode 100644 index 0000000000..3820705e5c --- /dev/null +++ b/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts @@ -0,0 +1,162 @@ +import * as fs from 'node:fs'; +import { Inject, Injectable } from '@nestjs/common'; +import { IsNull, MoreThan } from 'typeorm'; +import { format as dateFormat } from 'date-fns'; +import { DI } from '@/di-symbols.js'; +import type { NoteFavorite, NoteFavoritesRepository, NotesRepository, PollsRepository, User, UsersRepository } from '@/models/index.js'; +import type { Config } from '@/config.js'; +import type Logger from '@/logger.js'; +import { DriveService } from '@/core/DriveService.js'; +import { createTemp } from '@/misc/create-temp.js'; +import type { Poll } from '@/models/entities/Poll.js'; +import type { Note } from '@/models/entities/Note.js'; +import { bindThis } from '@/decorators.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type Bull from 'bull'; +import type { DbUserJobData } from '../types.js'; + +@Injectable() +export class ExportFavoritesProcessorService { + private logger: Logger; + + constructor( + @Inject(DI.config) + private config: Config, + + @Inject(DI.usersRepository) + private usersRepository: UsersRepository, + + @Inject(DI.pollsRepository) + private pollsRepository: PollsRepository, + + @Inject(DI.notesRepository) + private notesRepository: NotesRepository, + + @Inject(DI.noteFavoritesRepository) + private noteFavoritesRepository: NoteFavoritesRepository, + + private driveService: DriveService, + private queueLoggerService: QueueLoggerService, + ) { + this.logger = this.queueLoggerService.logger.createSubLogger('export-favorites'); + } + + @bindThis + public async process(job: Bull.Job, done: () => void): Promise { + this.logger.info(`Exporting favorites of ${job.data.user.id} ...`); + + const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); + if (user == null) { + done(); + return; + } + + // Create temp file + const [path, cleanup] = await createTemp(); + + this.logger.info(`Temp file is ${path}`); + + try { + const stream = fs.createWriteStream(path, { flags: 'a' }); + + const write = (text: string): Promise => { + return new Promise((res, rej) => { + stream.write(text, err => { + if (err) { + this.logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + }; + + await write('['); + + let exportedFavoritesCount = 0; + let cursor: NoteFavorite['id'] | null = null; + + while (true) { + const favorites = await this.noteFavoritesRepository.find({ + where: { + userId: user.id, + ...(cursor ? { id: MoreThan(cursor) } : {}), + }, + take: 100, + order: { + id: 1, + }, + relations: ['note', 'note.user'], + }) as (NoteFavorite & { note: Note & { user: User } })[]; + + if (favorites.length === 0) { + job.progress(100); + break; + } + + cursor = favorites[favorites.length - 1].id; + + for (const favorite of favorites) { + let poll: Poll | undefined; + if (favorite.note.hasPoll) { + poll = await this.pollsRepository.findOneByOrFail({ noteId: favorite.note.id }); + } + const content = JSON.stringify(serialize(favorite, poll)); + const isFirst = exportedFavoritesCount === 0; + await write(isFirst ? content : ',\n' + content); + exportedFavoritesCount++; + } + + const total = await this.noteFavoritesRepository.countBy({ + userId: user.id, + }); + + job.progress(exportedFavoritesCount / total); + } + + await write(']'); + + stream.end(); + this.logger.succ(`Exported to: ${path}`); + + const fileName = 'favorites-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json'; + const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true }); + + this.logger.succ(`Exported to: ${driveFile.id}`); + } finally { + cleanup(); + } + + done(); + } +} + +function serialize(favorite: NoteFavorite & { note: Note & { user: User } }, poll: Poll | null = null): Record { + return { + id: favorite.id, + createdAt: favorite.createdAt, + note: { + id: favorite.note.id, + text: favorite.note.text, + createdAt: favorite.note.createdAt, + fileIds: favorite.note.fileIds, + replyId: favorite.note.replyId, + renoteId: favorite.note.renoteId, + poll: poll, + cw: favorite.note.cw, + visibility: favorite.note.visibility, + visibleUserIds: favorite.note.visibleUserIds, + localOnly: favorite.note.localOnly, + uri: favorite.note.uri, + url: favorite.note.url, + user: { + id: favorite.note.user.id, + name: favorite.note.user.name, + username: favorite.note.user.username, + host: favorite.note.user.host, + uri: favorite.note.user.uri, + }, + }, + }; +} diff --git a/packages/backend/src/server/api/EndpointsModule.ts b/packages/backend/src/server/api/EndpointsModule.ts index 1f96647e7d..18ba16ac79 100644 --- a/packages/backend/src/server/api/EndpointsModule.ts +++ b/packages/backend/src/server/api/EndpointsModule.ts @@ -176,6 +176,7 @@ import * as ep___i_exportBlocking from './endpoints/i/export-blocking.js'; import * as ep___i_exportFollowing from './endpoints/i/export-following.js'; import * as ep___i_exportMute from './endpoints/i/export-mute.js'; import * as ep___i_exportNotes from './endpoints/i/export-notes.js'; +import * as ep___i_exportFavorites from './endpoints/i/export-favorites.js'; import * as ep___i_exportUserLists from './endpoints/i/export-user-lists.js'; import * as ep___i_favorites from './endpoints/i/favorites.js'; import * as ep___i_gallery_likes from './endpoints/i/gallery/likes.js'; @@ -495,6 +496,7 @@ const $i_exportBlocking: Provider = { provide: 'ep:i/export-blocking', useClass: const $i_exportFollowing: Provider = { provide: 'ep:i/export-following', useClass: ep___i_exportFollowing.default }; const $i_exportMute: Provider = { provide: 'ep:i/export-mute', useClass: ep___i_exportMute.default }; const $i_exportNotes: Provider = { provide: 'ep:i/export-notes', useClass: ep___i_exportNotes.default }; +const $i_exportFavorites: Provider = { provide: 'ep:i/export-favorites', useClass: ep___i_exportFavorites.default }; const $i_exportUserLists: Provider = { provide: 'ep:i/export-user-lists', useClass: ep___i_exportUserLists.default }; const $i_favorites: Provider = { provide: 'ep:i/favorites', useClass: ep___i_favorites.default }; const $i_gallery_likes: Provider = { provide: 'ep:i/gallery/likes', useClass: ep___i_gallery_likes.default }; @@ -818,6 +820,7 @@ const $retention: Provider = { provide: 'ep:retention', useClass: ep___retention $i_exportFollowing, $i_exportMute, $i_exportNotes, + $i_exportFavorites, $i_exportUserLists, $i_favorites, $i_gallery_likes, @@ -1135,6 +1138,7 @@ const $retention: Provider = { provide: 'ep:retention', useClass: ep___retention $i_exportFollowing, $i_exportMute, $i_exportNotes, + $i_exportFavorites, $i_exportUserLists, $i_favorites, $i_gallery_likes, diff --git a/packages/backend/src/server/api/endpoints.ts b/packages/backend/src/server/api/endpoints.ts index e8dc5abfa1..a09ffa832c 100644 --- a/packages/backend/src/server/api/endpoints.ts +++ b/packages/backend/src/server/api/endpoints.ts @@ -175,6 +175,7 @@ import * as ep___i_exportBlocking from './endpoints/i/export-blocking.js'; import * as ep___i_exportFollowing from './endpoints/i/export-following.js'; import * as ep___i_exportMute from './endpoints/i/export-mute.js'; import * as ep___i_exportNotes from './endpoints/i/export-notes.js'; +import * as ep___i_exportFavorites from './endpoints/i/export-favorites.js'; import * as ep___i_exportUserLists from './endpoints/i/export-user-lists.js'; import * as ep___i_favorites from './endpoints/i/favorites.js'; import * as ep___i_gallery_likes from './endpoints/i/gallery/likes.js'; @@ -492,6 +493,7 @@ const eps = [ ['i/export-following', ep___i_exportFollowing], ['i/export-mute', ep___i_exportMute], ['i/export-notes', ep___i_exportNotes], + ['i/export-favorites', ep___i_exportFavorites], ['i/export-user-lists', ep___i_exportUserLists], ['i/favorites', ep___i_favorites], ['i/gallery/likes', ep___i_gallery_likes], diff --git a/packages/backend/src/server/api/endpoints/i/export-favorites.ts b/packages/backend/src/server/api/endpoints/i/export-favorites.ts new file mode 100644 index 0000000000..b32f39d3e5 --- /dev/null +++ b/packages/backend/src/server/api/endpoints/i/export-favorites.ts @@ -0,0 +1,31 @@ +import { Inject, Injectable } from '@nestjs/common'; +import ms from 'ms'; +import { Endpoint } from '@/server/api/endpoint-base.js'; +import { QueueService } from '@/core/QueueService.js'; + +export const meta = { + secure: true, + requireCredential: true, + limit: { + duration: ms('1day'), + max: 1, + }, +} as const; + +export const paramDef = { + type: 'object', + properties: {}, + required: [], +} as const; + +// eslint-disable-next-line import/no-default-export +@Injectable() +export default class extends Endpoint { + constructor( + private queueService: QueueService, + ) { + super(meta, paramDef, async (ps, me) => { + this.queueService.createExportFavoritesJob(me); + }); + } +} diff --git a/packages/frontend/src/pages/settings/import-export.vue b/packages/frontend/src/pages/settings/import-export.vue index 7db267c142..ca3bea4d16 100644 --- a/packages/frontend/src/pages/settings/import-export.vue +++ b/packages/frontend/src/pages/settings/import-export.vue @@ -8,6 +8,14 @@ {{ i18n.ts.export }} + + + + + + {{ i18n.ts.export }} + + @@ -108,6 +116,10 @@ const exportNotes = () => { os.api('i/export-notes', {}).then(onExportSuccess).catch(onError); }; +const exportFavorites = () => { + os.api('i/export-favorites', {}).then(onExportSuccess).catch(onError); +}; + const exportFollowing = () => { os.api('i/export-following', { excludeMuting: excludeMutingUsers.value, -- cgit v1.2.3-freya