リモートファイルの削除が重い問題を修正

7 files changed, 116 insertions, 27 deletions

diff --git a/src/queue/index.ts b/src/queue/index.ts
index 70233d8318..a7e9b9814f 100644
--- a/src/queue/index.ts
+++ b/src/queue/index.ts
@@ -194,6 +194,13 @@ export function createDeleteObjectStorageFileJob(key: string) {
 	});
 }
 
+export function createCleanRemoteFilesJob() {
+	return objectStorageQueue.add('cleanRemoteFiles', {}, {
+		removeOnComplete: true,
+		removeOnFail: true
+	});
+}
+
 export default function() {
 	if (!program.onlyServer) {
 		deliverQueue.process(128, processDeliver);
diff --git a/src/queue/processors/db/delete-drive-files.ts b/src/queue/processors/db/delete-drive-files.ts
index 5ecee9139a..a2fd9050a9 100644
--- a/src/queue/processors/db/delete-drive-files.ts
+++ b/src/queue/processors/db/delete-drive-files.ts
@@ -1,7 +1,7 @@
 import * as Bull from 'bull';
 
 import { queueLogger } from '../../logger';
-import { deleteFile } from '../../../services/drive/delete-file';
+import { deleteFileSync } from '../../../services/drive/delete-file';
 import { Users, DriveFiles } from '../../../models';
 import { MoreThan } from 'typeorm';
 
@@ -39,7 +39,7 @@ export async function deleteDriveFiles(job: Bull.Job, done: any): Promise<void>
 		cursor = files[files.length - 1].id;
 
 		for (const file of files) {
-			await deleteFile(file);
+			await deleteFileSync(file);
 			deletedCount++;
 		}
 
diff --git a/src/queue/processors/object-storage/clean-remote-files.ts b/src/queue/processors/object-storage/clean-remote-files.ts
new file mode 100644
index 0000000000..7b34892e1f
--- /dev/null
+++ b/src/queue/processors/object-storage/clean-remote-files.ts
@@ -0,0 +1,50 @@
+import * as Bull from 'bull';
+
+import { queueLogger } from '../../logger';
+import { deleteFileSync } from '../../../services/drive/delete-file';
+import { DriveFiles } from '../../../models';
+import { MoreThan, Not, IsNull } from 'typeorm';
+
+const logger = queueLogger.createSubLogger('clean-remote-files');
+
+export default async function cleanRemoteFiles(job: Bull.Job, done: any): Promise<void> {
+	logger.info(`Deleting cached remote files...`);
+
+	let deletedCount = 0;
+	let cursor: any = null;
+
+	while (true) {
+		const files = await DriveFiles.find({
+			where: {
+				userHost: Not(IsNull()),
+				isLink: false,
+				...(cursor ? { id: MoreThan(cursor) } : {})
+			},
+			take: 8,
+			order: {
+				id: 1
+			}
+		});
+
+		if (files.length === 0) {
+			job.progress(100);
+			break;
+		}
+
+		cursor = files[files.length - 1].id;
+
+		await Promise.all(files.map(file => deleteFileSync(file, true)));
+
+		deletedCount += 8;
+
+		const total = await DriveFiles.count({
+			userHost: Not(IsNull()),
+			isLink: false,
+		});
+
+		job.progress(deletedCount / total);
+	}
+
+	logger.succ(`All cahced remote files has been deleted.`);
+	done();
+}
diff --git a/src/queue/processors/object-storage/delete-file.ts b/src/queue/processors/object-storage/delete-file.ts
index 8e6b5f959e..f899df7d2e 100644
--- a/src/queue/processors/object-storage/delete-file.ts
+++ b/src/queue/processors/object-storage/delete-file.ts
@@ -1,22 +1,10 @@
 import * as Bull from 'bull';
-import * as Minio from 'minio';
-import { fetchMeta } from '../../../misc/fetch-meta';
+import { deleteObjectStorageFile } from '../../../services/drive/delete-file';
 
 export default async (job: Bull.Job) => {
-	const meta = await fetchMeta();
-
-	const minio = new Minio.Client({
-		endPoint: meta.objectStorageEndpoint!,
-		region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined,
-		port: meta.objectStoragePort ? meta.objectStoragePort : undefined,
-		useSSL: meta.objectStorageUseSSL,
-		accessKey: meta.objectStorageAccessKey!,
-		secretKey: meta.objectStorageSecretKey!,
-	});
-
 	const key: string = job.data.key;
 
-	await minio.removeObject(meta.objectStorageBucket!, key);
+	await deleteObjectStorageFile(key);
 
 	return 'Success';
 };
diff --git a/src/queue/processors/object-storage/index.ts b/src/queue/processors/object-storage/index.ts
index 60f732ca64..33ef665b38 100644
--- a/src/queue/processors/object-storage/index.ts
+++ b/src/queue/processors/object-storage/index.ts
@@ -1,8 +1,10 @@
 import * as Bull from 'bull';
 import deleteFile from './delete-file';
+import cleanRemoteFiles from './clean-remote-files';
 
 const jobs = {
 	deleteFile,
+	cleanRemoteFiles,
 } as any;
 
 export default function(q: Bull.Queue) {
diff --git a/src/server/api/endpoints/admin/drive/clean-remote-files.ts b/src/server/api/endpoints/admin/drive/clean-remote-files.ts
index 69cfe0db94..e837ae1bb6 100644
--- a/src/server/api/endpoints/admin/drive/clean-remote-files.ts
+++ b/src/server/api/endpoints/admin/drive/clean-remote-files.ts
@@ -1,7 +1,5 @@
-import { Not, IsNull } from 'typeorm';
 import define from '../../../define';
-import { deleteFile } from '../../../../../services/drive/delete-file';
-import { DriveFiles } from '../../../../../models';
+import { createCleanRemoteFilesJob } from '../../../../../queue';
 
 export const meta = {
 	tags: ['admin'],
@@ -11,12 +9,5 @@ export const meta = {
 };
 
 export default define(meta, async (ps, me) => {
-	const files = await DriveFiles.find({
-		userHost: Not(IsNull()),
-		isLink: false,
-	});
-
-	for (const file of files) {
-		deleteFile(file, true);
-	}
+	createCleanRemoteFilesJob();
 });
diff --git a/src/services/drive/delete-file.ts b/src/services/drive/delete-file.ts
index 258c0853fb..5b44a0817a 100644
--- a/src/services/drive/delete-file.ts
+++ b/src/services/drive/delete-file.ts
@@ -1,8 +1,10 @@
+import * as Minio from 'minio';
 import { DriveFile } from '../../models/entities/drive-file';
 import { InternalStorage } from './internal-storage';
 import { DriveFiles, Instances, Notes } from '../../models';
 import { driveChart, perUserDriveChart, instanceChart } from '../chart';
 import { createDeleteObjectStorageFileJob } from '../../queue';
+import { fetchMeta } from '../../misc/fetch-meta';
 
 export async function deleteFile(file: DriveFile, isExpired = false) {
 	if (file.storedInternal) {
@@ -27,6 +29,40 @@ export async function deleteFile(file: DriveFile, isExpired = false) {
 		}
 	}
 
+	postProcess(file, isExpired);
+}
+
+export async function deleteFileSync(file: DriveFile, isExpired = false) {
+	if (file.storedInternal) {
+		InternalStorage.del(file.accessKey!);
+
+		if (file.thumbnailUrl) {
+			InternalStorage.del(file.thumbnailAccessKey!);
+		}
+
+		if (file.webpublicUrl) {
+			InternalStorage.del(file.webpublicAccessKey!);
+		}
+	} else if (!file.isLink) {
+		const promises = [];
+
+		promises.push(deleteObjectStorageFile(file.accessKey!));
+
+		if (file.thumbnailUrl) {
+			promises.push(deleteObjectStorageFile(file.thumbnailAccessKey!));
+		}
+
+		if (file.webpublicUrl) {
+			promises.push(deleteObjectStorageFile(file.webpublicAccessKey!));
+		}
+
+		await Promise.all(promises);
+	}
+
+	postProcess(file, isExpired);
+}
+
+function postProcess(file: DriveFile, isExpired = false) {
 	// リモートファイル期限切れ削除後は直リンクにする
 	if (isExpired && file.userHost !== null && file.uri != null) {
 		DriveFiles.update(file.id, {
@@ -53,3 +89,18 @@ export async function deleteFile(file: DriveFile, isExpired = false) {
 		Instances.decrement({ host: file.userHost }, 'driveFiles', 1);
 	}
 }
+
+export async function deleteObjectStorageFile(key: string) {
+	const meta = await fetchMeta();
+
+	const minio = new Minio.Client({
+		endPoint: meta.objectStorageEndpoint!,
+		region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined,
+		port: meta.objectStoragePort ? meta.objectStoragePort : undefined,
+		useSSL: meta.objectStorageUseSSL,
+		accessKey: meta.objectStorageAccessKey!,
+		secretKey: meta.objectStorageSecretKey!,
+	});
+
+	await minio.removeObject(meta.objectStorageBucket!, key);
+}