summaryrefslogtreecommitdiff
path: root/src/queue
diff options
context:
space:
mode:
Diffstat (limited to 'src/queue')
-rw-r--r--src/queue/index.ts7
-rw-r--r--src/queue/processors/db/delete-drive-files.ts4
-rw-r--r--src/queue/processors/object-storage/clean-remote-files.ts50
-rw-r--r--src/queue/processors/object-storage/delete-file.ts16
-rw-r--r--src/queue/processors/object-storage/index.ts2
5 files changed, 63 insertions, 16 deletions
diff --git a/src/queue/index.ts b/src/queue/index.ts
index 70233d8318..a7e9b9814f 100644
--- a/src/queue/index.ts
+++ b/src/queue/index.ts
@@ -194,6 +194,13 @@ export function createDeleteObjectStorageFileJob(key: string) {
});
}
+export function createCleanRemoteFilesJob() {
+ return objectStorageQueue.add('cleanRemoteFiles', {}, {
+ removeOnComplete: true,
+ removeOnFail: true
+ });
+}
+
export default function() {
if (!program.onlyServer) {
deliverQueue.process(128, processDeliver);
diff --git a/src/queue/processors/db/delete-drive-files.ts b/src/queue/processors/db/delete-drive-files.ts
index 5ecee9139a..a2fd9050a9 100644
--- a/src/queue/processors/db/delete-drive-files.ts
+++ b/src/queue/processors/db/delete-drive-files.ts
@@ -1,7 +1,7 @@
import * as Bull from 'bull';
import { queueLogger } from '../../logger';
-import { deleteFile } from '../../../services/drive/delete-file';
+import { deleteFileSync } from '../../../services/drive/delete-file';
import { Users, DriveFiles } from '../../../models';
import { MoreThan } from 'typeorm';
@@ -39,7 +39,7 @@ export async function deleteDriveFiles(job: Bull.Job, done: any): Promise<void>
cursor = files[files.length - 1].id;
for (const file of files) {
- await deleteFile(file);
+ await deleteFileSync(file);
deletedCount++;
}
diff --git a/src/queue/processors/object-storage/clean-remote-files.ts b/src/queue/processors/object-storage/clean-remote-files.ts
new file mode 100644
index 0000000000..7b34892e1f
--- /dev/null
+++ b/src/queue/processors/object-storage/clean-remote-files.ts
@@ -0,0 +1,50 @@
+import * as Bull from 'bull';
+
+import { queueLogger } from '../../logger';
+import { deleteFileSync } from '../../../services/drive/delete-file';
+import { DriveFiles } from '../../../models';
+import { MoreThan, Not, IsNull } from 'typeorm';
+
+const logger = queueLogger.createSubLogger('clean-remote-files');
+
+export default async function cleanRemoteFiles(job: Bull.Job, done: any): Promise<void> {
+ logger.info(`Deleting cached remote files...`);
+
+ let deletedCount = 0;
+ let cursor: any = null;
+
+ while (true) {
+ const files = await DriveFiles.find({
+ where: {
+ userHost: Not(IsNull()),
+ isLink: false,
+ ...(cursor ? { id: MoreThan(cursor) } : {})
+ },
+ take: 8,
+ order: {
+ id: 1
+ }
+ });
+
+ if (files.length === 0) {
+ job.progress(100);
+ break;
+ }
+
+ cursor = files[files.length - 1].id;
+
+ await Promise.all(files.map(file => deleteFileSync(file, true)));
+
+ deletedCount += 8;
+
+ const total = await DriveFiles.count({
+ userHost: Not(IsNull()),
+ isLink: false,
+ });
+
+ job.progress(deletedCount / total);
+ }
+
+ logger.succ(`All cahced remote files has been deleted.`);
+ done();
+}
diff --git a/src/queue/processors/object-storage/delete-file.ts b/src/queue/processors/object-storage/delete-file.ts
index 8e6b5f959e..f899df7d2e 100644
--- a/src/queue/processors/object-storage/delete-file.ts
+++ b/src/queue/processors/object-storage/delete-file.ts
@@ -1,22 +1,10 @@
import * as Bull from 'bull';
-import * as Minio from 'minio';
-import { fetchMeta } from '../../../misc/fetch-meta';
+import { deleteObjectStorageFile } from '../../../services/drive/delete-file';
export default async (job: Bull.Job) => {
- const meta = await fetchMeta();
-
- const minio = new Minio.Client({
- endPoint: meta.objectStorageEndpoint!,
- region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined,
- port: meta.objectStoragePort ? meta.objectStoragePort : undefined,
- useSSL: meta.objectStorageUseSSL,
- accessKey: meta.objectStorageAccessKey!,
- secretKey: meta.objectStorageSecretKey!,
- });
-
const key: string = job.data.key;
- await minio.removeObject(meta.objectStorageBucket!, key);
+ await deleteObjectStorageFile(key);
return 'Success';
};
diff --git a/src/queue/processors/object-storage/index.ts b/src/queue/processors/object-storage/index.ts
index 60f732ca64..33ef665b38 100644
--- a/src/queue/processors/object-storage/index.ts
+++ b/src/queue/processors/object-storage/index.ts
@@ -1,8 +1,10 @@
import * as Bull from 'bull';
import deleteFile from './delete-file';
+import cleanRemoteFiles from './clean-remote-files';
const jobs = {
deleteFile,
+ cleanRemoteFiles,
} as any;
export default function(q: Bull.Queue) {