リモートファイルの削除が重い問題を修正

src/queue/index.ts

@@ -194,6 +194,13 @@ export function createDeleteObjectStorageFileJob(key: string) {
 	});
 }
 
+export function createCleanRemoteFilesJob() {
+	return objectStorageQueue.add('cleanRemoteFiles', {}, {
+		removeOnComplete: true,
+		removeOnFail: true
+	});
+}
+
 export default function() {
 	if (!program.onlyServer) {
 		deliverQueue.process(128, processDeliver);

src/queue/processors/db/delete-drive-files.ts

@@ -1,7 +1,7 @@
 import * as Bull from 'bull';
 
 import { queueLogger } from '../../logger';
-import { deleteFile } from '../../../services/drive/delete-file';
+import { deleteFileSync } from '../../../services/drive/delete-file';
 import { Users, DriveFiles } from '../../../models';
 import { MoreThan } from 'typeorm';
 
@@ -39,7 +39,7 @@ export async function deleteDriveFiles(job: Bull.Job, done: any): Promise<void>
 		cursor = files[files.length - 1].id;
 
 		for (const file of files) {
-			await deleteFile(file);
+			await deleteFileSync(file);
 			deletedCount++;
 		}
 

src/queue/processors/object-storage/clean-remote-files.ts

@@ -0,0 +1,50 @@
+import * as Bull from 'bull';
+
+import { queueLogger } from '../../logger';
+import { deleteFileSync } from '../../../services/drive/delete-file';
+import { DriveFiles } from '../../../models';
+import { MoreThan, Not, IsNull } from 'typeorm';
+
+const logger = queueLogger.createSubLogger('clean-remote-files');
+
+export default async function cleanRemoteFiles(job: Bull.Job, done: any): Promise<void> {
+	logger.info(`Deleting cached remote files...`);
+
+	let deletedCount = 0;
+	let cursor: any = null;
+
+	while (true) {
+		const files = await DriveFiles.find({
+			where: {
+				userHost: Not(IsNull()),
+				isLink: false,
+				...(cursor ? { id: MoreThan(cursor) } : {})
+			},
+			take: 8,
+			order: {
+				id: 1
+			}
+		});
+
+		if (files.length === 0) {
+			job.progress(100);
+			break;
+		}
+
+		cursor = files[files.length - 1].id;
+
+		await Promise.all(files.map(file => deleteFileSync(file, true)));
+
+		deletedCount += 8;
+
+		const total = await DriveFiles.count({
+			userHost: Not(IsNull()),
+			isLink: false,
+		});
+
+		job.progress(deletedCount / total);
+	}
+
+	logger.succ(`All cahced remote files has been deleted.`);
+	done();
+}

src/queue/processors/object-storage/delete-file.ts

@@ -1,22 +1,10 @@
 import * as Bull from 'bull';
-import * as Minio from 'minio';
-import { fetchMeta } from '../../../misc/fetch-meta';
+import { deleteObjectStorageFile } from '../../../services/drive/delete-file';
 
 export default async (job: Bull.Job) => {
-	const meta = await fetchMeta();
-
-	const minio = new Minio.Client({
-		endPoint: meta.objectStorageEndpoint!,
-		region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined,
-		port: meta.objectStoragePort ? meta.objectStoragePort : undefined,
-		useSSL: meta.objectStorageUseSSL,
-		accessKey: meta.objectStorageAccessKey!,
-		secretKey: meta.objectStorageSecretKey!,
-	});
-
 	const key: string = job.data.key;
 
-	await minio.removeObject(meta.objectStorageBucket!, key);
+	await deleteObjectStorageFile(key);
 
 	return 'Success';
 };

src/queue/processors/object-storage/index.ts

@@ -1,8 +1,10 @@
 import * as Bull from 'bull';
 import deleteFile from './delete-file';
+import cleanRemoteFiles from './clean-remote-files';
 
 const jobs = {
 	deleteFile,
+	cleanRemoteFiles,
 } as any;
 
 export default function(q: Bull.Queue) {