feat: カスタム絵文字エクスポート

2021-12-10 18:24:26 +09:00
parent 0b038f6477
commit bd903cdbb5
10 changed files with 298 additions and 5 deletions
--- a/packages/backend/package.json
+++ b/packages/backend/package.json
@@ -74,6 +74,7 @@
 		"@typescript-eslint/eslint-plugin": "5.3.1",
 		"@typescript-eslint/parser": "5.1.0",
 		"abort-controller": "3.0.0",
+		"archiver": "5.3.0",
 		"autobind-decorator": "2.4.0",
 		"autosize": "4.0.4",
 		"autwh": "0.1.0",
@@ -131,6 +132,7 @@
 		"koa-views": "7.0.2",
 		"langmap": "0.0.16",
 		"mfm-js": "0.20.0",
+		"mime-types": "2.1.34",
 		"misskey-js": "0.0.8",
 		"mocha": "8.4.0",
 		"ms": "3.0.0-canary.1",
--- a/packages/backend/src/misc/download-url.ts
+++ b/packages/backend/src/misc/download-url.ts
@@ -11,7 +11,7 @@ const PrivateIp = require('private-ip');

 const pipeline = util.promisify(stream.pipeline);

-export async function downloadUrl(url: string, path: string) {
+export async function downloadUrl(url: string, path: string): Promise<void> {
 	const logger = new Logger('download');

 	logger.info(`Downloading ${chalk.cyan(url)} ...`);
--- a/packages/backend/src/queue/index.ts
+++ b/packages/backend/src/queue/index.ts
@@ -117,6 +117,15 @@ export function createDeleteDriveFilesJob(user: ThinUser) {
 	});
 }

+export function createExportCustomEmojisJob(user: ThinUser) {
+	return dbQueue.add('exportCustomEmojis', {
+		user: user,
+	}, {
+		removeOnComplete: true,
+		removeOnFail: true,
+	});
+}
+
 export function createExportNotesJob(user: ThinUser) {
 	return dbQueue.add('exportNotes', {
 		user: user,
--- a/packages/backend/src/queue/processors/db/export-custom-emojis.ts
+++ b/packages/backend/src/queue/processors/db/export-custom-emojis.ts
@@ -0,0 +1,131 @@
+import * as Bull from 'bull';
+import * as tmp from 'tmp';
+import * as fs from 'fs';
+
+import { ulid } from 'ulid';
+const mime = require('mime-types');
+const archiver = require('archiver');
+import { queueLogger } from '../../logger';
+import addFile from '@/services/drive/add-file';
+import * as dateFormat from 'dateformat';
+import { Users, Emojis } from '@/models/index';
+import {  } from '@/queue/types';
+import { downloadUrl } from '@/misc/download-url';
+
+const logger = queueLogger.createSubLogger('export-custom-emojis');
+
+export async function exportCustomEmojis(job: Bull.Job, done: () => void): Promise<void> {
+	logger.info(`Exporting custom emojis ...`);
+
+	const user = await Users.findOne(job.data.user.id);
+	if (user == null) {
+		done();
+		return;
+	}
+
+	// Create temp dir
+	const [path, cleanup] = await new Promise<[string, () => void]>((res, rej) => {
+		tmp.dir((e, path, cleanup) => {
+			if (e) return rej(e);
+			res([path, cleanup]);
+		});
+	});
+
+	logger.info(`Temp dir is ${path}`);
+
+	const metaPath = path + '/meta.json';
+
+	fs.writeFileSync(metaPath, '', 'utf-8');
+
+	const metaStream = fs.createWriteStream(metaPath, { flags: 'a' });
+
+	await new Promise<void>((res, rej) => {
+		metaStream.write('[', err => {
+			if (err) {
+				logger.error(err);
+				rej(err);
+			} else {
+				res();
+			}
+		});
+	});
+
+	const customEmojis = await Emojis.find({
+		where: {
+			host: null,
+		},
+		order: {
+			id: 'ASC',
+		},
+	});
+
+	for (const emoji of customEmojis) {
+		const exportId = ulid().toLowerCase();
+		const emojiPath = path + '/' + exportId + '.' + mime.extension(emoji.type);
+		fs.writeFileSync(emojiPath, '', 'binary');
+		let downloaded = false;
+
+		try {
+			await downloadUrl(emoji.url, emojiPath);
+			downloaded = true;
+		} catch (e) { // TODO: 何度か再試行
+			logger.error(e);
+		}
+
+		await new Promise<void>((res, rej) => {
+			const content = JSON.stringify({
+				id: exportId,
+				downloaded: downloaded,
+				emoji: emoji,
+			});
+			const isFirst = customEmojis.indexOf(emoji) === 0;
+			metaStream.write(isFirst ? content : ',\n' + content, err => {
+				if (err) {
+					logger.error(err);
+					rej(err);
+				} else {
+					res();
+				}
+			});
+		});
+	}
+
+	await new Promise<void>((res, rej) => {
+		metaStream.write(']', err => {
+			if (err) {
+				logger.error(err);
+				rej(err);
+			} else {
+				res();
+			}
+		});
+	});
+
+	metaStream.end();
+
+	// Create archive
+	const [archivePath, archiveCleanup] = await new Promise<[string, () => void]>((res, rej) => {
+		tmp.file((e, path, fd, cleanup) => {
+			if (e) return rej(e);
+			res([path, cleanup]);
+		});
+	});
+	const archiveStream = fs.createWriteStream(archivePath);
+	const archive = archiver('zip', {
+		zlib: { level: 0 },
+	});
+	archiveStream.on('close', async () => {
+		logger.succ(`Exported to: ${archivePath}`);
+
+		const fileName = 'custom-emojis-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.zip';
+		const driveFile = await addFile(user, archivePath, fileName, null, null, true);
+
+		logger.succ(`Exported to: ${driveFile.id}`);
+		cleanup();
+		archiveCleanup();
+		done();
+	});
+	archive.pipe(archiveStream);
+	archive.directory(path, false);
+	archive.finalize();
+}
--- a/packages/backend/src/queue/processors/db/export-notes.ts
+++ b/packages/backend/src/queue/processors/db/export-notes.ts
@@ -74,7 +74,8 @@ export async function exportNotes(job: Bull.Job<DbUserJobData>, done: any): Prom
 			}
 			const content = JSON.stringify(serialize(note, poll));
 			await new Promise<void>((res, rej) => {
-				stream.write(exportedNotesCount === 0 ? content : ',\n' + content, err => {
+				const isFirst = exportedNotesCount === 0;
+				stream.write(isFirst ? content : ',\n' + content, err => {
 					if (err) {
 						logger.error(err);
 						rej(err);
--- a/packages/backend/src/queue/processors/db/index.ts
+++ b/packages/backend/src/queue/processors/db/index.ts
@@ -1,6 +1,7 @@
 import * as Bull from 'bull';
 import { DbJobData } from '@/queue/types';
 import { deleteDriveFiles } from './delete-drive-files';
+import { exportCustomEmojis } from './export-custom-emojis';
 import { exportNotes } from './export-notes';
 import { exportFollowing } from './export-following';
 import { exportMute } from './export-mute';
@@ -14,6 +15,7 @@ import { importBlocking } from './import-blocking';

 const jobs = {
 	deleteDriveFiles,
+	exportCustomEmojis,
 	exportNotes,
 	exportFollowing,
 	exportMute,
--- a/packages/backend/src/server/api/endpoints/export-custom-emojis.ts
+++ b/packages/backend/src/server/api/endpoints/export-custom-emojis.ts
@@ -0,0 +1,17 @@
+import $ from 'cafy';
+import define from '../define';
+import { createExportCustomEmojisJob } from '@/queue/index';
+import ms from 'ms';
+
+export const meta = {
+	secure: true,
+	requireCredential: true as const,
+	limit: {
+		duration: ms('1hour'),
+		max: 1,
+	},
+};
+
+export default define(meta, async (ps, user) => {
+	createExportCustomEmojisJob(user);
+});
--- a/packages/backend/yarn.lock
+++ b/packages/backend/yarn.lock
@@ -1318,6 +1318,35 @@ aproba@^1.0.3:
  resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a"
  integrity sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==

+archiver-utils@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/archiver-utils/-/archiver-utils-2.1.0.tgz#e8a460e94b693c3e3da182a098ca6285ba9249e2"
+  integrity sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw==
+  dependencies:
+    glob "^7.1.4"
+    graceful-fs "^4.2.0"
+    lazystream "^1.0.0"
+    lodash.defaults "^4.2.0"
+    lodash.difference "^4.5.0"
+    lodash.flatten "^4.4.0"
+    lodash.isplainobject "^4.0.6"
+    lodash.union "^4.6.0"
+    normalize-path "^3.0.0"
+    readable-stream "^2.0.0"
+
+archiver@5.3.0:
+  version "5.3.0"
+  resolved "https://registry.yarnpkg.com/archiver/-/archiver-5.3.0.tgz#dd3e097624481741df626267564f7dd8640a45ba"
+  integrity sha512-iUw+oDwK0fgNpvveEsdQ0Ase6IIKztBJU2U0E9MzszMfmVVUyv1QJhS2ITW9ZCqx8dktAxVAjWWkKehuZE8OPg==
+  dependencies:
+    archiver-utils "^2.1.0"
+    async "^3.2.0"
+    buffer-crc32 "^0.2.1"
+    readable-stream "^3.6.0"
+    readdir-glob "^1.0.0"
+    tar-stream "^2.2.0"
+    zip-stream "^4.1.0"
+
 are-we-there-yet@~1.1.2:
  version "1.1.5"
  resolved "https://registry.yarnpkg.com/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz#4b35c2944f062a8bfcda66410760350fe9ddfc21"
@@ -1645,6 +1674,11 @@ buffer-alloc@^1.2.0:
    buffer-alloc-unsafe "^1.1.0"
    buffer-fill "^1.0.0"

+buffer-crc32@^0.2.1, buffer-crc32@^0.2.13:
+  version "0.2.13"
+  resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242"
+  integrity sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=
+
 buffer-equal-constant-time@1.0.1:
  version "1.0.1"
  resolved "https://registry.yarnpkg.com/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz#f8e71132f7ffe6e01a5c9697a4c6f3e48d5cc819"
@@ -2167,6 +2201,16 @@ compare-versions@3.6.0:
  resolved "https://registry.yarnpkg.com/compare-versions/-/compare-versions-3.6.0.tgz#1a5689913685e5a87637b8d3ffca75514ec41d62"
  integrity sha512-W6Af2Iw1z4CB7q4uU4hv646dW9GQuBM+YpC0UvUCWSD8w90SJjp+ujJuXaEMtAXBtSqGfMPuFOVn4/+FlaqfBA==

+compress-commons@^4.1.0:
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/compress-commons/-/compress-commons-4.1.1.tgz#df2a09a7ed17447642bad10a85cc9a19e5c42a7d"
+  integrity sha512-QLdDLCKNV2dtoTorqgxngQCMA+gWXkM/Nwu7FpeBhk/RdkzimqC3jueb/FDmaZeXh+uby1jkBqE3xArsLBE5wQ==
+  dependencies:
+    buffer-crc32 "^0.2.13"
+    crc32-stream "^4.0.2"
+    normalize-path "^3.0.0"
+    readable-stream "^3.6.0"
+
 concat-map@0.0.1:
  version "0.0.1"
  resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"
@@ -2249,7 +2293,7 @@ core-util-is@1.0.2, core-util-is@~1.0.0:
  resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"
  integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=

-crc-32@1.2.0:
+crc-32@1.2.0, crc-32@^1.2.0:
  version "1.2.0"
  resolved "https://registry.yarnpkg.com/crc-32/-/crc-32-1.2.0.tgz#cb2db6e29b88508e32d9dd0ec1693e7b41a18208"
  integrity sha512-1uBwHxF+Y/4yF5G48fwnKq6QsIXheor3ZLPT80yGBV1oEUwpPojlEhQbWKVw1VwcTQyMGHK1/XMmTjmlsmTTGA==
@@ -2257,6 +2301,14 @@ crc-32@1.2.0:
    exit-on-epipe "~1.0.1"
    printj "~1.1.0"

+crc32-stream@^4.0.2:
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/crc32-stream/-/crc32-stream-4.0.2.tgz#c922ad22b38395abe9d3870f02fa8134ed709007"
+  integrity sha512-DxFZ/Hk473b/muq1VJ///PMNLj0ZMnzye9thBpmjpJKCc5eMgB95aK8zCGrGfQ90cWo561Te6HK9D+j4KPdM6w==
+  dependencies:
+    crc-32 "^1.2.0"
+    readable-stream "^3.4.0"
+
 create-require@^1.1.0:
  version "1.1.1"
  resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333"
@@ -4901,6 +4953,13 @@ langmap@0.0.16:
  resolved "https://registry.yarnpkg.com/langmap/-/langmap-0.0.16.tgz#2fe3e98a531fec0fec546624ebe168c2855bab56"
  integrity sha512-AtYvBK7BsDvWwnSfmO7CfgeUy7GUT1wK3QX8eKH/Ey/eXodqoHuAtvdQ82hmWD9QVFVKnuiNjym9fGY4qSJeLA==

+lazystream@^1.0.0:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/lazystream/-/lazystream-1.0.1.tgz#494c831062f1f9408251ec44db1cba29242a2638"
+  integrity sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==
+  dependencies:
+    readable-stream "^2.0.5"
+
 lcid@^3.0.0:
  version "3.1.1"
  resolved "https://registry.yarnpkg.com/lcid/-/lcid-3.1.1.tgz#9030ec479a058fc36b5e8243ebaac8b6ac582fd0"
@@ -4981,6 +5040,11 @@ lodash.defaults@^4.0.1, lodash.defaults@^4.2.0:
  resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c"
  integrity sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=

+lodash.difference@^4.5.0:
+  version "4.5.0"
+  resolved "https://registry.yarnpkg.com/lodash.difference/-/lodash.difference-4.5.0.tgz#9ccb4e505d486b91651345772885a2df27fd017c"
+  integrity sha1-nMtOUF1Ia5FlE0V3KIWi3yf9AXw=
+
 lodash.filter@^4.4.0:
  version "4.6.0"
  resolved "https://registry.yarnpkg.com/lodash.filter/-/lodash.filter-4.6.0.tgz#668b1d4981603ae1cc5a6fa760143e480b4c4ace"
@@ -5006,6 +5070,11 @@ lodash.isfinite@^3.3.2:
  resolved "https://registry.yarnpkg.com/lodash.isfinite/-/lodash.isfinite-3.3.2.tgz#fb89b65a9a80281833f0b7478b3a5104f898ebb3"
  integrity sha1-+4m2WpqAKBgz8LdHizpRBPiY67M=

+lodash.isplainobject@^4.0.6:
+  version "4.0.6"
+  resolved "https://registry.yarnpkg.com/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz#7c526a52d89b45c45cc690b88163be0497f550cb"
+  integrity sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=
+
 lodash.isregexp@3.0.5:
  version "3.0.5"
  resolved "https://registry.yarnpkg.com/lodash.isregexp/-/lodash.isregexp-3.0.5.tgz#e0f596242f2fa228a840086b6c8ad82e4b71fd2d"
@@ -5051,6 +5120,11 @@ lodash.sortby@^4.7.0:
  resolved "https://registry.yarnpkg.com/lodash.sortby/-/lodash.sortby-4.7.0.tgz#edd14c824e2cc9c1e0b0a1b42bb5210516a42438"
  integrity sha1-7dFMgk4sycHgsKG0K7UhBRakJDg=

+lodash.union@^4.6.0:
+  version "4.6.0"
+  resolved "https://registry.yarnpkg.com/lodash.union/-/lodash.union-4.6.0.tgz#48bb5088409f16f1821666641c44dd1aaae3cd88"
+  integrity sha1-SLtQiECfFvGCFmZkHETdGqrjzYg=
+
 lodash.uniq@^4.5.0:
  version "4.5.0"
  resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773"
@@ -5198,6 +5272,18 @@ mime-db@1.44.0:
  resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.44.0.tgz#fa11c5eb0aca1334b4233cb4d52f10c5a6272f92"
  integrity sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg==

+mime-db@1.51.0:
+  version "1.51.0"
+  resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.51.0.tgz#d9ff62451859b18342d960850dc3cfb77e63fb0c"
+  integrity sha512-5y8A56jg7XVQx2mbv1lu49NR4dokRnhZYTtL+KGfaa27uq4pSTXkwQkFJl4pkRMyNFz/EtYDSkiiEHx3F7UN6g==
+
+mime-types@2.1.34:
+  version "2.1.34"
+  resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.34.tgz#5a712f9ec1503511a945803640fafe09d3793c24"
+  integrity sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A==
+  dependencies:
+    mime-db "1.51.0"
+
 mime-types@^2.1.12, mime-types@^2.1.18, mime-types@^2.1.27, mime-types@~2.1.19, mime-types@~2.1.24:
  version "2.1.27"
  resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.27.tgz#47949f98e279ea53119f5722e0f34e529bec009f"
@@ -6777,7 +6863,7 @@ readable-stream@1.1.x:
    isarray "0.0.1"
    string_decoder "~0.10.x"

-readable-stream@^2.0.6, readable-stream@^2.2.2:
+readable-stream@^2.0.0, readable-stream@^2.0.5, readable-stream@^2.0.6, readable-stream@^2.2.2:
  version "2.3.7"
  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57"
  integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==
@@ -6807,6 +6893,13 @@ readable-web-to-node-stream@^3.0.0:
    "@types/readable-stream" "^2.3.9"
    readable-stream "^3.6.0"

+readdir-glob@^1.0.0:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/readdir-glob/-/readdir-glob-1.1.1.tgz#f0e10bb7bf7bfa7e0add8baffdc54c3f7dbee6c4"
+  integrity sha512-91/k1EzZwDx6HbERR+zucygRFfiPl2zkIYZtv3Jjr6Mn7SkKcVct8aVO+sSRiGMc6fLf72du3d92/uY63YPdEA==
+  dependencies:
+    minimatch "^3.0.4"
+
 readdirp@~3.3.0:
  version "3.3.0"
  resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.3.0.tgz#984458d13a1e42e2e9f5841b129e162f369aff17"
@@ -7664,7 +7757,7 @@ tar-stream@^2.0.0:
    inherits "^2.0.3"
    readable-stream "^3.1.1"

-tar-stream@^2.1.4:
+tar-stream@^2.1.4, tar-stream@^2.2.0:
  version "2.2.0"
  resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287"
  integrity sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==
@@ -8583,3 +8676,12 @@ zen-observable@^0.8.15:
  version "0.8.15"
  resolved "https://registry.yarnpkg.com/zen-observable/-/zen-observable-0.8.15.tgz#96415c512d8e3ffd920afd3889604e30b9eaac15"
  integrity sha512-PQ2PC7R9rslx84ndNBZB/Dkv8V8fZEpk83RLgXtYd0fwUgEjseMn1Dgajh2x6S8QbZAFa9p2qVCEuYZNgve0dQ==
+
+zip-stream@^4.1.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/zip-stream/-/zip-stream-4.1.0.tgz#51dd326571544e36aa3f756430b313576dc8fc79"
+  integrity sha512-zshzwQW7gG7hjpBlgeQP9RuyPGNxvJdzR8SUM3QhxCnLjWN2E7j3dOvpeDcQoETfHx0urRS7EtmVToql7YpU4A==
+  dependencies:
+    archiver-utils "^2.1.0"
+    compress-commons "^4.1.0"
+    readable-stream "^3.6.0"
--- a/packages/client/src/pages/emojis.vue
+++ b/packages/client/src/pages/emojis.vue
@@ -21,10 +21,38 @@ export default defineComponent({
 				title: this.$ts.customEmojis,
 				icon: 'fas fa-laugh',
 				bg: 'var(--bg)',
+				actions: [{
+					icon: 'fas fa-ellipsis-h',
+					handler: this.menu
+				}],
 			})),
 			tab: 'category',
 		}
 	},
+
+	methods: {
+		menu(ev) {
+			os.popupMenu([{
+				icon: 'fas fa-download',
+				text: this.$ts.export,
+				action: async () => {
+					os.api('export-custom-emojis', {
+					})
+					.then(() => {
+						os.alert({
+							type: 'info',
+							text: this.$ts.exportRequested,
+						});
+					}).catch((e) => {
+						os.alert({
+							type: 'error',
+							text: e.message,
+						});
+					});
+				}
+			}], ev.currentTarget || ev.target);
+		}
+	}
 });
 </script>