fix: URLプレビューの動作改善＋動作設定を可能にする (#13579)

* wip

* support new version

* URLプレビュー無効化時、フロント側も非表示にしてリクエストをしないようにする

* fix lint

* fix lint

* tweak preview request error handles

* fix: CHANGELOG.md

* fix

* fix

---------

Co-authored-by: syuilo <4439005+syuilo@users.noreply.github.com>

packages/backend/src/core/entities/MetaEntityService.ts

@@ -111,6 +111,7 @@ export class MetaEntityService {
 			policies: { ...DEFAULT_POLICIES, ...instance.policies },
 
 			mediaProxy: this.config.mediaProxy,
+			enableUrlPreview: instance.urlPreviewEnabled,
 		};
 
 		return packed;

packages/backend/src/models/Meta.ts

@@ -277,12 +277,6 @@ export class MiMeta {
 	})
 	public enableSensitiveMediaDetectionForVideos: boolean;
 
-	@Column('varchar', {
-		length: 1024,
-		nullable: true,
-	})
-	public summalyProxy: string | null;
-
 	@Column('boolean', {
 		default: false,
 	})
@@ -588,4 +582,36 @@ export class MiMeta {
 		default: 0,
 	})
 	public notesPerOneAd: number;
+
+	@Column('boolean', {
+		default: true,
+	})
+	public urlPreviewEnabled: boolean;
+
+	@Column('integer', {
+		default: 10000,
+	})
+	public urlPreviewTimeout: number;
+
+	@Column('bigint', {
+		default: 1024 * 1024 * 10,
+	})
+	public urlPreviewMaximumContentLength: number;
+
+	@Column('boolean', {
+		default: true,
+	})
+	public urlPreviewRequireContentLength: boolean;
+
+	@Column('varchar', {
+		length: 1024,
+		nullable: true,
+	})
+	public urlPreviewSummaryProxyUrl: string | null;
+
+	@Column('varchar', {
+		length: 1024,
+		nullable: true,
+	})
+	public urlPreviewUserAgent: string | null;
 }

packages/backend/src/models/json-schema/meta.ts

@@ -207,6 +207,10 @@ export const packedMetaLiteSchema = {
 			type: 'string',
 			optional: false, nullable: false,
 		},
+		enableUrlPreview: {
+			type: 'boolean',
+			optional: false, nullable: false,
+		},
 		backgroundImageUrl: {
 			type: 'string',
 			optional: false, nullable: true,

packages/backend/src/server/api/endpoints/admin/meta.ts

@@ -434,6 +434,8 @@ export const meta = {
 			summalyProxy: {
 				type: 'string',
 				optional: false, nullable: true,
+				deprecated: true,
+				description: '[Deprecated] Use "urlPreviewSummaryProxyUrl" instead.',
 			},
 			themeColor: {
 				type: 'string',
@@ -451,6 +453,30 @@ export const meta = {
 				type: 'string',
 				optional: false, nullable: false,
 			},
+			urlPreviewEnabled: {
+				type: 'boolean',
+				optional: false, nullable: false,
+			},
+			urlPreviewTimeout: {
+				type: 'number',
+				optional: false, nullable: false,
+			},
+			urlPreviewMaximumContentLength: {
+				type: 'number',
+				optional: false, nullable: false,
+			},
+			urlPreviewRequireContentLength: {
+				type: 'boolean',
+				optional: false, nullable: false,
+			},
+			urlPreviewUserAgent: {
+				type: 'string',
+				optional: false, nullable: true,
+			},
+			urlPreviewSummaryProxyUrl: {
+				type: 'string',
+				optional: false, nullable: true,
+			},
 		},
 	},
 } as const;
@@ -533,7 +559,6 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				setSensitiveFlagAutomatically: instance.setSensitiveFlagAutomatically,
 				enableSensitiveMediaDetectionForVideos: instance.enableSensitiveMediaDetectionForVideos,
 				proxyAccountId: instance.proxyAccountId,
-				summalyProxy: instance.summalyProxy,
 				email: instance.email,
 				smtpSecure: instance.smtpSecure,
 				smtpHost: instance.smtpHost,
@@ -577,6 +602,13 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				perUserHomeTimelineCacheMax: instance.perUserHomeTimelineCacheMax,
 				perUserListTimelineCacheMax: instance.perUserListTimelineCacheMax,
 				notesPerOneAd: instance.notesPerOneAd,
+				summalyProxy: instance.urlPreviewSummaryProxyUrl,
+				urlPreviewEnabled: instance.urlPreviewEnabled,
+				urlPreviewTimeout: instance.urlPreviewTimeout,
+				urlPreviewMaximumContentLength: instance.urlPreviewMaximumContentLength,
+				urlPreviewRequireContentLength: instance.urlPreviewRequireContentLength,
+				urlPreviewUserAgent: instance.urlPreviewUserAgent,
+				urlPreviewSummaryProxyUrl: instance.urlPreviewSummaryProxyUrl,
 			};
 		});
 	}

packages/backend/src/server/api/endpoints/admin/update-meta.ts

@@ -90,7 +90,6 @@ export const paramDef = {
 				type: 'string',
 			},
 		},
-		summalyProxy: { type: 'string', nullable: true },
 		deeplAuthKey: { type: 'string', nullable: true },
 		deeplIsPro: { type: 'boolean' },
 		enableEmail: { type: 'boolean' },
@@ -150,6 +149,16 @@ export const paramDef = {
 				type: 'string',
 			},
 		},
+		summalyProxy: {
+			type: 'string', nullable: true,
+			description: '[Deprecated] Use "urlPreviewSummaryProxyUrl" instead.',
+		},
+		urlPreviewEnabled: { type: 'boolean' },
+		urlPreviewTimeout: { type: 'integer' },
+		urlPreviewMaximumContentLength: { type: 'integer' },
+		urlPreviewRequireContentLength: { type: 'boolean' },
+		urlPreviewUserAgent: { type: 'string', nullable: true },
+		urlPreviewSummaryProxyUrl: { type: 'string', nullable: true },
 	},
 	required: [],
 } as const;
@@ -353,10 +362,6 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				set.langs = ps.langs.filter(Boolean);
 			}
 
-			if (ps.summalyProxy !== undefined) {
-				set.summalyProxy = ps.summalyProxy;
-			}
-
 			if (ps.enableEmail !== undefined) {
 				set.enableEmail = ps.enableEmail;
 			}
@@ -581,6 +586,32 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				set.bannedEmailDomains = ps.bannedEmailDomains;
 			}
 
+			if (ps.urlPreviewEnabled !== undefined) {
+				set.urlPreviewEnabled = ps.urlPreviewEnabled;
+			}
+
+			if (ps.urlPreviewTimeout !== undefined) {
+				set.urlPreviewTimeout = ps.urlPreviewTimeout;
+			}
+
+			if (ps.urlPreviewMaximumContentLength !== undefined) {
+				set.urlPreviewMaximumContentLength = ps.urlPreviewMaximumContentLength;
+			}
+
+			if (ps.urlPreviewRequireContentLength !== undefined) {
+				set.urlPreviewRequireContentLength = ps.urlPreviewRequireContentLength;
+			}
+
+			if (ps.urlPreviewUserAgent !== undefined) {
+				const value = (ps.urlPreviewUserAgent ?? '').trim();
+				set.urlPreviewUserAgent = value === '' ? null : ps.urlPreviewUserAgent;
+			}
+
+			if (ps.summalyProxy !== undefined || ps.urlPreviewSummaryProxyUrl !== undefined) {
+				const value = ((ps.urlPreviewSummaryProxyUrl ?? ps.summalyProxy) ?? '').trim();
+				set.urlPreviewSummaryProxyUrl = value === '' ? null : value;
+			}
+
 			const before = await this.metaService.fetch(true);
 
 			await this.metaService.update(set);

packages/backend/src/server/web/UrlPreviewService.ts

@@ -5,6 +5,7 @@
 
 import { Inject, Injectable } from '@nestjs/common';
 import { summaly } from '@misskey-dev/summaly';
+import { SummalyResult } from '@misskey-dev/summaly/built/summary.js';
 import { DI } from '@/di-symbols.js';
 import type { Config } from '@/config.js';
 import { MetaService } from '@/core/MetaService.js';
@@ -14,6 +15,7 @@ import { query } from '@/misc/prelude/url.js';
 import { LoggerService } from '@/core/LoggerService.js';
 import { bindThis } from '@/decorators.js';
 import { ApiError } from '@/server/api/error.js';
+import { MiMeta } from '@/models/Meta.js';
 import type { FastifyRequest, FastifyReply } from 'fastify';
 
 @Injectable()
@@ -62,24 +64,25 @@ export class UrlPreviewService {
 
 		const meta = await this.metaService.fetch();
 
-		this.logger.info(meta.summalyProxy
+		if (!meta.urlPreviewEnabled) {
+			reply.code(403);
+			return {
+				error: new ApiError({
+					message: 'URL preview is disabled',
+					code: 'URL_PREVIEW_DISABLED',
+					id: '58b36e13-d2f5-0323-b0c6-76aa9dabefb8',
+				}),
+			};
+		}
+
+		this.logger.info(meta.urlPreviewSummaryProxyUrl
 			? `(Proxy) Getting preview of ${url}@${lang} ...`
 			: `Getting preview of ${url}@${lang} ...`);
+
 		try {
-			const summary = meta.summalyProxy ?
-				await this.httpRequestService.getJson<ReturnType<typeof summaly>>(`${meta.summalyProxy}?${query({
-					url: url,
-					lang: lang ?? 'ja-JP',
-				})}`)
-				:
-				await summaly(url, {
-					followRedirects: false,
-					lang: lang ?? 'ja-JP',
-					agent: this.config.proxy ? {
-						http: this.httpRequestService.httpAgent,
-						https: this.httpRequestService.httpsAgent,
-					} : undefined,
-				});
+			const summary = meta.urlPreviewSummaryProxyUrl
+				? await this.fetchSummaryFromProxy(url, meta, lang)
+				: await this.fetchSummary(url, meta, lang);
 
 			this.logger.succ(`Got preview of ${url}: ${summary.title}`);
 
@@ -100,6 +103,7 @@ export class UrlPreviewService {
 			return summary;
 		} catch (err) {
 			this.logger.warn(`Failed to get preview of ${url}: ${err}`);
+
 			reply.code(422);
 			reply.header('Cache-Control', 'max-age=86400, immutable');
 			return {
@@ -111,4 +115,37 @@ export class UrlPreviewService {
 			};
 		}
 	}
+
+	private fetchSummary(url: string, meta: MiMeta, lang?: string): Promise<SummalyResult> {
+		const agent = this.config.proxy
+			? {
+				http: this.httpRequestService.httpAgent,
+				https: this.httpRequestService.httpsAgent,
+			}
+			: undefined;
+
+		return summaly(url, {
+			followRedirects: false,
+			lang: lang ?? 'ja-JP',
+			agent: agent,
+			userAgent: meta.urlPreviewUserAgent ?? undefined,
+			operationTimeout: meta.urlPreviewTimeout,
+			contentLengthLimit: meta.urlPreviewMaximumContentLength,
+			contentLengthRequired: meta.urlPreviewRequireContentLength,
+		});
+	}
+
+	private fetchSummaryFromProxy(url: string, meta: MiMeta, lang?: string): Promise<SummalyResult> {
+		const proxy = meta.urlPreviewSummaryProxyUrl!;
+		const queryStr = query({
+			url: url,
+			lang: lang ?? 'ja-JP',
+			userAgent: meta.urlPreviewUserAgent ?? undefined,
+			operationTimeout: meta.urlPreviewTimeout,
+			contentLengthLimit: meta.urlPreviewMaximumContentLength,
+			contentLengthRequired: meta.urlPreviewRequireContentLength,
+		});
+
+		return this.httpRequestService.getJson<SummalyResult>(`${proxy}?${queryStr}`);
+	}
 }