sendnrw/misskey
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into feat-1714

Browse Source
This commit is contained in:
かっこかり
2024-06-06 17:40:57 +09:00
committed by GitHub
parent 8cdb6bbb9d 8f833d742f
commit 7ed69a8cde
13 changed files with 333 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/backend/src/NestLogger.ts
View File

@@ -7,7 +7,7 @@ import { LoggerService } from '@nestjs/common';
import Logger from '@/logger.js';
const logger = new Logger('core', 'cyan');
const nestLogger = logger.createSubLogger('nest', 'green', false);
const nestLogger = logger.createSubLogger('nest', 'green');
export class NestLogger implements LoggerService {
/**

2
packages/backend/src/boot/entry.ts
View File

@@ -25,7 +25,7 @@ Error.stackTraceLimit = Infinity;
EventEmitter.defaultMaxListeners = 128;
const logger = new Logger('core', 'cyan');
const clusterLogger = logger.createSubLogger('cluster', 'orange', false);
const clusterLogger = logger.createSubLogger('cluster', 'orange');
const ev = new Xev();
//#region Events

2
packages/backend/src/boot/master.ts
View File

@@ -25,7 +25,7 @@ const _dirname = dirname(_filename);
const meta = JSON.parse(fs.readFileSync(`${_dirname}/../../../../built/meta.json`, 'utf-8'));
const logger = new Logger('core', 'cyan');
const bootLogger = logger.createSubLogger('boot', 'magenta', false);
const bootLogger = logger.createSubLogger('boot', 'magenta');
const themeColor = chalk.hex('#86b300');

23
packages/backend/src/boot/worker.ts
View File

@@ -4,13 +4,36 @@
*/
import cluster from 'node:cluster';
import * as Sentry from '@sentry/node';
import { nodeProfilingIntegration } from '@sentry/profiling-node';
import { envOption } from '@/env.js';
import { loadConfig } from '@/config.js';
import { jobQueue, server } from './common.js';
/**
* Init worker process
*/
export async function workerMain() {
const config = loadConfig();
if (config.sentryForBackend) {
Sentry.init({
integrations: [
...(config.sentryForBackend.enableNodeProfiling ? [nodeProfilingIntegration()] : []),
],
// Performance Monitoring
tracesSampleRate: 1.0, // Capture 100% of the transactions
// Set sampling rate for profiling - this is relative to tracesSampleRate
profilesSampleRate: 1.0,
maxBreadcrumbs: 0,
...config.sentryForBackend.options,
});
}
if (envOption.onlyServer) {
await server();
} else if (envOption.onlyQueue) {

4
packages/backend/src/core/LoggerService.ts
View File

@@ -15,7 +15,7 @@ export class LoggerService {
}
@bindThis
public getLogger(domain: string, color?: KEYWORD | undefined, store?: boolean) {
return new Logger(domain, color, store);
public getLogger(domain: string, color?: KEYWORD | undefined) {
return new Logger(domain, color);
}
}

2
packages/backend/src/core/chart/ChartLoggerService.ts
View File

@@ -14,6 +14,6 @@ export class ChartLoggerService {
constructor(
private loggerService: LoggerService,
) {
this.logger = this.loggerService.getLogger('chart', 'white', process.env.NODE_ENV !== 'test');
this.logger = this.loggerService.getLogger('chart', 'white');
}
}

14
packages/backend/src/logger.ts
View File

@@ -22,31 +22,27 @@ type Level = 'error' | 'success' | 'warning' | 'debug' | 'info';
export default class Logger {
private context: Context;
private parentLogger: Logger | null = null;
private store: boolean;
constructor(context: string, color?: KEYWORD, store = true) {
constructor(context: string, color?: KEYWORD) {
this.context = {
name: context,
color: color,
};
this.store = store;
}
@bindThis
public createSubLogger(context: string, color?: KEYWORD, store = true): Logger {
const logger = new Logger(context, color, store);
public createSubLogger(context: string, color?: KEYWORD): Logger {
const logger = new Logger(context, color);
logger.parentLogger = this;
return logger;
}
@bindThis
private log(level: Level, message: string, data?: Record<string, any> | null, important = false, subContexts: Context[] = [], store = true): void {
private log(level: Level, message: string, data?: Record<string, any> | null, important = false, subContexts: Context[] = []): void {
if (envOption.quiet) return;
if (!this.store) store = false;
if (level === 'debug') store = false;
if (this.parentLogger) {
this.parentLogger.log(level, message, data, important, [this.context].concat(subContexts), store);
this.parentLogger.log(level, message, data, important, [this.context].concat(subContexts));
return;
}

2
packages/backend/src/postgres.ts
View File

@@ -85,7 +85,7 @@ import { bindThis } from '@/decorators.js';
export const dbLogger = new MisskeyLogger('db');
const sqlLogger = dbLogger.createSubLogger('sql', 'gray', false);
const sqlLogger = dbLogger.createSubLogger('sql', 'gray');
class MyCustomLogger implements Logger {
@bindThis

152
packages/backend/src/queue/QueueProcessorService.ts
View File

@@ -5,6 +5,7 @@
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
import * as Bull from 'bullmq';
import * as Sentry from '@sentry/node';
import type { Config } from '@/config.js';
import { DI } from '@/di-symbols.js';
import type Logger from '@/logger.js';
@@ -135,7 +136,8 @@ export class QueueProcessorService implements OnApplicationShutdown {
}
//#region system
this.systemQueueWorker = new Bull.Worker(QUEUE.SYSTEM, (job) => {
{
const processer = (job: Bull.Job) => {
switch (job.name) {
case 'tickCharts': return this.tickChartsProcessorService.process();
case 'resyncCharts': return this.resyncChartsProcessorService.process();
@@ -145,6 +147,14 @@ export class QueueProcessorService implements OnApplicationShutdown {
case 'clean': return this.cleanProcessorService.process();
default: throw new Error(`unrecognized job type ${job.name} for system`);
}
};
this.systemQueueWorker = new Bull.Worker(QUEUE.SYSTEM, (job) => {
if (this.config.sentryForBackend) {
return Sentry.startSpan({ name: 'Queue: System: ' + job.name }, () => processer(job));
} else {
return processer(job);
}
}, {
...baseQueueOptions(this.config, QUEUE.SYSTEM),
autorun: false,
@@ -155,13 +165,22 @@ export class QueueProcessorService implements OnApplicationShutdown {
this.systemQueueWorker
.on('active', (job) => systemLogger.debug(`active id=${job.id}`))
.on('completed', (job, result) => systemLogger.debug(`completed(${result}) id=${job.id}`))
.on('failed', (job, err) => systemLogger.warn(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }))
.on('failed', (job, err: Error) => {
systemLogger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) });
if (config.sentryForBackend) {
Sentry.captureMessage(`Queue: System: ${job?.name ?? '?'}: ${err.message}`, {
extra: { job, err },
});
}
})
.on('error', (err: Error) => systemLogger.error(`error ${err.stack}`, { e: renderError(err) }))
.on('stalled', (jobId) => systemLogger.warn(`stalled id=${jobId}`));
}
//#endregion
//#region db
this.dbQueueWorker = new Bull.Worker(QUEUE.DB, (job) => {
{
const processer = (job: Bull.Job) => {
switch (job.name) {
case 'deleteDriveFiles': return this.deleteDriveFilesProcessorService.process(job);
case 'exportCustomEmojis': return this.exportCustomEmojisProcessorService.process(job);
@@ -184,6 +203,14 @@ export class QueueProcessorService implements OnApplicationShutdown {
case 'deleteAccount': return this.deleteAccountProcessorService.process(job);
default: throw new Error(`unrecognized job type ${job.name} for db`);
}
};
this.dbQueueWorker = new Bull.Worker(QUEUE.DB, (job) => {
if (this.config.sentryForBackend) {
return Sentry.startSpan({ name: 'Queue: DB: ' + job.name }, () => processer(job));
} else {
return processer(job);
}
}, {
...baseQueueOptions(this.config, QUEUE.DB),
autorun: false,
@@ -194,13 +221,28 @@ export class QueueProcessorService implements OnApplicationShutdown {
this.dbQueueWorker
.on('active', (job) => dbLogger.debug(`active id=${job.id}`))
.on('completed', (job, result) => dbLogger.debug(`completed(${result}) id=${job.id}`))
.on('failed', (job, err) => dbLogger.warn(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }))
.on('failed', (job, err) => {
dbLogger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) });
if (config.sentryForBackend) {
Sentry.captureMessage(`Queue: DB: ${job?.name ?? '?'}: ${err.message}`, {
extra: { job, err },
});
}
})
.on('error', (err: Error) => dbLogger.error(`error ${err.stack}`, { e: renderError(err) }))
.on('stalled', (jobId) => dbLogger.warn(`stalled id=${jobId}`));
}
//#endregion
//#region deliver
this.deliverQueueWorker = new Bull.Worker(QUEUE.DELIVER, (job) => this.deliverProcessorService.process(job), {
{
this.deliverQueueWorker = new Bull.Worker(QUEUE.DELIVER, (job) => {
if (this.config.sentryForBackend) {
return Sentry.startSpan({ name: 'Queue: Deliver' }, () => this.deliverProcessorService.process(job));
} else {
return this.deliverProcessorService.process(job);
}
}, {
...baseQueueOptions(this.config, QUEUE.DELIVER),
autorun: false,
concurrency: this.config.deliverJobConcurrency ?? 128,
@@ -218,13 +260,28 @@ export class QueueProcessorService implements OnApplicationShutdown {
this.deliverQueueWorker
.on('active', (job) => deliverLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`))
.on('completed', (job, result) => deliverLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`))
.on('failed', (job, err) => deliverLogger.warn(`failed(${err.stack}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`))
.on('failed', (job, err) => {
deliverLogger.error(`failed(${err.stack}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`);
if (config.sentryForBackend) {
Sentry.captureMessage(`Queue: Deliver: ${err.message}`, {
extra: { job, err },
});
}
})
.on('error', (err: Error) => deliverLogger.error(`error ${err.stack}`, { e: renderError(err) }))
.on('stalled', (jobId) => deliverLogger.warn(`stalled id=${jobId}`));
}
//#endregion
//#region inbox
this.inboxQueueWorker = new Bull.Worker(QUEUE.INBOX, (job) => this.inboxProcessorService.process(job), {
{
this.inboxQueueWorker = new Bull.Worker(QUEUE.INBOX, (job) => {
if (this.config.sentryForBackend) {
return Sentry.startSpan({ name: 'Queue: Inbox' }, () => this.inboxProcessorService.process(job));
} else {
return this.inboxProcessorService.process(job);
}
}, {
...baseQueueOptions(this.config, QUEUE.INBOX),
autorun: false,
concurrency: this.config.inboxJobConcurrency ?? 16,
@@ -242,13 +299,28 @@ export class QueueProcessorService implements OnApplicationShutdown {
this.inboxQueueWorker
.on('active', (job) => inboxLogger.debug(`active ${getJobInfo(job, true)}`))
.on('completed', (job, result) => inboxLogger.debug(`completed(${result}) ${getJobInfo(job, true)}`))
.on('failed', (job, err) => inboxLogger.warn(`failed(${err.stack}) ${getJobInfo(job)} activity=${job ? (job.data.activity ? job.data.activity.id : 'none') : '-'}`, { job, e: renderError(err) }))
.on('failed', (job, err) => {
inboxLogger.error(`failed(${err.stack}) ${getJobInfo(job)} activity=${job ? (job.data.activity ? job.data.activity.id : 'none') : '-'}`, { job, e: renderError(err) });
if (config.sentryForBackend) {
Sentry.captureMessage(`Queue: Inbox: ${err.message}`, {
extra: { job, err },
});
}
})
.on('error', (err: Error) => inboxLogger.error(`error ${err.stack}`, { e: renderError(err) }))
.on('stalled', (jobId) => inboxLogger.warn(`stalled id=${jobId}`));
}
//#endregion
//#region webhook deliver
this.webhookDeliverQueueWorker = new Bull.Worker(QUEUE.WEBHOOK_DELIVER, (job) => this.webhookDeliverProcessorService.process(job), {
{
this.webhookDeliverQueueWorker = new Bull.Worker(QUEUE.WEBHOOK_DELIVER, (job) => {
if (this.config.sentryForBackend) {
return Sentry.startSpan({ name: 'Queue: WebhookDeliver' }, () => this.webhookDeliverProcessorService.process(job));
} else {
return this.webhookDeliverProcessorService.process(job);
}
}, {
...baseQueueOptions(this.config, QUEUE.WEBHOOK_DELIVER),
autorun: false,
concurrency: 64,
@@ -266,13 +338,22 @@ export class QueueProcessorService implements OnApplicationShutdown {
this.webhookDeliverQueueWorker
.on('active', (job) => webhookLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`))
.on('completed', (job, result) => webhookLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`))
.on('failed', (job, err) => webhookLogger.warn(`failed(${err.stack}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`))
.on('failed', (job, err) => {
webhookLogger.error(`failed(${err.stack}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`);
if (config.sentryForBackend) {
Sentry.captureMessage(`Queue: WebhookDeliver: ${err.message}`, {
extra: { job, err },
});
}
})
.on('error', (err: Error) => webhookLogger.error(`error ${err.stack}`, { e: renderError(err) }))
.on('stalled', (jobId) => webhookLogger.warn(`stalled id=${jobId}`));
}
//#endregion
//#region relationship
this.relationshipQueueWorker = new Bull.Worker(QUEUE.RELATIONSHIP, (job) => {
{
const processer = (job: Bull.Job) => {
switch (job.name) {
case 'follow': return this.relationshipProcessorService.processFollow(job);
case 'unfollow': return this.relationshipProcessorService.processUnfollow(job);
@@ -280,6 +361,14 @@ export class QueueProcessorService implements OnApplicationShutdown {
case 'unblock': return this.relationshipProcessorService.processUnblock(job);
default: throw new Error(`unrecognized job type ${job.name} for relationship`);
}
};
this.relationshipQueueWorker = new Bull.Worker(QUEUE.RELATIONSHIP, (job) => {
if (this.config.sentryForBackend) {
return Sentry.startSpan({ name: 'Queue: Relationship: ' + job.name }, () => processer(job));
} else {
return processer(job);
}
}, {
...baseQueueOptions(this.config, QUEUE.RELATIONSHIP),
autorun: false,
@@ -295,18 +384,35 @@ export class QueueProcessorService implements OnApplicationShutdown {
this.relationshipQueueWorker
.on('active', (job) => relationshipLogger.debug(`active id=${job.id}`))
.on('completed', (job, result) => relationshipLogger.debug(`completed(${result}) id=${job.id}`))
.on('failed', (job, err) => relationshipLogger.warn(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }))
.on('failed', (job, err) => {
relationshipLogger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) });
if (config.sentryForBackend) {
Sentry.captureMessage(`Queue: Relationship: ${job?.name ?? '?'}: ${err.message}`, {
extra: { job, err },
});
}
})
.on('error', (err: Error) => relationshipLogger.error(`error ${err.stack}`, { e: renderError(err) }))
.on('stalled', (jobId) => relationshipLogger.warn(`stalled id=${jobId}`));
}
//#endregion
//#region object storage
this.objectStorageQueueWorker = new Bull.Worker(QUEUE.OBJECT_STORAGE, (job) => {
{
const processer = (job: Bull.Job) => {
switch (job.name) {
case 'deleteFile': return this.deleteFileProcessorService.process(job);
case 'cleanRemoteFiles': return this.cleanRemoteFilesProcessorService.process(job);
default: throw new Error(`unrecognized job type ${job.name} for objectStorage`);
}
};
this.objectStorageQueueWorker = new Bull.Worker(QUEUE.OBJECT_STORAGE, (job) => {
if (this.config.sentryForBackend) {
return Sentry.startSpan({ name: 'Queue: ObjectStorage: ' + job.name }, () => processer(job));
} else {
return processer(job);
}
}, {
...baseQueueOptions(this.config, QUEUE.OBJECT_STORAGE),
autorun: false,
@@ -318,16 +424,32 @@ export class QueueProcessorService implements OnApplicationShutdown {
this.objectStorageQueueWorker
.on('active', (job) => objectStorageLogger.debug(`active id=${job.id}`))
.on('completed', (job, result) => objectStorageLogger.debug(`completed(${result}) id=${job.id}`))
.on('failed', (job, err) => objectStorageLogger.warn(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }))
.on('failed', (job, err) => {
objectStorageLogger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) });
if (config.sentryForBackend) {
Sentry.captureMessage(`Queue: ObjectStorage: ${job?.name ?? '?'}: ${err.message}`, {
extra: { job, err },
});
}
})
.on('error', (err: Error) => objectStorageLogger.error(`error ${err.stack}`, { e: renderError(err) }))
.on('stalled', (jobId) => objectStorageLogger.warn(`stalled id=${jobId}`));
}
//#endregion
//#region ended poll notification
this.endedPollNotificationQueueWorker = new Bull.Worker(QUEUE.ENDED_POLL_NOTIFICATION, (job) => this.endedPollNotificationProcessorService.process(job), {
{
this.endedPollNotificationQueueWorker = new Bull.Worker(QUEUE.ENDED_POLL_NOTIFICATION, (job) => {
if (this.config.sentryForBackend) {
return Sentry.startSpan({ name: 'Queue: EndedPollNotification' }, () => this.endedPollNotificationProcessorService.process(job));
} else {
return this.endedPollNotificationProcessorService.process(job);
}
}, {
...baseQueueOptions(this.config, QUEUE.ENDED_POLL_NOTIFICATION),
autorun: false,
});
}
//#endregion
}

2
packages/backend/src/server/FileServerService.ts
View File

@@ -53,7 +53,7 @@ export class FileServerService {
private internalStorageService: InternalStorageService,
private loggerService: LoggerService,
) {
this.logger = this.loggerService.getLogger('server', 'gray', false);
this.logger = this.loggerService.getLogger('server', 'gray');
//this.createServer = this.createServer.bind(this);
}

2
packages/backend/src/server/ServerService.ts
View File

@@ -68,7 +68,7 @@ export class ServerService implements OnApplicationShutdown {
private loggerService: LoggerService,
private oauth2ProviderService: OAuth2ProviderService,
) {
this.logger = this.loggerService.getLogger('server', 'gray', false);
this.logger = this.loggerService.getLogger('server', 'gray');
}
@bindThis

14
packages/backend/src/server/api/ApiCallService.ts
View File

@@ -93,7 +93,7 @@ export class ApiCallService implements OnApplicationShutdown {
}
}
#onExecError(ep: IEndpoint, data: any, err: Error): void {
#onExecError(ep: IEndpoint, data: any, err: Error, userId?: MiUser['id']): void {
if (err instanceof ApiError || err instanceof AuthenticationError) {
throw err;
} else {
@@ -108,10 +108,12 @@ export class ApiCallService implements OnApplicationShutdown {
id: errId,
},
});
console.error(err, errId);
if (this.config.sentryForBackend) {
Sentry.captureMessage(`Internal error occurred in ${ep.name}: ${err.message}`, {
user: {
id: userId,
},
extra: {
ep: ep.name,
ps: data,
@@ -410,9 +412,13 @@ export class ApiCallService implements OnApplicationShutdown {
// API invoking
if (this.config.sentryForBackend) {
return await Sentry.startSpan({ name: 'API: ' + ep.name }, () => ep.exec(data, user, token, file, request.ip, request.headers).catch((err: Error) => this.#onExecError(ep, data, err)));
return await Sentry.startSpan({
name: 'API: ' + ep.name,
}, () => ep.exec(data, user, token, file, request.ip, request.headers)
.catch((err: Error) => this.#onExecError(ep, data, err, user?.id)));
} else {
return await ep.exec(data, user, token, file, request.ip, request.headers).catch((err: Error) => this.#onExecError(ep, data, err));
return await ep.exec(data, user, token, file, request.ip, request.headers)
.catch((err: Error) => this.#onExecError(ep, data, err, user?.id));
}
}

9
packages/frontend/src/pages/about-misskey.vue
View File

@@ -102,13 +102,16 @@ SPDX-License-Identifier: AGPL-3.0-only
<template #label>Special thanks</template>
<div style="display:grid;grid-template-columns:repeat(auto-fill, minmax(130px, 1fr));grid-gap:24px;align-items:center;">
<div>
<a style="display: inline-block;" class="masknetwork" title="Mask Network" href="https://mask.io/" target="_blank"><img style="width: 100%;" src="https://misskey-hub.net/sponsors/masknetwork.png" alt="Mask Network"></a>
<a style="display: inline-block;" class="masknetwork" title="Mask Network" href="https://mask.io/" target="_blank"><img style="width: 100%;" src="https://assets.misskey-hub.net/sponsors/masknetwork.png" alt="Mask Network"></a>
</div>
<div>
<a style="display: inline-block;" class="xserver" title="XServer" href="https://www.xserver.ne.jp/" target="_blank"><img style="width: 100%;" src="https://misskey-hub.net/sponsors/xserver.png" alt="XServer"></a>
<a style="display: inline-block;" class="xserver" title="XServer" href="https://www.xserver.ne.jp/" target="_blank"><img style="width: 100%;" src="https://assets.misskey-hub.net/sponsors/xserver.png" alt="XServer"></a>
</div>
<div>
<a style="display: inline-block;" class="skeb" title="Skeb" href="https://skeb.jp/" target="_blank"><img style="width: 100%;" src="https://misskey-hub.net/sponsors/skeb.svg" alt="Skeb"></a>
<a style="display: inline-block;" class="skeb" title="Skeb" href="https://skeb.jp/" target="_blank"><img style="width: 100%;" src="https://assets.misskey-hub.net/sponsors/skeb.svg" alt="Skeb"></a>
</div>
<div>
<a style="display: inline-block;" class="pepabo" title="GMO Pepabo" href="https://pepabo.com/" target="_blank"><img style="width: 100%;" src="https://assets.misskey-hub.net/sponsors/gmo_pepabo.svg" alt="GMO Pepabo"></a>
</div>
</div>
</FormSection>