sendnrw/misskey
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve chart engine (#8253)

Browse Source 
* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update core.ts

* wip

* wip

* #7361

* delete network chart

* federationChart強化 apRequestChart追加

* tweak
This commit is contained in:
syuilo
2022-02-06 00:13:52 +09:00
committed by GitHub
parent 0b462feff6
commit c1b264e4e9
65 changed files with 1616 additions and 1756 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
packages/backend/src/services/chart/charts/active-users.ts
View File

@@ -1,51 +1,28 @@
import autobind from 'autobind-decorator';
import Chart, { Obj, DeepPartial } from '../core';
import Chart, { KVs } from '../core';
import { User } from '@/models/entities/user';
import { SchemaType } from '@/misc/schema';
import { Users } from '@/models/index';
import { name, schema } from './entities/active-users';
type ActiveUsersLog = SchemaType<typeof schema>;
/**
* アクティブユーザーに関するチャート
*/
// eslint-disable-next-line import/no-default-export
export default class ActiveUsersChart extends Chart<ActiveUsersLog> {
export default class ActiveUsersChart extends Chart<typeof schema> {
constructor() {
super(name, schema);
}
@autobind
protected genNewLog(latest: ActiveUsersLog): DeepPartial<ActiveUsersLog> {
return {};
}
@autobind
protected aggregate(logs: ActiveUsersLog[]): ActiveUsersLog {
return {
local: {
users: logs.reduce((a, b) => a.concat(b.local.users), [] as ActiveUsersLog['local']['users']),
},
remote: {
users: logs.reduce((a, b) => a.concat(b.remote.users), [] as ActiveUsersLog['remote']['users']),
},
};
}
@autobind
protected async fetchActual(): Promise<DeepPartial<ActiveUsersLog>> {
protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
@autobind
public async update(user: { id: User['id'], host: User['host'] }): Promise<void> {
const update: Obj = {
users: [user.id],
};
await this.inc({
[Users.isLocalUser(user) ? 'local' : 'remote']: update,
await this.commit({
'local.users': Users.isLocalUser(user) ? [user.id] : [],
'remote.users': Users.isLocalUser(user) ? [] : [user.id],
});
}
}

39
packages/backend/src/services/chart/charts/ap-request.ts Normal file
View File

@@ -0,0 +1,39 @@
import autobind from 'autobind-decorator';
import Chart, { KVs } from '../core';
import { name, schema } from './entities/ap-request';
/**
* Chart about ActivityPub requests
*/
// eslint-disable-next-line import/no-default-export
export default class ApRequestChart extends Chart<typeof schema> {
constructor() {
super(name, schema);
}
@autobind
protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
@autobind
public async deliverSucc(): Promise<void> {
await this.commit({
'deliverSucceeded': 1,
});
}
@autobind
public async deliverFail(): Promise<void> {
await this.commit({
'deliverFailed': 1,
});
}
@autobind
public async inbox(): Promise<void> {
await this.commit({
'inboxReceived': 1,
});
}
}

88
packages/backend/src/services/chart/charts/drive.ts
View File

@@ -1,95 +1,37 @@
import autobind from 'autobind-decorator';
import Chart, { Obj, DeepPartial } from '../core';
import { SchemaType } from '@/misc/schema';
import Chart, { KVs } from '../core';
import { DriveFiles } from '@/models/index';
import { Not, IsNull } from 'typeorm';
import { DriveFile } from '@/models/entities/drive-file';
import { name, schema } from './entities/drive';
type DriveLog = SchemaType<typeof schema>;
/**
* ドライブに関するチャート
*/
// eslint-disable-next-line import/no-default-export
export default class DriveChart extends Chart<DriveLog> {
export default class DriveChart extends Chart<typeof schema> {
constructor() {
super(name, schema);
}
@autobind
protected genNewLog(latest: DriveLog): DeepPartial<DriveLog> {
return {
local: {
totalCount: latest.local.totalCount,
totalSize: latest.local.totalSize,
},
remote: {
totalCount: latest.remote.totalCount,
totalSize: latest.remote.totalSize,
},
};
}
@autobind
protected aggregate(logs: DriveLog[]): DriveLog {
return {
local: {
totalCount: logs[0].local.totalCount,
totalSize: logs[0].local.totalSize,
incCount: logs.reduce((a, b) => a + b.local.incCount, 0),
incSize: logs.reduce((a, b) => a + b.local.incSize, 0),
decCount: logs.reduce((a, b) => a + b.local.decCount, 0),
decSize: logs.reduce((a, b) => a + b.local.decSize, 0),
},
remote: {
totalCount: logs[0].remote.totalCount,
totalSize: logs[0].remote.totalSize,
incCount: logs.reduce((a, b) => a + b.remote.incCount, 0),
incSize: logs.reduce((a, b) => a + b.remote.incSize, 0),
decCount: logs.reduce((a, b) => a + b.remote.decCount, 0),
decSize: logs.reduce((a, b) => a + b.remote.decSize, 0),
},
};
}
@autobind
protected async fetchActual(): Promise<DeepPartial<DriveLog>> {
const [localCount, remoteCount, localSize, remoteSize] = await Promise.all([
DriveFiles.count({ userHost: null }),
DriveFiles.count({ userHost: Not(IsNull()) }),
DriveFiles.calcDriveUsageOfLocal(),
DriveFiles.calcDriveUsageOfRemote(),
]);
return {
local: {
totalCount: localCount,
totalSize: localSize,
},
remote: {
totalCount: remoteCount,
totalSize: remoteSize,
},
};
protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
@autobind
public async update(file: DriveFile, isAdditional: boolean): Promise<void> {
const update: Obj = {};
update.totalCount = isAdditional ? 1 : -1;
update.totalSize = isAdditional ? file.size : -file.size;
if (isAdditional) {
update.incCount = 1;
update.incSize = file.size;
} else {
update.decCount = 1;
update.decSize = file.size;
}
await this.inc({
[file.userHost === null ? 'local' : 'remote']: update,
const fileSizeKb = file.size / 1000;
await this.commit(file.userHost === null ? {
'local.incCount': isAdditional ? 1 : 0,
'local.incSize': isAdditional ? fileSizeKb : 0,
'local.decCount': isAdditional ? 0 : 1,
'local.decSize': isAdditional ? 0 : fileSizeKb,
} : {
'remote.incCount': isAdditional ? 1 : 0,
'remote.incSize': isAdditional ? fileSizeKb : 0,
'remote.decCount': isAdditional ? 0 : 1,
'remote.decSize': isAdditional ? 0 : fileSizeKb,
});
}
}

32
packages/backend/src/services/chart/charts/entities/active-users.ts
View File

@@ -2,35 +2,9 @@ import Chart from '../../core';
export const name = 'activeUsers';
const logSchema = {
/**
* アクティブユーザー
*/
users: {
type: 'array' as const,
optional: false as const, nullable: false as const,
items: {
type: 'string' as const,
optional: false as const, nullable: false as const,
},
},
};
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
local: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema,
},
remote: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema,
},
},
};
'local.users': { uniqueIncrement: true },
'remote.users': { uniqueIncrement: true },
} as const;
export const entity = Chart.schemaToEntity(name, schema);

11
packages/backend/src/services/chart/charts/entities/ap-request.ts Normal file
View File

@@ -0,0 +1,11 @@
import Chart from '../../core';
export const name = 'apRequest';
export const schema = {
'deliverFailed': { },
'deliverSucceeded': { },
'inboxReceived': { },
} as const;
export const entity = Chart.schemaToEntity(name, schema);

74
packages/backend/src/services/chart/charts/entities/drive.ts
View File

@@ -2,71 +2,15 @@ import Chart from '../../core';
export const name = 'drive';
const logSchema = {
/**
* 集計期間時点での、全ドライブファイル数
*/
totalCount: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* 集計期間時点での、全ドライブファイルの合計サイズ
*/
totalSize: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* 増加したドライブファイル数
*/
incCount: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* 増加したドライブ使用量
*/
incSize: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* 減少したドライブファイル数
*/
decCount: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* 減少したドライブ使用量
*/
decSize: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
};
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
local: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema,
},
remote: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema,
},
},
};
'local.incCount': {},
'local.incSize': {}, // in kilobyte
'local.decCount': {},
'local.decSize': {}, // in kilobyte
'remote.incCount': {},
'remote.incSize': {}, // in kilobyte
'remote.decCount': {},
'remote.decSize': {}, // in kilobyte
} as const;
export const entity = Chart.schemaToEntity(name, schema);

29
packages/backend/src/services/chart/charts/entities/federation.ts
View File

@@ -3,28 +3,11 @@ import Chart from '../../core';
export const name = 'federation';
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
instance: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
total: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
inc: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
dec: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
},
},
},
};
'instance.total': { accumulate: true },
'instance.inc': { range: 'small' },
'instance.dec': { range: 'small' },
'deliveredInstances': { uniqueIncrement: true, range: 'small' },
'inboxInstances': { uniqueIncrement: true, range: 'small' },
} as const;
export const entity = Chart.schemaToEntity(name, schema);

32
packages/backend/src/services/chart/charts/entities/hashtag.ts
View File

@@ -2,35 +2,9 @@ import Chart from '../../core';
export const name = 'hashtag';
const logSchema = {
/**
* 投稿したユーザー
*/
users: {
type: 'array' as const,
optional: false as const, nullable: false as const,
items: {
type: 'string' as const,
optional: false as const, nullable: false as const,
},
},
};
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
local: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema,
},
remote: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema,
},
},
};
'local.users': { uniqueIncrement: true },
'remote.users': { uniqueIncrement: true },
} as const;
export const entity = Chart.schemaToEntity(name, schema, true);

175
packages/backend/src/services/chart/charts/entities/instance.ts
View File

@@ -3,156 +3,29 @@ import Chart from '../../core';
export const name = 'instance';
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
requests: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
failed: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
succeeded: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
received: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
},
},
notes: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
total: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
inc: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
dec: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
diffs: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
normal: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
reply: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
renote: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
},
},
},
},
users: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
total: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
inc: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
dec: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
},
},
following: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
total: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
inc: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
dec: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
},
},
followers: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
total: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
inc: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
dec: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
},
},
drive: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
totalFiles: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
totalUsage: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
incFiles: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
incUsage: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
decFiles: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
decUsage: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
},
},
},
};
'requests.failed': { range: 'small' },
'requests.succeeded': { range: 'small' },
'requests.received': { range: 'small' },
'notes.total': { accumulate: true },
'notes.inc': {},
'notes.dec': {},
'notes.diffs.normal': {},
'notes.diffs.reply': {},
'notes.diffs.renote': {},
'users.total': { accumulate: true },
'users.inc': { range: 'small' },
'users.dec': { range: 'small' },
'following.total': { accumulate: true },
'following.inc': { range: 'small' },
'following.dec': { range: 'small' },
'followers.total': { accumulate: true },
'followers.inc': { range: 'small' },
'followers.dec': { range: 'small' },
'drive.totalFiles': { accumulate: true },
'drive.incFiles': {},
'drive.decFiles': {},
'drive.incUsage': {}, // in kilobyte
'drive.decUsage': {}, // in kilobyte
} as const;
export const entity = Chart.schemaToEntity(name, schema, true);

32
packages/backend/src/services/chart/charts/entities/network.ts
View File

@@ -1,32 +0,0 @@
import Chart from '../../core';
export const name = 'network';
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
incomingRequests: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
outgoingRequests: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
totalTime: { // TIP: (totalTime / incomingRequests) でひとつのリクエストに平均でどれくらいの時間がかかったか知れる
type: 'number' as const,
optional: false as const, nullable: false as const,
},
incomingBytes: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
outgoingBytes: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
},
};
export const entity = Chart.schemaToEntity(name, schema);

66
packages/backend/src/services/chart/charts/entities/notes.ts
View File

@@ -2,59 +2,19 @@ import Chart from '../../core';
export const name = 'notes';
const logSchema = {
total: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
inc: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
dec: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
diffs: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
normal: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
reply: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
renote: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
},
},
};
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
local: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema,
},
remote: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema,
},
},
};
'local.total': { accumulate: true },
'local.inc': {},
'local.dec': {},
'local.diffs.normal': {},
'local.diffs.reply': {},
'local.diffs.renote': {},
'remote.total': { accumulate: true },
'remote.inc': {},
'remote.dec': {},
'remote.diffs.normal': {},
'remote.diffs.reply': {},
'remote.diffs.renote': {},
} as const;
export const entity = Chart.schemaToEntity(name, schema);

59
packages/backend/src/services/chart/charts/entities/per-user-drive.ts
View File

@@ -3,57 +3,12 @@ import Chart from '../../core';
export const name = 'perUserDrive';
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
/**
* 集計期間時点での、全ドライブファイル数
*/
totalCount: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* 集計期間時点での、全ドライブファイルの合計サイズ
*/
totalSize: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* 増加したドライブファイル数
*/
incCount: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* 増加したドライブ使用量
*/
incSize: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* 減少したドライブファイル数
*/
decCount: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* 減少したドライブ使用量
*/
decSize: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
},
};
'totalCount': { accumulate: true },
'totalSize': { accumulate: true }, // in kilobyte
'incCount': { range: 'small' },
'incSize': {}, // in kilobyte
'decCount': { range: 'small' },
'decSize': {}, // in kilobyte
} as const;
export const entity = Chart.schemaToEntity(name, schema, true);

96
packages/backend/src/services/chart/charts/entities/per-user-following.ts
View File

@@ -2,89 +2,19 @@ import Chart from '../../core';
export const name = 'perUserFollowing';
const logSchema = {
/**
* フォローしている
*/
followings: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
/**
* フォローしている合計
*/
total: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* フォローした数
*/
inc: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* フォロー解除した数
*/
dec: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
},
},
/**
* フォローされている
*/
followers: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
/**
* フォローされている合計
*/
total: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* フォローされた数
*/
inc: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* フォロー解除された数
*/
dec: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
},
},
};
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
local: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema,
},
remote: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema,
},
},
};
'local.followings.total': { accumulate: true },
'local.followings.inc': { range: 'small' },
'local.followings.dec': { range: 'small' },
'local.followers.total': { accumulate: true },
'local.followers.inc': { range: 'small' },
'local.followers.dec': { range: 'small' },
'remote.followings.total': { accumulate: true },
'remote.followings.inc': { range: 'small' },
'remote.followings.dec': { range: 'small' },
'remote.followers.total': { accumulate: true },
'remote.followers.inc': { range: 'small' },
'remote.followers.dec': { range: 'small' },
} as const;
export const entity = Chart.schemaToEntity(name, schema, true);

47
packages/backend/src/services/chart/charts/entities/per-user-notes.ts
View File

@@ -3,45 +3,12 @@ import Chart from '../../core';
export const name = 'perUserNotes';
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
total: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
inc: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
dec: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
diffs: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
normal: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
reply: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
renote: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
},
},
},
};
'total': { accumulate: true },
'inc': { range: 'small' },
'dec': { range: 'small' },
'diffs.normal': { range: 'small' },
'diffs.reply': { range: 'small' },
'diffs.renote': { range: 'small' },
} as const;
export const entity = Chart.schemaToEntity(name, schema, true);

28
packages/backend/src/services/chart/charts/entities/per-user-reactions.ts
View File

@@ -2,31 +2,9 @@ import Chart from '../../core';
export const name = 'perUserReaction';
const logSchema = {
/**
* 被リアクション数
*/
count: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
};
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
local: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema,
},
remote: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema,
},
},
};
'local.count': { range: 'small' },
'remote.count': { range: 'small' },
} as const;
export const entity = Chart.schemaToEntity(name, schema, true);

29
packages/backend/src/services/chart/charts/entities/test-grouped.ts
View File

@@ -3,30 +3,9 @@ import Chart from '../../core';
export const name = 'testGrouped';
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
foo: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
total: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
inc: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
dec: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
},
},
},
};
'foo.total': { accumulate: true },
'foo.inc': {},
'foo.dec': {},
} as const;
export const entity = Chart.schemaToEntity(name, schema, true);

15
packages/backend/src/services/chart/charts/entities/test-unique.ts
View File

@@ -3,18 +3,7 @@ import Chart from '../../core';
export const name = 'testUnique';
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
foo: {
type: 'array' as const,
optional: false as const, nullable: false as const,
items: {
type: 'string' as const,
optional: false as const, nullable: false as const,
},
},
},
};
'foo': { uniqueIncrement: true },
} as const;
export const entity = Chart.schemaToEntity(name, schema);

29
packages/backend/src/services/chart/charts/entities/test.ts
View File

@@ -3,30 +3,9 @@ import Chart from '../../core';
export const name = 'test';
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
foo: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
total: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
inc: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
dec: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
},
},
},
};
'foo.total': { accumulate: true },
'foo.inc': {},
'foo.dec': {},
} as const;
export const entity = Chart.schemaToEntity(name, schema);

48
packages/backend/src/services/chart/charts/entities/users.ts
View File

@@ -2,47 +2,13 @@ import Chart from '../../core';
export const name = 'users';
const logSchema = {
/**
* 集計期間時点での、全ユーザー数
*/
total: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* 増加したユーザー数
*/
inc: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
/**
* 減少したユーザー数
*/
dec: {
type: 'number' as const,
optional: false as const, nullable: false as const,
},
};
export const schema = {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: {
local: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema,
},
remote: {
type: 'object' as const,
optional: false as const, nullable: false as const,
properties: logSchema,
},
},
};
'local.total': { accumulate: true },
'local.inc': { range: 'small' },
'local.dec': { range: 'small' },
'remote.total': { accumulate: true },
'remote.inc': { range: 'small' },
'remote.dec': { range: 'small' },
} as const;
export const entity = Chart.schemaToEntity(name, schema);

58
packages/backend/src/services/chart/charts/federation.ts
View File

@@ -1,66 +1,48 @@
import autobind from 'autobind-decorator';
import Chart, { Obj, DeepPartial } from '../core';
import { SchemaType } from '@/misc/schema';
import Chart, { KVs } from '../core';
import { Instances } from '@/models/index';
import { name, schema } from './entities/federation';
type FederationLog = SchemaType<typeof schema>;
/**
* フェデレーションに関するチャート
*/
// eslint-disable-next-line import/no-default-export
export default class FederationChart extends Chart<FederationLog> {
export default class FederationChart extends Chart<typeof schema> {
constructor() {
super(name, schema);
}
@autobind
protected genNewLog(latest: FederationLog): DeepPartial<FederationLog> {
return {
instance: {
total: latest.instance.total,
},
};
}
@autobind
protected aggregate(logs: FederationLog[]): FederationLog {
return {
instance: {
total: logs[0].instance.total,
inc: logs.reduce((a, b) => a + b.instance.inc, 0),
dec: logs.reduce((a, b) => a + b.instance.dec, 0),
},
};
}
@autobind
protected async fetchActual(): Promise<DeepPartial<FederationLog>> {
protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
const [total] = await Promise.all([
Instances.count({}),
]);
return {
instance: {
total: total,
},
'instance.total': total,
};
}
@autobind
public async update(isAdditional: boolean): Promise<void> {
const update: Obj = {};
await this.commit({
'instance.total': isAdditional ? 1 : -1,
'instance.inc': isAdditional ? 1 : 0,
'instance.dec': isAdditional ? 0 : 1,
});
}
update.total = isAdditional ? 1 : -1;
if (isAdditional) {
update.inc = 1;
} else {
update.dec = 1;
}
@autobind
public async deliverd(host: string): Promise<void> {
await this.commit({
'deliveredInstances': [host],
});
}
await this.inc({
instance: update,
@autobind
public async inbox(host: string): Promise<void> {
await this.commit({
'inboxInstances': [host],
});
}
}

35
packages/backend/src/services/chart/charts/hashtag.ts
View File

@@ -1,51 +1,28 @@
import autobind from 'autobind-decorator';
import Chart, { Obj, DeepPartial } from '../core';
import Chart, { KVs } from '../core';
import { User } from '@/models/entities/user';
import { SchemaType } from '@/misc/schema';
import { Users } from '@/models/index';
import { name, schema } from './entities/hashtag';
type HashtagLog = SchemaType<typeof schema>;
/**
* ハッシュタグに関するチャート
*/
// eslint-disable-next-line import/no-default-export
export default class HashtagChart extends Chart<HashtagLog> {
export default class HashtagChart extends Chart<typeof schema> {
constructor() {
super(name, schema, true);
}
@autobind
protected genNewLog(latest: HashtagLog): DeepPartial<HashtagLog> {
return {};
}
@autobind
protected aggregate(logs: HashtagLog[]): HashtagLog {
return {
local: {
users: logs.reduce((a, b) => a.concat(b.local.users), [] as HashtagLog['local']['users']),
},
remote: {
users: logs.reduce((a, b) => a.concat(b.remote.users), [] as HashtagLog['remote']['users']),
},
};
}
@autobind
protected async fetchActual(): Promise<DeepPartial<HashtagLog>> {
protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
@autobind
public async update(hashtag: string, user: { id: User['id'], host: User['host'] }): Promise<void> {
const update: Obj = {
users: [user.id],
};
await this.inc({
[Users.isLocalUser(user) ? 'local' : 'remote']: update,
await this.commit({
'local.users': Users.isLocalUser(user) ? [user.id] : [],
'remote.users': Users.isLocalUser(user) ? [] : [user.id],
}, hashtag);
}
}

182
packages/backend/src/services/chart/charts/instance.ts
View File

@@ -1,158 +1,67 @@
import autobind from 'autobind-decorator';
import Chart, { Obj, DeepPartial } from '../core';
import { SchemaType } from '@/misc/schema';
import Chart, { KVs } from '../core';
import { DriveFiles, Followings, Users, Notes } from '@/models/index';
import { DriveFile } from '@/models/entities/drive-file';
import { Note } from '@/models/entities/note';
import { toPuny } from '@/misc/convert-host';
import { name, schema } from './entities/instance';
type InstanceLog = SchemaType<typeof schema>;
/**
* インスタンスごとのチャート
*/
// eslint-disable-next-line import/no-default-export
export default class InstanceChart extends Chart<InstanceLog> {
export default class InstanceChart extends Chart<typeof schema> {
constructor() {
super(name, schema, true);
}
@autobind
protected genNewLog(latest: InstanceLog): DeepPartial<InstanceLog> {
return {
notes: {
total: latest.notes.total,
},
users: {
total: latest.users.total,
},
following: {
total: latest.following.total,
},
followers: {
total: latest.followers.total,
},
drive: {
totalFiles: latest.drive.totalFiles,
totalUsage: latest.drive.totalUsage,
},
};
}
@autobind
protected aggregate(logs: InstanceLog[]): InstanceLog {
return {
requests: {
failed: logs.reduce((a, b) => a + b.requests.failed, 0),
succeeded: logs.reduce((a, b) => a + b.requests.succeeded, 0),
received: logs.reduce((a, b) => a + b.requests.received, 0),
},
notes: {
total: logs[0].notes.total,
inc: logs.reduce((a, b) => a + b.notes.inc, 0),
dec: logs.reduce((a, b) => a + b.notes.dec, 0),
diffs: {
reply: logs.reduce((a, b) => a + b.notes.diffs.reply, 0),
renote: logs.reduce((a, b) => a + b.notes.diffs.renote, 0),
normal: logs.reduce((a, b) => a + b.notes.diffs.normal, 0),
},
},
users: {
total: logs[0].users.total,
inc: logs.reduce((a, b) => a + b.users.inc, 0),
dec: logs.reduce((a, b) => a + b.users.dec, 0),
},
following: {
total: logs[0].following.total,
inc: logs.reduce((a, b) => a + b.following.inc, 0),
dec: logs.reduce((a, b) => a + b.following.dec, 0),
},
followers: {
total: logs[0].followers.total,
inc: logs.reduce((a, b) => a + b.followers.inc, 0),
dec: logs.reduce((a, b) => a + b.followers.dec, 0),
},
drive: {
totalFiles: logs[0].drive.totalFiles,
totalUsage: logs[0].drive.totalUsage,
incFiles: logs.reduce((a, b) => a + b.drive.incFiles, 0),
incUsage: logs.reduce((a, b) => a + b.drive.incUsage, 0),
decFiles: logs.reduce((a, b) => a + b.drive.decFiles, 0),
decUsage: logs.reduce((a, b) => a + b.drive.decUsage, 0),
},
};
}
@autobind
protected async fetchActual(group: string): Promise<DeepPartial<InstanceLog>> {
protected async queryCurrentState(group: string): Promise<Partial<KVs<typeof schema>>> {
const [
notesCount,
usersCount,
followingCount,
followersCount,
driveFiles,
driveUsage,
//driveUsage,
] = await Promise.all([
Notes.count({ userHost: group }),
Users.count({ host: group }),
Followings.count({ followerHost: group }),
Followings.count({ followeeHost: group }),
DriveFiles.count({ userHost: group }),
DriveFiles.calcDriveUsageOfHost(group),
//DriveFiles.calcDriveUsageOfHost(group),
]);
return {
notes: {
total: notesCount,
},
users: {
total: usersCount,
},
following: {
total: followingCount,
},
followers: {
total: followersCount,
},
drive: {
totalFiles: driveFiles,
totalUsage: driveUsage,
},
'notes.total': notesCount,
'users.total': usersCount,
'following.total': followingCount,
'followers.total': followersCount,
'drive.totalFiles': driveFiles,
};
}
@autobind
public async requestReceived(host: string): Promise<void> {
await this.inc({
requests: {
received: 1,
},
await this.commit({
'requests.received': 1,
}, toPuny(host));
}
@autobind
public async requestSent(host: string, isSucceeded: boolean): Promise<void> {
const update: Obj = {};
if (isSucceeded) {
update.succeeded = 1;
} else {
update.failed = 1;
}
await this.inc({
requests: update,
await this.commit({
'requests.succeeded': isSucceeded ? 1 : 0,
'requests.failed': isSucceeded ? 0 : 1,
}, toPuny(host));
}
@autobind
public async newUser(host: string): Promise<void> {
await this.inc({
users: {
total: 1,
inc: 1,
},
await this.commit({
'users.total': 1,
'users.inc': 1,
}, toPuny(host));
}
@@ -168,54 +77,43 @@ export default class InstanceChart extends Chart<InstanceLog> {
diffs.normal = isAdditional ? 1 : -1;
}
await this.inc({
notes: {
total: isAdditional ? 1 : -1,
inc: isAdditional ? 1 : 0,
dec: isAdditional ? 0 : 1,
diffs: diffs,
},
await this.commit({
'notes.total': isAdditional ? 1 : -1,
'notes.inc': isAdditional ? 1 : 0,
'notes.dec': isAdditional ? 0 : 1,
'notes.diffs.normal': note.replyId == null && note.renoteId == null ? (isAdditional ? 1 : -1) : 0,
'notes.diffs.renote': note.renoteId != null ? (isAdditional ? 1 : -1) : 0,
'notes.diffs.reply': note.replyId != null ? (isAdditional ? 1 : -1) : 0,
}, toPuny(host));
}
@autobind
public async updateFollowing(host: string, isAdditional: boolean): Promise<void> {
await this.inc({
following: {
total: isAdditional ? 1 : -1,
inc: isAdditional ? 1 : 0,
dec: isAdditional ? 0 : 1,
},
await this.commit({
'following.total': isAdditional ? 1 : -1,
'following.inc': isAdditional ? 1 : 0,
'following.dec': isAdditional ? 0 : 1,
}, toPuny(host));
}
@autobind
public async updateFollowers(host: string, isAdditional: boolean): Promise<void> {
await this.inc({
followers: {
total: isAdditional ? 1 : -1,
inc: isAdditional ? 1 : 0,
dec: isAdditional ? 0 : 1,
},
await this.commit({
'followers.total': isAdditional ? 1 : -1,
'followers.inc': isAdditional ? 1 : 0,
'followers.dec': isAdditional ? 0 : 1,
}, toPuny(host));
}
@autobind
public async updateDrive(file: DriveFile, isAdditional: boolean): Promise<void> {
const update: Obj = {};
update.totalFiles = isAdditional ? 1 : -1;
update.totalUsage = isAdditional ? file.size : -file.size;
if (isAdditional) {
update.incFiles = 1;
update.incUsage = file.size;
} else {
update.decFiles = 1;
update.decUsage = file.size;
}
await this.inc({
drive: update,
const fileSizeKb = file.size / 1000;
await this.commit({
'drive.totalFiles': isAdditional ? 1 : -1,
'drive.incFiles': isAdditional ? 1 : 0,
'drive.incUsage': isAdditional ? fileSizeKb : 0,
'drive.decFiles': isAdditional ? 1 : 0,
'drive.decUsage': isAdditional ? fileSizeKb : 0,
}, file.userHost);
}
}

49
packages/backend/src/services/chart/charts/network.ts
View File

@@ -1,49 +0,0 @@
import autobind from 'autobind-decorator';
import Chart, { DeepPartial } from '../core';
import { SchemaType } from '@/misc/schema';
import { name, schema } from './entities/network';
type NetworkLog = SchemaType<typeof schema>;
/**
* ネットワークに関するチャート
*/
// eslint-disable-next-line import/no-default-export
export default class NetworkChart extends Chart<NetworkLog> {
constructor() {
super(name, schema);
}
@autobind
protected genNewLog(latest: NetworkLog): DeepPartial<NetworkLog> {
return {};
}
@autobind
protected aggregate(logs: NetworkLog[]): NetworkLog {
return {
incomingRequests: logs.reduce((a, b) => a + b.incomingRequests, 0),
outgoingRequests: logs.reduce((a, b) => a + b.outgoingRequests, 0),
totalTime: logs.reduce((a, b) => a + b.totalTime, 0),
incomingBytes: logs.reduce((a, b) => a + b.incomingBytes, 0),
outgoingBytes: logs.reduce((a, b) => a + b.outgoingBytes, 0),
};
}
@autobind
protected async fetchActual(): Promise<DeepPartial<NetworkLog>> {
return {};
}
@autobind
public async update(incomingRequests: number, time: number, incomingBytes: number, outgoingBytes: number): Promise<void> {
const inc: DeepPartial<NetworkLog> = {
incomingRequests: incomingRequests,
totalTime: time,
incomingBytes: incomingBytes,
outgoingBytes: outgoingBytes,
};
await this.inc(inc);
}
}

84
packages/backend/src/services/chart/charts/notes.ts
View File

@@ -1,101 +1,43 @@
import autobind from 'autobind-decorator';
import Chart, { Obj, DeepPartial } from '../core';
import { SchemaType } from '@/misc/schema';
import Chart, { KVs } from '../core';
import { Notes } from '@/models/index';
import { Not, IsNull } from 'typeorm';
import { Note } from '@/models/entities/note';
import { name, schema } from './entities/notes';
type NotesLog = SchemaType<typeof schema>;
/**
* ノートに関するチャート
*/
// eslint-disable-next-line import/no-default-export
export default class NotesChart extends Chart<NotesLog> {
export default class NotesChart extends Chart<typeof schema> {
constructor() {
super(name, schema);
}
@autobind
protected genNewLog(latest: NotesLog): DeepPartial<NotesLog> {
return {
local: {
total: latest.local.total,
},
remote: {
total: latest.remote.total,
},
};
}
@autobind
protected aggregate(logs: NotesLog[]): NotesLog {
return {
local: {
total: logs[0].local.total,
inc: logs.reduce((a, b) => a + b.local.inc, 0),
dec: logs.reduce((a, b) => a + b.local.dec, 0),
diffs: {
reply: logs.reduce((a, b) => a + b.local.diffs.reply, 0),
renote: logs.reduce((a, b) => a + b.local.diffs.renote, 0),
normal: logs.reduce((a, b) => a + b.local.diffs.normal, 0),
},
},
remote: {
total: logs[0].remote.total,
inc: logs.reduce((a, b) => a + b.remote.inc, 0),
dec: logs.reduce((a, b) => a + b.remote.dec, 0),
diffs: {
reply: logs.reduce((a, b) => a + b.remote.diffs.reply, 0),
renote: logs.reduce((a, b) => a + b.remote.diffs.renote, 0),
normal: logs.reduce((a, b) => a + b.remote.diffs.normal, 0),
},
},
};
}
@autobind
protected async fetchActual(): Promise<DeepPartial<NotesLog>> {
protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
const [localCount, remoteCount] = await Promise.all([
Notes.count({ userHost: null }),
Notes.count({ userHost: Not(IsNull()) }),
]);
return {
local: {
total: localCount,
},
remote: {
total: remoteCount,
},
'local.total': localCount,
'remote.total': remoteCount,
};
}
@autobind
public async update(note: Note, isAdditional: boolean): Promise<void> {
const update: Obj = {
diffs: {},
};
const prefix = note.userHost === null ? 'local' : 'remote';
update.total = isAdditional ? 1 : -1;
if (isAdditional) {
update.inc = 1;
} else {
update.dec = 1;
}
if (note.replyId != null) {
update.diffs.reply = isAdditional ? 1 : -1;
} else if (note.renoteId != null) {
update.diffs.renote = isAdditional ? 1 : -1;
} else {
update.diffs.normal = isAdditional ? 1 : -1;
}
await this.inc({
[note.userHost === null ? 'local' : 'remote']: update,
await this.commit({
[`${prefix}.total`]: isAdditional ? 1 : -1,
[`${prefix}.inc`]: isAdditional ? 1 : 0,
[`${prefix}.dec`]: isAdditional ? 0 : 1,
[`${prefix}.diffs.normal`]: note.replyId == null && note.renoteId == null ? (isAdditional ? 1 : -1) : 0,
[`${prefix}.diffs.renote`]: note.renoteId != null ? (isAdditional ? 1 : -1) : 0,
[`${prefix}.diffs.reply`]: note.replyId != null ? (isAdditional ? 1 : -1) : 0,
});
}
}

55
packages/backend/src/services/chart/charts/per-user-drive.ts
View File

@@ -1,68 +1,41 @@
import autobind from 'autobind-decorator';
import Chart, { Obj, DeepPartial } from '../core';
import { SchemaType } from '@/misc/schema';
import Chart, { KVs } from '../core';
import { DriveFiles } from '@/models/index';
import { DriveFile } from '@/models/entities/drive-file';
import { name, schema } from './entities/per-user-drive';
type PerUserDriveLog = SchemaType<typeof schema>;
/**
* ユーザーごとのドライブに関するチャート
*/
// eslint-disable-next-line import/no-default-export
export default class PerUserDriveChart extends Chart<PerUserDriveLog> {
export default class PerUserDriveChart extends Chart<typeof schema> {
constructor() {
super(name, schema, true);
}
@autobind
protected genNewLog(latest: PerUserDriveLog): DeepPartial<PerUserDriveLog> {
return {
totalCount: latest.totalCount,
totalSize: latest.totalSize,
};
}
@autobind
protected aggregate(logs: PerUserDriveLog[]): PerUserDriveLog {
return {
totalCount: logs[0].totalCount,
totalSize: logs[0].totalSize,
incCount: logs.reduce((a, b) => a + b.incCount, 0),
incSize: logs.reduce((a, b) => a + b.incSize, 0),
decCount: logs.reduce((a, b) => a + b.decCount, 0),
decSize: logs.reduce((a, b) => a + b.decSize, 0),
};
}
@autobind
protected async fetchActual(group: string): Promise<DeepPartial<PerUserDriveLog>> {
protected async queryCurrentState(group: string): Promise<Partial<KVs<typeof schema>>> {
const [count, size] = await Promise.all([
DriveFiles.count({ userId: group }),
DriveFiles.calcDriveUsageOf(group),
]);
return {
totalCount: count,
totalSize: size,
'totalCount': count,
'totalSize': size,
};
}
@autobind
public async update(file: DriveFile, isAdditional: boolean): Promise<void> {
const update: Obj = {};
update.totalCount = isAdditional ? 1 : -1;
update.totalSize = isAdditional ? file.size : -file.size;
if (isAdditional) {
update.incCount = 1;
update.incSize = file.size;
} else {
update.decCount = 1;
update.decSize = file.size;
}
await this.inc(update, file.userId);
const fileSizeKb = file.size / 1000;
await this.commit({
'totalCount': isAdditional ? 1 : -1,
'totalSize': isAdditional ? fileSizeKb : -fileSizeKb,
'incCount': isAdditional ? 1 : 0,
'incSize': isAdditional ? fileSizeKb : 0,
'decCount': isAdditional ? 0 : 1,
'decSize': isAdditional ? 0 : fileSizeKb,
}, file.userId);
}
}

104
packages/backend/src/services/chart/charts/per-user-following.ts
View File

@@ -1,76 +1,21 @@
import autobind from 'autobind-decorator';
import Chart, { Obj, DeepPartial } from '../core';
import { SchemaType } from '@/misc/schema';
import Chart, { KVs } from '../core';
import { Followings, Users } from '@/models/index';
import { Not, IsNull } from 'typeorm';
import { User } from '@/models/entities/user';
import { name, schema } from './entities/per-user-following';
type PerUserFollowingLog = SchemaType<typeof schema>;
/**
* ユーザーごとのフォローに関するチャート
*/
// eslint-disable-next-line import/no-default-export
export default class PerUserFollowingChart extends Chart<PerUserFollowingLog> {
export default class PerUserFollowingChart extends Chart<typeof schema> {
constructor() {
super(name, schema, true);
}
@autobind
protected genNewLog(latest: PerUserFollowingLog): DeepPartial<PerUserFollowingLog> {
return {
local: {
followings: {
total: latest.local.followings.total,
},
followers: {
total: latest.local.followers.total,
},
},
remote: {
followings: {
total: latest.remote.followings.total,
},
followers: {
total: latest.remote.followers.total,
},
},
};
}
@autobind
protected aggregate(logs: PerUserFollowingLog[]): PerUserFollowingLog {
return {
local: {
followings: {
total: logs[0].local.followings.total,
inc: logs.reduce((a, b) => a + b.local.followings.inc, 0),
dec: logs.reduce((a, b) => a + b.local.followings.dec, 0),
},
followers: {
total: logs[0].local.followers.total,
inc: logs.reduce((a, b) => a + b.local.followers.inc, 0),
dec: logs.reduce((a, b) => a + b.local.followers.dec, 0),
},
},
remote: {
followings: {
total: logs[0].remote.followings.total,
inc: logs.reduce((a, b) => a + b.remote.followings.inc, 0),
dec: logs.reduce((a, b) => a + b.remote.followings.dec, 0),
},
followers: {
total: logs[0].remote.followers.total,
inc: logs.reduce((a, b) => a + b.remote.followers.inc, 0),
dec: logs.reduce((a, b) => a + b.remote.followers.dec, 0),
},
},
};
}
@autobind
protected async fetchActual(group: string): Promise<DeepPartial<PerUserFollowingLog>> {
protected async queryCurrentState(group: string): Promise<Partial<KVs<typeof schema>>> {
const [
localFollowingsCount,
localFollowersCount,
@@ -84,42 +29,27 @@ export default class PerUserFollowingChart extends Chart<PerUserFollowingLog> {
]);
return {
local: {
followings: {
total: localFollowingsCount,
},
followers: {
total: localFollowersCount,
},
},
remote: {
followings: {
total: remoteFollowingsCount,
},
followers: {
total: remoteFollowersCount,
},
},
'local.followings.total': localFollowingsCount,
'local.followers.total': localFollowersCount,
'remote.followings.total': remoteFollowingsCount,
'remote.followers.total': remoteFollowersCount,
};
}
@autobind
public async update(follower: { id: User['id']; host: User['host']; }, followee: { id: User['id']; host: User['host']; }, isFollow: boolean): Promise<void> {
const update: Obj = {};
const prefixFollower = Users.isLocalUser(follower) ? 'local' : 'remote';
const prefixFollowee = Users.isLocalUser(followee) ? 'local' : 'remote';
update.total = isFollow ? 1 : -1;
if (isFollow) {
update.inc = 1;
} else {
update.dec = 1;
}
this.inc({
[Users.isLocalUser(follower) ? 'local' : 'remote']: { followings: update },
this.commit({
[`${prefixFollower}.followings.total`]: isFollow ? 1 : -1,
[`${prefixFollower}.followings.inc`]: isFollow ? 1 : 0,
[`${prefixFollower}.followings.dec`]: isFollow ? 0 : 1,
}, follower.id);
this.inc({
[Users.isLocalUser(followee) ? 'local' : 'remote']: { followers: update },
this.commit({
[`${prefixFollowee}.followers.total`]: isFollow ? 1 : -1,
[`${prefixFollowee}.followers.inc`]: isFollow ? 1 : 0,
[`${prefixFollowee}.followers.dec`]: isFollow ? 0 : 1,
}, followee.id);
}
}

59
packages/backend/src/services/chart/charts/per-user-notes.ts
View File

@@ -1,45 +1,21 @@
import autobind from 'autobind-decorator';
import Chart, { Obj, DeepPartial } from '../core';
import Chart, { KVs } from '../core';
import { User } from '@/models/entities/user';
import { SchemaType } from '@/misc/schema';
import { Notes } from '@/models/index';
import { Note } from '@/models/entities/note';
import { name, schema } from './entities/per-user-notes';
type PerUserNotesLog = SchemaType<typeof schema>;
/**
* ユーザーごとのノートに関するチャート
*/
// eslint-disable-next-line import/no-default-export
export default class PerUserNotesChart extends Chart<PerUserNotesLog> {
export default class PerUserNotesChart extends Chart<typeof schema> {
constructor() {
super(name, schema, true);
}
@autobind
protected genNewLog(latest: PerUserNotesLog): DeepPartial<PerUserNotesLog> {
return {
total: latest.total,
};
}
@autobind
protected aggregate(logs: PerUserNotesLog[]): PerUserNotesLog {
return {
total: logs[0].total,
inc: logs.reduce((a, b) => a + b.inc, 0),
dec: logs.reduce((a, b) => a + b.dec, 0),
diffs: {
reply: logs.reduce((a, b) => a + b.diffs.reply, 0),
renote: logs.reduce((a, b) => a + b.diffs.renote, 0),
normal: logs.reduce((a, b) => a + b.diffs.normal, 0),
},
};
}
@autobind
protected async fetchActual(group: string): Promise<DeepPartial<PerUserNotesLog>> {
protected async queryCurrentState(group: string): Promise<Partial<KVs<typeof schema>>> {
const [count] = await Promise.all([
Notes.count({ userId: group }),
]);
@@ -51,26 +27,13 @@ export default class PerUserNotesChart extends Chart<PerUserNotesLog> {
@autobind
public async update(user: { id: User['id'] }, note: Note, isAdditional: boolean): Promise<void> {
const update: Obj = {
diffs: {},
};
update.total = isAdditional ? 1 : -1;
if (isAdditional) {
update.inc = 1;
} else {
update.dec = 1;
}
if (note.replyId != null) {
update.diffs.reply = isAdditional ? 1 : -1;
} else if (note.renoteId != null) {
update.diffs.renote = isAdditional ? 1 : -1;
} else {
update.diffs.normal = isAdditional ? 1 : -1;
}
await this.inc(update, user.id);
await this.commit({
'total': isAdditional ? 1 : -1,
'inc': isAdditional ? 1 : 0,
'dec': isAdditional ? 0 : 1,
'diffs.normal': note.replyId == null && note.renoteId == null ? (isAdditional ? 1 : -1) : 0,
'diffs.renote': note.renoteId != null ? (isAdditional ? 1 : -1) : 0,
'diffs.reply': note.replyId != null ? (isAdditional ? 1 : -1) : 0,
}, user.id);
}
}

31
packages/backend/src/services/chart/charts/per-user-reactions.ts
View File

@@ -1,48 +1,29 @@
import autobind from 'autobind-decorator';
import Chart, { DeepPartial } from '../core';
import Chart, { KVs } from '../core';
import { User } from '@/models/entities/user';
import { Note } from '@/models/entities/note';
import { SchemaType } from '@/misc/schema';
import { Users } from '@/models/index';
import { name, schema } from './entities/per-user-reactions';
type PerUserReactionsLog = SchemaType<typeof schema>;
/**
* ユーザーごとのリアクションに関するチャート
*/
// eslint-disable-next-line import/no-default-export
export default class PerUserReactionsChart extends Chart<PerUserReactionsLog> {
export default class PerUserReactionsChart extends Chart<typeof schema> {
constructor() {
super(name, schema, true);
}
@autobind
protected genNewLog(latest: PerUserReactionsLog): DeepPartial<PerUserReactionsLog> {
return {};
}
@autobind
protected aggregate(logs: PerUserReactionsLog[]): PerUserReactionsLog {
return {
local: {
count: logs.reduce((a, b) => a + b.local.count, 0),
},
remote: {
count: logs.reduce((a, b) => a + b.remote.count, 0),
},
};
}
@autobind
protected async fetchActual(group: string): Promise<DeepPartial<PerUserReactionsLog>> {
protected async queryCurrentState(group: string): Promise<Partial<KVs<typeof schema>>> {
return {};
}
@autobind
public async update(user: { id: User['id'], host: User['host'] }, note: Note): Promise<void> {
this.inc({
[Users.isLocalUser(user) ? 'local' : 'remote']: { count: 1 },
const prefix = Users.isLocalUser(user) ? 'local' : 'remote';
this.commit({
[`${prefix}.count`]: 1,
}, note.userId);
}
}

42
packages/backend/src/services/chart/charts/test-grouped.ts
View File

@@ -1,15 +1,12 @@
import autobind from 'autobind-decorator';
import Chart, { Obj, DeepPartial } from '../core';
import { SchemaType } from '@/misc/schema';
import Chart, { KVs } from '../core';
import { name, schema } from './entities/test-grouped';
type TestGroupedLog = SchemaType<typeof schema>;
/**
* For testing
*/
// eslint-disable-next-line import/no-default-export
export default class TestGroupedChart extends Chart<TestGroupedLog> {
export default class TestGroupedChart extends Chart<typeof schema> {
private total = {} as Record<string, number>;
constructor() {
@@ -17,31 +14,9 @@ export default class TestGroupedChart extends Chart<TestGroupedLog> {
}
@autobind
protected genNewLog(latest: TestGroupedLog): DeepPartial<TestGroupedLog> {
protected async queryCurrentState(group: string): Promise<Partial<KVs<typeof schema>>> {
return {
foo: {
total: latest.foo.total,
},
};
}
@autobind
protected aggregate(logs: TestGroupedLog[]): TestGroupedLog {
return {
foo: {
total: logs[0].foo.total,
inc: logs.reduce((a, b) => a + b.foo.inc, 0),
dec: logs.reduce((a, b) => a + b.foo.dec, 0),
},
};
}
@autobind
protected async fetchActual(group: string): Promise<DeepPartial<TestGroupedLog>> {
return {
foo: {
total: this.total[group],
},
'foo.total': this.total[group],
};
}
@@ -49,14 +24,11 @@ export default class TestGroupedChart extends Chart<TestGroupedLog> {
public async increment(group: string): Promise<void> {
if (this.total[group] == null) this.total[group] = 0;
const update: Obj = {};
update.total = 1;
update.inc = 1;
this.total[group]++;
await this.inc({
foo: update,
await this.commit({
'foo.total': 1,
'foo.inc': 1,
}, group);
}
}

23
packages/backend/src/services/chart/charts/test-unique.ts
View File

@@ -1,39 +1,24 @@
import autobind from 'autobind-decorator';
import Chart, { DeepPartial } from '../core';
import { SchemaType } from '@/misc/schema';
import Chart, { KVs } from '../core';
import { name, schema } from './entities/test-unique';
type TestUniqueLog = SchemaType<typeof schema>;
/**
* For testing
*/
// eslint-disable-next-line import/no-default-export
export default class TestUniqueChart extends Chart<TestUniqueLog> {
export default class TestUniqueChart extends Chart<typeof schema> {
constructor() {
super(name, schema);
}
@autobind
protected genNewLog(latest: TestUniqueLog): DeepPartial<TestUniqueLog> {
return {};
}
@autobind
protected aggregate(logs: TestUniqueLog[]): TestUniqueLog {
return {
foo: logs.reduce((a, b) => a.concat(b.foo), [] as TestUniqueLog['foo']),
};
}
@autobind
protected async fetchActual(): Promise<DeepPartial<TestUniqueLog>> {
protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
@autobind
public async uniqueIncrement(key: string): Promise<void> {
await this.inc({
await this.commit({
foo: [key],
});
}

51
packages/backend/src/services/chart/charts/test.ts
View File

@@ -1,15 +1,12 @@
import autobind from 'autobind-decorator';
import Chart, { Obj, DeepPartial } from '../core';
import { SchemaType } from '@/misc/schema';
import Chart, { KVs } from '../core';
import { name, schema } from './entities/test';
type TestLog = SchemaType<typeof schema>;
/**
* For testing
*/
// eslint-disable-next-line import/no-default-export
export default class TestChart extends Chart<TestLog> {
export default class TestChart extends Chart<typeof schema> {
public total = 0; // publicにするのはテストのため
constructor() {
@@ -17,57 +14,29 @@ export default class TestChart extends Chart<TestLog> {
}
@autobind
protected genNewLog(latest: TestLog): DeepPartial<TestLog> {
protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
return {
foo: {
total: latest.foo.total,
},
};
}
@autobind
protected aggregate(logs: TestLog[]): TestLog {
return {
foo: {
total: logs[0].foo.total,
inc: logs.reduce((a, b) => a + b.foo.inc, 0),
dec: logs.reduce((a, b) => a + b.foo.dec, 0),
},
};
}
@autobind
protected async fetchActual(): Promise<DeepPartial<TestLog>> {
return {
foo: {
total: this.total,
},
'foo.total': this.total,
};
}
@autobind
public async increment(): Promise<void> {
const update: Obj = {};
update.total = 1;
update.inc = 1;
this.total++;
await this.inc({
foo: update,
await this.commit({
'foo.total': 1,
'foo.inc': 1,
});
}
@autobind
public async decrement(): Promise<void> {
const update: Obj = {};
update.total = -1;
update.dec = 1;
this.total--;
await this.inc({
foo: update,
await this.commit({
'foo.total': -1,
'foo.dec': 1,
});
}
}

60
packages/backend/src/services/chart/charts/users.ts
View File

@@ -1,80 +1,40 @@
import autobind from 'autobind-decorator';
import Chart, { Obj, DeepPartial } from '../core';
import { SchemaType } from '@/misc/schema';
import Chart, { KVs } from '../core';
import { Users } from '@/models/index';
import { Not, IsNull } from 'typeorm';
import { User } from '@/models/entities/user';
import { name, schema } from './entities/users';
type UsersLog = SchemaType<typeof schema>;
/**
* ユーザー数に関するチャート
*/
// eslint-disable-next-line import/no-default-export
export default class UsersChart extends Chart<UsersLog> {
export default class UsersChart extends Chart<typeof schema> {
constructor() {
super(name, schema);
}
@autobind
protected genNewLog(latest: UsersLog): DeepPartial<UsersLog> {
return {
local: {
total: latest.local.total,
},
remote: {
total: latest.remote.total,
},
};
}
@autobind
protected aggregate(logs: UsersLog[]): UsersLog {
return {
local: {
total: logs[0].local.total,
inc: logs.reduce((a, b) => a + b.local.inc, 0),
dec: logs.reduce((a, b) => a + b.local.dec, 0),
},
remote: {
total: logs[0].remote.total,
inc: logs.reduce((a, b) => a + b.remote.inc, 0),
dec: logs.reduce((a, b) => a + b.remote.dec, 0),
},
};
}
@autobind
protected async fetchActual(): Promise<DeepPartial<UsersLog>> {
protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
const [localCount, remoteCount] = await Promise.all([
Users.count({ host: null }),
Users.count({ host: Not(IsNull()) }),
]);
return {
local: {
total: localCount,
},
remote: {
total: remoteCount,
},
'local.total': localCount,
'remote.total': remoteCount,
};
}
@autobind
public async update(user: { id: User['id'], host: User['host'] }, isAdditional: boolean): Promise<void> {
const update: Obj = {};
const prefix = Users.isLocalUser(user) ? 'local' : 'remote';
update.total = isAdditional ? 1 : -1;
if (isAdditional) {
update.inc = 1;
} else {
update.dec = 1;
}
await this.inc({
[Users.isLocalUser(user) ? 'local' : 'remote']: update,
await this.commit({
[`${prefix}.total`]: isAdditional ? 1 : -1,
[`${prefix}.inc`]: isAdditional ? 1 : 0,
[`${prefix}.dec`]: isAdditional ? 0 : 1,
});
}
}

376
packages/backend/src/services/chart/core.ts
View File

@@ -7,24 +7,19 @@
import * as nestedProperty from 'nested-property';
import autobind from 'autobind-decorator';
import Logger from '../logger';
import { Schema } from '@/misc/schema';
import { EntitySchema, getRepository, Repository, LessThan, Between } from 'typeorm';
import { dateUTC, isTimeSame, isTimeBefore, subtractTime, addTime } from '@/prelude/time';
import { getChartInsertLock } from '@/misc/app-lock';
const logger = new Logger('chart', 'white', process.env.NODE_ENV !== 'test');
export type Obj = { [key: string]: any };
const columnPrefix = '___' as const;
const uniqueTempColumnPrefix = 'unique_temp___' as const;
const columnDot = '_' as const;
export type DeepPartial<T> = {
[P in keyof T]?: DeepPartial<T[P]>;
};
type KeyToColumnName<T extends string> = T extends `${infer R1}.${infer R2}` ? `${R1}${typeof columnDot}${KeyToColumnName<R2>}` : T;
type ArrayValue<T> = {
[P in keyof T]: T[P] extends number ? T[P][] : ArrayValue<T[P]>;
};
type Log = {
type RawRecord<S extends Schema> = {
id: number;
/**
@@ -36,6 +31,10 @@ type Log = {
* 集計日時のUnixタイムスタンプ(秒)
*/
date: number;
} & {
[K in keyof S as `${typeof uniqueTempColumnPrefix}${KeyToColumnName<string & K>}`]: S[K]['uniqueIncrement'] extends true ? string[] : never;
} & {
[K in keyof S as `${typeof columnPrefix}${KeyToColumnName<string & K>}`]: number;
};
const camelToSnake = (str: string): string => {
@@ -44,123 +43,72 @@ const camelToSnake = (str: string): string => {
const removeDuplicates = (array: any[]) => Array.from(new Set(array));
type Schema = Record<string, {
uniqueIncrement?: boolean;
range?: 'big' | 'small' | 'medium';
// previousな値を引き継ぐかどうか
accumulate?: boolean;
}>;
type Commit<S extends Schema> = {
[K in keyof S]?: S[K]['uniqueIncrement'] extends true ? string[] : number;
};
export type KVs<S extends Schema> = {
[K in keyof S]: number;
};
type ChartResult<T extends Schema> = {
[P in keyof T]: number[];
};
/**
* 様々なチャートの管理を司るクラス
*/
// eslint-disable-next-line import/no-default-export
export default abstract class Chart<T extends Record<string, any>> {
private static readonly columnPrefix = '___';
private static readonly columnDot = '_';
export default abstract class Chart<T extends Schema> {
public schema: T;
private name: string;
private buffer: {
diff: DeepPartial<T>;
diff: Commit<T>;
group: string | null;
}[] = [];
public schema: Schema;
protected repositoryForHour: Repository<Log>;
protected repositoryForDay: Repository<Log>;
protected repositoryForHour: Repository<RawRecord<T>>;
protected repositoryForDay: Repository<RawRecord<T>>;
protected abstract genNewLog(latest: T): DeepPartial<T>;
/**
* @param logs 日時が新しい方が先頭
*/
protected abstract aggregate(logs: T[]): T;
protected abstract fetchActual(group: string | null): Promise<DeepPartial<T>>;
protected abstract queryCurrentState(group: string | null): Promise<Partial<KVs<T>>>;
@autobind
private static convertSchemaToFlatColumnDefinitions(schema: Schema) {
const columns = {} as Record<string, unknown>;
const flatColumns = (x: Obj, path?: string) => {
for (const [k, v] of Object.entries(x)) {
const p = path ? `${path}${this.columnDot}${k}` : k;
if (v.type === 'object') {
flatColumns(v.properties, p);
} else if (v.type === 'number') {
columns[this.columnPrefix + p] = {
type: 'bigint',
};
} else if (v.type === 'array' && v.items.type === 'string') {
columns[this.columnPrefix + p] = {
type: 'varchar',
array: true,
};
}
private static convertSchemaToColumnDefinitions(schema: Schema): Record<string, { type: string; array?: boolean; default?: any; }> {
const columns = {} as Record<string, { type: string; array?: boolean; default?: any; }>;
for (const [k, v] of Object.entries(schema)) {
const name = k.replaceAll('.', columnDot);
const type = v.range === 'big' ? 'bigint' : v.range === 'small' ? 'smallint' : 'integer';
if (v.uniqueIncrement) {
columns[uniqueTempColumnPrefix + name] = {
type: 'varchar',
array: true,
default: '{}',
};
columns[columnPrefix + name] = {
type,
default: 0,
};
} else {
columns[columnPrefix + name] = {
type,
default: 0,
};
}
};
flatColumns(schema.properties!);
}
return columns;
}
@autobind
private static convertFlattenColumnsToObject(x: Record<string, unknown>): Record<string, unknown> {
const obj = {} as Record<string, unknown>;
for (const k of Object.keys(x).filter(k => k.startsWith(Chart.columnPrefix))) {
// now k is ___x_y_z
const path = k.substr(Chart.columnPrefix.length).split(Chart.columnDot).join('.');
nestedProperty.set(obj, path, x[k]);
}
return obj;
}
@autobind
private static convertObjectToFlattenColumns(x: Record<string, unknown>) {
const columns = {} as Record<string, number | unknown[]>;
const flatten = (x: Obj, path?: string) => {
for (const [k, v] of Object.entries(x)) {
const p = path ? `${path}${this.columnDot}${k}` : k;
if (typeof v === 'object' && !Array.isArray(v)) {
flatten(v, p);
} else {
columns[this.columnPrefix + p] = v;
}
}
};
flatten(x);
return columns;
}
@autobind
private static countUniqueFields(x: Record<string, unknown>) {
const exec = (x: Obj) => {
const res = {} as Record<string, unknown>;
for (const [k, v] of Object.entries(x)) {
if (typeof v === 'object' && !Array.isArray(v)) {
res[k] = exec(v);
} else if (Array.isArray(v)) {
res[k] = Array.from(new Set(v)).length;
} else {
res[k] = v;
}
}
return res;
};
return exec(x);
}
@autobind
private static convertQuery(diff: Record<string, number | unknown[]>) {
const query: Record<string, () => string> = {};
for (const [k, v] of Object.entries(diff)) {
if (typeof v === 'number') {
if (v > 0) query[k] = () => `"${k}" + ${v}`;
if (v < 0) query[k] = () => `"${k}" - ${Math.abs(v)}`;
} else if (Array.isArray(v)) {
// TODO: item が文字列以外の場合も対応
// TODO: item をSQLエスケープ
const items = v.map(item => `"${item}"`).join(',');
query[k] = () => `array_cat("${k}", '{${items}}'::varchar[])`;
}
}
return query;
}
@autobind
private static dateToTimestamp(x: Date): Log['date'] {
private static dateToTimestamp(x: Date): number {
return Math.floor(x.getTime() / 1000);
}
@@ -207,7 +155,7 @@ export default abstract class Chart<T extends Record<string, any>> {
length: 128,
},
} : {}),
...Chart.convertSchemaToFlatColumnDefinitions(schema),
...Chart.convertSchemaToColumnDefinitions(schema),
},
indices: [{
columns: grouped ? ['date', 'group'] : ['date'],
@@ -233,37 +181,39 @@ export default abstract class Chart<T extends Record<string, any>> {
};
}
constructor(name: string, schema: Schema, grouped = false) {
constructor(name: string, schema: T, grouped = false) {
this.name = name;
this.schema = schema;
const { hour, day } = Chart.schemaToEntity(name, schema, grouped);
this.repositoryForHour = getRepository<Log>(hour);
this.repositoryForDay = getRepository<Log>(day);
this.repositoryForHour = getRepository<RawRecord<T>>(hour);
this.repositoryForDay = getRepository<RawRecord<T>>(day);
}
@autobind
private getNewLog(latest: T | null): T {
const log = latest ? this.genNewLog(latest) : {};
const flatColumns = (x: Obj, path?: string) => {
for (const [k, v] of Object.entries(x)) {
const p = path ? `${path}.${k}` : k;
if (v.type === 'object') {
flatColumns(v.properties, p);
} else {
if (nestedProperty.get(log, p) == null) {
const emptyValue = v.type === 'number' ? 0 : [];
nestedProperty.set(log, p, emptyValue);
}
}
private convertRawRecord(x: RawRecord<T>): KVs<T> {
const kvs = {} as KVs<T>;
for (const k of Object.keys(x).filter(k => k.startsWith(columnPrefix))) {
kvs[k.substr(columnPrefix.length).split(columnDot).join('.')] = x[k];
}
return kvs;
}
@autobind
private getNewLog(latest: KVs<T> | null): KVs<T> {
const log = {} as Record<keyof T, number>;
for (const [k, v] of Object.entries(this.schema)) {
if (v.accumulate && latest) {
log[k] = latest[k];
} else {
log[k] = 0;
}
};
flatColumns(this.schema.properties!);
return log as T;
}
return log as KVs<T>;
}
@autobind
private getLatestLog(group: string | null, span: 'hour' | 'day'): Promise<Log | null> {
private getLatestLog(group: string | null, span: 'hour' | 'day'): Promise<RawRecord<T> | null> {
const repository =
span === 'hour' ? this.repositoryForHour :
span === 'day' ? this.repositoryForDay :
@@ -282,7 +232,7 @@ export default abstract class Chart<T extends Record<string, any>> {
* 現在(=今のHour or Day)のログをデータベースから探して、あればそれを返し、なければ作成して返します。
*/
@autobind
private async claimCurrentLog(group: string | null, span: 'hour' | 'day'): Promise<Log> {
private async claimCurrentLog(group: string | null, span: 'hour' | 'day'): Promise<RawRecord<T>> {
const [y, m, d, h] = Chart.getCurrentDate();
const current = dateUTC(
@@ -306,8 +256,8 @@ export default abstract class Chart<T extends Record<string, any>> {
return currentLog;
}
let log: Log;
let data: T;
let log: RawRecord<T>;
let data: KVs<T>;
// 集計期間が変わってから、初めてのチャート更新なら
// 最も最近のログを持ってくる
@@ -318,10 +268,8 @@ export default abstract class Chart<T extends Record<string, any>> {
const latest = await this.getLatestLog(group, span);
if (latest != null) {
const obj = Chart.convertFlattenColumnsToObject(latest) as T;
// 空ログデータを作成
data = this.getNewLog(obj);
data = this.getNewLog(this.convertRawRecord(latest));
} else {
// ログが存在しなかったら
// (Misskeyインスタンスを建てて初めてのチャート更新時など)
@@ -346,11 +294,17 @@ export default abstract class Chart<T extends Record<string, any>> {
// ログがあればそれを返して終了
if (currentLog != null) return currentLog;
const columns = {} as Record<string, number | unknown[]>;
for (const [k, v] of Object.entries(data)) {
const name = k.replaceAll('.', columnDot);
columns[columnPrefix + name] = v;
}
// 新規ログ挿入
log = await repository.insert({
date: date,
...(group ? { group: group } : {}),
...Chart.convertObjectToFlattenColumns(data),
...columns,
}).then(x => repository.findOneOrFail(x.identifiers[0]));
logger.info(`${this.name + (group ? `:${group}` : '')}(${span}): New commit created`);
@@ -362,7 +316,10 @@ export default abstract class Chart<T extends Record<string, any>> {
}
@autobind
protected commit(diff: DeepPartial<T>, group: string | null = null): void {
protected commit(diff: Commit<T>, group: string | null = null): void {
for (const [k, v] of Object.entries(diff)) {
if (v == null || v === 0 || (Array.isArray(v) && v.length === 0)) delete diff[k];
}
this.buffer.push({
diff, group,
});
@@ -381,13 +338,11 @@ export default abstract class Chart<T extends Record<string, any>> {
// そのログは本来は 01:00~ のログとしてDBに保存されて欲しいのに、02:00~ のログ扱いになってしまう。
// これを回避するための実装は複雑になりそうなため、一旦保留。
const update = async (logHour: Log, logDay: Log): Promise<void> => {
const update = async (logHour: RawRecord<T>, logDay: RawRecord<T>): Promise<void> => {
const finalDiffs = {} as Record<string, number | unknown[]>;
for (const diff of this.buffer.filter(q => q.group == null || (q.group === logHour.group)).map(q => q.diff)) {
const columns = Chart.convertObjectToFlattenColumns(diff);
for (const [k, v] of Object.entries(columns)) {
for (const [k, v] of Object.entries(diff)) {
if (finalDiffs[k] == null) {
finalDiffs[k] = v;
} else {
@@ -400,18 +355,45 @@ export default abstract class Chart<T extends Record<string, any>> {
}
}
const query = Chart.convertQuery(finalDiffs);
const queryForHour: Record<string, number | (() => string)> = {};
const queryForDay: Record<string, number | (() => string)> = {};
for (const [k, v] of Object.entries(finalDiffs)) {
if (typeof v === 'number') {
const name = columnPrefix + k.replaceAll('.', columnDot);
if (v > 0) queryForHour[name] = () => `"${name}" + ${v}`;
if (v < 0) queryForHour[name] = () => `"${name}" - ${Math.abs(v)}`;
if (v > 0) queryForDay[name] = () => `"${name}" + ${v}`;
if (v < 0) queryForDay[name] = () => `"${name}" - ${Math.abs(v)}`;
} else if (Array.isArray(v) && v.length > 0) { // ユニークインクリメント
const name = uniqueTempColumnPrefix + k.replaceAll('.', columnDot);
// TODO: item が文字列以外の場合も対応
// TODO: item をSQLエスケープ
// TODO: 値が重複しないようにしたい
const items = v.map(item => `"${item}"`).join(',');
queryForHour[name] = () => `array_cat("${name}", '{${items}}'::varchar[])`;
queryForDay[name] = () => `array_cat("${name}", '{${items}}'::varchar[])`;
}
}
for (const [k, v] of Object.entries(this.schema)) {
const name = columnPrefix + k.replaceAll('.', columnDot);
if (v.uniqueIncrement) {
const tempColumnName = uniqueTempColumnPrefix + k.replaceAll('.', columnDot);
queryForHour[name] = new Set([...finalDiffs[k], ...logHour[tempColumnName]]).size;
queryForDay[name] = new Set([...finalDiffs[k], ...logDay[tempColumnName]]).size;
}
}
// ログ更新
await Promise.all([
this.repositoryForHour.createQueryBuilder()
.update()
.set(query)
.set(queryForHour)
.where('id = :id', { id: logHour.id })
.execute(),
this.repositoryForDay.createQueryBuilder()
.update()
.set(query)
.set(queryForDay)
.where('id = :id', { id: logDay.id })
.execute(),
]);
@@ -435,18 +417,24 @@ export default abstract class Chart<T extends Record<string, any>> {
@autobind
public async resync(group: string | null = null): Promise<void> {
const data = await this.fetchActual(group);
const data = await this.queryCurrentState(group);
const update = async (logHour: Log, logDay: Log): Promise<void> => {
const columns = {} as Record<string, number>;
for (const [k, v] of Object.entries(data)) {
const name = k.replaceAll('.', columnDot);
columns[columnPrefix + name] = v;
}
const update = async (logHour: RawRecord<T>, logDay: RawRecord<T>): Promise<void> => {
await Promise.all([
this.repositoryForHour.createQueryBuilder()
.update()
.set(Chart.convertObjectToFlattenColumns(data))
.set(columns as any)
.where('id = :id', { id: logHour.id })
.execute(),
this.repositoryForDay.createQueryBuilder()
.update()
.set(Chart.convertObjectToFlattenColumns(data))
.set(columns as any)
.where('id = :id', { id: logDay.id })
.execute(),
]);
@@ -460,12 +448,39 @@ export default abstract class Chart<T extends Record<string, any>> {
}
@autobind
protected async inc(inc: DeepPartial<T>, group: string | null = null): Promise<void> {
await this.commit(inc, group);
public async clean(): Promise<void> {
const current = dateUTC(Chart.getCurrentDate());
// 一日以上前かつ三日以内
const gt = Chart.dateToTimestamp(current) - (1000 * 60 * 60 * 24 * 3);
const lt = Chart.dateToTimestamp(current) - (1000 * 60 * 60 * 24);
const columns = {} as Record<string, number>;
for (const [k, v] of Object.entries(this.schema)) {
if (v.uniqueIncrement) {
const name = k.replaceAll('.', columnDot);
columns[uniqueTempColumnPrefix + name] = [];
}
}
await Promise.all([
this.repositoryForHour.createQueryBuilder()
.update()
.set(columns as any)
.where('date > :gt', { gt })
.andWhere('date < :lt', { lt })
.execute(),
this.repositoryForDay.createQueryBuilder()
.update()
.set(columns as any)
.where('date > :gt', { gt })
.andWhere('date < :lt', { lt })
.execute(),
]);
}
@autobind
public async getChart(span: 'hour' | 'day', amount: number, cursor: Date | null, group: string | null = null): Promise<ArrayValue<T>> {
public async getChartRaw(span: 'hour' | 'day', amount: number, cursor: Date | null, group: string | null = null): Promise<ChartResult<T>> {
const [y, m, d, h, _m, _s, _ms] = cursor ? Chart.parseDate(subtractTime(addTime(cursor, 1, span), 1)) : Chart.getCurrentDate();
const [y2, m2, d2, h2] = cursor ? Chart.parseDate(addTime(cursor, 1, span)) : [] as never;
@@ -526,7 +541,7 @@ export default abstract class Chart<T extends Record<string, any>> {
}
}
const chart: T[] = [];
const chart: KVs<T>[] = [];
for (let i = (amount - 1); i >= 0; i--) {
const current =
@@ -537,17 +552,16 @@ export default abstract class Chart<T extends Record<string, any>> {
const log = logs.find(l => isTimeSame(new Date(l.date * 1000), current));
if (log) {
const data = Chart.convertFlattenColumnsToObject(log);
chart.unshift(Chart.countUniqueFields(data) as T);
chart.unshift(this.convertRawRecord(log));
} else {
// 隙間埋め
const latest = logs.find(l => isTimeBefore(new Date(l.date * 1000), current));
const data = latest ? Chart.convertFlattenColumnsToObject(latest) as T : null;
chart.unshift(Chart.countUniqueFields(this.getNewLog(data)) as T);
const data = latest ? this.convertRawRecord(latest) : null;
chart.unshift(this.getNewLog(data));
}
}
const res = {} as Record<string, unknown>;
const res = {} as ChartResult<T>;
/**
* [{ foo: 1, bar: 5 }, { foo: 2, bar: 6 }, { foo: 3, bar: 7 }]
@@ -555,36 +569,26 @@ export default abstract class Chart<T extends Record<string, any>> {
* { foo: [1, 2, 3], bar: [5, 6, 7] }
* にする
*/
const compact = (x: Obj, path?: string): void => {
for (const [k, v] of Object.entries(x)) {
const p = path ? `${path}.${k}` : k;
if (typeof v === 'object' && !Array.isArray(v)) {
compact(v, p);
for (const record of chart) {
for (const [k, v] of Object.entries(record)) {
if (res[k]) {
res[k].push(v);
} else {
const values = chart.map(s => nestedProperty.get(s, p));
nestedProperty.set(res, p, values);
res[k] = [v];
}
}
};
compact(chart[0]);
return res as ArrayValue<T>;
}
}
export function convertLog(logSchema: Schema): Schema {
const v: Schema = JSON.parse(JSON.stringify(logSchema)); // copy
if (v.type === 'number') {
v.type = 'array';
v.items = {
type: 'number' as const,
optional: false as const, nullable: false as const,
};
} else if (v.type === 'object') {
for (const k of Object.keys(v.properties!)) {
v.properties![k] = convertLog(v.properties![k]);
}
return res;
}
@autobind
public async getChart(span: 'hour' | 'day', amount: number, cursor: Date | null, group: string | null = null): Promise<Record<string, unknown>> {
const result = await this.getChartRaw(span, amount, cursor, group);
const object = {};
for (const [k, v] of Object.entries(result)) {
nestedProperty.set(object, k, v);
}
return object;
}
return v;
}

4
packages/backend/src/services/chart/entities.ts
View File

@@ -1,7 +1,6 @@
import { entity as FederationChart } from './charts/entities/federation';
import { entity as NotesChart } from './charts/entities/notes';
import { entity as UsersChart } from './charts/entities/users';
import { entity as NetworkChart } from './charts/entities/network';
import { entity as ActiveUsersChart } from './charts/entities/active-users';
import { entity as InstanceChart } from './charts/entities/instance';
import { entity as PerUserNotesChart } from './charts/entities/per-user-notes';
@@ -10,12 +9,12 @@ import { entity as PerUserReactionsChart } from './charts/entities/per-user-reac
import { entity as HashtagChart } from './charts/entities/hashtag';
import { entity as PerUserFollowingChart } from './charts/entities/per-user-following';
import { entity as PerUserDriveChart } from './charts/entities/per-user-drive';
import { entity as ApRequestChart } from './charts/entities/ap-request';
export const entities = [
FederationChart.hour, FederationChart.day,
NotesChart.hour, NotesChart.day,
UsersChart.hour, UsersChart.day,
NetworkChart.hour, NetworkChart.day,
ActiveUsersChart.hour, ActiveUsersChart.day,
InstanceChart.hour, InstanceChart.day,
PerUserNotesChart.hour, PerUserNotesChart.day,
@@ -24,4 +23,5 @@ export const entities = [
HashtagChart.hour, HashtagChart.day,
PerUserFollowingChart.hour, PerUserFollowingChart.day,
PerUserDriveChart.hour, PerUserDriveChart.day,
ApRequestChart.hour, ApRequestChart.day,
];

6
packages/backend/src/services/chart/index.ts
View File

@@ -3,7 +3,6 @@ import { beforeShutdown } from '@/misc/before-shutdown';
import FederationChart from './charts/federation';
import NotesChart from './charts/notes';
import UsersChart from './charts/users';
import NetworkChart from './charts/network';
import ActiveUsersChart from './charts/active-users';
import InstanceChart from './charts/instance';
import PerUserNotesChart from './charts/per-user-notes';
@@ -12,11 +11,11 @@ import PerUserReactionsChart from './charts/per-user-reactions';
import HashtagChart from './charts/hashtag';
import PerUserFollowingChart from './charts/per-user-following';
import PerUserDriveChart from './charts/per-user-drive';
import ApRequestChart from './charts/ap-request';
export const federationChart = new FederationChart();
export const notesChart = new NotesChart();
export const usersChart = new UsersChart();
export const networkChart = new NetworkChart();
export const activeUsersChart = new ActiveUsersChart();
export const instanceChart = new InstanceChart();
export const perUserNotesChart = new PerUserNotesChart();
@@ -25,12 +24,12 @@ export const perUserReactionsChart = new PerUserReactionsChart();
export const hashtagChart = new HashtagChart();
export const perUserFollowingChart = new PerUserFollowingChart();
export const perUserDriveChart = new PerUserDriveChart();
export const apRequestChart = new ApRequestChart();
const charts = [
federationChart,
notesChart,
usersChart,
networkChart,
activeUsersChart,
instanceChart,
perUserNotesChart,
@@ -39,6 +38,7 @@ const charts = [
hashtagChart,
perUserFollowingChart,
perUserDriveChart,
apRequestChart,
];
// 20分おきにメモリ情報をDBに書き込み