Use PostgreSQL instead of MongoDB (#4572)
* wip * Update note.ts * Update timeline.ts * Update core.ts * wip * Update generate-visibility-query.ts * wip * wip * wip * wip * wip * Update global-timeline.ts * wip * wip * wip * Update vote.ts * wip * wip * Update create.ts * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * Update files.ts * wip * wip * Update CONTRIBUTING.md * wip * wip * wip * wip * wip * wip * wip * wip * Update read-notification.ts * wip * wip * wip * wip * wip * wip * wip * Update cancel.ts * wip * wip * wip * Update show.ts * wip * wip * Update gen-id.ts * Update create.ts * Update id.ts * wip * wip * wip * wip * wip * wip * wip * Docker: Update files about Docker (#4599) * Docker: Use cache if files used by `yarn install` was not updated This patch reduces the number of times to installing node_modules. For example, `yarn install` step will be skipped when only ".config/default.yml" is updated. * Docker: Migrate MongoDB to Postgresql Misskey uses Postgresql as a database instead of Mongodb since version 11. * Docker: Uncomment about data persistence This patch will save a lot of databases. * wip * wip * wip * Update activitypub.ts * wip * wip * wip * Update logs.ts * wip * Update drive-file.ts * Update register.ts * wip * wip * Update mentions.ts * wip * wip * wip * Update recommendation.ts * wip * Update index.ts * wip * Update recommendation.ts * Doc: Update docker.ja.md and docker.en.md (#1) (#4608) Update how to set up misskey. * wip * ✌️ * wip * Update note.ts * Update postgre.ts * wip * wip * wip * wip * Update add-file.ts * wip * wip * wip * Clean up * Update logs.ts * wip * 🍕 * wip * Ad notes * wip * Update api-visibility.ts * Update note.ts * Update add-file.ts * tests * tests * Update postgre.ts * Update utils.ts * wip * wip * Refactor * wip * Refactor * wip * wip * Update show-users.ts * Update update-instance.ts * wip * Update feed.ts * Update outbox.ts * Update outbox.ts * Update user.ts * wip * Update list.ts * Update update-hashtag.ts * wip * Update update-hashtag.ts * Refactor * Update update.ts * wip * wip * ✌️ * clean up * docs * Update push.ts * wip * Update api.ts * wip * ✌️ * Update make-pagination-query.ts * ✌️ * Delete hashtags.ts * Update instances.ts * Update instances.ts * Update create.ts * Update search.ts * Update reversi-game.ts * Update signup.ts * Update user.ts * id * Update example.yml * 🎨 * objectid * fix * reversi * reversi * Fix bug of chart engine * Add test of chart engine * Improve test * Better testing * Improve chart engine * Refactor * Add test of chart engine * Refactor * Add chart test * Fix bug * コミットし忘れ * Refactoring * ✌️ * Add tests * Add test * Extarct note tests * Refactor * 存在しないユーザーにメンションできなくなっていた問題を修正 * Fix bug * Update update-meta.ts * Fix bug * Update mention.vue * Fix bug * Update meta.ts * Update CONTRIBUTING.md * Fix bug * Fix bug * Fix bug * Clean up * Clean up * Update notification.ts * Clean up * Add mute tests * Add test * Refactor * Add test * Fix test * Refactor * Refactor * Add tests * Update utils.ts * Update utils.ts * Fix test * Update package.json * Update update.ts * Update manifest.ts * Fix bug * Fix bug * Add test * 🎨 * Update endpoint permissions * Updaye permisison * Update person.ts #4299 * データベースと同期しないように * Fix bug * Fix bug * Update reversi-game.ts * Use a feature of Node v11.7.0 to extract a public key (#4644) * wip * wip * ✌️ * Refactoring #1540 * test * test * test * test * test * test * test * Fix bug * Fix test * 🍣 * wip * #4471 * Add test for #4335 * Refactor * Fix test * Add tests * 🕓 * Fix bug * Add test * Add test * rename * Fix bug
This commit is contained in:
syuilo
@@ -2,14 +2,14 @@ import * as Queue from 'bull';
|
||||
import * as httpSignature from 'http-signature';
|
||||
|
||||
import config from '../config';
|
||||
import { ILocalUser } from '../models/user';
|
||||
import { ILocalUser } from '../models/entities/user';
|
||||
import { program } from '../argv';
|
||||
|
||||
import processDeliver from './processors/deliver';
|
||||
import processInbox from './processors/inbox';
|
||||
import processDb from './processors/db';
|
||||
import { queueLogger } from './logger';
|
||||
import { IDriveFile } from '../models/drive-file';
|
||||
import { DriveFile } from '../models/entities/drive-file';
|
||||
|
||||
function initializeQueue(name: string) {
|
||||
return new Queue(name, config.redis != null ? {
|
||||
@@ -83,15 +83,6 @@ export function inbox(activity: any, signature: httpSignature.IParsedSignature)
|
||||
});
|
||||
}
|
||||
|
||||
export function createDeleteNotesJob(user: ILocalUser) {
|
||||
return dbQueue.add('deleteNotes', {
|
||||
user: user
|
||||
}, {
|
||||
removeOnComplete: true,
|
||||
removeOnFail: true
|
||||
});
|
||||
}
|
||||
|
||||
export function createDeleteDriveFilesJob(user: ILocalUser) {
|
||||
return dbQueue.add('deleteDriveFiles', {
|
||||
user: user
|
||||
@@ -146,7 +137,7 @@ export function createExportUserListsJob(user: ILocalUser) {
|
||||
});
|
||||
}
|
||||
|
||||
export function createImportFollowingJob(user: ILocalUser, fileId: IDriveFile['_id']) {
|
||||
export function createImportFollowingJob(user: ILocalUser, fileId: DriveFile['id']) {
|
||||
return dbQueue.add('importFollowing', {
|
||||
user: user,
|
||||
fileId: fileId
|
||||
@@ -156,7 +147,7 @@ export function createImportFollowingJob(user: ILocalUser, fileId: IDriveFile['_
|
||||
});
|
||||
}
|
||||
|
||||
export function createImportUserListsJob(user: ILocalUser, fileId: IDriveFile['_id']) {
|
||||
export function createImportUserListsJob(user: ILocalUser, fileId: DriveFile['id']) {
|
||||
return dbQueue.add('importUserLists', {
|
||||
user: user,
|
||||
fileId: fileId
|
||||
|
||||
@@ -1,18 +1,17 @@
|
||||
import * as Bull from 'bull';
|
||||
import * as mongo from 'mongodb';
|
||||
|
||||
import { queueLogger } from '../../logger';
|
||||
import User from '../../../models/user';
|
||||
import DriveFile from '../../../models/drive-file';
|
||||
import deleteFile from '../../../services/drive/delete-file';
|
||||
import { Users, DriveFiles } from '../../../models';
|
||||
import { MoreThan } from 'typeorm';
|
||||
|
||||
const logger = queueLogger.createSubLogger('delete-drive-files');
|
||||
|
||||
export async function deleteDriveFiles(job: Bull.Job, done: any): Promise<void> {
|
||||
logger.info(`Deleting drive files of ${job.data.user._id} ...`);
|
||||
logger.info(`Deleting drive files of ${job.data.user.id} ...`);
|
||||
|
||||
const user = await User.findOne({
|
||||
_id: new mongo.ObjectID(job.data.user._id.toString())
|
||||
const user = await Users.findOne({
|
||||
id: job.data.user.id
|
||||
});
|
||||
|
||||
let deletedCount = 0;
|
||||
@@ -20,13 +19,14 @@ export async function deleteDriveFiles(job: Bull.Job, done: any): Promise<void>
|
||||
let cursor: any = null;
|
||||
|
||||
while (!ended) {
|
||||
const files = await DriveFile.find({
|
||||
userId: user._id,
|
||||
...(cursor ? { _id: { $gt: cursor } } : {})
|
||||
}, {
|
||||
limit: 100,
|
||||
sort: {
|
||||
_id: 1
|
||||
const files = await DriveFiles.find({
|
||||
where: {
|
||||
userId: user.id,
|
||||
...(cursor ? { id: MoreThan(cursor) } : {})
|
||||
},
|
||||
take: 100,
|
||||
order: {
|
||||
id: 1
|
||||
}
|
||||
});
|
||||
|
||||
@@ -36,20 +36,20 @@ export async function deleteDriveFiles(job: Bull.Job, done: any): Promise<void>
|
||||
break;
|
||||
}
|
||||
|
||||
cursor = files[files.length - 1]._id;
|
||||
cursor = files[files.length - 1].id;
|
||||
|
||||
for (const file of files) {
|
||||
await deleteFile(file);
|
||||
deletedCount++;
|
||||
}
|
||||
|
||||
const total = await DriveFile.count({
|
||||
userId: user._id,
|
||||
const total = await DriveFiles.count({
|
||||
userId: user.id,
|
||||
});
|
||||
|
||||
job.progress(deletedCount / total);
|
||||
}
|
||||
|
||||
logger.succ(`All drive files (${deletedCount}) of ${user._id} has been deleted.`);
|
||||
logger.succ(`All drive files (${deletedCount}) of ${user.id} has been deleted.`);
|
||||
done();
|
||||
}
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
import * as Bull from 'bull';
|
||||
import * as mongo from 'mongodb';
|
||||
|
||||
import { queueLogger } from '../../logger';
|
||||
import Note from '../../../models/note';
|
||||
import deleteNote from '../../../services/note/delete';
|
||||
import User from '../../../models/user';
|
||||
|
||||
const logger = queueLogger.createSubLogger('delete-notes');
|
||||
|
||||
export async function deleteNotes(job: Bull.Job, done: any): Promise<void> {
|
||||
logger.info(`Deleting notes of ${job.data.user._id} ...`);
|
||||
|
||||
const user = await User.findOne({
|
||||
_id: new mongo.ObjectID(job.data.user._id.toString())
|
||||
});
|
||||
|
||||
let deletedCount = 0;
|
||||
let ended = false;
|
||||
let cursor: any = null;
|
||||
|
||||
while (!ended) {
|
||||
const notes = await Note.find({
|
||||
userId: user._id,
|
||||
...(cursor ? { _id: { $gt: cursor } } : {})
|
||||
}, {
|
||||
limit: 100,
|
||||
sort: {
|
||||
_id: 1
|
||||
}
|
||||
});
|
||||
|
||||
if (notes.length === 0) {
|
||||
ended = true;
|
||||
job.progress(100);
|
||||
break;
|
||||
}
|
||||
|
||||
cursor = notes[notes.length - 1]._id;
|
||||
|
||||
for (const note of notes) {
|
||||
await deleteNote(user, note, true);
|
||||
deletedCount++;
|
||||
}
|
||||
|
||||
const total = await Note.count({
|
||||
userId: user._id,
|
||||
});
|
||||
|
||||
job.progress(deletedCount / total);
|
||||
}
|
||||
|
||||
logger.succ(`All notes (${deletedCount}) of ${user._id} has been deleted.`);
|
||||
done();
|
||||
}
|
||||
@@ -1,22 +1,21 @@
|
||||
import * as Bull from 'bull';
|
||||
import * as tmp from 'tmp';
|
||||
import * as fs from 'fs';
|
||||
import * as mongo from 'mongodb';
|
||||
|
||||
import { queueLogger } from '../../logger';
|
||||
import addFile from '../../../services/drive/add-file';
|
||||
import User from '../../../models/user';
|
||||
import dateFormat = require('dateformat');
|
||||
import Blocking from '../../../models/blocking';
|
||||
import { getFullApAccount } from '../../../misc/convert-host';
|
||||
import { Users, Blockings } from '../../../models';
|
||||
import { MoreThan } from 'typeorm';
|
||||
|
||||
const logger = queueLogger.createSubLogger('export-blocking');
|
||||
|
||||
export async function exportBlocking(job: Bull.Job, done: any): Promise<void> {
|
||||
logger.info(`Exporting blocking of ${job.data.user._id} ...`);
|
||||
logger.info(`Exporting blocking of ${job.data.user.id} ...`);
|
||||
|
||||
const user = await User.findOne({
|
||||
_id: new mongo.ObjectID(job.data.user._id.toString())
|
||||
const user = await Users.findOne({
|
||||
id: job.data.user.id
|
||||
});
|
||||
|
||||
// Create temp file
|
||||
@@ -36,13 +35,14 @@ export async function exportBlocking(job: Bull.Job, done: any): Promise<void> {
|
||||
let cursor: any = null;
|
||||
|
||||
while (!ended) {
|
||||
const blockings = await Blocking.find({
|
||||
blockerId: user._id,
|
||||
...(cursor ? { _id: { $gt: cursor } } : {})
|
||||
}, {
|
||||
limit: 100,
|
||||
sort: {
|
||||
_id: 1
|
||||
const blockings = await Blockings.find({
|
||||
where: {
|
||||
blockerId: user.id,
|
||||
...(cursor ? { id: MoreThan(cursor) } : {})
|
||||
},
|
||||
take: 100,
|
||||
order: {
|
||||
id: 1
|
||||
}
|
||||
});
|
||||
|
||||
@@ -52,10 +52,10 @@ export async function exportBlocking(job: Bull.Job, done: any): Promise<void> {
|
||||
break;
|
||||
}
|
||||
|
||||
cursor = blockings[blockings.length - 1]._id;
|
||||
cursor = blockings[blockings.length - 1].id;
|
||||
|
||||
for (const block of blockings) {
|
||||
const u = await User.findOne({ _id: block.blockeeId }, { fields: { username: true, host: true } });
|
||||
const u = await Users.findOne({ id: block.blockeeId });
|
||||
const content = getFullApAccount(u.username, u.host);
|
||||
await new Promise((res, rej) => {
|
||||
stream.write(content + '\n', err => {
|
||||
@@ -70,8 +70,8 @@ export async function exportBlocking(job: Bull.Job, done: any): Promise<void> {
|
||||
exportedCount++;
|
||||
}
|
||||
|
||||
const total = await Blocking.count({
|
||||
blockerId: user._id,
|
||||
const total = await Blockings.count({
|
||||
blockerId: user.id,
|
||||
});
|
||||
|
||||
job.progress(exportedCount / total);
|
||||
@@ -83,7 +83,7 @@ export async function exportBlocking(job: Bull.Job, done: any): Promise<void> {
|
||||
const fileName = 'blocking-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.csv';
|
||||
const driveFile = await addFile(user, path, fileName);
|
||||
|
||||
logger.succ(`Exported to: ${driveFile._id}`);
|
||||
logger.succ(`Exported to: ${driveFile.id}`);
|
||||
cleanup();
|
||||
done();
|
||||
}
|
||||
|
||||
@@ -1,22 +1,21 @@
|
||||
import * as Bull from 'bull';
|
||||
import * as tmp from 'tmp';
|
||||
import * as fs from 'fs';
|
||||
import * as mongo from 'mongodb';
|
||||
|
||||
import { queueLogger } from '../../logger';
|
||||
import addFile from '../../../services/drive/add-file';
|
||||
import User from '../../../models/user';
|
||||
import dateFormat = require('dateformat');
|
||||
import Following from '../../../models/following';
|
||||
import { getFullApAccount } from '../../../misc/convert-host';
|
||||
import { Users, Followings } from '../../../models';
|
||||
import { MoreThan } from 'typeorm';
|
||||
|
||||
const logger = queueLogger.createSubLogger('export-following');
|
||||
|
||||
export async function exportFollowing(job: Bull.Job, done: any): Promise<void> {
|
||||
logger.info(`Exporting following of ${job.data.user._id} ...`);
|
||||
logger.info(`Exporting following of ${job.data.user.id} ...`);
|
||||
|
||||
const user = await User.findOne({
|
||||
_id: new mongo.ObjectID(job.data.user._id.toString())
|
||||
const user = await Users.findOne({
|
||||
id: job.data.user.id
|
||||
});
|
||||
|
||||
// Create temp file
|
||||
@@ -36,13 +35,14 @@ export async function exportFollowing(job: Bull.Job, done: any): Promise<void> {
|
||||
let cursor: any = null;
|
||||
|
||||
while (!ended) {
|
||||
const followings = await Following.find({
|
||||
followerId: user._id,
|
||||
...(cursor ? { _id: { $gt: cursor } } : {})
|
||||
}, {
|
||||
limit: 100,
|
||||
sort: {
|
||||
_id: 1
|
||||
const followings = await Followings.find({
|
||||
where: {
|
||||
followerId: user.id,
|
||||
...(cursor ? { id: MoreThan(cursor) } : {})
|
||||
},
|
||||
take: 100,
|
||||
order: {
|
||||
id: 1
|
||||
}
|
||||
});
|
||||
|
||||
@@ -52,10 +52,10 @@ export async function exportFollowing(job: Bull.Job, done: any): Promise<void> {
|
||||
break;
|
||||
}
|
||||
|
||||
cursor = followings[followings.length - 1]._id;
|
||||
cursor = followings[followings.length - 1].id;
|
||||
|
||||
for (const following of followings) {
|
||||
const u = await User.findOne({ _id: following.followeeId }, { fields: { username: true, host: true } });
|
||||
const u = await Users.findOne({ id: following.followeeId });
|
||||
const content = getFullApAccount(u.username, u.host);
|
||||
await new Promise((res, rej) => {
|
||||
stream.write(content + '\n', err => {
|
||||
@@ -70,8 +70,8 @@ export async function exportFollowing(job: Bull.Job, done: any): Promise<void> {
|
||||
exportedCount++;
|
||||
}
|
||||
|
||||
const total = await Following.count({
|
||||
followerId: user._id,
|
||||
const total = await Followings.count({
|
||||
followerId: user.id,
|
||||
});
|
||||
|
||||
job.progress(exportedCount / total);
|
||||
@@ -83,7 +83,7 @@ export async function exportFollowing(job: Bull.Job, done: any): Promise<void> {
|
||||
const fileName = 'following-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.csv';
|
||||
const driveFile = await addFile(user, path, fileName);
|
||||
|
||||
logger.succ(`Exported to: ${driveFile._id}`);
|
||||
logger.succ(`Exported to: ${driveFile.id}`);
|
||||
cleanup();
|
||||
done();
|
||||
}
|
||||
|
||||
@@ -1,22 +1,21 @@
|
||||
import * as Bull from 'bull';
|
||||
import * as tmp from 'tmp';
|
||||
import * as fs from 'fs';
|
||||
import * as mongo from 'mongodb';
|
||||
|
||||
import { queueLogger } from '../../logger';
|
||||
import addFile from '../../../services/drive/add-file';
|
||||
import User from '../../../models/user';
|
||||
import dateFormat = require('dateformat');
|
||||
import Mute from '../../../models/mute';
|
||||
import { getFullApAccount } from '../../../misc/convert-host';
|
||||
import { Users, Mutings } from '../../../models';
|
||||
import { MoreThan } from 'typeorm';
|
||||
|
||||
const logger = queueLogger.createSubLogger('export-mute');
|
||||
|
||||
export async function exportMute(job: Bull.Job, done: any): Promise<void> {
|
||||
logger.info(`Exporting mute of ${job.data.user._id} ...`);
|
||||
logger.info(`Exporting mute of ${job.data.user.id} ...`);
|
||||
|
||||
const user = await User.findOne({
|
||||
_id: new mongo.ObjectID(job.data.user._id.toString())
|
||||
const user = await Users.findOne({
|
||||
id: job.data.user.id
|
||||
});
|
||||
|
||||
// Create temp file
|
||||
@@ -36,13 +35,14 @@ export async function exportMute(job: Bull.Job, done: any): Promise<void> {
|
||||
let cursor: any = null;
|
||||
|
||||
while (!ended) {
|
||||
const mutes = await Mute.find({
|
||||
muterId: user._id,
|
||||
...(cursor ? { _id: { $gt: cursor } } : {})
|
||||
}, {
|
||||
limit: 100,
|
||||
sort: {
|
||||
_id: 1
|
||||
const mutes = await Mutings.find({
|
||||
where: {
|
||||
muterId: user.id,
|
||||
...(cursor ? { id: MoreThan(cursor) } : {})
|
||||
},
|
||||
take: 100,
|
||||
order: {
|
||||
id: 1
|
||||
}
|
||||
});
|
||||
|
||||
@@ -52,10 +52,10 @@ export async function exportMute(job: Bull.Job, done: any): Promise<void> {
|
||||
break;
|
||||
}
|
||||
|
||||
cursor = mutes[mutes.length - 1]._id;
|
||||
cursor = mutes[mutes.length - 1].id;
|
||||
|
||||
for (const mute of mutes) {
|
||||
const u = await User.findOne({ _id: mute.muteeId }, { fields: { username: true, host: true } });
|
||||
const u = await Users.findOne({ id: mute.muteeId });
|
||||
const content = getFullApAccount(u.username, u.host);
|
||||
await new Promise((res, rej) => {
|
||||
stream.write(content + '\n', err => {
|
||||
@@ -70,8 +70,8 @@ export async function exportMute(job: Bull.Job, done: any): Promise<void> {
|
||||
exportedCount++;
|
||||
}
|
||||
|
||||
const total = await Mute.count({
|
||||
muterId: user._id,
|
||||
const total = await Mutings.count({
|
||||
muterId: user.id,
|
||||
});
|
||||
|
||||
job.progress(exportedCount / total);
|
||||
@@ -83,7 +83,7 @@ export async function exportMute(job: Bull.Job, done: any): Promise<void> {
|
||||
const fileName = 'mute-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.csv';
|
||||
const driveFile = await addFile(user, path, fileName);
|
||||
|
||||
logger.succ(`Exported to: ${driveFile._id}`);
|
||||
logger.succ(`Exported to: ${driveFile.id}`);
|
||||
cleanup();
|
||||
done();
|
||||
}
|
||||
|
||||
@@ -1,21 +1,22 @@
|
||||
import * as Bull from 'bull';
|
||||
import * as tmp from 'tmp';
|
||||
import * as fs from 'fs';
|
||||
import * as mongo from 'mongodb';
|
||||
|
||||
import { queueLogger } from '../../logger';
|
||||
import Note, { INote } from '../../../models/note';
|
||||
import addFile from '../../../services/drive/add-file';
|
||||
import User from '../../../models/user';
|
||||
import dateFormat = require('dateformat');
|
||||
import { Users, Notes, Polls } from '../../../models';
|
||||
import { MoreThan } from 'typeorm';
|
||||
import { Note } from '../../../models/entities/note';
|
||||
import { Poll } from '../../../models/entities/poll';
|
||||
|
||||
const logger = queueLogger.createSubLogger('export-notes');
|
||||
|
||||
export async function exportNotes(job: Bull.Job, done: any): Promise<void> {
|
||||
logger.info(`Exporting notes of ${job.data.user._id} ...`);
|
||||
logger.info(`Exporting notes of ${job.data.user.id} ...`);
|
||||
|
||||
const user = await User.findOne({
|
||||
_id: new mongo.ObjectID(job.data.user._id.toString())
|
||||
const user = await Users.findOne({
|
||||
id: job.data.user.id
|
||||
});
|
||||
|
||||
// Create temp file
|
||||
@@ -46,13 +47,14 @@ export async function exportNotes(job: Bull.Job, done: any): Promise<void> {
|
||||
let cursor: any = null;
|
||||
|
||||
while (!ended) {
|
||||
const notes = await Note.find({
|
||||
userId: user._id,
|
||||
...(cursor ? { _id: { $gt: cursor } } : {})
|
||||
}, {
|
||||
limit: 100,
|
||||
sort: {
|
||||
_id: 1
|
||||
const notes = await Notes.find({
|
||||
where: {
|
||||
userId: user.id,
|
||||
...(cursor ? { id: MoreThan(cursor) } : {})
|
||||
},
|
||||
take: 100,
|
||||
order: {
|
||||
id: 1
|
||||
}
|
||||
});
|
||||
|
||||
@@ -62,10 +64,14 @@ export async function exportNotes(job: Bull.Job, done: any): Promise<void> {
|
||||
break;
|
||||
}
|
||||
|
||||
cursor = notes[notes.length - 1]._id;
|
||||
cursor = notes[notes.length - 1].id;
|
||||
|
||||
for (const note of notes) {
|
||||
const content = JSON.stringify(serialize(note));
|
||||
let poll: Poll;
|
||||
if (note.hasPoll) {
|
||||
poll = await Polls.findOne({ noteId: note.id });
|
||||
}
|
||||
const content = JSON.stringify(serialize(note, poll));
|
||||
await new Promise((res, rej) => {
|
||||
stream.write(exportedNotesCount === 0 ? content : ',\n' + content, err => {
|
||||
if (err) {
|
||||
@@ -79,8 +85,8 @@ export async function exportNotes(job: Bull.Job, done: any): Promise<void> {
|
||||
exportedNotesCount++;
|
||||
}
|
||||
|
||||
const total = await Note.count({
|
||||
userId: user._id,
|
||||
const total = await Notes.count({
|
||||
userId: user.id,
|
||||
});
|
||||
|
||||
job.progress(exportedNotesCount / total);
|
||||
@@ -103,20 +109,20 @@ export async function exportNotes(job: Bull.Job, done: any): Promise<void> {
|
||||
const fileName = 'notes-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.json';
|
||||
const driveFile = await addFile(user, path, fileName);
|
||||
|
||||
logger.succ(`Exported to: ${driveFile._id}`);
|
||||
logger.succ(`Exported to: ${driveFile.id}`);
|
||||
cleanup();
|
||||
done();
|
||||
}
|
||||
|
||||
function serialize(note: INote): any {
|
||||
function serialize(note: Note, poll: Poll): any {
|
||||
return {
|
||||
id: note._id,
|
||||
id: note.id,
|
||||
text: note.text,
|
||||
createdAt: note.createdAt,
|
||||
fileIds: note.fileIds,
|
||||
replyId: note.replyId,
|
||||
renoteId: note.renoteId,
|
||||
poll: note.poll,
|
||||
poll: poll,
|
||||
cw: note.cw,
|
||||
viaMobile: note.viaMobile,
|
||||
visibility: note.visibility,
|
||||
|
||||
@@ -1,26 +1,25 @@
|
||||
import * as Bull from 'bull';
|
||||
import * as tmp from 'tmp';
|
||||
import * as fs from 'fs';
|
||||
import * as mongo from 'mongodb';
|
||||
|
||||
import { queueLogger } from '../../logger';
|
||||
import addFile from '../../../services/drive/add-file';
|
||||
import User from '../../../models/user';
|
||||
import dateFormat = require('dateformat');
|
||||
import UserList from '../../../models/user-list';
|
||||
import { getFullApAccount } from '../../../misc/convert-host';
|
||||
import { Users, UserLists, UserListJoinings } from '../../../models';
|
||||
import { In } from 'typeorm';
|
||||
|
||||
const logger = queueLogger.createSubLogger('export-user-lists');
|
||||
|
||||
export async function exportUserLists(job: Bull.Job, done: any): Promise<void> {
|
||||
logger.info(`Exporting user lists of ${job.data.user._id} ...`);
|
||||
logger.info(`Exporting user lists of ${job.data.user.id} ...`);
|
||||
|
||||
const user = await User.findOne({
|
||||
_id: new mongo.ObjectID(job.data.user._id.toString())
|
||||
const user = await Users.findOne({
|
||||
id: job.data.user.id
|
||||
});
|
||||
|
||||
const lists = await UserList.find({
|
||||
userId: user._id
|
||||
const lists = await UserLists.find({
|
||||
userId: user.id
|
||||
});
|
||||
|
||||
// Create temp file
|
||||
@@ -36,18 +35,14 @@ export async function exportUserLists(job: Bull.Job, done: any): Promise<void> {
|
||||
const stream = fs.createWriteStream(path, { flags: 'a' });
|
||||
|
||||
for (const list of lists) {
|
||||
const users = await User.find({
|
||||
_id: { $in: list.userIds }
|
||||
}, {
|
||||
fields: {
|
||||
username: true,
|
||||
host: true
|
||||
}
|
||||
const joinings = await UserListJoinings.find({ userListId: list.id });
|
||||
const users = await Users.find({
|
||||
id: In(joinings.map(j => j.userId))
|
||||
});
|
||||
|
||||
for (const u of users) {
|
||||
const acct = getFullApAccount(u.username, u.host);
|
||||
const content = `${list.title},${acct}`;
|
||||
const content = `${list.name},${acct}`;
|
||||
await new Promise((res, rej) => {
|
||||
stream.write(content + '\n', err => {
|
||||
if (err) {
|
||||
@@ -67,7 +62,7 @@ export async function exportUserLists(job: Bull.Job, done: any): Promise<void> {
|
||||
const fileName = 'user-lists-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.csv';
|
||||
const driveFile = await addFile(user, path, fileName);
|
||||
|
||||
logger.succ(`Exported to: ${driveFile._id}`);
|
||||
logger.succ(`Exported to: ${driveFile.id}`);
|
||||
cleanup();
|
||||
done();
|
||||
}
|
||||
|
||||
@@ -1,32 +1,27 @@
|
||||
import * as Bull from 'bull';
|
||||
import * as mongo from 'mongodb';
|
||||
|
||||
import { queueLogger } from '../../logger';
|
||||
import User from '../../../models/user';
|
||||
import follow from '../../../services/following/create';
|
||||
import DriveFile from '../../../models/drive-file';
|
||||
import { getOriginalUrl } from '../../../misc/get-drive-file-url';
|
||||
import parseAcct from '../../../misc/acct/parse';
|
||||
import resolveUser from '../../../remote/resolve-user';
|
||||
import { downloadTextFile } from '../../../misc/download-text-file';
|
||||
import { isSelfHost, toDbHost } from '../../../misc/convert-host';
|
||||
import { Users, DriveFiles } from '../../../models';
|
||||
|
||||
const logger = queueLogger.createSubLogger('import-following');
|
||||
|
||||
export async function importFollowing(job: Bull.Job, done: any): Promise<void> {
|
||||
logger.info(`Importing following of ${job.data.user._id} ...`);
|
||||
logger.info(`Importing following of ${job.data.user.id} ...`);
|
||||
|
||||
const user = await User.findOne({
|
||||
_id: new mongo.ObjectID(job.data.user._id.toString())
|
||||
const user = await Users.findOne({
|
||||
id: job.data.user.id
|
||||
});
|
||||
|
||||
const file = await DriveFile.findOne({
|
||||
_id: new mongo.ObjectID(job.data.fileId.toString())
|
||||
const file = await DriveFiles.findOne({
|
||||
id: job.data.fileId
|
||||
});
|
||||
|
||||
const url = getOriginalUrl(file);
|
||||
|
||||
const csv = await downloadTextFile(url);
|
||||
const csv = await downloadTextFile(file.url);
|
||||
|
||||
let linenum = 0;
|
||||
|
||||
@@ -36,10 +31,10 @@ export async function importFollowing(job: Bull.Job, done: any): Promise<void> {
|
||||
try {
|
||||
const { username, host } = parseAcct(line.trim());
|
||||
|
||||
let target = isSelfHost(host) ? await User.findOne({
|
||||
let target = isSelfHost(host) ? await Users.findOne({
|
||||
host: null,
|
||||
usernameLower: username.toLowerCase()
|
||||
}) : await User.findOne({
|
||||
}) : await Users.findOne({
|
||||
host: toDbHost(host),
|
||||
usernameLower: username.toLowerCase()
|
||||
});
|
||||
@@ -55,9 +50,9 @@ export async function importFollowing(job: Bull.Job, done: any): Promise<void> {
|
||||
}
|
||||
|
||||
// skip myself
|
||||
if (target._id.equals(job.data.user._id)) continue;
|
||||
if (target.id === job.data.user.id) continue;
|
||||
|
||||
logger.info(`Follow[${linenum}] ${target._id} ...`);
|
||||
logger.info(`Follow[${linenum}] ${target.id} ...`);
|
||||
|
||||
follow(user, target);
|
||||
} catch (e) {
|
||||
|
||||
@@ -1,62 +1,59 @@
|
||||
import * as Bull from 'bull';
|
||||
import * as mongo from 'mongodb';
|
||||
|
||||
import { queueLogger } from '../../logger';
|
||||
import User from '../../../models/user';
|
||||
import UserList from '../../../models/user-list';
|
||||
import DriveFile from '../../../models/drive-file';
|
||||
import { getOriginalUrl } from '../../../misc/get-drive-file-url';
|
||||
import parseAcct from '../../../misc/acct/parse';
|
||||
import resolveUser from '../../../remote/resolve-user';
|
||||
import { pushUserToUserList } from '../../../services/user-list/push';
|
||||
import { downloadTextFile } from '../../../misc/download-text-file';
|
||||
import { isSelfHost, toDbHost } from '../../../misc/convert-host';
|
||||
import { DriveFiles, Users, UserLists, UserListJoinings } from '../../../models';
|
||||
import { genId } from '../../../misc/gen-id';
|
||||
|
||||
const logger = queueLogger.createSubLogger('import-user-lists');
|
||||
|
||||
export async function importUserLists(job: Bull.Job, done: any): Promise<void> {
|
||||
logger.info(`Importing user lists of ${job.data.user._id} ...`);
|
||||
logger.info(`Importing user lists of ${job.data.user.id} ...`);
|
||||
|
||||
const user = await User.findOne({
|
||||
_id: new mongo.ObjectID(job.data.user._id.toString())
|
||||
const user = await Users.findOne({
|
||||
id: job.data.user.id
|
||||
});
|
||||
|
||||
const file = await DriveFile.findOne({
|
||||
_id: new mongo.ObjectID(job.data.fileId.toString())
|
||||
const file = await DriveFiles.findOne({
|
||||
id: job.data.fileId
|
||||
});
|
||||
|
||||
const url = getOriginalUrl(file);
|
||||
|
||||
const csv = await downloadTextFile(url);
|
||||
const csv = await downloadTextFile(file.url);
|
||||
|
||||
for (const line of csv.trim().split('\n')) {
|
||||
const listName = line.split(',')[0].trim();
|
||||
const { username, host } = parseAcct(line.split(',')[1].trim());
|
||||
|
||||
let list = await UserList.findOne({
|
||||
userId: user._id,
|
||||
title: listName
|
||||
let list = await UserLists.findOne({
|
||||
userId: user.id,
|
||||
name: listName
|
||||
});
|
||||
|
||||
if (list == null) {
|
||||
list = await UserList.insert({
|
||||
list = await UserLists.save({
|
||||
id: genId(),
|
||||
createdAt: new Date(),
|
||||
userId: user._id,
|
||||
title: listName,
|
||||
userId: user.id,
|
||||
name: listName,
|
||||
userIds: []
|
||||
});
|
||||
}
|
||||
|
||||
let target = isSelfHost(host) ? await User.findOne({
|
||||
let target = isSelfHost(host) ? await Users.findOne({
|
||||
host: null,
|
||||
usernameLower: username.toLowerCase()
|
||||
}) : await User.findOne({
|
||||
}) : await Users.findOne({
|
||||
host: toDbHost(host),
|
||||
usernameLower: username.toLowerCase()
|
||||
});
|
||||
|
||||
if (host == null && target == null) continue;
|
||||
if (list.userIds.some(id => id.equals(target._id))) continue;
|
||||
|
||||
if (await UserListJoinings.findOne({ userListId: list.id, userId: target.id }) != null) continue;
|
||||
|
||||
if (target == null) {
|
||||
target = await resolveUser(username, host);
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import * as Bull from 'bull';
|
||||
import { deleteNotes } from './delete-notes';
|
||||
import { deleteDriveFiles } from './delete-drive-files';
|
||||
import { exportNotes } from './export-notes';
|
||||
import { exportFollowing } from './export-following';
|
||||
@@ -10,7 +9,6 @@ import { importFollowing } from './import-following';
|
||||
import { importUserLists } from './import-user-lists';
|
||||
|
||||
const jobs = {
|
||||
deleteNotes,
|
||||
deleteDriveFiles,
|
||||
exportNotes,
|
||||
exportFollowing,
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import * as Bull from 'bull';
|
||||
import request from '../../remote/activitypub/request';
|
||||
import { registerOrFetchInstanceDoc } from '../../services/register-or-fetch-instance-doc';
|
||||
import Instance from '../../models/instance';
|
||||
import instanceChart from '../../services/chart/instance';
|
||||
import Logger from '../../services/logger';
|
||||
import { Instances } from '../../models';
|
||||
import { instanceChart } from '../../services/chart';
|
||||
|
||||
const logger = new Logger('deliver');
|
||||
|
||||
@@ -21,13 +21,11 @@ export default async (job: Bull.Job) => {
|
||||
|
||||
// Update stats
|
||||
registerOrFetchInstanceDoc(host).then(i => {
|
||||
Instance.update({ _id: i._id }, {
|
||||
$set: {
|
||||
latestRequestSentAt: new Date(),
|
||||
latestStatus: 200,
|
||||
lastCommunicatedAt: new Date(),
|
||||
isNotResponding: false
|
||||
}
|
||||
Instances.update(i.id, {
|
||||
latestRequestSentAt: new Date(),
|
||||
latestStatus: 200,
|
||||
lastCommunicatedAt: new Date(),
|
||||
isNotResponding: false
|
||||
});
|
||||
|
||||
instanceChart.requestSent(i.host, true);
|
||||
@@ -37,12 +35,10 @@ export default async (job: Bull.Job) => {
|
||||
} catch (res) {
|
||||
// Update stats
|
||||
registerOrFetchInstanceDoc(host).then(i => {
|
||||
Instance.update({ _id: i._id }, {
|
||||
$set: {
|
||||
latestRequestSentAt: new Date(),
|
||||
latestStatus: res != null && res.hasOwnProperty('statusCode') ? res.statusCode : null,
|
||||
isNotResponding: true
|
||||
}
|
||||
Instances.update(i.id, {
|
||||
latestRequestSentAt: new Date(),
|
||||
latestStatus: res != null && res.hasOwnProperty('statusCode') ? res.statusCode : null,
|
||||
isNotResponding: true
|
||||
});
|
||||
|
||||
instanceChart.requestSent(i.host, false);
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import * as Bull from 'bull';
|
||||
import * as httpSignature from 'http-signature';
|
||||
import parseAcct from '../../misc/acct/parse';
|
||||
import User, { IRemoteUser } from '../../models/user';
|
||||
import { IRemoteUser } from '../../models/entities/user';
|
||||
import perform from '../../remote/activitypub/perform';
|
||||
import { resolvePerson, updatePerson } from '../../remote/activitypub/models/person';
|
||||
import { toUnicode } from 'punycode';
|
||||
@@ -9,8 +9,10 @@ import { URL } from 'url';
|
||||
import { publishApLogStream } from '../../services/stream';
|
||||
import Logger from '../../services/logger';
|
||||
import { registerOrFetchInstanceDoc } from '../../services/register-or-fetch-instance-doc';
|
||||
import Instance from '../../models/instance';
|
||||
import instanceChart from '../../services/chart/instance';
|
||||
import { Instances, Users, UserPublickeys } from '../../models';
|
||||
import { instanceChart } from '../../services/chart';
|
||||
import { UserPublickey } from '../../models/entities/user-publickey';
|
||||
import fetchMeta from '../../misc/fetch-meta';
|
||||
|
||||
const logger = new Logger('inbox');
|
||||
|
||||
@@ -28,6 +30,7 @@ export default async (job: Bull.Job): Promise<void> => {
|
||||
|
||||
const keyIdLower = signature.keyId.toLowerCase();
|
||||
let user: IRemoteUser;
|
||||
let key: UserPublickey;
|
||||
|
||||
if (keyIdLower.startsWith('acct:')) {
|
||||
const { username, host } = parseAcct(keyIdLower.slice('acct:'.length));
|
||||
@@ -46,13 +49,17 @@ export default async (job: Bull.Job): Promise<void> => {
|
||||
|
||||
// ブロックしてたら中断
|
||||
// TODO: いちいちデータベースにアクセスするのはコスト高そうなのでどっかにキャッシュしておく
|
||||
const instance = await Instance.findOne({ host: host.toLowerCase() });
|
||||
if (instance && instance.isBlocked) {
|
||||
const meta = await fetchMeta();
|
||||
if (meta.blockedHosts.includes(host.toLowerCase())) {
|
||||
logger.info(`Blocked request: ${host}`);
|
||||
return;
|
||||
}
|
||||
|
||||
user = await User.findOne({ usernameLower: username, host: host.toLowerCase() }) as IRemoteUser;
|
||||
user = await Users.findOne({ usernameLower: username, host: host.toLowerCase() }) as IRemoteUser;
|
||||
|
||||
key = await UserPublickeys.findOne({
|
||||
userId: user.id
|
||||
});
|
||||
} else {
|
||||
// アクティビティ内のホストの検証
|
||||
const host = toUnicode(new URL(signature.keyId).hostname.toLowerCase());
|
||||
@@ -65,16 +72,17 @@ export default async (job: Bull.Job): Promise<void> => {
|
||||
|
||||
// ブロックしてたら中断
|
||||
// TODO: いちいちデータベースにアクセスするのはコスト高そうなのでどっかにキャッシュしておく
|
||||
const instance = await Instance.findOne({ host: host.toLowerCase() });
|
||||
if (instance && instance.isBlocked) {
|
||||
logger.warn(`Blocked request: ${host}`);
|
||||
const meta = await fetchMeta();
|
||||
if (meta.blockedHosts.includes(host.toLowerCase())) {
|
||||
logger.info(`Blocked request: ${host}`);
|
||||
return;
|
||||
}
|
||||
|
||||
user = await User.findOne({
|
||||
host: { $ne: null },
|
||||
'publicKey.id': signature.keyId
|
||||
}) as IRemoteUser;
|
||||
key = await UserPublickeys.findOne({
|
||||
keyId: signature.keyId
|
||||
});
|
||||
|
||||
user = await Users.findOne(key.userId) as IRemoteUser;
|
||||
}
|
||||
|
||||
// Update Person activityの場合は、ここで署名検証/更新処理まで実施して終了
|
||||
@@ -82,7 +90,7 @@ export default async (job: Bull.Job): Promise<void> => {
|
||||
if (activity.object && activity.object.type === 'Person') {
|
||||
if (user == null) {
|
||||
logger.warn('Update activity received, but user not registed.');
|
||||
} else if (!httpSignature.verifySignature(signature, user.publicKey.publicKeyPem)) {
|
||||
} else if (!httpSignature.verifySignature(signature, key.keyPem)) {
|
||||
logger.warn('Update activity received, but signature verification failed.');
|
||||
} else {
|
||||
updatePerson(activity.actor, null, activity.object);
|
||||
@@ -92,15 +100,15 @@ export default async (job: Bull.Job): Promise<void> => {
|
||||
}
|
||||
|
||||
// アクティビティを送信してきたユーザーがまだMisskeyサーバーに登録されていなかったら登録する
|
||||
if (user === null) {
|
||||
if (user == null) {
|
||||
user = await resolvePerson(activity.actor) as IRemoteUser;
|
||||
}
|
||||
|
||||
if (user === null) {
|
||||
if (user == null) {
|
||||
throw new Error('failed to resolve user');
|
||||
}
|
||||
|
||||
if (!httpSignature.verifySignature(signature, user.publicKey.publicKeyPem)) {
|
||||
if (!httpSignature.verifySignature(signature, key.keyPem)) {
|
||||
logger.error('signature verification failed');
|
||||
return;
|
||||
}
|
||||
@@ -116,12 +124,10 @@ export default async (job: Bull.Job): Promise<void> => {
|
||||
|
||||
// Update stats
|
||||
registerOrFetchInstanceDoc(user.host).then(i => {
|
||||
Instance.update({ _id: i._id }, {
|
||||
$set: {
|
||||
latestRequestReceivedAt: new Date(),
|
||||
lastCommunicatedAt: new Date(),
|
||||
isNotResponding: false
|
||||
}
|
||||
Instances.update(i.id, {
|
||||
latestRequestReceivedAt: new Date(),
|
||||
lastCommunicatedAt: new Date(),
|
||||
isNotResponding: false
|
||||
});
|
||||
|
||||
instanceChart.requestReceived(i.host);
|
||||
|
||||
Reference in New Issue
Block a user