Implement remote account resolution

This commit is contained in:
Akihiko Odaki
2018-03-27 16:51:12 +09:00
parent bee892d446
commit 68ce6d5748
73 changed files with 735 additions and 334 deletions

View File

@@ -1,10 +1,11 @@
import * as EventEmitter from 'events';
import * as bcrypt from 'bcryptjs';
import User, { IUser, init as initUser } from '../models/user';
import User, { ILocalAccount, IUser, init as initUser } from '../models/user';
import getPostSummary from '../../common/get-post-summary';
import getUserSummary from '../../common/get-user-summary';
import getUserSummary from '../../common/user/get-summary';
import parseAcct from '../../common/user/parse-acct';
import getNotificationSummary from '../../common/get-notification-summary';
const hmm = [
@@ -163,9 +164,7 @@ export default class BotCore extends EventEmitter {
public async showUserCommand(q: string): Promise<string> {
try {
const user = await require('../endpoints/users/show')({
username: q.substr(1)
}, this.user);
const user = await require('../endpoints/users/show')(parseAcct(q.substr(1)), this.user);
const text = getUserSummary(user);
@@ -209,7 +208,8 @@ class SigninContext extends Context {
if (this.temporaryUser == null) {
// Fetch user
const user: IUser = await User.findOne({
username_lower: query.toLowerCase()
username_lower: query.toLowerCase(),
host: null
}, {
fields: {
data: false
@@ -225,7 +225,7 @@ class SigninContext extends Context {
}
} else {
// Compare password
const same = await bcrypt.compare(query, this.temporaryUser.account.password);
const same = await bcrypt.compare(query, (this.temporaryUser.account as ILocalAccount).password);
if (same) {
this.bot.signin(this.temporaryUser);

View File

@@ -7,6 +7,8 @@ import config from '../../../conf';
import BotCore from '../core';
import _redis from '../../../db/redis';
import prominence = require('prominence');
import getAcct from '../../../common/user/get-acct';
import parseAcct from '../../../common/user/parse-acct';
import getPostSummary from '../../../common/get-post-summary';
const redis = prominence(_redis);
@@ -98,10 +100,9 @@ class LineBot extends BotCore {
}
public async showUserCommand(q: string) {
const user = await require('../../endpoints/users/show')({
username: q.substr(1)
}, this.user);
const user = await require('../../endpoints/users/show')(parseAcct(q.substr(1)), this.user);
const acct = getAcct(user);
const actions = [];
actions.push({
@@ -121,7 +122,7 @@ class LineBot extends BotCore {
actions.push({
type: 'uri',
label: 'Webで見る',
uri: `${config.url}/@${user.username}`
uri: `${config.url}/@${acct}`
});
this.reply([{
@@ -130,7 +131,7 @@ class LineBot extends BotCore {
template: {
type: 'buttons',
thumbnailImageUrl: `${user.avatar_url}?thumbnail&size=1024`,
title: `${user.name} (@${user.username})`,
title: `${user.name} (@${acct})`,
text: user.description || '(no description)',
actions: actions
}
@@ -171,6 +172,7 @@ module.exports = async (app: express.Application) => {
if (session == null) {
const user = await User.findOne({
host: null,
'account.line': {
user_id: sourceId
}

View File

@@ -10,17 +10,18 @@ import * as debug from 'debug';
import fileType = require('file-type');
import prominence = require('prominence');
import DriveFile, { getGridFSBucket } from '../models/drive-file';
import DriveFolder from '../models/drive-folder';
import { pack } from '../models/drive-file';
import event, { publishDriveStream } from '../event';
import config from '../../conf';
import DriveFile, { getGridFSBucket } from '../../models/drive-file';
import DriveFolder from '../../models/drive-folder';
import { pack } from '../../models/drive-file';
import event, { publishDriveStream } from '../../event';
import getAcct from '../../../common/user/get-acct';
import config from '../../../conf';
const gm = _gm.subClass({
imageMagick: true
});
const log = debug('misskey:register-drive-file');
const log = debug('misskey:drive:add-file');
const tmpFile = (): Promise<string> => new Promise((resolve, reject) => {
tmp.file((e, path) => {
@@ -46,7 +47,7 @@ const addFile = async (
folderId: mongodb.ObjectID = null,
force: boolean = false
) => {
log(`registering ${name} (user: ${user.username}, path: ${path})`);
log(`registering ${name} (user: ${getAcct(user)}, path: ${path})`);
// Calculate hash, get content type and get file size
const [hash, [mime, ext], size] = await Promise.all([

View File

@@ -0,0 +1,46 @@
import * as URL from 'url';
import { IDriveFile, validateFileName } from '../../models/drive-file';
import create from './add-file';
import * as debug from 'debug';
import * as tmp from 'tmp';
import * as fs from 'fs';
import * as request from 'request';
const log = debug('misskey:common:drive:upload_from_url');
export default async (url, user, folderId = null): Promise<IDriveFile> => {
let name = URL.parse(url).pathname.split('/').pop();
if (!validateFileName(name)) {
name = null;
}
// Create temp file
const path = await new Promise((res: (string) => void, rej) => {
tmp.file((e, path) => {
if (e) return rej(e);
res(path);
});
});
// write content at URL to temp file
await new Promise((res, rej) => {
const writable = fs.createWriteStream(path);
request(url)
.on('error', rej)
.on('end', () => {
writable.close();
res(path);
})
.pipe(writable)
.on('error', rej);
});
const driveFile = await create(user, path, name, null, folderId);
// clean-up
fs.unlink(path, (e) => {
if (e) log(e.stack);
});
return driveFile;
};

View File

@@ -0,0 +1,5 @@
import { toUnicode } from 'punycode';
export default host => {
return toUnicode(host).replace(/[A-Z]+/, match => match.toLowerCase());
};

View File

@@ -1,14 +1,17 @@
/**
* Mention
*/
import parseAcct from '../../../../common/user/parse-acct';
module.exports = text => {
const match = text.match(/^@[a-zA-Z0-9\-]+/);
const match = text.match(/^(?:@[a-zA-Z0-9\-]+){1,2}/);
if (!match) return null;
const mention = match[0];
const { username, host } = parseAcct(mention.substr(1));
return {
type: 'mention',
content: mention,
username: mention.substr(1)
username,
host
};
};

View File

@@ -3,7 +3,7 @@
*/
import $ from 'cafy';
import { validateFileName, pack } from '../../../models/drive-file';
import create from '../../../common/add-file-to-drive';
import create from '../../../common/drive/add-file';
/**
* Create a file

View File

@@ -1,16 +1,9 @@
/**
* Module dependencies
*/
import * as URL from 'url';
import $ from 'cafy';
import { validateFileName, pack } from '../../../models/drive-file';
import create from '../../../common/add-file-to-drive';
import * as debug from 'debug';
import * as tmp from 'tmp';
import * as fs from 'fs';
import * as request from 'request';
const log = debug('misskey:endpoint:upload_from_url');
import { pack } from '../../../models/drive-file';
import uploadFromUrl from '../../../common/drive/upload_from_url';
/**
* Create a file from a URL
@@ -25,42 +18,9 @@ module.exports = async (params, user): Promise<any> => {
const [url, urlErr] = $(params.url).string().$;
if (urlErr) throw 'invalid url param';
let name = URL.parse(url).pathname.split('/').pop();
if (!validateFileName(name)) {
name = null;
}
// Get 'folder_id' parameter
const [folderId = null, folderIdErr] = $(params.folder_id).optional.nullable.id().$;
if (folderIdErr) throw 'invalid folder_id param';
// Create temp file
const path = await new Promise((res: (string) => void, rej) => {
tmp.file((e, path) => {
if (e) return rej(e);
res(path);
});
});
// write content at URL to temp file
await new Promise((res, rej) => {
const writable = fs.createWriteStream(path);
request(url)
.on('error', rej)
.on('end', () => {
writable.close();
res(path);
})
.pipe(writable)
.on('error', rej);
});
const driveFile = await create(user, path, name, null, folderId);
// clean-up
fs.unlink(path, (e) => {
if (e) log(e.stack);
});
return pack(driveFile);
return pack(await uploadFromUrl(url, user, folderId));
};

View File

@@ -5,7 +5,7 @@ import $ from 'cafy';
import deepEqual = require('deep-equal');
import parse from '../../common/text';
import { default as Post, IPost, isValidText } from '../../models/post';
import { default as User, IUser } from '../../models/user';
import { default as User, ILocalAccount, IUser } from '../../models/user';
import { default as Channel, IChannel } from '../../models/channel';
import Following from '../../models/following';
import Mute from '../../models/mute';
@@ -16,6 +16,8 @@ import { pack } from '../../models/post';
import notify from '../../common/notify';
import watch from '../../common/watch-post';
import event, { pushSw, publishChannelStream } from '../../event';
import getAcct from '../../../common/user/get-acct';
import parseAcct from '../../../common/user/parse-acct';
import config from '../../../conf';
/**
@@ -390,7 +392,7 @@ module.exports = (params, user: IUser, app) => new Promise(async (res, rej) => {
});
// この投稿をWatchする
if (user.account.settings.auto_watch !== false) {
if ((user.account as ILocalAccount).settings.auto_watch !== false) {
watch(user._id, reply);
}
@@ -477,7 +479,7 @@ module.exports = (params, user: IUser, app) => new Promise(async (res, rej) => {
// Extract an '@' mentions
const atMentions = tokens
.filter(t => t.type == 'mention')
.map(m => m.username)
.map(getAcct)
// Drop dupulicates
.filter((v, i, s) => s.indexOf(v) == i);
@@ -486,9 +488,7 @@ module.exports = (params, user: IUser, app) => new Promise(async (res, rej) => {
// Fetch mentioned user
// SELECT _id
const mentionee = await User
.findOne({
username_lower: mention.toLowerCase()
}, { _id: true });
.findOne(parseAcct(mention), { _id: true });
// When mentioned user not found
if (mentionee == null) return;

View File

@@ -19,6 +19,7 @@ module.exports = async (params) => new Promise(async (res, rej) => {
// Get exist
const exist = await User
.count({
host: null,
username_lower: username.toLowerCase()
}, {
limit: 1

View File

@@ -2,6 +2,7 @@
* Module dependencies
*/
import $ from 'cafy';
import getHostLower from '../../common/get-host-lower';
import Post, { pack } from '../../models/post';
import User from '../../models/user';
@@ -22,7 +23,15 @@ module.exports = (params, me) => new Promise(async (res, rej) => {
if (usernameErr) return rej('invalid username param');
if (userId === undefined && username === undefined) {
return rej('user_id or username is required');
return rej('user_id or pair of username and host is required');
}
// Get 'host' parameter
const [host, hostErr] = $(params.host).optional.string().$;
if (hostErr) return rej('invalid host param');
if (userId === undefined && host === undefined) {
return rej('user_id or pair of username and host is required');
}
// Get 'include_replies' parameter
@@ -60,7 +69,7 @@ module.exports = (params, me) => new Promise(async (res, rej) => {
const q = userId !== undefined
? { _id: userId }
: { username_lower: username.toLowerCase() } ;
: { username_lower: username.toLowerCase(), host_lower: getHostLower(host) } ;
// Lookup user
const user = await User.findOne(q, {

View File

@@ -30,9 +30,15 @@ module.exports = (params, me) => new Promise(async (res, rej) => {
_id: {
$nin: followingIds
},
'account.last_used_at': {
$gte: new Date(Date.now() - ms('7days'))
}
$or: [
{
'account.last_used_at': {
$gte: new Date(Date.now() - ms('7days'))
}
}, {
host: { $not: null }
}
]
}, {
limit: limit,
skip: offset,

View File

@@ -2,7 +2,49 @@
* Module dependencies
*/
import $ from 'cafy';
import User, { pack } from '../../models/user';
import { JSDOM } from 'jsdom';
import { toUnicode, toASCII } from 'punycode';
import uploadFromUrl from '../../common/drive/upload_from_url';
import User, { pack, validateUsername, isValidName, isValidDescription } from '../../models/user';
const request = require('request-promise-native');
const WebFinger = require('webfinger.js');
const webFinger = new WebFinger({});
async function getCollectionCount(url) {
if (!url) {
return null;
}
try {
const collection = await request({ url, json: true });
return collection ? collection.totalItems : null;
} catch (exception) {
return null;
}
}
function findUser(q) {
return User.findOne(q, {
fields: {
data: false
}
});
}
function webFingerAndVerify(query, verifier) {
return new Promise((res, rej) => webFinger.lookup(query, (error, result) => {
if (error) {
return rej(error);
}
if (result.object.subject.toLowerCase().replace(/^acct:/, '') !== verifier) {
return rej('WebFinger verfification failed');
}
res(result.object);
}));
}
/**
* Show a user
@@ -12,6 +54,8 @@ import User, { pack } from '../../models/user';
* @return {Promise<any>}
*/
module.exports = (params, me) => new Promise(async (res, rej) => {
let user;
// Get 'user_id' parameter
const [userId, userIdErr] = $(params.user_id).optional.id().$;
if (userIdErr) return rej('invalid user_id param');
@@ -20,23 +64,142 @@ module.exports = (params, me) => new Promise(async (res, rej) => {
const [username, usernameErr] = $(params.username).optional.string().$;
if (usernameErr) return rej('invalid username param');
if (userId === undefined && username === undefined) {
return rej('user_id or username is required');
// Get 'host' parameter
const [host, hostErr] = $(params.host).optional.string().$;
if (hostErr) return rej('invalid username param');
if (userId === undefined && typeof username !== 'string') {
return rej('user_id or pair of username and host is required');
}
const q = userId !== undefined
? { _id: userId }
: { username_lower: username.toLowerCase() };
// Lookup user
const user = await User.findOne(q, {
fields: {
data: false
}
});
if (typeof host === 'string') {
const username_lower = username.toLowerCase();
const host_lower_ascii = toASCII(host).toLowerCase();
const host_lower = toUnicode(host_lower_ascii);
if (user === null) {
return rej('user not found');
user = await findUser({ username_lower, host_lower });
if (user === null) {
const acct_lower = `${username_lower}@${host_lower_ascii}`;
let activityStreams;
let finger;
let followers_count;
let following_count;
let likes_count;
let posts_count;
if (!validateUsername(username)) {
return rej('username validation failed');
}
try {
finger = await webFingerAndVerify(acct_lower, acct_lower);
} catch (exception) {
return rej('WebFinger lookup failed');
}
const self = finger.links.find(link => link.rel && link.rel.toLowerCase() === 'self');
if (!self) {
return rej('WebFinger has no reference to self representation');
}
try {
activityStreams = await request({
url: self.href,
headers: {
Accept: 'application/activity+json, application/ld+json'
},
json: true
});
} catch (exception) {
return rej('failed to retrieve ActivityStreams representation');
}
if (!(activityStreams &&
(Array.isArray(activityStreams['@context']) ?
activityStreams['@context'].includes('https://www.w3.org/ns/activitystreams') :
activityStreams['@context'] === 'https://www.w3.org/ns/activitystreams') &&
activityStreams.type === 'Person' &&
typeof activityStreams.preferredUsername === 'string' &&
activityStreams.preferredUsername.toLowerCase() === username_lower &&
isValidName(activityStreams.name) &&
isValidDescription(activityStreams.summary)
)) {
return rej('failed ActivityStreams validation');
}
try {
[followers_count, following_count, likes_count, posts_count] = await Promise.all([
getCollectionCount(activityStreams.followers),
getCollectionCount(activityStreams.following),
getCollectionCount(activityStreams.liked),
getCollectionCount(activityStreams.outbox),
webFingerAndVerify(activityStreams.id, acct_lower),
]);
} catch (exception) {
return rej('failed to fetch assets');
}
const summaryDOM = JSDOM.fragment(activityStreams.summary);
// Create user
user = await User.insert({
avatar_id: null,
banner_id: null,
created_at: new Date(),
description: summaryDOM.textContent,
followers_count,
following_count,
name: activityStreams.name,
posts_count,
likes_count,
liked_count: 0,
drive_capacity: 1073741824, // 1GB
username: username,
username_lower,
host: toUnicode(finger.subject.replace(/^.*?@/, '')),
host_lower,
account: {
uri: activityStreams.id,
},
});
const [icon, image] = await Promise.all([
activityStreams.icon,
activityStreams.image,
].map(async image => {
if (!image || image.type !== 'Image') {
return { _id: null };
}
try {
return await uploadFromUrl(image.url, user);
} catch (exception) {
return { _id: null };
}
}));
User.update({ _id: user._id }, {
$set: {
avatar_id: icon._id,
banner_id: image._id,
},
});
user.avatar_id = icon._id;
user.banner_id = icon._id;
}
} else {
const q = userId !== undefined
? { _id: userId }
: { username_lower: username.toLowerCase(), host: null };
user = await findUser(q);
if (user === null) {
return rej('user not found');
}
}
// Send response

View File

@@ -3,6 +3,7 @@ import * as debug from 'debug';
import limiterDB from '../db/redis';
import { Endpoint } from './endpoints';
import { IAuthContext } from './authenticate';
import getAcct from '../common/user/get-acct';
const log = debug('misskey:limitter');
@@ -42,7 +43,7 @@ export default (endpoint: Endpoint, ctx: IAuthContext) => new Promise((ok, rejec
return reject('ERR');
}
log(`@${ctx.user.username} ${endpoint.name} min remaining: ${info.remaining}`);
log(`@${getAcct(ctx.user)} ${endpoint.name} min remaining: ${info.remaining}`);
if (info.remaining === 0) {
reject('BRIEF_REQUEST_INTERVAL');
@@ -70,7 +71,7 @@ export default (endpoint: Endpoint, ctx: IAuthContext) => new Promise((ok, rejec
return reject('ERR');
}
log(`@${ctx.user.username} ${endpoint.name} max remaining: ${info.remaining}`);
log(`@${getAcct(ctx.user)} ${endpoint.name} max remaining: ${info.remaining}`);
if (info.remaining === 0) {
reject('RATE_LIMIT_EXCEEDED');

View File

@@ -39,6 +39,39 @@ export function isValidBirthday(birthday: string): boolean {
return typeof birthday == 'string' && /^([0-9]{4})\-([0-9]{2})-([0-9]{2})$/.test(birthday);
}
export type ILocalAccount = {
keypair: string;
email: string;
links: string[];
password: string;
token: string;
twitter: {
access_token: string;
access_token_secret: string;
user_id: string;
screen_name: string;
};
line: {
user_id: string;
};
profile: {
location: string;
birthday: string; // 'YYYY-MM-DD'
tags: string[];
};
last_used_at: Date;
is_bot: boolean;
is_pro: boolean;
two_factor_secret: string;
two_factor_enabled: boolean;
client_settings: any;
settings: any;
};
export type IRemoteAccount = {
uri: string;
};
export type IUser = {
_id: mongo.ObjectID;
created_at: Date;
@@ -60,34 +93,7 @@ export type IUser = {
keywords: string[];
host: string;
host_lower: string;
account: {
keypair: string;
email: string;
links: string[];
password: string;
token: string;
twitter: {
access_token: string;
access_token_secret: string;
user_id: string;
screen_name: string;
};
line: {
user_id: string;
};
profile: {
location: string;
birthday: string; // 'YYYY-MM-DD'
tags: string[];
};
last_used_at: Date;
is_bot: boolean;
is_pro: boolean;
two_factor_secret: string;
two_factor_enabled: boolean;
client_settings: any;
settings: any;
};
account: ILocalAccount | IRemoteAccount;
};
export function init(user): IUser {
@@ -162,28 +168,30 @@ export const pack = (
// Remove needless properties
delete _user.latest_post;
// Remove private properties
delete _user.account.keypair;
delete _user.account.password;
delete _user.account.token;
delete _user.account.two_factor_temp_secret;
delete _user.account.two_factor_secret;
delete _user.username_lower;
if (_user.account.twitter) {
delete _user.account.twitter.access_token;
delete _user.account.twitter.access_token_secret;
}
delete _user.account.line;
if (!_user.host) {
// Remove private properties
delete _user.account.keypair;
delete _user.account.password;
delete _user.account.token;
delete _user.account.two_factor_temp_secret;
delete _user.account.two_factor_secret;
delete _user.username_lower;
if (_user.account.twitter) {
delete _user.account.twitter.access_token;
delete _user.account.twitter.access_token_secret;
}
delete _user.account.line;
// Visible via only the official client
if (!opts.includeSecrets) {
delete _user.account.email;
delete _user.account.settings;
delete _user.account.client_settings;
}
// Visible via only the official client
if (!opts.includeSecrets) {
delete _user.account.email;
delete _user.account.settings;
delete _user.account.client_settings;
}
if (!opts.detail) {
delete _user.account.two_factor_enabled;
if (!opts.detail) {
delete _user.account.two_factor_enabled;
}
}
_user.avatar_url = _user.avatar_id != null

View File

@@ -1,7 +1,7 @@
import * as express from 'express';
import * as bcrypt from 'bcryptjs';
import * as speakeasy from 'speakeasy';
import { default as User, IUser } from '../models/user';
import { default as User, ILocalAccount, IUser } from '../models/user';
import Signin, { pack } from '../models/signin';
import event from '../event';
import signin from '../common/signin';
@@ -32,7 +32,8 @@ export default async (req: express.Request, res: express.Response) => {
// Fetch user
const user: IUser = await User.findOne({
username_lower: username.toLowerCase()
username_lower: username.toLowerCase(),
host: null
}, {
fields: {
data: false,
@@ -47,13 +48,15 @@ export default async (req: express.Request, res: express.Response) => {
return;
}
const account = user.account as ILocalAccount;
// Compare password
const same = await bcrypt.compare(password, user.account.password);
const same = await bcrypt.compare(password, account.password);
if (same) {
if (user.account.two_factor_enabled) {
if (account.two_factor_enabled) {
const verified = (speakeasy as any).totp.verify({
secret: user.account.two_factor_secret,
secret: account.two_factor_secret,
encoding: 'base32',
token: token
});

View File

@@ -64,7 +64,8 @@ export default async (req: express.Request, res: express.Response) => {
// Fetch exist user that same username
const usernameExist = await User
.count({
username_lower: username.toLowerCase()
username_lower: username.toLowerCase(),
host: null
}, {
limit: 1
});

View File

@@ -39,6 +39,7 @@ module.exports = (app: express.Application) => {
if (userToken == null) return res.send('plz signin');
const user = await User.findOneAndUpdate({
host: null,
'account.token': userToken
}, {
$set: {
@@ -126,6 +127,7 @@ module.exports = (app: express.Application) => {
const result = await twAuth.done(JSON.parse(ctx), req.query.oauth_verifier);
const user = await User.findOne({
host: null,
'account.twitter.user_id': result.userId
});
@@ -148,6 +150,7 @@ module.exports = (app: express.Application) => {
const result = await twAuth.done(JSON.parse(ctx), verifier);
const user = await User.findOneAndUpdate({
host: null,
'account.token': userToken
}, {
$set: {

View File

@@ -94,6 +94,7 @@ function authenticate(token: string): Promise<IUser> {
// Fetch user
const user: IUser = await User
.findOne({
host: null,
'account.token': token
});