sendnrw/misskey
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge commit from fork

Browse Source 
* fix(backend): check target IP before sending HTTP request

* fix(backend): allow accessing private IP when testing

* Apply suggestions from code review

Co-authored-by: anatawa12 <anatawa12@icloud.com>

* fix(backend): lint and typecheck

* fix(backend): add isLocalAddressAllowed option to getAgentByUrl and send (HttpRequestService)

* fix(backend): allow fetchSummaryFromProxy, trueMail to access local addresses

---------

Co-authored-by: anatawa12 <anatawa12@icloud.com>
Co-authored-by: syuilo <4439005+syuilo@users.noreply.github.com>
This commit is contained in:
rectcoordsystem
2024-11-21 08:27:09 +09:00
committed by GitHub
parent b9cb949eb1
commit 090e9392cd
4 changed files with 123 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
packages/backend/src/core/DownloadService.ts
View File

@@ -6,7 +6,6 @@
import * as fs from 'node:fs';
import * as stream from 'node:stream/promises';
import { Inject, Injectable } from '@nestjs/common';
import ipaddr from 'ipaddr.js';
import chalk from 'chalk';
import got, * as Got from 'got';
import { parse } from 'content-disposition';
@@ -70,13 +69,6 @@ export class DownloadService {
},
enableUnixSockets: false,
}).on('response', (res: Got.Response) => {
if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !this.config.proxy && res.ip) {
if (this.isPrivateIp(res.ip)) {
this.logger.warn(`Blocked address: ${res.ip}`);
req.destroy();
}
}
const contentLength = res.headers['content-length'];
if (contentLength != null) {
const size = Number(contentLength);
@@ -139,18 +131,4 @@ export class DownloadService {
cleanup();
}
}
@bindThis
private isPrivateIp(ip: string): boolean {
const parsedIp = ipaddr.parse(ip);
for (const net of this.config.allowedPrivateNetworks ?? []) {
const cidr = ipaddr.parseCIDR(net);
if (cidr[0].kind() === parsedIp.kind() && parsedIp.match(ipaddr.parseCIDR(net))) {
return false;
}
}
return parsedIp.range() !== 'unicast';
}
}

1
packages/backend/src/core/EmailService.ts
View File

@@ -312,6 +312,7 @@ export class EmailService {
Accept: 'application/json',
Authorization: truemailAuthKey,
},
isLocalAddressAllowed: true,
});
const json = (await res.json()) as {

134
packages/backend/src/core/HttpRequestService.ts
View File

@@ -6,6 +6,7 @@
import * as http from 'node:http';
import * as https from 'node:https';
import * as net from 'node:net';
import ipaddr from 'ipaddr.js';
import CacheableLookup from 'cacheable-lookup';
import fetch from 'node-fetch';
import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
@@ -25,8 +26,102 @@ export type HttpRequestSendOptions = {
validators?: ((res: Response) => void)[];
};
declare module 'node:http' {
interface Agent {
createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket;
}
}
class HttpRequestServiceAgent extends http.Agent {
constructor(
private config: Config,
options?: http.AgentOptions,
) {
super(options);
}
@bindThis
public createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket {
const socket = super.createConnection(options, callback)
.on('connect', () => {
const address = socket.remoteAddress;
if (process.env.NODE_ENV === 'production') {
if (address && ipaddr.isValid(address)) {
if (this.isPrivateIp(address)) {
socket.destroy(new Error(`Blocked address: ${address}`));
}
}
}
});
return socket;
};
@bindThis
private isPrivateIp(ip: string): boolean {
const parsedIp = ipaddr.parse(ip);
for (const net of this.config.allowedPrivateNetworks ?? []) {
const cidr = ipaddr.parseCIDR(net);
if (cidr[0].kind() === parsedIp.kind() && parsedIp.match(ipaddr.parseCIDR(net))) {
return false;
}
}
return parsedIp.range() !== 'unicast';
}
}
class HttpsRequestServiceAgent extends https.Agent {
constructor(
private config: Config,
options?: https.AgentOptions,
) {
super(options);
}
@bindThis
public createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket {
const socket = super.createConnection(options, callback)
.on('connect', () => {
const address = socket.remoteAddress;
if (process.env.NODE_ENV === 'production') {
if (address && ipaddr.isValid(address)) {
if (this.isPrivateIp(address)) {
socket.destroy(new Error(`Blocked address: ${address}`));
}
}
}
});
return socket;
};
@bindThis
private isPrivateIp(ip: string): boolean {
const parsedIp = ipaddr.parse(ip);
for (const net of this.config.allowedPrivateNetworks ?? []) {
const cidr = ipaddr.parseCIDR(net);
if (cidr[0].kind() === parsedIp.kind() && parsedIp.match(ipaddr.parseCIDR(net))) {
return false;
}
}
return parsedIp.range() !== 'unicast';
}
}
@Injectable()
export class HttpRequestService {
/**
* Get http non-proxy agent (without local address filtering)
*/
private httpNative: http.Agent;
/**
* Get https non-proxy agent (without local address filtering)
*/
private httpsNative: https.Agent;
/**
* Get http non-proxy agent
*/
@@ -57,19 +152,20 @@ export class HttpRequestService {
lookup: false, // nativeのdns.lookupにfallbackしない
});
this.http = new http.Agent({
const agentOption = {
keepAlive: true,
keepAliveMsecs: 30 * 1000,
lookup: cache.lookup as unknown as net.LookupFunction,
localAddress: config.outgoingAddress,
});
};
this.https = new https.Agent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
lookup: cache.lookup as unknown as net.LookupFunction,
localAddress: config.outgoingAddress,
});
this.httpNative = new http.Agent(agentOption);
this.httpsNative = new https.Agent(agentOption);
this.http = new HttpRequestServiceAgent(config, agentOption);
this.https = new HttpsRequestServiceAgent(config, agentOption);
const maxSockets = Math.max(256, config.deliverJobConcurrency ?? 128);
@@ -104,16 +200,22 @@ export class HttpRequestService {
* @param bypassProxy Allways bypass proxy
*/
@bindThis
public getAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent {
public getAgentByUrl(url: URL, bypassProxy = false, isLocalAddressAllowed = false): http.Agent | https.Agent {
if (bypassProxy || (this.config.proxyBypassHosts ?? []).includes(url.hostname)) {
if (isLocalAddressAllowed) {
return url.protocol === 'http:' ? this.httpNative : this.httpsNative;
}
return url.protocol === 'http:' ? this.http : this.https;
} else {
if (isLocalAddressAllowed && (!this.config.proxy)) {
return url.protocol === 'http:' ? this.httpNative : this.httpsNative;
}
return url.protocol === 'http:' ? this.httpAgent : this.httpsAgent;
}
}
@bindThis
public async getActivityJson(url: string): Promise<IObject> {
public async getActivityJson(url: string, isLocalAddressAllowed = false): Promise<IObject> {
const res = await this.send(url, {
method: 'GET',
headers: {
@@ -121,6 +223,7 @@ export class HttpRequestService {
},
timeout: 5000,
size: 1024 * 256,
isLocalAddressAllowed: isLocalAddressAllowed,
}, {
throwErrorWhenResponseNotOk: true,
validators: [validateContentTypeSetAsActivityPub],
@@ -135,7 +238,7 @@ export class HttpRequestService {
}
@bindThis
public async getJson<T = unknown>(url: string, accept = 'application/json, */*', headers?: Record<string, string>): Promise<T> {
public async getJson<T = unknown>(url: string, accept = 'application/json, */*', headers?: Record<string, string>, isLocalAddressAllowed = false): Promise<T> {
const res = await this.send(url, {
method: 'GET',
headers: Object.assign({
@@ -143,19 +246,21 @@ export class HttpRequestService {
}, headers ?? {}),
timeout: 5000,
size: 1024 * 256,
isLocalAddressAllowed: isLocalAddressAllowed,
});
return await res.json() as T;
}
@bindThis
public async getHtml(url: string, accept = 'text/html, */*', headers?: Record<string, string>): Promise<string> {
public async getHtml(url: string, accept = 'text/html, */*', headers?: Record<string, string>, isLocalAddressAllowed = false): Promise<string> {
const res = await this.send(url, {
method: 'GET',
headers: Object.assign({
Accept: accept,
}, headers ?? {}),
timeout: 5000,
isLocalAddressAllowed: isLocalAddressAllowed,
});
return await res.text();
@@ -170,6 +275,7 @@ export class HttpRequestService {
headers?: Record<string, string>,
timeout?: number,
size?: number,
isLocalAddressAllowed?: boolean,
} = {},
extra: HttpRequestSendOptions = {
throwErrorWhenResponseNotOk: true,
@@ -183,6 +289,8 @@ export class HttpRequestService {
controller.abort();
}, timeout);
const isLocalAddressAllowed = args.isLocalAddressAllowed ?? false;
const res = await fetch(url, {
method: args.method ?? 'GET',
headers: {
@@ -191,7 +299,7 @@ export class HttpRequestService {
},
body: args.body,
size: args.size ?? 10 * 1024 * 1024,
agent: (url) => this.getAgentByUrl(url),
agent: (url) => this.getAgentByUrl(url, false, isLocalAddressAllowed),
signal: controller.signal,
});

2
packages/backend/src/server/web/UrlPreviewService.ts
View File

@@ -145,6 +145,6 @@ export class UrlPreviewService {
contentLengthRequired: meta.urlPreviewRequireContentLength,
});
return this.httpRequestService.getJson<SummalyResult>(`${proxy}?${queryStr}`);
return this.httpRequestService.getJson<SummalyResult>(`${proxy}?${queryStr}`, 'application/json, */*', undefined, true);
}
}