diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31a3bdbc..8c5a472c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,9 @@ name: Node CI on: push: - branches: [ master ] + branches: [ "*" ] pull_request: - branches: [ master ] + branches: [ "*" ] jobs: build: diff --git a/README.md b/README.md index aa6cd348..cc5acab7 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@

Better understand your network routing, fix anycast issues, monitor your CDN and DNS performance,
do uptime monitoring and build your own network tools for personal or public use. -
+

diff --git a/config/default.cjs b/config/default.cjs index a0a8e283..72ab53a7 100644 --- a/config/default.cjs +++ b/config/default.cjs @@ -15,7 +15,7 @@ module.exports = { }, geoip: { cache: { - ttl: 3 * 24 * 60 * 60 * 1000, // 24hrs + ttl: 3 * 24 * 60 * 60 * 1000, // 3 days }, }, maxmind: { @@ -29,7 +29,8 @@ module.exports = { fetchSocketsCacheTTL: 1000, }, measurement: { - rateLimit: 300, + rateLimit: 100000, + rateLimitReset: 3600, maxInProgressProbes: 5, // Timeout after which measurement will be marked as finished even if not all probes respond timeout: 30, // 30 seconds diff --git a/config/test.cjs b/config/test.cjs index c0912512..d10d5eb3 100644 --- a/config/test.cjs +++ b/config/test.cjs @@ -7,6 +7,11 @@ module.exports = { admin: { key: 'admin', }, + geoip: { + cache: { + ttl: 1, // 1 ms ttl here to disable redis cache in tests + }, + }, ws: { fetchSocketsCacheTTL: 0, }, diff --git a/src/lib/cache/redis-cache.ts b/src/lib/cache/redis-cache.ts index df1177df..fcebd361 100644 --- a/src/lib/cache/redis-cache.ts +++ b/src/lib/cache/redis-cache.ts @@ -5,7 +5,7 @@ export default class RedisCache implements CacheInterface { constructor (private readonly redis: RedisClient) {} async set (key: string, value: unknown, ttl?: number): Promise { - await this.redis.set(this.buildCacheKey(key), JSON.stringify(value), { EX: ttl ? ttl / 1000 : 0 }); + await this.redis.set(this.buildCacheKey(key), JSON.stringify(value), { PX: ttl ?? 0 }); } async get (key: string): Promise { diff --git a/src/lib/geoip/client.ts b/src/lib/geoip/client.ts index 7c7473c1..cd6170e2 100644 --- a/src/lib/geoip/client.ts +++ b/src/lib/geoip/client.ts @@ -63,7 +63,7 @@ export default class GeoipClient { const resultsWithCities = results.filter(s => s.city); - if (resultsWithCities.length < 2 && resultsWithCities[0]?.provider === 'fastly') { + if (resultsWithCities.length === 0 || (resultsWithCities.length === 1 && resultsWithCities[0]?.provider === 'fastly')) { throw new InternalError(`unresolvable geoip: ${addr}`, true); } diff --git a/src/lib/geoip/providers/maxmind.ts b/src/lib/geoip/providers/maxmind.ts index 03bc5701..adfcf9bf 100644 --- a/src/lib/geoip/providers/maxmind.ts +++ b/src/lib/geoip/providers/maxmind.ts @@ -15,7 +15,8 @@ export const isMaxmindError = (error: unknown): error is WebServiceClientError = const query = async (addr: string, retryCounter = 0): Promise => { try { - return await client.city(addr); + const city = await client.city(addr); + return city; } catch (error: unknown) { if (isMaxmindError(error)) { if (error.code === 'SERVER_ERROR' && retryCounter < 3) { @@ -27,7 +28,7 @@ const query = async (addr: string, retryCounter = 0): Promise => { } } - throw new Error('no maxmind data'); + throw error; } }; diff --git a/src/lib/http/middleware/ratelimit.ts b/src/lib/http/middleware/ratelimit.ts index 75310b72..b05028e9 100644 --- a/src/lib/http/middleware/ratelimit.ts +++ b/src/lib/http/middleware/ratelimit.ts @@ -1,8 +1,9 @@ import type { Context, Next } from 'koa'; import requestIp from 'request-ip'; import type { RateLimiterRes } from 'rate-limiter-flexible'; - -import rateLimiter from '../../ratelimiter.js'; +import createHttpError from 'http-errors'; +import rateLimiter, { defaultState } from '../../ratelimiter.js'; +import type { MeasurementRequest } from '../../../measurement/types.js'; const setResponseHeaders = (ctx: Context, response: RateLimiterRes) => { ctx.set('X-RateLimit-Reset', `${Math.round(response.msBeforeNext / 1000)}`); @@ -10,24 +11,30 @@ const setResponseHeaders = (ctx: Context, response: RateLimiterRes) => { ctx.set('X-RateLimit-Remaining', `${response.remainingPoints}`); }; -const methodsWhitelist = new Set([ 'GET', 'HEAD', 'OPTIONS' ]); - export const rateLimitHandler = () => async (ctx: Context, next: Next) => { - const { method, isAdmin } = ctx; + const { isAdmin } = ctx; + const clientIp = requestIp.getClientIp(ctx.req) ?? ''; + const request = ctx.request.body as MeasurementRequest; + const limit = request.locations.some(l => l.limit) ? request.locations.reduce((sum, { limit = 1 }) => sum + limit, 0) : request.limit; - if (methodsWhitelist.has(method) || isAdmin) { + if (isAdmin) { return next(); } - try { - const response = await rateLimiter.consume(requestIp.getClientIp(ctx.req) ?? ''); - setResponseHeaders(ctx, response); - } catch (error: unknown) { // Ts requires 'unknown' for errors - setResponseHeaders(ctx, error as RateLimiterRes); - ctx.status = 429; - ctx.body = 'Too Many Requests'; - return; + const currentState = await rateLimiter.get(clientIp) ?? defaultState as RateLimiterRes; + + if (currentState.remainingPoints < limit) { + setResponseHeaders(ctx, currentState); + throw createHttpError(429, 'Too Many Probes Requested', { type: 'too_many_probes' }); } await next(); + const response = ctx.response.body as object & { probesCount?: number }; + + if (!('probesCount' in response) || typeof response.probesCount !== 'number') { + throw new Error('Missing probesCount field in response object'); + } + + const newState = await rateLimiter.penalty(clientIp, response.probesCount); + setResponseHeaders(ctx, newState); }; diff --git a/src/lib/http/server.ts b/src/lib/http/server.ts index e4549797..e091c192 100644 --- a/src/lib/http/server.ts +++ b/src/lib/http/server.ts @@ -15,7 +15,6 @@ import { registerGetMeasurementRoute } from '../../measurement/route/get-measure import { registerCreateMeasurementRoute } from '../../measurement/route/create-measurement.js'; import { registerHealthRoute } from '../../health/route/get.js'; import { errorHandler } from './error-handler.js'; -import { rateLimitHandler } from './middleware/ratelimit.js'; import { errorHandlerMw } from './middleware/error-handler.js'; import { corsHandler } from './middleware/cors.js'; import { isAdminMw } from './middleware/is-admin.js'; @@ -40,8 +39,7 @@ rootRouter.get('/', (ctx) => { const apiRouter = new Router({ strict: true, sensitive: true }); apiRouter.prefix('/v1') - .use(isAdminMw) - .use(rateLimitHandler()); + .use(isAdminMw); // POST /measurements registerCreateMeasurementRoute(apiRouter); diff --git a/src/lib/ratelimiter.ts b/src/lib/ratelimiter.ts index 8715eb26..f82871fd 100644 --- a/src/lib/ratelimiter.ts +++ b/src/lib/ratelimiter.ts @@ -4,11 +4,18 @@ import { createRedisClient } from './redis/client.js'; const redisClient = await createRedisClient({ legacyMode: true }); -export const rateLimiter = new RateLimiterRedis({ +export const defaultState = { + remainingPoints: config.get('measurement.rateLimit'), + msBeforeNext: config.get('measurement.rateLimitReset') * 1000, + consumedPoints: 0, + isFirstInDuration: true, +}; + +const rateLimiter = new RateLimiterRedis({ storeClient: redisClient, keyPrefix: 'rate', points: config.get('measurement.rateLimit'), - duration: 60, + duration: config.get('measurement.rateLimitReset'), }); export default rateLimiter; diff --git a/src/lib/redis/scripts.ts b/src/lib/redis/scripts.ts index 67886e36..ea841b69 100644 --- a/src/lib/redis/scripts.ts +++ b/src/lib/redis/scripts.ts @@ -4,8 +4,8 @@ import type { MeasurementRecord, MeasurementResultMessage } from '../../measurem type CountScript = { NUMBER_OF_KEYS: number; SCRIPT: string; - transformArguments (this: void, key: string): Array; - transformReply (this: void, reply: number): number; + transformArguments (key: string): Array; + transformReply (reply: number): number; } & { SHA1: string; }; @@ -13,8 +13,8 @@ type CountScript = { export type RecordResultScript = { NUMBER_OF_KEYS: number; SCRIPT: string; - transformArguments (this: void, measurementId: string, testId: string, data: MeasurementResultMessage['result']): string[]; - transformReply (this: void, reply: string): MeasurementRecord | null; + transformArguments (measurementId: string, testId: string, data: MeasurementResultMessage['result']): string[]; + transformReply (reply: string): MeasurementRecord | null; } & { SHA1: string; }; diff --git a/src/lib/ws/helper/error-handler.ts b/src/lib/ws/helper/error-handler.ts index e6151fd5..d6ca3833 100644 --- a/src/lib/ws/helper/error-handler.ts +++ b/src/lib/ws/helper/error-handler.ts @@ -19,7 +19,7 @@ type NextArgument = NextConnectArgument | NextMwArgument; const isError = (error: unknown): error is Error => Boolean(error as Error['message']); // eslint-disable-next-line @typescript-eslint/no-explicit-any -export const errorHandler = (next: NextArgument) => (socket: Socket, mwNext?: (error?: any) => void | undefined) => { +export const errorHandler = (next: NextArgument) => (socket: Socket, mwNext?: (error?: any) => void) => { next(socket, mwNext!).catch((error) => { // eslint-disable-line @typescript-eslint/no-non-null-assertion const clientIp = getProbeIp(socket.request) ?? ''; const reason = isError(error) ? error.message : 'unknown'; diff --git a/src/lib/ws/helper/reconnect-probes.ts b/src/lib/ws/helper/reconnect-probes.ts index ae989ddb..89bf6be1 100644 --- a/src/lib/ws/helper/reconnect-probes.ts +++ b/src/lib/ws/helper/reconnect-probes.ts @@ -1,8 +1,8 @@ -import { fetchSockets } from '../server.js'; +import type { ThrottledFetchSockets } from '../server'; const TIME_TO_RECONNECT_PROBES = 60_000; -export const reconnectProbes = async () => { +export const reconnectProbes = async (fetchSockets: ThrottledFetchSockets) => { // passing fetchSockets in arguments to avoid cycle dependency const sockets = await fetchSockets(); for (const socket of sockets) { diff --git a/src/lib/ws/server.ts b/src/lib/ws/server.ts index 0de9945d..50c9cf1d 100644 --- a/src/lib/ws/server.ts +++ b/src/lib/ws/server.ts @@ -46,7 +46,7 @@ export const initWsServer = async () => { ); setTimeout(() => { - reconnectProbes().catch(error => logger.error(error)); + reconnectProbes(fetchSockets).catch(error => logger.error(error)); }, TIME_UNTIL_VM_BECOMES_HEALTHY); }; @@ -67,3 +67,5 @@ export const fetchSockets = async () => { return sockets; }; + +export type ThrottledFetchSockets = typeof fetchSockets; diff --git a/src/measurement/route/create-measurement.ts b/src/measurement/route/create-measurement.ts index f0d4a9ae..6f66adad 100644 --- a/src/measurement/route/create-measurement.ts +++ b/src/measurement/route/create-measurement.ts @@ -6,6 +6,7 @@ import type { MeasurementRequest } from '../types.js'; import { bodyParser } from '../../lib/http/middleware/body-parser.js'; import { validate } from '../../lib/http/middleware/validate.js'; import { schema } from '../schema/global-schema.js'; +import { rateLimitHandler } from '../../lib/http/middleware/ratelimit.js'; const hostConfig = config.get('server.host'); const runner = getMeasurementRunner(); @@ -24,5 +25,5 @@ const handle = async (ctx: Context): Promise => { }; export const registerCreateMeasurementRoute = (router: Router): void => { - router.post('/measurements', bodyParser(), validate(schema), handle); + router.post('/measurements', bodyParser(), validate(schema), rateLimitHandler(), handle); }; diff --git a/src/measurement/schema/location-schema.ts b/src/measurement/schema/location-schema.ts index eac65e80..2a18c0c0 100644 --- a/src/measurement/schema/location-schema.ts +++ b/src/measurement/schema/location-schema.ts @@ -29,5 +29,6 @@ export const schema = Joi.array().items(Joi.object().keys({ limit: Joi.number().min(1).max(measurementConfig.limits.location).when(Joi.ref('/limit'), { is: Joi.exist(), then: Joi.forbidden().messages({ 'any.unknown': 'limit per location is not allowed when a global limit is set' }), + otherwise: Joi.number().default(1), }), }).or('continent', 'region', 'country', 'state', 'city', 'network', 'asn', 'magic', 'tags')).default(GLOBAL_DEFAULTS.locations); diff --git a/src/probe/builder.ts b/src/probe/builder.ts index 40f33cb5..bf60a5e9 100644 --- a/src/probe/builder.ts +++ b/src/probe/builder.ts @@ -18,18 +18,20 @@ import getProbeIp from '../lib/get-probe-ip.js'; import { getRegion } from '../lib/ip-ranges.js'; import type { Probe, ProbeLocation, Tag } from './types.js'; -const fakeIpForDebug = () => _.sample([ - '18.200.0.1', // aws-eu-west-1 - '34.140.0.10', // gcp-europe-west1 - '95.155.94.127', - '65.49.22.66', - '185.229.226.83', - '51.158.22.211', - '131.255.7.26', - '213.136.174.80', - '94.214.253.78', - '79.205.97.254', -]); +const fakeIpForDebug = () => { + return _.sample([ + '18.200.0.1', // aws-eu-west-1 + '34.140.0.10', // gcp-europe-west1 + '95.155.94.127', + '65.49.22.66', + '185.229.226.83', + '51.158.22.211', + '131.255.7.26', + '213.136.174.80', + '94.214.253.78', + '79.205.97.254', + ]); +}; const geoipClient = createGeoipClient(); diff --git a/src/probe/handler/dns.ts b/src/probe/handler/dns.ts index d4817d31..00719ac1 100644 --- a/src/probe/handler/dns.ts +++ b/src/probe/handler/dns.ts @@ -1,4 +1,4 @@ -import type { Probe } from '../../probe/types.js'; +import type { Probe } from '../types.js'; export const handleDnsUpdate = (probe: Probe) => (list: string[]): void => { probe.resolvers = list; diff --git a/src/probe/handler/stats.ts b/src/probe/handler/stats.ts index c6bc4df5..39e8ab9d 100644 --- a/src/probe/handler/stats.ts +++ b/src/probe/handler/stats.ts @@ -1,7 +1,4 @@ -import type { - Probe, - ProbeStats, -} from '../../probe/types.js'; +import type { Probe, ProbeStats } from '../types.js'; export const handleStatsReport = (probe: Probe) => (report: ProbeStats): void => { probe.stats = report; diff --git a/src/probe/handler/status.ts b/src/probe/handler/status.ts index 282c610d..100abc4b 100644 --- a/src/probe/handler/status.ts +++ b/src/probe/handler/status.ts @@ -1,4 +1,4 @@ -import type { Probe } from '../../probe/types.js'; +import type { Probe } from '../types.js'; export const handleStatusUpdate = (probe: Probe) => (status: Probe['status']): void => { probe.status = status; diff --git a/test/mocks/redis-cache.ts b/test/mocks/redis-cache.ts deleted file mode 100644 index 779b8816..00000000 --- a/test/mocks/redis-cache.ts +++ /dev/null @@ -1,5 +0,0 @@ -export default class RedisCacheMock { - set = () => Promise.resolve(); - get = () => Promise.resolve(); - delete = () => Promise.resolve(); -} diff --git a/test/tests/integration/measurement/create-measurement.test.ts b/test/tests/integration/measurement/create-measurement.test.ts index f6e6fa95..a1c3609c 100644 --- a/test/tests/integration/measurement/create-measurement.test.ts +++ b/test/tests/integration/measurement/create-measurement.test.ts @@ -5,7 +5,6 @@ import request, { type SuperTest, type Test } from 'supertest'; import * as td from 'testdouble'; import nock from 'nock'; import type { Socket } from 'socket.io-client'; -import RedisCacheMock from '../../../mocks/redis-cache.js'; const nockMocks = JSON.parse(fs.readFileSync('./test/mocks/nock-geoip.json').toString()) as Record; @@ -16,7 +15,6 @@ describe('Create measurement', () => { let requestAgent: SuperTest; before(async () => { - await td.replaceEsm('../../../../src/lib/cache/redis-cache.ts', {}, RedisCacheMock); await td.replaceEsm('../../../../src/lib/ip-ranges.ts', { getRegion: () => 'gcp-us-west4', populateMemList: () => Promise.resolve() }); ({ getTestServer, addFakeProbe, deleteFakeProbe } = await import('../../../utils/server.js')); const app = await getTestServer(); diff --git a/test/tests/integration/measurement/probe-communication.test.ts b/test/tests/integration/measurement/probe-communication.test.ts index d2222b06..85b1fa9d 100644 --- a/test/tests/integration/measurement/probe-communication.test.ts +++ b/test/tests/integration/measurement/probe-communication.test.ts @@ -5,7 +5,6 @@ import nock from 'nock'; import type { Socket } from 'socket.io-client'; import * as sinon from 'sinon'; import { expect } from 'chai'; -import RedisCacheMock from '../../../mocks/redis-cache.js'; const nockMocks = JSON.parse(fs.readFileSync('./test/mocks/nock-geoip.json').toString()) as Record; @@ -22,7 +21,6 @@ describe('Create measurement request', () => { before(async () => { await td.replaceEsm('crypto-random-string', {}, cryptoRandomString); - await td.replaceEsm('../../../../src/lib/cache/redis-cache.ts', {}, RedisCacheMock); await td.replaceEsm('../../../../src/lib/ip-ranges.ts', { getRegion: () => 'gcp-us-west4', populateMemList: () => Promise.resolve() }); ({ getTestServer, addFakeProbe, deleteFakeProbe } = await import('../../../utils/server.js')); const app = await getTestServer(); diff --git a/test/tests/integration/measurement/timeout-result.test.ts b/test/tests/integration/measurement/timeout-result.test.ts index e2446029..659f80e6 100644 --- a/test/tests/integration/measurement/timeout-result.test.ts +++ b/test/tests/integration/measurement/timeout-result.test.ts @@ -5,7 +5,6 @@ import nock from 'nock'; import type { Socket } from 'socket.io-client'; import * as sinon from 'sinon'; import { expect } from 'chai'; -import RedisCacheMock from '../../../mocks/redis-cache.js'; const nockMocks = JSON.parse(fs.readFileSync('./test/mocks/nock-geoip.json').toString()) as Record; @@ -23,7 +22,6 @@ describe('Timeout results', () => { sandbox = sinon.createSandbox({ useFakeTimers: true }); await td.replaceEsm('@jcoreio/async-throttle', null, (f: any) => f); await td.replaceEsm('crypto-random-string', {}, cryptoRandomString); - await td.replaceEsm('../../../../src/lib/cache/redis-cache.ts', {}, RedisCacheMock); await td.replaceEsm('../../../../src/lib/ip-ranges.ts', { getRegion: () => 'gcp-us-west4', populateMemList: () => Promise.resolve() }); ({ getTestServer, addFakeProbe, deleteFakeProbe } = await import('../../../utils/server.js')); const app = await getTestServer(); diff --git a/test/tests/integration/middleware/compress.test.ts b/test/tests/integration/middleware/compress.test.ts index 16029dc5..97ec4485 100644 --- a/test/tests/integration/middleware/compress.test.ts +++ b/test/tests/integration/middleware/compress.test.ts @@ -1,26 +1,19 @@ import fs from 'node:fs'; import request, { type Response } from 'supertest'; import { expect } from 'chai'; -import * as td from 'testdouble'; import nock from 'nock'; import type { Socket } from 'socket.io-client'; -import RedisCacheMock from '../../../mocks/redis-cache.js'; +import { getTestServer, addFakeProbe, deleteFakeProbe } from '../../../utils/server.js'; const nockMocks = JSON.parse(fs.readFileSync('./test/mocks/nock-geoip.json').toString()) as Record; describe('compression', () => { - let addFakeProbe: () => Promise; - let deleteFakeProbe: (socket: Socket) => Promise; let requestAgent: any; let probes: Socket[] = []; describe('headers', () => { before(async () => { - await td.replaceEsm('../../../../src/lib/cache/redis-cache.ts', {}, RedisCacheMock); - const http = await import('../../../utils/server.js'); - addFakeProbe = http.addFakeProbe; - deleteFakeProbe = http.deleteFakeProbe; - const app = await http.getTestServer(); + const app = await getTestServer(); requestAgent = request(app); }); @@ -39,7 +32,6 @@ describe('compression', () => { probe.emit('probe:status:update', 'ready'); } - // eslint-disable-next-line @typescript-eslint/no-unsafe-call const response = await requestAgent .get('/v1/probes') .set('accept-encoding', '*') diff --git a/test/tests/integration/middleware/cors.test.ts b/test/tests/integration/middleware/cors.test.ts index 47051665..d7fe9de1 100644 --- a/test/tests/integration/middleware/cors.test.ts +++ b/test/tests/integration/middleware/cors.test.ts @@ -15,14 +15,12 @@ describe('cors', () => { describe('Access-Control-Allow-Origin header', () => { it('should include the header with value of *', async () => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-call const response = await requestAgent.get('/v1/').set('Origin', 'elocast.com').send() as Response; expect(response.headers['access-control-allow-origin']).to.equal('*'); }); it('should include the header at root', async () => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-call const response = await requestAgent.get('/').send() as Response; expect(response.headers['access-control-allow-origin']).to.equal('*'); diff --git a/test/tests/integration/middleware/domain-redirect.test.ts b/test/tests/integration/middleware/domain-redirect.test.ts index 3d1da517..81cbd01c 100644 --- a/test/tests/integration/middleware/domain-redirect.test.ts +++ b/test/tests/integration/middleware/domain-redirect.test.ts @@ -15,7 +15,6 @@ describe('domain redirect', () => { describe('http requests', () => { it('should be redirected to "https://jsdelivr.com/globalping" if Host is "globalping.io"', async () => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-call await requestAgent .get('/v1/probes') .send() @@ -27,7 +26,6 @@ describe('domain redirect', () => { }); it('should not be redirected to if Host is not "globalping.io"', async () => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-call await requestAgent .get('/v1/probes') .send() diff --git a/test/tests/integration/middleware/etag.test.ts b/test/tests/integration/middleware/etag.test.ts index 4487cfe3..aa514b11 100644 --- a/test/tests/integration/middleware/etag.test.ts +++ b/test/tests/integration/middleware/etag.test.ts @@ -15,7 +15,6 @@ describe('etag', () => { describe('ETag header', () => { it('should include the header', async () => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-call const response = await requestAgent.get('/v1/probes').send() as Response; expect(response.headers.etag).to.exist; @@ -24,7 +23,6 @@ describe('etag', () => { describe('conditional get', () => { it('should redirect to cache', async () => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-call const response = await requestAgent .get('/v1/probes') .set('if-none-match', 'W/"2-l9Fw4VUO7kr8CvBlt4zaMCqXZ0w"') diff --git a/test/tests/integration/middleware/ratelimit.test.ts b/test/tests/integration/middleware/ratelimit.test.ts index e1c95447..8acb4b16 100644 --- a/test/tests/integration/middleware/ratelimit.test.ts +++ b/test/tests/integration/middleware/ratelimit.test.ts @@ -1,21 +1,26 @@ +import fs from 'node:fs'; import type { Server } from 'node:http'; import request, { type Response } from 'supertest'; import requestIp from 'request-ip'; import type { RateLimiterRedis } from 'rate-limiter-flexible'; import { expect } from 'chai'; -import { getTestServer } from '../../../utils/server.js'; +import nock from 'nock'; +import type { Socket } from 'socket.io-client'; +import { getTestServer, addFakeProbe, deleteFakeProbe } from '../../../utils/server.js'; + +const nockMocks = JSON.parse(fs.readFileSync('./test/mocks/nock-geoip.json').toString()) as Record; describe('rate limiter', () => { let app: Server; let requestAgent: any; let clientIpv6: string; let rateLimiterInstance: RateLimiterRedis; + let probe: Socket; before(async () => { app = await getTestServer(); requestAgent = request(app); - // eslint-disable-next-line @typescript-eslint/no-unsafe-call const httpResponse = await requestAgent.post('/v1/').send() as Response & {req: any}; // Supertest renders request as ipv4 const clientIp = requestIp.getClientIp(httpResponse.req); @@ -24,15 +29,25 @@ describe('rate limiter', () => { const rateLimiter = await import('../../../../src/lib/ratelimiter.js'); rateLimiterInstance = rateLimiter.default; + + nock('https://globalping-geoip.global.ssl.fastly.net').get(/.*/).reply(200, nockMocks['01.00'].fastly); + nock('https://ipinfo.io').get(/.*/).reply(200, nockMocks['01.00'].ipinfo); + nock('https://geoip.maxmind.com/geoip/v2.1/city/').get(/.*/).reply(200, nockMocks['01.00'].maxmind); + probe = await addFakeProbe(); + probe.emit('probe:status:update', 'ready'); }); + afterEach(async () => { await rateLimiterInstance.delete(clientIpv6); }); + after(async () => { + await deleteFakeProbe(probe); + }); + describe('headers', () => { it('should NOT include headers (GET)', async () => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-call const response = await requestAgent.get('/v1/').send() as Response; expect(response.headers['x-ratelimit-limit']).to.not.exist; @@ -40,25 +55,43 @@ describe('rate limiter', () => { expect(response.headers['x-ratelimit-reset']).to.not.exist; }); - it('should include headers (POST)', async () => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-call + it('should NOT include headers if body is not valid (POST)', async () => { const response = await requestAgent.post('/v1/measurements').send() as Response; + expect(response.headers['x-ratelimit-limit']).to.not.exist; + expect(response.headers['x-ratelimit-remaining']).to.not.exist; + expect(response.headers['x-ratelimit-reset']).to.not.exist; + }); + + it('should include headers (POST)', async () => { + const response = await requestAgent.post('/v1/measurements').send({ + type: 'ping', + target: 'jsdelivr.com', + }) as Response; + expect(response.headers['x-ratelimit-limit']).to.exist; expect(response.headers['x-ratelimit-remaining']).to.exist; expect(response.headers['x-ratelimit-reset']).to.exist; }); - it('should change values on next request (5) (POST)', async () => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-call - const requestPromise = () => requestAgent.post('/v1/measurements').send() as Response; - const responseList = await Promise.all(Array.from({ length: 5 }).map(() => requestPromise())); + it('should change values on multiple requests (POST)', async () => { + const response = await requestAgent.post('/v1/measurements').send({ + type: 'ping', + target: 'jsdelivr.com', + }) as Response; + + expect(response.headers['x-ratelimit-limit']).to.equal('100000'); + expect(response.headers['x-ratelimit-remaining']).to.equal('99999'); + expect(response.headers['x-ratelimit-reset']).to.equal('3600'); - const firstResponse = responseList[0]; - const lastResponse = responseList[4]; + const response2 = await requestAgent.post('/v1/measurements').send({ + type: 'ping', + target: 'jsdelivr.com', + }) as Response; - expect(responseList).to.have.lengthOf(5); - expect(firstResponse?.headers?.['x-ratelimit-remaining']).to.not.equal(lastResponse?.headers?.['x-ratelimit-remaining']); + expect(response2.headers['x-ratelimit-limit']).to.equal('100000'); + expect(response2.headers['x-ratelimit-remaining']).to.equal('99998'); + expect(response2.headers['x-ratelimit-reset']).to.equal('3600'); }); }); @@ -66,17 +99,21 @@ describe('rate limiter', () => { it('should succeed (limit not reached)', async () => { await rateLimiterInstance.set(clientIpv6, 0, 0); - // eslint-disable-next-line @typescript-eslint/no-unsafe-call - const response = await requestAgent.post('/v1/measurements').send() as Response; + const response = await requestAgent.post('/v1/measurements').send({ + type: 'ping', + target: 'jsdelivr.com', + }) as Response; - expect(Number(response.headers['x-ratelimit-remaining'])).to.equal(299); + expect(Number(response.headers['x-ratelimit-remaining'])).to.equal(99999); }); it('should fail (limit reached) (start at 100)', async () => { - await rateLimiterInstance.set(clientIpv6, 300, 0); + await rateLimiterInstance.set(clientIpv6, 100000, 0); - // eslint-disable-next-line @typescript-eslint/no-unsafe-call - const response = await requestAgent.post('/v1/measurements').send() as Response; + const response = await requestAgent.post('/v1/measurements').send({ + type: 'ping', + target: 'jsdelivr.com', + }) as Response; expect(Number(response.headers['x-ratelimit-remaining'])).to.equal(0); expect(response.statusCode).to.equal(429); diff --git a/test/tests/integration/middleware/responsetime.test.ts b/test/tests/integration/middleware/responsetime.test.ts index 70826606..2d0937dc 100644 --- a/test/tests/integration/middleware/responsetime.test.ts +++ b/test/tests/integration/middleware/responsetime.test.ts @@ -16,7 +16,6 @@ describe('response time', () => { describe('X-Response-Time header', () => { describe('should include the header', (): void => { it('should succeed', async () => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-call const response = await requestAgent.get('/v1/').send() as Response; expect(response.headers['x-response-time']).to.exist; diff --git a/test/tests/integration/probes/get-probes.test.ts b/test/tests/integration/probes/get-probes.test.ts index 1a1def5b..409d06cb 100644 --- a/test/tests/integration/probes/get-probes.test.ts +++ b/test/tests/integration/probes/get-probes.test.ts @@ -5,28 +5,23 @@ import { expect } from 'chai'; import request, { type SuperTest, type Test } from 'supertest'; import * as td from 'testdouble'; import type { Socket } from 'socket.io-client'; -import RedisCacheMock from '../../../mocks/redis-cache.js'; +import { getTestServer, addFakeProbe as addProbe, deleteFakeProbe } from '../../../utils/server.js'; const nockMocks = JSON.parse(fs.readFileSync('./test/mocks/nock-geoip.json').toString()) as Record; describe('Get Probes', () => { - let addFakeProbe: () => Promise; - let deleteFakeProbe: (socket: Socket) => Promise; let requestAgent: SuperTest; + let addFakeProbe: () => Promise; const probes: Socket[] = []; before(async () => { - await td.replaceEsm('../../../../src/lib/cache/redis-cache.ts', {}, RedisCacheMock); - const http = await import('../../../utils/server.js'); - deleteFakeProbe = http.deleteFakeProbe; - addFakeProbe = async () => { - const probe = await http.addFakeProbe(); + const probe = await addProbe(); probes.push(probe); return probe; }; - const app = await http.getTestServer(); + const app = await getTestServer(); requestAgent = request(app); }); diff --git a/test/tests/unit/middleware/ratelimit.test.ts b/test/tests/unit/middleware/ratelimit.test.ts new file mode 100644 index 00000000..e6a0ff30 --- /dev/null +++ b/test/tests/unit/middleware/ratelimit.test.ts @@ -0,0 +1,159 @@ +import * as sinon from 'sinon'; +import { expect } from 'chai'; +import rateLimiter from '../../../../src/lib/ratelimiter.js'; +import { rateLimitHandler } from '../../../../src/lib/http/middleware/ratelimit.js'; +import createHttpError from 'http-errors'; + +describe('rate limit middleware', () => { + const defaultCtx: any = { + set: sinon.stub(), + req: {}, + request: { + body: {}, + }, + response: {}, + }; + let ctx = { ...defaultCtx }; + + beforeEach(async () => { + defaultCtx.set.reset(); + ctx = { ...defaultCtx }; + await rateLimiter.delete(''); + }); + + it('should set rate limit headers based on the "probesCount" field value', async () => { + ctx.request.body = { + limit: 10, + locations: [], + }; + + const next: any = () => { + ctx.response.body = { + id: 'id', + probesCount: 5, + }; + }; + + await rateLimitHandler()(ctx, next); + + expect(ctx.set.callCount).to.equal(3); + expect(ctx.set.firstCall.args[0]).to.equal('X-RateLimit-Reset'); + expect(ctx.set.secondCall.args).to.deep.equal([ 'X-RateLimit-Limit', '100000' ]); + expect(ctx.set.thirdCall.args).to.deep.equal([ 'X-RateLimit-Remaining', '99995' ]); + }); + + it('should throw an error if response body doesn\'t have "probesCount" field', async () => { + ctx.request.body = { + limit: 10, + locations: [], + }; + + const next: any = () => { + ctx.response.body = {}; + }; + + const err = await rateLimitHandler()(ctx, next).catch(err => err); + expect(err).to.deep.equal(new Error('Missing probesCount field in response object')); + }); + + it('should NOT set rate limit headers for admin', async () => { + ctx.request.body = { + limit: 10, + locations: [], + }; + + ctx.isAdmin = true; + + const next: any = () => { + ctx.response.body = { + id: 'id', + probesCount: 10, + }; + }; + + await rateLimitHandler()(ctx, next); + expect(ctx.set.callCount).to.equal(0); + }); + + it('should validate request based on the "limit" field value', async () => { + ctx.request.body = { + limit: 60000, + locations: [], + }; + + const next: any = () => { + ctx.response.body = { + id: 'id', + probesCount: 60000, + }; + }; + + await rateLimitHandler()(ctx, next); + expect(ctx.set.args[2]).to.deep.equal([ 'X-RateLimit-Remaining', '40000' ]); + + const err = await rateLimitHandler()(ctx, next).catch(err => err); // 60000 > 40000 so another request with the same body fails + expect(err).to.deep.equal(createHttpError(429, 'Too Many Probes Requested', { type: 'too_many_probes' })); + expect(ctx.set.args[5]).to.deep.equal([ 'X-RateLimit-Remaining', '40000' ]); + + ctx.request.body = { + limit: 40000, + locations: [], + }; + + const next2: any = () => { + ctx.response.body = { + id: 'id', + probesCount: 40000, + }; + }; + + await rateLimitHandler()(ctx, next2); // 40000 === 40000 so request with the updated body works + expect(ctx.set.args[8]).to.deep.equal([ 'X-RateLimit-Remaining', '0' ]); + }); + + it('should validate request based on the "location.limit" field value', async () => { + ctx.request.body = { + locations: [{ + continent: 'EU', + limit: 45000, + }, { + continent: 'NA', + limit: 45000, + }], + }; + + const next: any = () => { + ctx.response.body = { + id: 'id', + probesCount: 90000, + }; + }; + + await rateLimitHandler()(ctx, next); + expect(ctx.set.args[2]).to.deep.equal([ 'X-RateLimit-Remaining', '10000' ]); + + const err = await rateLimitHandler()(ctx, next).catch(err => err); // only 10000 points remaining so another request with the same body fails + expect(err).to.deep.equal(createHttpError(429, 'Too Many Probes Requested', { type: 'too_many_probes' })); + expect(ctx.set.args[5]).to.deep.equal([ 'X-RateLimit-Remaining', '10000' ]); + + ctx.request.body = { + locations: [{ + continent: 'EU', + limit: 5000, + }, { + continent: 'NA', + limit: 5000, + }], + }; + + const next2: any = () => { + ctx.response.body = { + id: 'id', + probesCount: 10000, + }; + }; + + await rateLimitHandler()(ctx, next2); // request with 10000 probes will work fine + expect(ctx.set.args[8]).to.deep.equal([ 'X-RateLimit-Remaining', '0' ]); + }); +}); diff --git a/test/utils/server.ts b/test/utils/server.ts index af36b9f4..edec9f5d 100644 --- a/test/utils/server.ts +++ b/test/utils/server.ts @@ -2,6 +2,7 @@ import type { Server } from 'node:http'; import type { AddressInfo } from 'node:net'; import { io, type Socket } from 'socket.io-client'; import { createServer } from '../../src/lib/server.js'; +import { getRedisClient } from '../../src/lib/redis/client.js'; let app: Server; let url: string; @@ -12,6 +13,8 @@ export const getTestServer = async (): Promise => { app.listen(0); const { port } = app.address() as AddressInfo; url = `http://127.0.0.1:${port}/probes`; + const redis = getRedisClient(); + await redis.flushDb(); } return app;