From f55cfb466fe4e78effd55fdc30c2716b9b5235fc Mon Sep 17 00:00:00 2001 From: Alexey Yarmosh Date: Wed, 7 Jun 2023 18:28:49 +0200 Subject: [PATCH 01/15] feat: update rate limit logic --- src/lib/http/middleware/ratelimit.ts | 23 +++++++++++++++------ src/lib/http/server.ts | 4 +--- src/measurement/route/create-measurement.ts | 3 ++- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/lib/http/middleware/ratelimit.ts b/src/lib/http/middleware/ratelimit.ts index 75310b72..a920eddd 100644 --- a/src/lib/http/middleware/ratelimit.ts +++ b/src/lib/http/middleware/ratelimit.ts @@ -1,3 +1,4 @@ +import config from 'config'; import type { Context, Next } from 'koa'; import requestIp from 'request-ip'; import type { RateLimiterRes } from 'rate-limiter-flexible'; @@ -19,15 +20,25 @@ export const rateLimitHandler = () => async (ctx: Context, next: Next) => { return next(); } - try { - const response = await rateLimiter.consume(requestIp.getClientIp(ctx.req) ?? ''); - setResponseHeaders(ctx, response); - } catch (error: unknown) { // Ts requires 'unknown' for errors - setResponseHeaders(ctx, error as RateLimiterRes); + const defaultState = { + remainingPoints: config.get('measurement.rateLimit'), + msBeforeNext: config.get('measurement.rateLimitResetMs'), + consumedPoints: 0, + isFirstInDuration: true, + }; + const currentState = await rateLimiter.get(requestIp.getClientIp(ctx.req) ?? '') ?? defaultState; + + if (currentState.remainingPoints >= ctx.request.body.limit) { + await next(); + const newState = await rateLimiter.penalty(requestIp.getClientIp(ctx.req) ?? '', ctx.response.body.probesCount); + setResponseHeaders(ctx, newState); + } else { + setResponseHeaders(ctx, currentState); ctx.status = 429; ctx.body = 'Too Many Requests'; return; } - await next(); + const data = await rateLimiter.get(requestIp.getClientIp(ctx.req) ?? ''); + console.log('data', data); }; diff --git a/src/lib/http/server.ts b/src/lib/http/server.ts index e4549797..e091c192 100644 --- a/src/lib/http/server.ts +++ b/src/lib/http/server.ts @@ -15,7 +15,6 @@ import { registerGetMeasurementRoute } from '../../measurement/route/get-measure import { registerCreateMeasurementRoute } from '../../measurement/route/create-measurement.js'; import { registerHealthRoute } from '../../health/route/get.js'; import { errorHandler } from './error-handler.js'; -import { rateLimitHandler } from './middleware/ratelimit.js'; import { errorHandlerMw } from './middleware/error-handler.js'; import { corsHandler } from './middleware/cors.js'; import { isAdminMw } from './middleware/is-admin.js'; @@ -40,8 +39,7 @@ rootRouter.get('/', (ctx) => { const apiRouter = new Router({ strict: true, sensitive: true }); apiRouter.prefix('/v1') - .use(isAdminMw) - .use(rateLimitHandler()); + .use(isAdminMw); // POST /measurements registerCreateMeasurementRoute(apiRouter); diff --git a/src/measurement/route/create-measurement.ts b/src/measurement/route/create-measurement.ts index f0d4a9ae..6f66adad 100644 --- a/src/measurement/route/create-measurement.ts +++ b/src/measurement/route/create-measurement.ts @@ -6,6 +6,7 @@ import type { MeasurementRequest } from '../types.js'; import { bodyParser } from '../../lib/http/middleware/body-parser.js'; import { validate } from '../../lib/http/middleware/validate.js'; import { schema } from '../schema/global-schema.js'; +import { rateLimitHandler } from '../../lib/http/middleware/ratelimit.js'; const hostConfig = config.get('server.host'); const runner = getMeasurementRunner(); @@ -24,5 +25,5 @@ const handle = async (ctx: Context): Promise => { }; export const registerCreateMeasurementRoute = (router: Router): void => { - router.post('/measurements', bodyParser(), validate(schema), handle); + router.post('/measurements', bodyParser(), validate(schema), rateLimitHandler(), handle); }; From 95472414d6930cf49582c6859049676424fd1923 Mon Sep 17 00:00:00 2001 From: Alexey Yarmosh Date: Fri, 9 Jun 2023 13:53:15 +0300 Subject: [PATCH 02/15] feat: update ratelimit logic --- src/lib/http/middleware/ratelimit.ts | 40 +++++++++++++--------------- src/lib/ratelimiter.ts | 11 ++++++-- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/src/lib/http/middleware/ratelimit.ts b/src/lib/http/middleware/ratelimit.ts index a920eddd..7648b5f7 100644 --- a/src/lib/http/middleware/ratelimit.ts +++ b/src/lib/http/middleware/ratelimit.ts @@ -1,9 +1,9 @@ -import config from 'config'; import type { Context, Next } from 'koa'; import requestIp from 'request-ip'; import type { RateLimiterRes } from 'rate-limiter-flexible'; - -import rateLimiter from '../../ratelimiter.js'; +import createHttpError from 'http-errors'; +import rateLimiter, { defaultState } from '../../ratelimiter.js'; +import type { MeasurementRequest } from '../../../measurement/types.js'; const setResponseHeaders = (ctx: Context, response: RateLimiterRes) => { ctx.set('X-RateLimit-Reset', `${Math.round(response.msBeforeNext / 1000)}`); @@ -15,30 +15,28 @@ const methodsWhitelist = new Set([ 'GET', 'HEAD', 'OPTIONS' ]); export const rateLimitHandler = () => async (ctx: Context, next: Next) => { const { method, isAdmin } = ctx; + const clientIp = requestIp.getClientIp(ctx.req) ?? ''; + const request = ctx.request.body as MeasurementRequest; if (methodsWhitelist.has(method) || isAdmin) { return next(); } - const defaultState = { - remainingPoints: config.get('measurement.rateLimit'), - msBeforeNext: config.get('measurement.rateLimitResetMs'), - consumedPoints: 0, - isFirstInDuration: true, - }; - const currentState = await rateLimiter.get(requestIp.getClientIp(ctx.req) ?? '') ?? defaultState; - - if (currentState.remainingPoints >= ctx.request.body.limit) { - await next(); - const newState = await rateLimiter.penalty(requestIp.getClientIp(ctx.req) ?? '', ctx.response.body.probesCount); - setResponseHeaders(ctx, newState); - } else { + const currentState = await rateLimiter.get(clientIp) ?? defaultState as RateLimiterRes; + setResponseHeaders(ctx, currentState); + + if (currentState.remainingPoints < request.limit) { setResponseHeaders(ctx, currentState); - ctx.status = 429; - ctx.body = 'Too Many Requests'; - return; + throw createHttpError(429, 'Too Many Probes Requested', { type: 'too_many_probes' }); + } + + await next(); + const response = ctx.response.body as object; + + if (!('probesCount' in response) || typeof response.probesCount !== 'number') { + throw new Error('Missing probesCount field in response object'); } - const data = await rateLimiter.get(requestIp.getClientIp(ctx.req) ?? ''); - console.log('data', data); + const newState = await rateLimiter.penalty(clientIp, response.probesCount as number); + setResponseHeaders(ctx, newState); }; diff --git a/src/lib/ratelimiter.ts b/src/lib/ratelimiter.ts index 8715eb26..f82871fd 100644 --- a/src/lib/ratelimiter.ts +++ b/src/lib/ratelimiter.ts @@ -4,11 +4,18 @@ import { createRedisClient } from './redis/client.js'; const redisClient = await createRedisClient({ legacyMode: true }); -export const rateLimiter = new RateLimiterRedis({ +export const defaultState = { + remainingPoints: config.get('measurement.rateLimit'), + msBeforeNext: config.get('measurement.rateLimitReset') * 1000, + consumedPoints: 0, + isFirstInDuration: true, +}; + +const rateLimiter = new RateLimiterRedis({ storeClient: redisClient, keyPrefix: 'rate', points: config.get('measurement.rateLimit'), - duration: 60, + duration: config.get('measurement.rateLimitReset'), }); export default rateLimiter; From 4948ad9727a91a7a226a7674a8818f21005160e2 Mon Sep 17 00:00:00 2001 From: Alexey Yarmosh Date: Fri, 9 Jun 2023 14:39:27 +0300 Subject: [PATCH 03/15] feat: add support for limits in location --- src/lib/http/middleware/ratelimit.ts | 3 ++- src/measurement/schema/location-schema.ts | 1 + src/measurement/types.ts | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib/http/middleware/ratelimit.ts b/src/lib/http/middleware/ratelimit.ts index 7648b5f7..77698e74 100644 --- a/src/lib/http/middleware/ratelimit.ts +++ b/src/lib/http/middleware/ratelimit.ts @@ -17,6 +17,7 @@ export const rateLimitHandler = () => async (ctx: Context, next: Next) => { const { method, isAdmin } = ctx; const clientIp = requestIp.getClientIp(ctx.req) ?? ''; const request = ctx.request.body as MeasurementRequest; + const limit = request.locations.some(l => l.limit) ? request.locations.reduce((sum, { limit }) => sum + limit, 0) : request.limit; if (methodsWhitelist.has(method) || isAdmin) { return next(); @@ -25,7 +26,7 @@ export const rateLimitHandler = () => async (ctx: Context, next: Next) => { const currentState = await rateLimiter.get(clientIp) ?? defaultState as RateLimiterRes; setResponseHeaders(ctx, currentState); - if (currentState.remainingPoints < request.limit) { + if (currentState.remainingPoints < limit) { setResponseHeaders(ctx, currentState); throw createHttpError(429, 'Too Many Probes Requested', { type: 'too_many_probes' }); } diff --git a/src/measurement/schema/location-schema.ts b/src/measurement/schema/location-schema.ts index eac65e80..2a18c0c0 100644 --- a/src/measurement/schema/location-schema.ts +++ b/src/measurement/schema/location-schema.ts @@ -29,5 +29,6 @@ export const schema = Joi.array().items(Joi.object().keys({ limit: Joi.number().min(1).max(measurementConfig.limits.location).when(Joi.ref('/limit'), { is: Joi.exist(), then: Joi.forbidden().messages({ 'any.unknown': 'limit per location is not allowed when a global limit is set' }), + otherwise: Joi.number().default(1), }), }).or('continent', 'region', 'country', 'state', 'city', 'network', 'asn', 'magic', 'tags')).default(GLOBAL_DEFAULTS.locations); diff --git a/src/measurement/types.ts b/src/measurement/types.ts index 49a64d80..f84dc0ba 100644 --- a/src/measurement/types.ts +++ b/src/measurement/types.ts @@ -163,7 +163,7 @@ type TestProgress = { export type RequestType = 'ping' | 'traceroute' | 'dns' | 'http' | 'mtr'; export type MeasurementOptions = PingTest | TracerouteTest | MtrTest | DnsTest | HttpTest; -export type LocationWithLimit = Location & {limit?: number}; +export type LocationWithLimit = Location & {limit: number}; /** * Measurement Objects From 5d0800d7a141fbad9b1f2f7d307ec0668bf80eda Mon Sep 17 00:00:00 2001 From: Alexey Yarmosh Date: Fri, 9 Jun 2023 14:41:03 +0300 Subject: [PATCH 04/15] feat: update default config --- config/default.cjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/default.cjs b/config/default.cjs index a0a8e283..0eeff8bc 100644 --- a/config/default.cjs +++ b/config/default.cjs @@ -29,7 +29,8 @@ module.exports = { fetchSocketsCacheTTL: 1000, }, measurement: { - rateLimit: 300, + rateLimit: 100000, + rateLimitReset: 3600, maxInProgressProbes: 5, // Timeout after which measurement will be marked as finished even if not all probes respond timeout: 30, // 30 seconds From 8ab760ec278d3785787e0ecc6c36220f1c67473a Mon Sep 17 00:00:00 2001 From: Alexey Yarmosh Date: Fri, 9 Jun 2023 16:19:36 +0300 Subject: [PATCH 05/15] fix: fix tests --- .../integration/middleware/compress.test.ts | 1 - .../tests/integration/middleware/cors.test.ts | 2 - .../middleware/domain-redirect.test.ts | 2 - .../tests/integration/middleware/etag.test.ts | 2 - .../integration/middleware/ratelimit.test.ts | 66 +++++++++++++------ .../middleware/responsetime.test.ts | 1 - 6 files changed, 47 insertions(+), 27 deletions(-) diff --git a/test/tests/integration/middleware/compress.test.ts b/test/tests/integration/middleware/compress.test.ts index 16029dc5..2b6eeac8 100644 --- a/test/tests/integration/middleware/compress.test.ts +++ b/test/tests/integration/middleware/compress.test.ts @@ -39,7 +39,6 @@ describe('compression', () => { probe.emit('probe:status:update', 'ready'); } - // eslint-disable-next-line @typescript-eslint/no-unsafe-call const response = await requestAgent .get('/v1/probes') .set('accept-encoding', '*') diff --git a/test/tests/integration/middleware/cors.test.ts b/test/tests/integration/middleware/cors.test.ts index 47051665..d7fe9de1 100644 --- a/test/tests/integration/middleware/cors.test.ts +++ b/test/tests/integration/middleware/cors.test.ts @@ -15,14 +15,12 @@ describe('cors', () => { describe('Access-Control-Allow-Origin header', () => { it('should include the header with value of *', async () => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-call const response = await requestAgent.get('/v1/').set('Origin', 'elocast.com').send() as Response; expect(response.headers['access-control-allow-origin']).to.equal('*'); }); it('should include the header at root', async () => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-call const response = await requestAgent.get('/').send() as Response; expect(response.headers['access-control-allow-origin']).to.equal('*'); diff --git a/test/tests/integration/middleware/domain-redirect.test.ts b/test/tests/integration/middleware/domain-redirect.test.ts index 3d1da517..81cbd01c 100644 --- a/test/tests/integration/middleware/domain-redirect.test.ts +++ b/test/tests/integration/middleware/domain-redirect.test.ts @@ -15,7 +15,6 @@ describe('domain redirect', () => { describe('http requests', () => { it('should be redirected to "https://jsdelivr.com/globalping" if Host is "globalping.io"', async () => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-call await requestAgent .get('/v1/probes') .send() @@ -27,7 +26,6 @@ describe('domain redirect', () => { }); it('should not be redirected to if Host is not "globalping.io"', async () => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-call await requestAgent .get('/v1/probes') .send() diff --git a/test/tests/integration/middleware/etag.test.ts b/test/tests/integration/middleware/etag.test.ts index 4487cfe3..aa514b11 100644 --- a/test/tests/integration/middleware/etag.test.ts +++ b/test/tests/integration/middleware/etag.test.ts @@ -15,7 +15,6 @@ describe('etag', () => { describe('ETag header', () => { it('should include the header', async () => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-call const response = await requestAgent.get('/v1/probes').send() as Response; expect(response.headers.etag).to.exist; @@ -24,7 +23,6 @@ describe('etag', () => { describe('conditional get', () => { it('should redirect to cache', async () => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-call const response = await requestAgent .get('/v1/probes') .set('if-none-match', 'W/"2-l9Fw4VUO7kr8CvBlt4zaMCqXZ0w"') diff --git a/test/tests/integration/middleware/ratelimit.test.ts b/test/tests/integration/middleware/ratelimit.test.ts index e1c95447..78546324 100644 --- a/test/tests/integration/middleware/ratelimit.test.ts +++ b/test/tests/integration/middleware/ratelimit.test.ts @@ -3,19 +3,20 @@ import request, { type Response } from 'supertest'; import requestIp from 'request-ip'; import type { RateLimiterRedis } from 'rate-limiter-flexible'; import { expect } from 'chai'; -import { getTestServer } from '../../../utils/server.js'; +import { Socket } from 'socket.io-client'; +import { getTestServer, addFakeProbe, deleteFakeProbe } from '../../../utils/server.js'; describe('rate limiter', () => { let app: Server; let requestAgent: any; let clientIpv6: string; let rateLimiterInstance: RateLimiterRedis; + let probe: Socket; before(async () => { app = await getTestServer(); requestAgent = request(app); - // eslint-disable-next-line @typescript-eslint/no-unsafe-call const httpResponse = await requestAgent.post('/v1/').send() as Response & {req: any}; // Supertest renders request as ipv4 const clientIp = requestIp.getClientIp(httpResponse.req); @@ -24,6 +25,12 @@ describe('rate limiter', () => { const rateLimiter = await import('../../../../src/lib/ratelimiter.js'); rateLimiterInstance = rateLimiter.default; + probe = await addFakeProbe(); + probe.emit('probe:status:update', 'ready'); + }); + + after(async () => { + await deleteFakeProbe(probe); }); afterEach(async () => { @@ -32,7 +39,6 @@ describe('rate limiter', () => { describe('headers', () => { it('should NOT include headers (GET)', async () => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-call const response = await requestAgent.get('/v1/').send() as Response; expect(response.headers['x-ratelimit-limit']).to.not.exist; @@ -40,25 +46,43 @@ describe('rate limiter', () => { expect(response.headers['x-ratelimit-reset']).to.not.exist; }); - it('should include headers (POST)', async () => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-call + it('should NOT include headers if body is not valid (POST)', async () => { const response = await requestAgent.post('/v1/measurements').send() as Response; + expect(response.headers['x-ratelimit-limit']).to.not.exist; + expect(response.headers['x-ratelimit-remaining']).to.not.exist; + expect(response.headers['x-ratelimit-reset']).to.not.exist; + }); + + it('should include headers (POST)', async () => { + const response = await requestAgent.post('/v1/measurements').send({ + type: 'ping', + target: 'jsdelivr.com', + }) as Response; + expect(response.headers['x-ratelimit-limit']).to.exist; expect(response.headers['x-ratelimit-remaining']).to.exist; expect(response.headers['x-ratelimit-reset']).to.exist; }); - it('should change values on next request (5) (POST)', async () => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-call - const requestPromise = () => requestAgent.post('/v1/measurements').send() as Response; - const responseList = await Promise.all(Array.from({ length: 5 }).map(() => requestPromise())); + it('should change values on multiple requests (POST)', async () => { + const response = await requestAgent.post('/v1/measurements').send({ + type: 'ping', + target: 'jsdelivr.com', + }) as Response; - const firstResponse = responseList[0]; - const lastResponse = responseList[4]; + expect(response.headers['x-ratelimit-limit']).to.equal('100000'); + expect(response.headers['x-ratelimit-remaining']).to.equal('99999'); + expect(response.headers['x-ratelimit-reset']).to.equal('3600'); - expect(responseList).to.have.lengthOf(5); - expect(firstResponse?.headers?.['x-ratelimit-remaining']).to.not.equal(lastResponse?.headers?.['x-ratelimit-remaining']); + const response2 = await requestAgent.post('/v1/measurements').send({ + type: 'ping', + target: 'jsdelivr.com', + }) as Response; + + expect(response2.headers['x-ratelimit-limit']).to.equal('100000'); + expect(response2.headers['x-ratelimit-remaining']).to.equal('99998'); + expect(response2.headers['x-ratelimit-reset']).to.equal('3600'); }); }); @@ -66,17 +90,21 @@ describe('rate limiter', () => { it('should succeed (limit not reached)', async () => { await rateLimiterInstance.set(clientIpv6, 0, 0); - // eslint-disable-next-line @typescript-eslint/no-unsafe-call - const response = await requestAgent.post('/v1/measurements').send() as Response; + const response = await requestAgent.post('/v1/measurements').send({ + type: 'ping', + target: 'jsdelivr.com', + }) as Response; - expect(Number(response.headers['x-ratelimit-remaining'])).to.equal(299); + expect(Number(response.headers['x-ratelimit-remaining'])).to.equal(99999); }); it('should fail (limit reached) (start at 100)', async () => { - await rateLimiterInstance.set(clientIpv6, 300, 0); + await rateLimiterInstance.set(clientIpv6, 100000, 0); - // eslint-disable-next-line @typescript-eslint/no-unsafe-call - const response = await requestAgent.post('/v1/measurements').send() as Response; + const response = await requestAgent.post('/v1/measurements').send({ + type: 'ping', + target: 'jsdelivr.com', + }) as Response; expect(Number(response.headers['x-ratelimit-remaining'])).to.equal(0); expect(response.statusCode).to.equal(429); diff --git a/test/tests/integration/middleware/responsetime.test.ts b/test/tests/integration/middleware/responsetime.test.ts index 70826606..2d0937dc 100644 --- a/test/tests/integration/middleware/responsetime.test.ts +++ b/test/tests/integration/middleware/responsetime.test.ts @@ -16,7 +16,6 @@ describe('response time', () => { describe('X-Response-Time header', () => { describe('should include the header', (): void => { it('should succeed', async () => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-call const response = await requestAgent.get('/v1/').send() as Response; expect(response.headers['x-response-time']).to.exist; From 6ebc40e5cc36a69f85e05e50bb5271f28b229405 Mon Sep 17 00:00:00 2001 From: Alexey Yarmosh Date: Fri, 9 Jun 2023 18:09:17 +0300 Subject: [PATCH 06/15] feat: add ratelimit unit tests --- src/lib/http/middleware/ratelimit.ts | 13 +- src/measurement/types.ts | 2 +- .../integration/middleware/ratelimit.test.ts | 2 +- test/tests/unit/middleware/ratelimit.test.ts | 159 ++++++++++++++++++ 4 files changed, 166 insertions(+), 10 deletions(-) create mode 100644 test/tests/unit/middleware/ratelimit.test.ts diff --git a/src/lib/http/middleware/ratelimit.ts b/src/lib/http/middleware/ratelimit.ts index 77698e74..b05028e9 100644 --- a/src/lib/http/middleware/ratelimit.ts +++ b/src/lib/http/middleware/ratelimit.ts @@ -11,20 +11,17 @@ const setResponseHeaders = (ctx: Context, response: RateLimiterRes) => { ctx.set('X-RateLimit-Remaining', `${response.remainingPoints}`); }; -const methodsWhitelist = new Set([ 'GET', 'HEAD', 'OPTIONS' ]); - export const rateLimitHandler = () => async (ctx: Context, next: Next) => { - const { method, isAdmin } = ctx; + const { isAdmin } = ctx; const clientIp = requestIp.getClientIp(ctx.req) ?? ''; const request = ctx.request.body as MeasurementRequest; - const limit = request.locations.some(l => l.limit) ? request.locations.reduce((sum, { limit }) => sum + limit, 0) : request.limit; + const limit = request.locations.some(l => l.limit) ? request.locations.reduce((sum, { limit = 1 }) => sum + limit, 0) : request.limit; - if (methodsWhitelist.has(method) || isAdmin) { + if (isAdmin) { return next(); } const currentState = await rateLimiter.get(clientIp) ?? defaultState as RateLimiterRes; - setResponseHeaders(ctx, currentState); if (currentState.remainingPoints < limit) { setResponseHeaders(ctx, currentState); @@ -32,12 +29,12 @@ export const rateLimitHandler = () => async (ctx: Context, next: Next) => { } await next(); - const response = ctx.response.body as object; + const response = ctx.response.body as object & { probesCount?: number }; if (!('probesCount' in response) || typeof response.probesCount !== 'number') { throw new Error('Missing probesCount field in response object'); } - const newState = await rateLimiter.penalty(clientIp, response.probesCount as number); + const newState = await rateLimiter.penalty(clientIp, response.probesCount); setResponseHeaders(ctx, newState); }; diff --git a/src/measurement/types.ts b/src/measurement/types.ts index f84dc0ba..49a64d80 100644 --- a/src/measurement/types.ts +++ b/src/measurement/types.ts @@ -163,7 +163,7 @@ type TestProgress = { export type RequestType = 'ping' | 'traceroute' | 'dns' | 'http' | 'mtr'; export type MeasurementOptions = PingTest | TracerouteTest | MtrTest | DnsTest | HttpTest; -export type LocationWithLimit = Location & {limit: number}; +export type LocationWithLimit = Location & {limit?: number}; /** * Measurement Objects diff --git a/test/tests/integration/middleware/ratelimit.test.ts b/test/tests/integration/middleware/ratelimit.test.ts index 78546324..596ff603 100644 --- a/test/tests/integration/middleware/ratelimit.test.ts +++ b/test/tests/integration/middleware/ratelimit.test.ts @@ -3,7 +3,7 @@ import request, { type Response } from 'supertest'; import requestIp from 'request-ip'; import type { RateLimiterRedis } from 'rate-limiter-flexible'; import { expect } from 'chai'; -import { Socket } from 'socket.io-client'; +import type { Socket } from 'socket.io-client'; import { getTestServer, addFakeProbe, deleteFakeProbe } from '../../../utils/server.js'; describe('rate limiter', () => { diff --git a/test/tests/unit/middleware/ratelimit.test.ts b/test/tests/unit/middleware/ratelimit.test.ts new file mode 100644 index 00000000..e6a0ff30 --- /dev/null +++ b/test/tests/unit/middleware/ratelimit.test.ts @@ -0,0 +1,159 @@ +import * as sinon from 'sinon'; +import { expect } from 'chai'; +import rateLimiter from '../../../../src/lib/ratelimiter.js'; +import { rateLimitHandler } from '../../../../src/lib/http/middleware/ratelimit.js'; +import createHttpError from 'http-errors'; + +describe('rate limit middleware', () => { + const defaultCtx: any = { + set: sinon.stub(), + req: {}, + request: { + body: {}, + }, + response: {}, + }; + let ctx = { ...defaultCtx }; + + beforeEach(async () => { + defaultCtx.set.reset(); + ctx = { ...defaultCtx }; + await rateLimiter.delete(''); + }); + + it('should set rate limit headers based on the "probesCount" field value', async () => { + ctx.request.body = { + limit: 10, + locations: [], + }; + + const next: any = () => { + ctx.response.body = { + id: 'id', + probesCount: 5, + }; + }; + + await rateLimitHandler()(ctx, next); + + expect(ctx.set.callCount).to.equal(3); + expect(ctx.set.firstCall.args[0]).to.equal('X-RateLimit-Reset'); + expect(ctx.set.secondCall.args).to.deep.equal([ 'X-RateLimit-Limit', '100000' ]); + expect(ctx.set.thirdCall.args).to.deep.equal([ 'X-RateLimit-Remaining', '99995' ]); + }); + + it('should throw an error if response body doesn\'t have "probesCount" field', async () => { + ctx.request.body = { + limit: 10, + locations: [], + }; + + const next: any = () => { + ctx.response.body = {}; + }; + + const err = await rateLimitHandler()(ctx, next).catch(err => err); + expect(err).to.deep.equal(new Error('Missing probesCount field in response object')); + }); + + it('should NOT set rate limit headers for admin', async () => { + ctx.request.body = { + limit: 10, + locations: [], + }; + + ctx.isAdmin = true; + + const next: any = () => { + ctx.response.body = { + id: 'id', + probesCount: 10, + }; + }; + + await rateLimitHandler()(ctx, next); + expect(ctx.set.callCount).to.equal(0); + }); + + it('should validate request based on the "limit" field value', async () => { + ctx.request.body = { + limit: 60000, + locations: [], + }; + + const next: any = () => { + ctx.response.body = { + id: 'id', + probesCount: 60000, + }; + }; + + await rateLimitHandler()(ctx, next); + expect(ctx.set.args[2]).to.deep.equal([ 'X-RateLimit-Remaining', '40000' ]); + + const err = await rateLimitHandler()(ctx, next).catch(err => err); // 60000 > 40000 so another request with the same body fails + expect(err).to.deep.equal(createHttpError(429, 'Too Many Probes Requested', { type: 'too_many_probes' })); + expect(ctx.set.args[5]).to.deep.equal([ 'X-RateLimit-Remaining', '40000' ]); + + ctx.request.body = { + limit: 40000, + locations: [], + }; + + const next2: any = () => { + ctx.response.body = { + id: 'id', + probesCount: 40000, + }; + }; + + await rateLimitHandler()(ctx, next2); // 40000 === 40000 so request with the updated body works + expect(ctx.set.args[8]).to.deep.equal([ 'X-RateLimit-Remaining', '0' ]); + }); + + it('should validate request based on the "location.limit" field value', async () => { + ctx.request.body = { + locations: [{ + continent: 'EU', + limit: 45000, + }, { + continent: 'NA', + limit: 45000, + }], + }; + + const next: any = () => { + ctx.response.body = { + id: 'id', + probesCount: 90000, + }; + }; + + await rateLimitHandler()(ctx, next); + expect(ctx.set.args[2]).to.deep.equal([ 'X-RateLimit-Remaining', '10000' ]); + + const err = await rateLimitHandler()(ctx, next).catch(err => err); // only 10000 points remaining so another request with the same body fails + expect(err).to.deep.equal(createHttpError(429, 'Too Many Probes Requested', { type: 'too_many_probes' })); + expect(ctx.set.args[5]).to.deep.equal([ 'X-RateLimit-Remaining', '10000' ]); + + ctx.request.body = { + locations: [{ + continent: 'EU', + limit: 5000, + }, { + continent: 'NA', + limit: 5000, + }], + }; + + const next2: any = () => { + ctx.response.body = { + id: 'id', + probesCount: 10000, + }; + }; + + await rateLimitHandler()(ctx, next2); // request with 10000 probes will work fine + expect(ctx.set.args[8]).to.deep.equal([ 'X-RateLimit-Remaining', '0' ]); + }); +}); From 24ce5ef2e3ea6a7a97827bcb6534d895a55c8f86 Mon Sep 17 00:00:00 2001 From: Alexey Yarmosh Date: Fri, 9 Jun 2023 18:31:56 +0300 Subject: [PATCH 07/15] fix: fix test --- .../integration/middleware/ratelimit.test.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/test/tests/integration/middleware/ratelimit.test.ts b/test/tests/integration/middleware/ratelimit.test.ts index 596ff603..8acb4b16 100644 --- a/test/tests/integration/middleware/ratelimit.test.ts +++ b/test/tests/integration/middleware/ratelimit.test.ts @@ -1,11 +1,15 @@ +import fs from 'node:fs'; import type { Server } from 'node:http'; import request, { type Response } from 'supertest'; import requestIp from 'request-ip'; import type { RateLimiterRedis } from 'rate-limiter-flexible'; import { expect } from 'chai'; +import nock from 'nock'; import type { Socket } from 'socket.io-client'; import { getTestServer, addFakeProbe, deleteFakeProbe } from '../../../utils/server.js'; +const nockMocks = JSON.parse(fs.readFileSync('./test/mocks/nock-geoip.json').toString()) as Record; + describe('rate limiter', () => { let app: Server; let requestAgent: any; @@ -25,18 +29,23 @@ describe('rate limiter', () => { const rateLimiter = await import('../../../../src/lib/ratelimiter.js'); rateLimiterInstance = rateLimiter.default; + + nock('https://globalping-geoip.global.ssl.fastly.net').get(/.*/).reply(200, nockMocks['01.00'].fastly); + nock('https://ipinfo.io').get(/.*/).reply(200, nockMocks['01.00'].ipinfo); + nock('https://geoip.maxmind.com/geoip/v2.1/city/').get(/.*/).reply(200, nockMocks['01.00'].maxmind); probe = await addFakeProbe(); probe.emit('probe:status:update', 'ready'); }); - after(async () => { - await deleteFakeProbe(probe); - }); afterEach(async () => { await rateLimiterInstance.delete(clientIpv6); }); + after(async () => { + await deleteFakeProbe(probe); + }); + describe('headers', () => { it('should NOT include headers (GET)', async () => { const response = await requestAgent.get('/v1/').send() as Response; From a11ec000665aa129ba3b580015c3f81786135810 Mon Sep 17 00:00:00 2001 From: Alexey Yarmosh Date: Mon, 12 Jun 2023 13:11:50 +0300 Subject: [PATCH 08/15] feat: update unresolvable geoip logic --- src/lib/geoip/client.ts | 2 +- src/lib/geoip/providers/maxmind.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/geoip/client.ts b/src/lib/geoip/client.ts index 7c7473c1..cd6170e2 100644 --- a/src/lib/geoip/client.ts +++ b/src/lib/geoip/client.ts @@ -63,7 +63,7 @@ export default class GeoipClient { const resultsWithCities = results.filter(s => s.city); - if (resultsWithCities.length < 2 && resultsWithCities[0]?.provider === 'fastly') { + if (resultsWithCities.length === 0 || (resultsWithCities.length === 1 && resultsWithCities[0]?.provider === 'fastly')) { throw new InternalError(`unresolvable geoip: ${addr}`, true); } diff --git a/src/lib/geoip/providers/maxmind.ts b/src/lib/geoip/providers/maxmind.ts index 03bc5701..3587ee63 100644 --- a/src/lib/geoip/providers/maxmind.ts +++ b/src/lib/geoip/providers/maxmind.ts @@ -27,7 +27,7 @@ const query = async (addr: string, retryCounter = 0): Promise => { } } - throw new Error('no maxmind data'); + throw error; } }; From b6e5befbb809c979e3a1295391f89f5a152965e8 Mon Sep 17 00:00:00 2001 From: Alexey Yarmosh Date: Mon, 12 Jun 2023 13:41:53 +0300 Subject: [PATCH 09/15] test: disable redis cache in tests --- config/default.cjs | 2 +- config/test.cjs | 5 +++++ src/lib/cache/redis-cache.ts | 2 +- test/mocks/redis-cache.ts | 5 ----- .../measurement/create-measurement.test.ts | 2 -- .../measurement/probe-communication.test.ts | 2 -- .../integration/measurement/timeout-result.test.ts | 2 -- test/tests/integration/middleware/compress.test.ts | 11 ++--------- test/tests/integration/probes/get-probes.test.ts | 13 ++++--------- 9 files changed, 13 insertions(+), 31 deletions(-) delete mode 100644 test/mocks/redis-cache.ts diff --git a/config/default.cjs b/config/default.cjs index 0eeff8bc..72ab53a7 100644 --- a/config/default.cjs +++ b/config/default.cjs @@ -15,7 +15,7 @@ module.exports = { }, geoip: { cache: { - ttl: 3 * 24 * 60 * 60 * 1000, // 24hrs + ttl: 3 * 24 * 60 * 60 * 1000, // 3 days }, }, maxmind: { diff --git a/config/test.cjs b/config/test.cjs index c0912512..d10d5eb3 100644 --- a/config/test.cjs +++ b/config/test.cjs @@ -7,6 +7,11 @@ module.exports = { admin: { key: 'admin', }, + geoip: { + cache: { + ttl: 1, // 1 ms ttl here to disable redis cache in tests + }, + }, ws: { fetchSocketsCacheTTL: 0, }, diff --git a/src/lib/cache/redis-cache.ts b/src/lib/cache/redis-cache.ts index df1177df..fcebd361 100644 --- a/src/lib/cache/redis-cache.ts +++ b/src/lib/cache/redis-cache.ts @@ -5,7 +5,7 @@ export default class RedisCache implements CacheInterface { constructor (private readonly redis: RedisClient) {} async set (key: string, value: unknown, ttl?: number): Promise { - await this.redis.set(this.buildCacheKey(key), JSON.stringify(value), { EX: ttl ? ttl / 1000 : 0 }); + await this.redis.set(this.buildCacheKey(key), JSON.stringify(value), { PX: ttl ?? 0 }); } async get (key: string): Promise { diff --git a/test/mocks/redis-cache.ts b/test/mocks/redis-cache.ts deleted file mode 100644 index 779b8816..00000000 --- a/test/mocks/redis-cache.ts +++ /dev/null @@ -1,5 +0,0 @@ -export default class RedisCacheMock { - set = () => Promise.resolve(); - get = () => Promise.resolve(); - delete = () => Promise.resolve(); -} diff --git a/test/tests/integration/measurement/create-measurement.test.ts b/test/tests/integration/measurement/create-measurement.test.ts index f6e6fa95..a1c3609c 100644 --- a/test/tests/integration/measurement/create-measurement.test.ts +++ b/test/tests/integration/measurement/create-measurement.test.ts @@ -5,7 +5,6 @@ import request, { type SuperTest, type Test } from 'supertest'; import * as td from 'testdouble'; import nock from 'nock'; import type { Socket } from 'socket.io-client'; -import RedisCacheMock from '../../../mocks/redis-cache.js'; const nockMocks = JSON.parse(fs.readFileSync('./test/mocks/nock-geoip.json').toString()) as Record; @@ -16,7 +15,6 @@ describe('Create measurement', () => { let requestAgent: SuperTest; before(async () => { - await td.replaceEsm('../../../../src/lib/cache/redis-cache.ts', {}, RedisCacheMock); await td.replaceEsm('../../../../src/lib/ip-ranges.ts', { getRegion: () => 'gcp-us-west4', populateMemList: () => Promise.resolve() }); ({ getTestServer, addFakeProbe, deleteFakeProbe } = await import('../../../utils/server.js')); const app = await getTestServer(); diff --git a/test/tests/integration/measurement/probe-communication.test.ts b/test/tests/integration/measurement/probe-communication.test.ts index d2222b06..85b1fa9d 100644 --- a/test/tests/integration/measurement/probe-communication.test.ts +++ b/test/tests/integration/measurement/probe-communication.test.ts @@ -5,7 +5,6 @@ import nock from 'nock'; import type { Socket } from 'socket.io-client'; import * as sinon from 'sinon'; import { expect } from 'chai'; -import RedisCacheMock from '../../../mocks/redis-cache.js'; const nockMocks = JSON.parse(fs.readFileSync('./test/mocks/nock-geoip.json').toString()) as Record; @@ -22,7 +21,6 @@ describe('Create measurement request', () => { before(async () => { await td.replaceEsm('crypto-random-string', {}, cryptoRandomString); - await td.replaceEsm('../../../../src/lib/cache/redis-cache.ts', {}, RedisCacheMock); await td.replaceEsm('../../../../src/lib/ip-ranges.ts', { getRegion: () => 'gcp-us-west4', populateMemList: () => Promise.resolve() }); ({ getTestServer, addFakeProbe, deleteFakeProbe } = await import('../../../utils/server.js')); const app = await getTestServer(); diff --git a/test/tests/integration/measurement/timeout-result.test.ts b/test/tests/integration/measurement/timeout-result.test.ts index e2446029..659f80e6 100644 --- a/test/tests/integration/measurement/timeout-result.test.ts +++ b/test/tests/integration/measurement/timeout-result.test.ts @@ -5,7 +5,6 @@ import nock from 'nock'; import type { Socket } from 'socket.io-client'; import * as sinon from 'sinon'; import { expect } from 'chai'; -import RedisCacheMock from '../../../mocks/redis-cache.js'; const nockMocks = JSON.parse(fs.readFileSync('./test/mocks/nock-geoip.json').toString()) as Record; @@ -23,7 +22,6 @@ describe('Timeout results', () => { sandbox = sinon.createSandbox({ useFakeTimers: true }); await td.replaceEsm('@jcoreio/async-throttle', null, (f: any) => f); await td.replaceEsm('crypto-random-string', {}, cryptoRandomString); - await td.replaceEsm('../../../../src/lib/cache/redis-cache.ts', {}, RedisCacheMock); await td.replaceEsm('../../../../src/lib/ip-ranges.ts', { getRegion: () => 'gcp-us-west4', populateMemList: () => Promise.resolve() }); ({ getTestServer, addFakeProbe, deleteFakeProbe } = await import('../../../utils/server.js')); const app = await getTestServer(); diff --git a/test/tests/integration/middleware/compress.test.ts b/test/tests/integration/middleware/compress.test.ts index 2b6eeac8..97ec4485 100644 --- a/test/tests/integration/middleware/compress.test.ts +++ b/test/tests/integration/middleware/compress.test.ts @@ -1,26 +1,19 @@ import fs from 'node:fs'; import request, { type Response } from 'supertest'; import { expect } from 'chai'; -import * as td from 'testdouble'; import nock from 'nock'; import type { Socket } from 'socket.io-client'; -import RedisCacheMock from '../../../mocks/redis-cache.js'; +import { getTestServer, addFakeProbe, deleteFakeProbe } from '../../../utils/server.js'; const nockMocks = JSON.parse(fs.readFileSync('./test/mocks/nock-geoip.json').toString()) as Record; describe('compression', () => { - let addFakeProbe: () => Promise; - let deleteFakeProbe: (socket: Socket) => Promise; let requestAgent: any; let probes: Socket[] = []; describe('headers', () => { before(async () => { - await td.replaceEsm('../../../../src/lib/cache/redis-cache.ts', {}, RedisCacheMock); - const http = await import('../../../utils/server.js'); - addFakeProbe = http.addFakeProbe; - deleteFakeProbe = http.deleteFakeProbe; - const app = await http.getTestServer(); + const app = await getTestServer(); requestAgent = request(app); }); diff --git a/test/tests/integration/probes/get-probes.test.ts b/test/tests/integration/probes/get-probes.test.ts index 1a1def5b..409d06cb 100644 --- a/test/tests/integration/probes/get-probes.test.ts +++ b/test/tests/integration/probes/get-probes.test.ts @@ -5,28 +5,23 @@ import { expect } from 'chai'; import request, { type SuperTest, type Test } from 'supertest'; import * as td from 'testdouble'; import type { Socket } from 'socket.io-client'; -import RedisCacheMock from '../../../mocks/redis-cache.js'; +import { getTestServer, addFakeProbe as addProbe, deleteFakeProbe } from '../../../utils/server.js'; const nockMocks = JSON.parse(fs.readFileSync('./test/mocks/nock-geoip.json').toString()) as Record; describe('Get Probes', () => { - let addFakeProbe: () => Promise; - let deleteFakeProbe: (socket: Socket) => Promise; let requestAgent: SuperTest; + let addFakeProbe: () => Promise; const probes: Socket[] = []; before(async () => { - await td.replaceEsm('../../../../src/lib/cache/redis-cache.ts', {}, RedisCacheMock); - const http = await import('../../../utils/server.js'); - deleteFakeProbe = http.deleteFakeProbe; - addFakeProbe = async () => { - const probe = await http.addFakeProbe(); + const probe = await addProbe(); probes.push(probe); return probe; }; - const app = await http.getTestServer(); + const app = await getTestServer(); requestAgent = request(app); }); From bb9776a5fc36f887f7118fad189f8512a6c00e4f Mon Sep 17 00:00:00 2001 From: Alexey Yarmosh Date: Mon, 12 Jun 2023 16:42:51 +0300 Subject: [PATCH 10/15] test: separate dev and test fake ips --- src/probe/builder.ts | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/src/probe/builder.ts b/src/probe/builder.ts index 40f33cb5..3ebb12c0 100644 --- a/src/probe/builder.ts +++ b/src/probe/builder.ts @@ -18,18 +18,26 @@ import getProbeIp from '../lib/get-probe-ip.js'; import { getRegion } from '../lib/ip-ranges.js'; import type { Probe, ProbeLocation, Tag } from './types.js'; -const fakeIpForDebug = () => _.sample([ - '18.200.0.1', // aws-eu-west-1 - '34.140.0.10', // gcp-europe-west1 - '95.155.94.127', - '65.49.22.66', - '185.229.226.83', - '51.158.22.211', - '131.255.7.26', - '213.136.174.80', - '94.214.253.78', - '79.205.97.254', -]); +const fakeIpForDebug = () => { + /** + * Ips for test and dev should be separated so redis will not return dev data during the tests + */ + if (process.env['NODE_ENV'] === 'test') { + return '95.155.94.127'; + } + + return _.sample([ + '18.200.0.1', // aws-eu-west-1 + '34.140.0.10', // gcp-europe-west1 + '65.49.22.66', + '185.229.226.83', + '51.158.22.211', + '131.255.7.26', + '213.136.174.80', + '94.214.253.78', + '79.205.97.254', + ]); +}; const geoipClient = createGeoipClient(); From 1bee98bcc6e38e7af1987750bf3f211622ee42bb Mon Sep 17 00:00:00 2001 From: Alexey Yarmosh Date: Mon, 12 Jun 2023 16:45:19 +0300 Subject: [PATCH 11/15] fix: fix deepsource warnings --- src/lib/geoip/providers/maxmind.ts | 3 ++- src/lib/redis/scripts.ts | 8 ++++---- src/lib/ws/helper/error-handler.ts | 2 +- src/lib/ws/helper/reconnect-probes.ts | 4 ++-- src/lib/ws/server.ts | 4 +++- src/probe/handler/dns.ts | 2 +- src/probe/handler/stats.ts | 5 +---- src/probe/handler/status.ts | 2 +- 8 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/lib/geoip/providers/maxmind.ts b/src/lib/geoip/providers/maxmind.ts index 3587ee63..adfcf9bf 100644 --- a/src/lib/geoip/providers/maxmind.ts +++ b/src/lib/geoip/providers/maxmind.ts @@ -15,7 +15,8 @@ export const isMaxmindError = (error: unknown): error is WebServiceClientError = const query = async (addr: string, retryCounter = 0): Promise => { try { - return await client.city(addr); + const city = await client.city(addr); + return city; } catch (error: unknown) { if (isMaxmindError(error)) { if (error.code === 'SERVER_ERROR' && retryCounter < 3) { diff --git a/src/lib/redis/scripts.ts b/src/lib/redis/scripts.ts index 67886e36..ea841b69 100644 --- a/src/lib/redis/scripts.ts +++ b/src/lib/redis/scripts.ts @@ -4,8 +4,8 @@ import type { MeasurementRecord, MeasurementResultMessage } from '../../measurem type CountScript = { NUMBER_OF_KEYS: number; SCRIPT: string; - transformArguments (this: void, key: string): Array; - transformReply (this: void, reply: number): number; + transformArguments (key: string): Array; + transformReply (reply: number): number; } & { SHA1: string; }; @@ -13,8 +13,8 @@ type CountScript = { export type RecordResultScript = { NUMBER_OF_KEYS: number; SCRIPT: string; - transformArguments (this: void, measurementId: string, testId: string, data: MeasurementResultMessage['result']): string[]; - transformReply (this: void, reply: string): MeasurementRecord | null; + transformArguments (measurementId: string, testId: string, data: MeasurementResultMessage['result']): string[]; + transformReply (reply: string): MeasurementRecord | null; } & { SHA1: string; }; diff --git a/src/lib/ws/helper/error-handler.ts b/src/lib/ws/helper/error-handler.ts index e6151fd5..d6ca3833 100644 --- a/src/lib/ws/helper/error-handler.ts +++ b/src/lib/ws/helper/error-handler.ts @@ -19,7 +19,7 @@ type NextArgument = NextConnectArgument | NextMwArgument; const isError = (error: unknown): error is Error => Boolean(error as Error['message']); // eslint-disable-next-line @typescript-eslint/no-explicit-any -export const errorHandler = (next: NextArgument) => (socket: Socket, mwNext?: (error?: any) => void | undefined) => { +export const errorHandler = (next: NextArgument) => (socket: Socket, mwNext?: (error?: any) => void) => { next(socket, mwNext!).catch((error) => { // eslint-disable-line @typescript-eslint/no-non-null-assertion const clientIp = getProbeIp(socket.request) ?? ''; const reason = isError(error) ? error.message : 'unknown'; diff --git a/src/lib/ws/helper/reconnect-probes.ts b/src/lib/ws/helper/reconnect-probes.ts index ae989ddb..89bf6be1 100644 --- a/src/lib/ws/helper/reconnect-probes.ts +++ b/src/lib/ws/helper/reconnect-probes.ts @@ -1,8 +1,8 @@ -import { fetchSockets } from '../server.js'; +import type { ThrottledFetchSockets } from '../server'; const TIME_TO_RECONNECT_PROBES = 60_000; -export const reconnectProbes = async () => { +export const reconnectProbes = async (fetchSockets: ThrottledFetchSockets) => { // passing fetchSockets in arguments to avoid cycle dependency const sockets = await fetchSockets(); for (const socket of sockets) { diff --git a/src/lib/ws/server.ts b/src/lib/ws/server.ts index 0de9945d..50c9cf1d 100644 --- a/src/lib/ws/server.ts +++ b/src/lib/ws/server.ts @@ -46,7 +46,7 @@ export const initWsServer = async () => { ); setTimeout(() => { - reconnectProbes().catch(error => logger.error(error)); + reconnectProbes(fetchSockets).catch(error => logger.error(error)); }, TIME_UNTIL_VM_BECOMES_HEALTHY); }; @@ -67,3 +67,5 @@ export const fetchSockets = async () => { return sockets; }; + +export type ThrottledFetchSockets = typeof fetchSockets; diff --git a/src/probe/handler/dns.ts b/src/probe/handler/dns.ts index d4817d31..00719ac1 100644 --- a/src/probe/handler/dns.ts +++ b/src/probe/handler/dns.ts @@ -1,4 +1,4 @@ -import type { Probe } from '../../probe/types.js'; +import type { Probe } from '../types.js'; export const handleDnsUpdate = (probe: Probe) => (list: string[]): void => { probe.resolvers = list; diff --git a/src/probe/handler/stats.ts b/src/probe/handler/stats.ts index c6bc4df5..39e8ab9d 100644 --- a/src/probe/handler/stats.ts +++ b/src/probe/handler/stats.ts @@ -1,7 +1,4 @@ -import type { - Probe, - ProbeStats, -} from '../../probe/types.js'; +import type { Probe, ProbeStats } from '../types.js'; export const handleStatsReport = (probe: Probe) => (report: ProbeStats): void => { probe.stats = report; diff --git a/src/probe/handler/status.ts b/src/probe/handler/status.ts index 282c610d..100abc4b 100644 --- a/src/probe/handler/status.ts +++ b/src/probe/handler/status.ts @@ -1,4 +1,4 @@ -import type { Probe } from '../../probe/types.js'; +import type { Probe } from '../types.js'; export const handleStatusUpdate = (probe: Probe) => (status: Probe['status']): void => { probe.status = status; From e3c9f5bf0128db3116992e10ab21e3525591c742 Mon Sep 17 00:00:00 2001 From: Alexey Yarmosh Date: Mon, 12 Jun 2023 17:08:39 +0300 Subject: [PATCH 12/15] ci: trigger pipeline From 33730a28063d5a40a9f2c628e68687ca1acb5d26 Mon Sep 17 00:00:00 2001 From: Dmitriy Akulov Date: Mon, 12 Jun 2023 16:15:08 +0200 Subject: [PATCH 13/15] test --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index aa6cd348..cc5acab7 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@

Better understand your network routing, fix anycast issues, monitor your CDN and DNS performance,
do uptime monitoring and build your own network tools for personal or public use. -
+

From d5267c452265ba5da8f8d1a07a95e139b3943da8 Mon Sep 17 00:00:00 2001 From: Alexey Yarmosh Date: Mon, 12 Jun 2023 17:22:21 +0300 Subject: [PATCH 14/15] fix: fix build of PRs to non-master branches --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31a3bdbc..8c5a472c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,9 @@ name: Node CI on: push: - branches: [ master ] + branches: [ "*" ] pull_request: - branches: [ master ] + branches: [ "*" ] jobs: build: From 2b00917f099936dbe92fbf6049804d8bdc85f2ae Mon Sep 17 00:00:00 2001 From: Alexey Yarmosh Date: Mon, 19 Jun 2023 11:07:05 +0300 Subject: [PATCH 15/15] feat: clear redis before test run --- src/probe/builder.ts | 8 +------- test/utils/server.ts | 3 +++ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/probe/builder.ts b/src/probe/builder.ts index 3ebb12c0..bf60a5e9 100644 --- a/src/probe/builder.ts +++ b/src/probe/builder.ts @@ -19,16 +19,10 @@ import { getRegion } from '../lib/ip-ranges.js'; import type { Probe, ProbeLocation, Tag } from './types.js'; const fakeIpForDebug = () => { - /** - * Ips for test and dev should be separated so redis will not return dev data during the tests - */ - if (process.env['NODE_ENV'] === 'test') { - return '95.155.94.127'; - } - return _.sample([ '18.200.0.1', // aws-eu-west-1 '34.140.0.10', // gcp-europe-west1 + '95.155.94.127', '65.49.22.66', '185.229.226.83', '51.158.22.211', diff --git a/test/utils/server.ts b/test/utils/server.ts index af36b9f4..edec9f5d 100644 --- a/test/utils/server.ts +++ b/test/utils/server.ts @@ -2,6 +2,7 @@ import type { Server } from 'node:http'; import type { AddressInfo } from 'node:net'; import { io, type Socket } from 'socket.io-client'; import { createServer } from '../../src/lib/server.js'; +import { getRedisClient } from '../../src/lib/redis/client.js'; let app: Server; let url: string; @@ -12,6 +13,8 @@ export const getTestServer = async (): Promise => { app.listen(0); const { port } = app.address() as AddressInfo; url = `http://127.0.0.1:${port}/probes`; + const redis = getRedisClient(); + await redis.flushDb(); } return app;