diff --git a/migrations/create-tables.js.sql b/migrations/create-tables.js.sql index b6d12090..536b29bb 100644 --- a/migrations/create-tables.js.sql +++ b/migrations/create-tables.js.sql @@ -54,3 +54,17 @@ CREATE TABLE IF NOT EXISTS gp_credits ( CONSTRAINT gp_credits_user_id_unique UNIQUE (user_id), CONSTRAINT gp_credits_amount_positive CHECK (`amount` >= 0) ); + +CREATE TABLE IF NOT EXISTS gp_location_overrides ( + id INT(10) UNSIGNED AUTO_INCREMENT PRIMARY KEY, + user_created CHAR(36), + date_created TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + user_updated CHAR(36), + date_updated TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + ip_range VARCHAR(255) NOT NULL, + city VARCHAR(255) NOT NULL, + state VARCHAR(255), + country VARCHAR(255), + latitude FLOAT(10, 5), + longitude FLOAT(10, 5) +); diff --git a/src/lib/location/location.ts b/src/lib/location/location.ts index 8ee2035f..432e454f 100644 --- a/src/lib/location/location.ts +++ b/src/lib/location/location.ts @@ -8,6 +8,7 @@ import { } from './countries.js'; import { aliases as networkAliases } from './networks.js'; import { aliases as continentAliases } from './continents.js'; +import type { ProbeLocation, Tag } from '../../probe/types.js'; const countryToRegionMap = new Map(_.flatMap(regions, (v, r) => v.map(c => [ c, r ]))); @@ -95,3 +96,26 @@ export const getRegionAliases = (key: string): string[] => { return array ?? []; }; +export const getIndex = (location: ProbeLocation, tags: Tag[]) => { + // Storing index as string[][] so every category will have it's exact position in the index array across all probes + const index = [ + [ location.country ], + [ getCountryIso3ByIso2(location.country) ], + [ getCountryByIso(location.country) ], + getCountryAliases(location.country), + [ location.normalizedCity ], + location.state ? [ location.state ] : [], + location.state ? [ getStateNameByIso(location.state) ] : [], + [ location.continent ], + getContinentAliases(location.continent), + [ location.region ], + getRegionAliases(location.region), + [ `as${location.asn}` ], + tags.filter(tag => tag.type === 'system').map(tag => tag.value), + [ location.normalizedNetwork ], + getNetworkAliases(location.normalizedNetwork), + ].map(category => category.map(s => s.toLowerCase().replaceAll('-', ' '))); + + return index; +}; + diff --git a/src/lib/override/admin-data.ts b/src/lib/override/admin-data.ts new file mode 100644 index 00000000..280f7480 --- /dev/null +++ b/src/lib/override/admin-data.ts @@ -0,0 +1,137 @@ +/* eslint-disable camelcase */ +import ipaddr from 'ipaddr.js'; +import type { Knex } from 'knex'; +import type { Probe, ProbeLocation } from '../../probe/types.js'; +import { normalizeFromPublicName } from '../geoip/utils.js'; +import { getContinentByCountry, getRegionByCountry } from '../location/location.js'; +import { scopedLogger } from '../logger.js'; + +const logger = scopedLogger('admin-data'); + +const LOCATION_OVERRIDES_TABLE = 'gp_location_overrides'; + +type ParsedIpRange = [ipaddr.IPv4 | ipaddr.IPv6, number]; + +type LocationOverride = { + date_created: Date; + date_updated: Date | null; + ip_range: string; + city: string; + country: string; + state: string | null; + latitude: number; + longitude: number; +} + +type UpdatedFields = { + continent: string; + region: string; + city: string; + normalizedCity: string; + country: string; + state: string | null; + latitude: number; + longitude: number; +} + +export class AdminData { + private rangesToUpdatedFields: Map = new Map(); + + private ipsToUpdatedFields: Map = new Map(); + + private lastUpdate: Date = new Date('01-01-1970'); + + private lastOverridesLength: number = 0; + + constructor (private readonly sql: Knex) {} + + scheduleSync () { + setTimeout(() => { + this.syncDashboardData() + .finally(() => this.scheduleSync()) + .catch(error => logger.error(error)); + }, 60_000).unref(); + } + + async syncDashboardData () { + const overrides = await this.sql(LOCATION_OVERRIDES_TABLE).select(); + + this.rangesToUpdatedFields = new Map(overrides.map(override => [ ipaddr.parseCIDR(override.ip_range), { + continent: getContinentByCountry(override.country), + region: getRegionByCountry(override.country), + city: override.city, + normalizedCity: normalizeFromPublicName(override.city), + country: override.country, + state: override.state, + latitude: override.latitude, + longitude: override.longitude, + }])); + + const newLastUpdate = overrides.reduce((lastUpdate, { date_created, date_updated }) => { + lastUpdate = date_created > lastUpdate ? date_created : lastUpdate; + lastUpdate = (date_updated && date_updated > lastUpdate) ? date_updated : lastUpdate; + return lastUpdate; + }, new Date('01-01-1970')); + + if (newLastUpdate > this.lastUpdate || overrides.length !== this.lastOverridesLength) { + this.ipsToUpdatedFields.clear(); + this.lastUpdate = newLastUpdate; + this.lastOverridesLength = overrides.length; + } + } + + getUpdatedProbes (probes: Probe[]) { + return probes.map((probe) => { + const updatedFields = this.getUpdatedFields(probe); + + if (!updatedFields) { + return probe; + } + + return { + ...probe, + location: { + ...probe.location, + ...updatedFields, + }, + }; + }); + } + + getUpdatedLocation (probe: Probe): ProbeLocation | null { + const updatedFields = this.getUpdatedFields(probe); + + if (!updatedFields) { + return null; + } + + return { + ...probe.location, + ...updatedFields, + }; + } + + private getUpdatedFields (probe: Probe): UpdatedFields | null { + const updatedFields = this.ipsToUpdatedFields.get(probe.ipAddress); + + if (updatedFields !== undefined) { + return updatedFields; + } + + const newUpdatedFields = this.findUpdatedFields(probe); + this.ipsToUpdatedFields.set(probe.ipAddress, newUpdatedFields); + return newUpdatedFields; + } + + findUpdatedFields (probe: Probe): UpdatedFields | null { + for (const [ range, updatedFields ] of this.rangesToUpdatedFields) { + const ip = ipaddr.parse(probe.ipAddress); + + if (ip.kind() === range[0].kind() && ip.match(range)) { + return updatedFields; + } + } + + return null; + } +} diff --git a/src/lib/adopted-probes.ts b/src/lib/override/adopted-probes.ts similarity index 88% rename from src/lib/adopted-probes.ts rename to src/lib/override/adopted-probes.ts index 2835c168..a1267c8b 100644 --- a/src/lib/adopted-probes.ts +++ b/src/lib/override/adopted-probes.ts @@ -1,10 +1,11 @@ import type { Knex } from 'knex'; import Bluebird from 'bluebird'; import _ from 'lodash'; -import { scopedLogger } from './logger.js'; -import type { fetchRawProbes as serverFetchRawProbes } from './ws/server.js'; -import type { Probe } from '../probe/types.js'; -import { normalizeFromPublicName } from './geoip/utils.js'; +import { scopedLogger } from '../logger.js'; +import type { fetchProbesWithAdminData as serverFetchProbesWithAdminData } from '../ws/server.js'; +import type { Probe, ProbeLocation } from '../../probe/types.js'; +import { normalizeFromPublicName } from '../geoip/utils.js'; +import { getIndex } from '../location/location.js'; const logger = scopedLogger('adopted-probes'); @@ -88,27 +89,27 @@ export class AdoptedProbes { constructor ( private readonly sql: Knex, - private readonly fetchRawProbes: typeof serverFetchRawProbes, + private readonly fetchProbesWithAdminData: typeof serverFetchProbesWithAdminData, ) {} getByIp (ip: string) { return this.adoptedIpToProbe.get(ip); } - getUpdatedLocation (probe: Probe) { + getUpdatedLocation (probe: Probe): ProbeLocation | null { const adoptedProbe = this.getByIp(probe.ipAddress); if (!adoptedProbe || !adoptedProbe.isCustomCity || adoptedProbe.countryOfCustomCity !== probe.location.country) { - return probe.location; + return null; } return { ...probe.location, city: adoptedProbe.city!, normalizedCity: normalizeFromPublicName(adoptedProbe.city!), + state: adoptedProbe.state, latitude: adoptedProbe.latitude!, longitude: adoptedProbe.longitude!, - ...(adoptedProbe.state && { state: adoptedProbe.state }), }; } @@ -125,6 +126,34 @@ export class AdoptedProbes { ]; } + getUpdatedProbes (probes: Probe[]) { + return probes.map((probe) => { + const adopted = this.getByIp(probe.ipAddress); + + if (!adopted) { + return probe; + } + + const isCustomCity = adopted.isCustomCity; + const hasUserTags = adopted.tags && adopted.tags.length; + + if (!isCustomCity && !hasUserTags) { + return probe; + } + + const newLocation = this.getUpdatedLocation(probe) || probe.location; + + const newTags = this.getUpdatedTags(probe); + + return { + ...probe, + location: newLocation, + tags: newTags, + index: getIndex(newLocation, newTags), + }; + }); + } + scheduleSync () { setTimeout(() => { this.syncDashboardData() @@ -134,7 +163,7 @@ export class AdoptedProbes { } async syncDashboardData () { - const allProbes = await this.fetchRawProbes(); + const allProbes = await this.fetchProbesWithAdminData(); this.connectedIpToProbe = new Map(allProbes.map(probe => [ probe.ipAddress, probe ])); this.connectedUuidToIp = new Map(allProbes.map(probe => [ probe.uuid, probe.ipAddress ])); diff --git a/src/lib/override/probe-override.ts b/src/lib/override/probe-override.ts new file mode 100644 index 00000000..e4f8fef3 --- /dev/null +++ b/src/lib/override/probe-override.ts @@ -0,0 +1,36 @@ +import type { Probe } from '../../probe/types.js'; +import type { AdoptedProbes } from './adopted-probes.js'; +import type { AdminData } from './admin-data.js'; + +export class ProbeOverride { + constructor ( + private readonly adoptedProbes: AdoptedProbes, + private readonly adminData: AdminData, + ) {} + + async syncDashboardData () { + await Promise.all([ + this.adoptedProbes.syncDashboardData(), + this.adminData.syncDashboardData(), + ]); + } + + scheduleSync () { + this.adoptedProbes.scheduleSync(); + this.adminData.scheduleSync(); + } + + getUpdatedLocation (probe: Probe) { + const adminLocation = this.adminData.getUpdatedLocation(probe); + const adoptedLocation = this.adoptedProbes.getUpdatedLocation(probe); + return { ...probe.location, ...adminLocation, ...adoptedLocation }; + } + + addAdminData (probes: Probe[]) { + return this.adminData.getUpdatedProbes(probes); + } + + addAdoptedData (probes: Probe[]) { + return this.adoptedProbes.getUpdatedProbes(probes); + } +} diff --git a/src/lib/server.ts b/src/lib/server.ts index ec95ad24..e4b395e6 100644 --- a/src/lib/server.ts +++ b/src/lib/server.ts @@ -1,6 +1,6 @@ import type { Server } from 'node:http'; import { initRedisClient } from './redis/client.js'; -import { adoptedProbes, probeIpLimit, initWsServer } from './ws/server.js'; +import { probeOverride, probeIpLimit, initWsServer } from './ws/server.js'; import { getMetricsAgent } from './metrics.js'; import { populateMemList as populateMemMalwareList } from './malware/client.js'; import { populateMemList as populateMemIpRangesList } from './ip-ranges.js'; @@ -27,8 +27,8 @@ export const createServer = async (): Promise => { await initWsServer(); - await adoptedProbes.syncDashboardData(); - adoptedProbes.scheduleSync(); + await probeOverride.syncDashboardData(); + probeOverride.scheduleSync(); await auth.syncTokens(); auth.scheduleSync(); diff --git a/src/lib/ws/gateway.ts b/src/lib/ws/gateway.ts index 89b1ecac..27ee52c7 100644 --- a/src/lib/ws/gateway.ts +++ b/src/lib/ws/gateway.ts @@ -6,7 +6,7 @@ import { handleStatusUpdate } from '../../probe/handler/status.js'; import { handleDnsUpdate } from '../../probe/handler/dns.js'; import { handleStatsReport } from '../../probe/handler/stats.js'; import { scopedLogger } from '../logger.js'; -import { adoptedProbes, getWsServer, PROBES_NAMESPACE, ServerSocket } from './server.js'; +import { probeOverride, getWsServer, PROBES_NAMESPACE, ServerSocket } from './server.js'; import { probeMetadata } from './middleware/probe-metadata.js'; import { errorHandler } from './helper/error-handler.js'; import { subscribeWithHandler } from './helper/subscribe-handler.js'; @@ -20,7 +20,7 @@ io .use(probeMetadata) .on('connect', errorHandler(async (socket: ServerSocket) => { const probe = socket.data.probe; - const location = adoptedProbes.getUpdatedLocation(probe); + const location = probeOverride.getUpdatedLocation(probe); socket.emit('api:connect:location', location); logger.info(`ws client ${socket.id} connected from ${location.city}, ${location.country} [${probe.ipAddress} - ${location.network}]`); diff --git a/src/lib/ws/server.ts b/src/lib/ws/server.ts index baf0a6ed..db467197 100644 --- a/src/lib/ws/server.ts +++ b/src/lib/ws/server.ts @@ -6,8 +6,10 @@ import type { Probe } from '../../probe/types.js'; import { getRedisClient } from '../redis/client.js'; import { SyncedProbeList } from './synced-probe-list.js'; import { client } from '../sql/client.js'; -import { AdoptedProbes } from '../adopted-probes.js'; +import { ProbeOverride } from '../override/probe-override.js'; import { ProbeIpLimit } from './helper/probe-ip-limit.js'; +import { AdoptedProbes } from '../override/adopted-probes.js'; +import { AdminData } from '../override/admin-data.js'; export type SocketData = { probe: Probe; @@ -45,7 +47,7 @@ export const initWsServer = async () => { dynamicPrivateChannels: true, })); - syncedProbeList = new SyncedProbeList(redis, io.of(PROBES_NAMESPACE), adoptedProbes); + syncedProbeList = new SyncedProbeList(redis, io.of(PROBES_NAMESPACE), probeOverride); await syncedProbeList.sync(); syncedProbeList.scheduleSync(); @@ -75,22 +77,34 @@ export const fetchRawSockets = async () => { return io.of(PROBES_NAMESPACE).fetchSockets(); }; -export const fetchProbes = async ({ allowStale = true } = {}): Promise => { +export const fetchRawProbes = async (): Promise => { if (!syncedProbeList) { throw new Error('WS server not initialized yet'); } - return allowStale ? syncedProbeList.getProbes() : syncedProbeList.fetchProbes(); + return syncedProbeList.getRawProbes(); }; -export const fetchRawProbes = async (): Promise => { +export const fetchProbesWithAdminData = async (): Promise => { if (!syncedProbeList) { throw new Error('WS server not initialized yet'); } - return syncedProbeList.getRawProbes(); + return syncedProbeList.getProbesWithAdminData(); +}; + +export const fetchProbes = async ({ allowStale = true } = {}): Promise => { + if (!syncedProbeList) { + throw new Error('WS server not initialized yet'); + } + + return allowStale ? syncedProbeList.getProbes() : syncedProbeList.fetchProbes(); }; export const adoptedProbes = new AdoptedProbes(client, fetchRawProbes); +export const adminData = new AdminData(client); + +export const probeOverride = new ProbeOverride(adoptedProbes, adminData); + export const probeIpLimit = new ProbeIpLimit(fetchProbes, fetchRawSockets); diff --git a/src/lib/ws/synced-probe-list.ts b/src/lib/ws/synced-probe-list.ts index a6d2f552..35168815 100644 --- a/src/lib/ws/synced-probe-list.ts +++ b/src/lib/ws/synced-probe-list.ts @@ -6,8 +6,7 @@ import { scopedLogger } from '../logger.js'; import type { WsServerNamespace } from './server.js'; import type { Probe, ProbeStats } from '../../probe/types.js'; import type winston from 'winston'; -import type { AdoptedProbes } from '../adopted-probes.js'; -import { getIndex } from '../../probe/builder.js'; +import type { ProbeOverride } from '../override/probe-override.js'; import type { RedisClient } from '../redis/shared.js'; type NodeData = { @@ -46,6 +45,7 @@ export class SyncedProbeList extends EventEmitter { private logger: winston.Logger; private rawProbes: Probe[]; + private probesWithAdminData: Probe[]; private probes: Probe[]; private oldest: number; private pushTimer: NodeJS.Timeout | undefined; @@ -55,11 +55,12 @@ export class SyncedProbeList extends EventEmitter { private readonly nodeId: string; private readonly nodeData: TTLCache; - constructor (private readonly redis: RedisClient, private readonly ioNamespace: WsServerNamespace, private readonly adoptedProbes: AdoptedProbes) { + constructor (private readonly redis: RedisClient, private readonly ioNamespace: WsServerNamespace, private readonly probeOverride: ProbeOverride) { super(); this.nodeId = randomBytes(8).toString('hex'); this.logger = scopedLogger('synced-probe-list', this.nodeId); this.rawProbes = []; + this.probesWithAdminData = []; this.probes = []; this.oldest = Infinity; this.lastReadEventId = Date.now().toString(); @@ -73,6 +74,18 @@ export class SyncedProbeList extends EventEmitter { }); } + getRawProbes (): Probe[] { + return this.rawProbes.slice(); + } + + getProbesWithAdminData (): Probe[] { + return this.probesWithAdminData.slice(); + } + + getProbes (): Probe[] { + return this.probes.slice(); + } + async fetchProbes (): Promise { const start = Date.now(); @@ -88,14 +101,6 @@ export class SyncedProbeList extends EventEmitter { }); } - getProbes (): Probe[] { - return this.probes.slice(); - } - - getRawProbes (): Probe[] { - return this.rawProbes.slice(); - } - private updateProbes () { const probes = []; let oldest = Infinity; @@ -109,7 +114,8 @@ export class SyncedProbeList extends EventEmitter { } this.rawProbes = probes; - this.probes = addAdoptedProbesData(this.adoptedProbes, probes); + this.probesWithAdminData = this.probeOverride.addAdminData(probes); + this.probes = this.probeOverride.addAdoptedData(this.probesWithAdminData); this.oldest = oldest; this.emit(this.localUpdateEvent); @@ -393,7 +399,7 @@ export class SyncedProbeList extends EventEmitter { } } - const newNodeData = { + const newNodeData: NodeData = { ...nodeData, ...changes.revalidateTimestamp ? { revalidateTimestamp: changes.revalidateTimestamp } : {}, probesById, @@ -442,31 +448,3 @@ export class SyncedProbeList extends EventEmitter { clearTimeout(this.pullTimer); } } - -const addAdoptedProbesData = (adoptedProbes: AdoptedProbes, probes: Probe[]) => { - return probes.map((probe) => { - const adopted = adoptedProbes.getByIp(probe.ipAddress); - - if (!adopted) { - return probe; - } - - const isCustomCity = adopted.isCustomCity; - const hasUserTags = adopted.tags && adopted.tags.length; - - if (!isCustomCity && !hasUserTags) { - return probe; - } - - const newLocation = adoptedProbes.getUpdatedLocation(probe); - - const newTags = adoptedProbes.getUpdatedTags(probe); - - return { - ...probe, - location: newLocation, - tags: newTags, - index: getIndex(newLocation, newTags), - }; - }); -}; diff --git a/src/probe/builder.ts b/src/probe/builder.ts index 74c285d2..ce47d641 100644 --- a/src/probe/builder.ts +++ b/src/probe/builder.ts @@ -2,15 +2,7 @@ import * as process from 'node:process'; import type { Socket } from 'socket.io'; import { isIpPrivate } from '../lib/private-ip.js'; import semver from 'semver'; -import { - getStateNameByIso, - getCountryByIso, - getCountryIso3ByIso2, - getCountryAliases, - getNetworkAliases, - getContinentAliases, - getRegionAliases, -} from '../lib/location/location.js'; +import { getIndex } from '../lib/location/location.js'; import { ProbeError } from '../lib/probe-error.js'; import { createGeoipClient, LocationInfo } from '../lib/geoip/client.js'; import type GeoipClient from '../lib/geoip/client.js'; @@ -95,29 +87,6 @@ export const buildProbe = async (socket: Socket): Promise => { }; }; -export const getIndex = (location: ProbeLocation, tags: Tag[]) => { - // Storing index as string[][] so every category will have it's exact position in the index array across all probes - const index = [ - [ location.country ], - [ getCountryIso3ByIso2(location.country) ], - [ getCountryByIso(location.country) ], - getCountryAliases(location.country), - [ location.normalizedCity ], - location.state ? [ location.state ] : [], - location.state ? [ getStateNameByIso(location.state) ] : [], - [ location.continent ], - getContinentAliases(location.continent), - [ location.region ], - getRegionAliases(location.region), - [ `as${location.asn}` ], - tags.filter(tag => tag.type === 'system').map(tag => tag.value), - [ location.normalizedNetwork ], - getNetworkAliases(location.normalizedNetwork), - ].map(category => category.map(s => s.toLowerCase().replaceAll('-', ' '))); - - return index; -}; - const getLocation = (ipInfo: LocationInfo): ProbeLocation => ({ continent: ipInfo.continent, region: ipInfo.region, diff --git a/test/e2e/cases/adopted-probes.test.ts b/test/e2e/cases/adopted-probes.test.ts index 53397392..2c8586b7 100644 --- a/test/e2e/cases/adopted-probes.test.ts +++ b/test/e2e/cases/adopted-probes.test.ts @@ -22,8 +22,8 @@ describe('adopted probe', () => { country: 'FR', countryOfCustomCity: 'FR', city: 'Marseille', - latitude: '43.29695', - longitude: '5.38107', + latitude: 43.29695, + longitude: 5.38107, network: 'InterBS S.R.L. (BAEHOST)', asn: 61004, }); diff --git a/test/e2e/cases/location-overrides.test.ts b/test/e2e/cases/location-overrides.test.ts new file mode 100644 index 00000000..2c1c32ce --- /dev/null +++ b/test/e2e/cases/location-overrides.test.ts @@ -0,0 +1,71 @@ +import got from 'got'; +import { expect } from 'chai'; +import { client } from '../../../src/lib/sql/client.js'; +import { waitProbeInCity } from '../utils.js'; + +const LOCATION_OVERRIDES_TABLE = 'gp_location_overrides'; + +describe('location overrides', () => { + before(async function () { + this.timeout(80000); + + await client(LOCATION_OVERRIDES_TABLE).insert({ + user_created: '89da69bd-a236-4ab7-9c5d-b5f52ce09959', + date_created: new Date(), + user_updated: null, + date_updated: null, + ip_range: '51.158.22.0/24', + country: 'US', + state: 'FL', + city: 'Miami', + latitude: 25.7743, + longitude: -80.1937, + }); + + await waitProbeInCity('Miami'); + }); + + after(async function () { + this.timeout(80000); + await client(LOCATION_OVERRIDES_TABLE).where({ city: 'Miami' }).delete(); + await waitProbeInCity('Paris'); + }); + + it('should return probe list with updated location', async () => { + const probes = await got('http://localhost:80/v1/probes').json(); + + expect(probes[0].location).to.include({ + continent: 'NA', + region: 'Northern America', + country: 'US', + state: 'FL', + city: 'Miami', + latitude: 25.7743, + longitude: -80.1937, + }); + }); + + it('should create measurement by its new location', async () => { + const response = await got.post('http://localhost:80/v1/measurements', { json: { + target: 'www.jsdelivr.com', + type: 'ping', + locations: [{ + city: 'Miami', + }], + } }); + + expect(response.statusCode).to.equal(202); + }); + + it('should not create measurement by its old location', async () => { + const response = await got.post('http://localhost:80/v1/measurements', { json: { + target: 'www.jsdelivr.com', + type: 'ping', + locations: [{ + city: 'Paris', + }], + }, throwHttpErrors: false }); + + expect(response.statusCode).to.equal(422); + }); +}); diff --git a/test/e2e/docker.ts b/test/e2e/docker.ts index 1ba3d33a..5e380887 100644 --- a/test/e2e/docker.ts +++ b/test/e2e/docker.ts @@ -13,19 +13,10 @@ class DockerManager { } public async createApiContainer () { - let networkMode = 'host'; - let redisUrl = config.get('redis.url'); - let dbConnectionHost = config.get('db.connection.host'); + const redisUrl = config.get('redis.url').replace('localhost', 'host.docker.internal'); + const dbConnectionHost = config.get('db.connection.host').replace('localhost', 'host.docker.internal'); const processes = config.get('server.processes'); - const isLinux = await this.isLinuxHost(); - - if (!isLinux) { - networkMode = 'bridge'; - redisUrl = redisUrl.replace('localhost', 'host.docker.internal'); - dbConnectionHost = dbConnectionHost.replace('localhost', 'host.docker.internal'); - } - // docker run -e NODE_ENV=test -e TEST_MODE=e2e -e NEW_RELIC_ENABLED=false -e REDIS_URL=redis://host.docker.internal:6379 -e DB_CONNECTION_HOST=host.docker.internal --name globalping-api-e2e globalping-api-e2e const container = await this.docker.createContainer({ Image: 'globalping-api-e2e', @@ -42,7 +33,7 @@ class DockerManager { PortBindings: { '80/tcp': [{ HostPort: '80' }], }, - NetworkMode: networkMode, + ExtraHosts: [ 'host.docker.internal:host-gateway' ], }, }); @@ -51,25 +42,15 @@ class DockerManager { } public async createProbeContainer () { - let networkMode = 'host'; - let apiHost = 'ws://localhost:80'; - - const isLinux = await this.isLinuxHost(); - - if (!isLinux) { - networkMode = 'bridge'; - apiHost = apiHost.replace('localhost', 'host.docker.internal'); - } - // docker run -e API_HOST=ws://host.docker.internal:80 --name globalping-probe-e2e globalping-probe-e2e const container = await this.docker.createContainer({ Image: 'globalping-probe-e2e', name: 'globalping-probe-e2e', Env: [ - `API_HOST=${apiHost}`, + 'API_HOST=ws://host.docker.internal:80', ], HostConfig: { - NetworkMode: networkMode, + ExtraHosts: [ 'host.docker.internal:host-gateway' ], }, }); @@ -117,12 +98,6 @@ class DockerManager { await container.start({}); } - private async isLinuxHost (): Promise { - const versionInfo = await this.docker.version(); - const platformName = versionInfo.Platform.Name.toLowerCase(); - return platformName.includes('engine'); - } - private async attachLogs (container: Docker.Container) { const stream = await container.logs({ follow: true, diff --git a/test/tests/integration/measurement/create-measurement.test.ts b/test/tests/integration/measurement/create-measurement.test.ts index c061a264..27de2b14 100644 --- a/test/tests/integration/measurement/create-measurement.test.ts +++ b/test/tests/integration/measurement/create-measurement.test.ts @@ -5,7 +5,7 @@ import nock from 'nock'; import type { Socket } from 'socket.io-client'; import nockGeoIpProviders from '../../../utils/nock-geo-ip.js'; import { client } from '../../../../src/lib/sql/client.js'; -import type { AdoptedProbes } from '../../../../src/lib/adopted-probes.js'; +import type { ProbeOverride } from '../../../../src/lib/override/probe-override.js'; import { waitForProbesUpdate } from '../../../utils/server.js'; describe('Create measurement', () => { @@ -13,14 +13,14 @@ describe('Create measurement', () => { let deleteFakeProbes: () => Promise; let getTestServer; let requestAgent: Agent; - let adoptedProbes: AdoptedProbes; + let probeOverride: ProbeOverride; let ADOPTED_PROBES_TABLE: string; before(async () => { await td.replaceEsm('../../../../src/lib/ip-ranges.ts', { getRegion: () => 'gcp-us-west4', populateMemList: () => Promise.resolve() }); ({ getTestServer, addFakeProbe, deleteFakeProbes } = await import('../../../utils/server.js')); - ({ ADOPTED_PROBES_TABLE } = await import('../../../../src/lib/adopted-probes.js')); - ({ adoptedProbes } = await import('../../../../src/lib/ws/server.js')); + ({ ADOPTED_PROBES_TABLE } = await import('../../../../src/lib/override/adopted-probes.js')); + ({ probeOverride } = await import('../../../../src/lib/ws/server.js')); const app = await getTestServer(); requestAgent = request(app); }); @@ -660,13 +660,13 @@ describe('Create measurement', () => { country: 'US', countryOfCustomCity: 'US', city: 'Oklahoma City', - latitude: '35.46756', - longitude: '-97.51643', + latitude: 35.46756, + longitude: -97.51643, network: 'InterBS S.R.L. (BAEHOST)', asn: 61004, }); - await adoptedProbes.syncDashboardData(); + await probeOverride.syncDashboardData(); await waitForProbesUpdate(); }); diff --git a/test/tests/integration/probes/get-probes.test.ts b/test/tests/integration/probes/get-probes.test.ts index 3cf3de11..3e43394d 100644 --- a/test/tests/integration/probes/get-probes.test.ts +++ b/test/tests/integration/probes/get-probes.test.ts @@ -4,8 +4,8 @@ import { expect } from 'chai'; import request, { type Agent } from 'supertest'; import { getTestServer, addFakeProbe, deleteFakeProbes, waitForProbesUpdate } from '../../../utils/server.js'; import nockGeoIpProviders from '../../../utils/nock-geo-ip.js'; -import { ADOPTED_PROBES_TABLE } from '../../../../src/lib/adopted-probes.js'; -import { adoptedProbes } from '../../../../src/lib/ws/server.js'; +import { ADOPTED_PROBES_TABLE } from '../../../../src/lib/override/adopted-probes.js'; +import { probeOverride } from '../../../../src/lib/ws/server.js'; import { client } from '../../../../src/lib/sql/client.js'; describe('Get Probes', () => { @@ -284,13 +284,13 @@ describe('Get Probes', () => { country: 'AR', countryOfCustomCity: 'AR', city: 'Cordoba', - latitude: '-31.4135', - longitude: '-64.18105', + latitude: -31.4135, + longitude: -64.18105, network: 'InterBS S.R.L. (BAEHOST)', asn: 61004, }); - await adoptedProbes.syncDashboardData(); + await probeOverride.syncDashboardData(); }); after(async () => { diff --git a/test/tests/unit/override/admin-data.test.ts b/test/tests/unit/override/admin-data.test.ts new file mode 100644 index 00000000..17a06bc3 --- /dev/null +++ b/test/tests/unit/override/admin-data.test.ts @@ -0,0 +1,123 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { AdminData } from '../../../../src/lib/override/admin-data.js'; +import type { Knex } from 'knex'; +import type { Probe } from '../../../../src/probe/types.js'; + +describe('AdminData', () => { + const sandbox = sinon.createSandbox(); + const select = sandbox.stub(); + const sql = sandbox.stub().returns({ select }) as unknown as Knex; + + let adminData = new AdminData(sql); + + const overrideLocation = { + id: 1, + user_created: '00b7776d-3bbc-4348-aab4-cf282b298a1b', + date_created: new Date('2024-04-09'), + user_updated: '00b7776d-3bbc-4348-aab4-cf282b298a1b', + date_updated: new Date('2024-04-09'), + ip_range: '1.1.1.1/32', + city: 'Bydgoszcz', + state: null, + country: 'PL', + latitude: 53.1235, + longitude: 18.00762, + }; + + beforeEach(() => { + select.reset(); + adminData = new AdminData(sql); + }); + + it('syncDashboardData method should populate admin location overrides', async () => { + select.resolves([ overrideLocation ]); + + await adminData.syncDashboardData(); + const updatedProbes = adminData.getUpdatedProbes([ + { ipAddress: '1.1.1.1', location: { city: 'Miami', state: 'FL' } } as Probe, + { ipAddress: '2.2.2.2', location: { city: 'Miami', state: 'FL' } } as Probe, + ]); + + expect(updatedProbes[0]).to.deep.equal({ + ipAddress: '1.1.1.1', + location: { + city: 'Bydgoszcz', + continent: 'EU', + region: 'Eastern Europe', + normalizedCity: 'bydgoszcz', + country: 'PL', + state: null, + latitude: 53.1235, + longitude: 18.00762, + }, + }); + + expect(updatedProbes[1]).to.deep.equal({ ipAddress: '2.2.2.2', location: { city: 'Miami', state: 'FL' } }); + }); + + it('syncDashboardData method should work for ip ranges', async () => { + select.resolves([{ ...overrideLocation, ip_range: '1.1.1.1/8' }]); + + await adminData.syncDashboardData(); + const updatedProbes = adminData.getUpdatedProbes([ + { ipAddress: '1.200.210.220', location: { city: 'Miami', state: 'FL' } } as Probe, + { ipAddress: '2.200.210.220', location: { city: 'Miami', state: 'FL' } } as Probe, + ]); + + expect(updatedProbes[0]?.location.city).to.deep.equal('Bydgoszcz'); + expect(updatedProbes[1]?.location.city).to.deep.equal('Miami'); + }); + + it('getUpdatedLocation should return null if no override found', async () => { + select.resolves([]); + await adminData.syncDashboardData(); + const updatedLocation = adminData.getUpdatedLocation({ ipAddress: '1.1.1.1' } as Probe); + expect(updatedLocation).to.equal(null); + }); + + it('getUpdatedProbes method should use cached search results', async () => { + select.resolves([ overrideLocation ]); + + const findUpdatedFieldsSpy = sandbox.spy(adminData, 'findUpdatedFields'); + await adminData.syncDashboardData(); + + adminData.getUpdatedProbes([ { ipAddress: '1.1.1.1', location: { city: 'Miami', state: 'FL' } } as Probe ]); + adminData.getUpdatedProbes([ { ipAddress: '1.1.1.1', location: { city: 'Miami', state: 'FL' } } as Probe ]); + await adminData.syncDashboardData(); + adminData.getUpdatedProbes([ { ipAddress: '1.1.1.1', location: { city: 'Miami', state: 'FL' } } as Probe ]); + expect(findUpdatedFieldsSpy.callCount).to.equal(1); + }); + + it('syncDashboardData method should reset cache if there is an update', async () => { + select.resolves([ overrideLocation ]); + + const adminData = new AdminData(sql); + const findUpdatedFieldsSpy = sandbox.spy(adminData, 'findUpdatedFields'); + await adminData.syncDashboardData(); + + adminData.getUpdatedProbes([ { ipAddress: '1.1.1.1', location: { city: 'Miami', state: 'FL' } } as Probe ]); + adminData.getUpdatedProbes([ { ipAddress: '1.1.1.1', location: { city: 'Miami', state: 'FL' } } as Probe ]); + select.resolves([{ ...overrideLocation, date_updated: new Date('2024-04-10') }]); + await adminData.syncDashboardData(); + adminData.getUpdatedProbes([ { ipAddress: '1.1.1.1', location: { city: 'Miami', state: 'FL' } } as Probe ]); + adminData.getUpdatedProbes([ { ipAddress: '1.1.1.1', location: { city: 'Miami', state: 'FL' } } as Probe ]); + expect(findUpdatedFieldsSpy.callCount).to.equal(2); + }); + + it('syncDashboardData method should reset cache if override was deleted', async () => { + select.resolves([ overrideLocation ]); + + const adminData = new AdminData(sql); + const findUpdatedFieldsSpy = sandbox.spy(adminData, 'findUpdatedFields'); + await adminData.syncDashboardData(); + + adminData.getUpdatedProbes([ { ipAddress: '1.1.1.1', location: { city: 'Miami', state: 'FL' } } as Probe ]); + adminData.getUpdatedProbes([ { ipAddress: '1.1.1.1', location: { city: 'Miami', state: 'FL' } } as Probe ]); + select.resolves([]); + await adminData.syncDashboardData(); + adminData.getUpdatedProbes([ { ipAddress: '1.1.1.1', location: { city: 'Miami', state: 'FL' } } as Probe ]); + adminData.getUpdatedProbes([ { ipAddress: '1.1.1.1', location: { city: 'Miami', state: 'FL' } } as Probe ]); + expect(findUpdatedFieldsSpy.callCount).to.equal(2); + }); +}); diff --git a/test/tests/unit/adopted-probes.test.ts b/test/tests/unit/override/adopted-probes.test.ts similarity index 96% rename from test/tests/unit/adopted-probes.test.ts rename to test/tests/unit/override/adopted-probes.test.ts index 12562229..07e33bf0 100644 --- a/test/tests/unit/adopted-probes.test.ts +++ b/test/tests/unit/override/adopted-probes.test.ts @@ -2,8 +2,8 @@ import { expect } from 'chai'; import type { Knex } from 'knex'; import * as sinon from 'sinon'; import relativeDayUtc from 'relative-day-utc'; -import { AdoptedProbes } from '../../../src/lib/adopted-probes.js'; -import type { Probe } from '../../../src/probe/types.js'; +import { AdoptedProbes } from '../../../../src/lib/override/adopted-probes.js'; +import type { Probe } from '../../../../src/probe/types.js'; describe('AdoptedProbes', () => { const defaultAdoptedProbe = { @@ -393,7 +393,7 @@ describe('AdoptedProbes', () => { }); }); - it('getUpdatedLocation method should return same location object if connected.country !== adopted.countryOfCustomCity', async () => { + it('getUpdatedLocation method should return null if connected.country !== adopted.countryOfCustomCity', async () => { const adoptedProbes = new AdoptedProbes(sqlStub as unknown as Knex, fetchSocketsStub); selectStub.resolves([{ ...defaultAdoptedProbe, @@ -407,10 +407,10 @@ describe('AdoptedProbes', () => { await adoptedProbes.syncDashboardData(); const updatedLocation = adoptedProbes.getUpdatedLocation(defaultConnectedProbe); - expect(updatedLocation).to.equal(defaultConnectedProbe.location); + expect(updatedLocation).to.equal(null); }); - it('getUpdatedLocation method should return same location object if "isCustomCity: false"', async () => { + it('getUpdatedLocation method should return null if "isCustomCity: false"', async () => { const adoptedProbes = new AdoptedProbes(sqlStub as unknown as Knex, fetchSocketsStub); selectStub.resolves([{ ...defaultAdoptedProbe, @@ -422,7 +422,7 @@ describe('AdoptedProbes', () => { await adoptedProbes.syncDashboardData(); const updatedLocation = adoptedProbes.getUpdatedLocation(defaultConnectedProbe); - expect(updatedLocation).to.equal(defaultConnectedProbe.location); + expect(updatedLocation).to.equal(null); }); it('getUpdatedTags method should return updated tags', async () => { diff --git a/test/tests/unit/ws/synced-probe-list.test.ts b/test/tests/unit/ws/synced-probe-list.test.ts index 50464e30..d6c249f6 100644 --- a/test/tests/unit/ws/synced-probe-list.test.ts +++ b/test/tests/unit/ws/synced-probe-list.test.ts @@ -3,10 +3,10 @@ import { expect } from 'chai'; import type { WsServerNamespace } from '../../../../src/lib/ws/server.js'; import { SyncedProbeList } from '../../../../src/lib/ws/synced-probe-list.js'; -import { type AdoptedProbe, AdoptedProbes } from '../../../../src/lib/adopted-probes.js'; import type { Probe } from '../../../../src/probe/types.js'; import { getRegionByCountry } from '../../../../src/lib/location/location.js'; import { getRedisClient } from '../../../../src/lib/redis/client.js'; +import { ProbeOverride } from '../../../../src/lib/override/probe-override.js'; describe('SyncedProbeList', () => { const sandbox = sinon.createSandbox(); @@ -34,7 +34,7 @@ describe('SyncedProbeList', () => { }, } as unknown as WsServerNamespace; - const adoptedProbes = sandbox.createStubInstance(AdoptedProbes); + const probeOverride = sandbox.createStubInstance(ProbeOverride); let syncedProbeList: SyncedProbeList; @@ -43,10 +43,10 @@ describe('SyncedProbeList', () => { redisJsonGet.callThrough(); redisPExpire.callThrough(); localFetchSocketsStub.resolves([]); - adoptedProbes.getUpdatedLocation.callThrough(); - adoptedProbes.getUpdatedTags.callThrough(); + probeOverride.addAdminData.returnsArg(0); + probeOverride.addAdoptedData.returnsArg(0); - syncedProbeList = new SyncedProbeList(redisClient, ioNamespace, adoptedProbes); + syncedProbeList = new SyncedProbeList(redisClient, ioNamespace, probeOverride); }); afterEach(() => { @@ -56,8 +56,8 @@ describe('SyncedProbeList', () => { it('updates and emits local probes during sync', async () => { const sockets = [ - { data: { probe: { client: 'A' } } }, - { data: { probe: { client: 'B' } } }, + { data: { probe: { client: 'A', location: {} } } }, + { data: { probe: { client: 'B', location: {} } } }, ]; localFetchSocketsStub.resolves(sockets); @@ -95,9 +95,9 @@ describe('SyncedProbeList', () => { it('emits stats in the message on change', async () => { const sockets = [ - { data: { probe: { client: 'A', stats: { cpu: { count: 1, load: [{ idle: 0, usage: 0 }] }, jobs: { count: 0 } } } } }, - { data: { probe: { client: 'B', stats: { cpu: { count: 1, load: [{ idle: 0, usage: 0 }] }, jobs: { count: 0 } } } } }, - { data: { probe: { client: 'C', stats: { cpu: { count: 1, load: [{ idle: 0, usage: 0 }] }, jobs: { count: 0 } } } } }, + { data: { probe: { client: 'A', location: {}, stats: { cpu: { count: 1, load: [{ idle: 0, usage: 0 }] }, jobs: { count: 0 } } } } }, + { data: { probe: { client: 'B', location: {}, stats: { cpu: { count: 1, load: [{ idle: 0, usage: 0 }] }, jobs: { count: 0 } } } } }, + { data: { probe: { client: 'C', location: {}, stats: { cpu: { count: 1, load: [{ idle: 0, usage: 0 }] }, jobs: { count: 0 } } } } }, ]; localFetchSocketsStub.resolves(sockets); @@ -189,9 +189,9 @@ describe('SyncedProbeList', () => { it('reads remote stats updates', async () => { const probes = { - A: { client: 'A', stats: { cpu: { count: 1, load: [{ idle: 0, usage: 0 }] }, jobs: { count: 0 } } }, - B: { client: 'B', stats: { cpu: { count: 1, load: [{ idle: 0, usage: 0 }] }, jobs: { count: 0 } } }, - C: { client: 'C', stats: { cpu: { count: 1, load: [{ idle: 0, usage: 0 }] }, jobs: { count: 0 } } }, + A: { client: 'A', location: {}, stats: { cpu: { count: 1, load: [{ idle: 0, usage: 0 }] }, jobs: { count: 0 } } }, + B: { client: 'B', location: {}, stats: { cpu: { count: 1, load: [{ idle: 0, usage: 0 }] }, jobs: { count: 0 } } }, + C: { client: 'C', location: {}, stats: { cpu: { count: 1, load: [{ idle: 0, usage: 0 }] }, jobs: { count: 0 } } }, } as unknown as Record; redisXRange.resolves([ @@ -252,8 +252,8 @@ describe('SyncedProbeList', () => { it('expires remote probes after the timeout', async () => { const probes = { - A: { client: 'A' }, - B: { client: 'B' }, + A: { client: 'A', location: {} }, + B: { client: 'B', location: {} }, } as unknown as Record; redisXRange.resolves([ @@ -277,17 +277,17 @@ describe('SyncedProbeList', () => { expect(syncedProbeList.getProbes()).to.be.empty; }); - it('applies adoption data to getProbes()/fetchProbes() but not to getRawProbes()', async () => { - const sockets = [ - { data: { probe: { client: 'A', location: { ...location }, tags: [], ipAddress: '1.1.1.1' } } }, - { data: { probe: { client: 'B', location: { ...location }, tags: [] } } }, - ]; + it('applies adoption data to getProbes()/fetchProbes() but not to getRawProbes()/getProbesWithAdminData()', async () => { + const probe1 = { client: 'A', location: { ...location }, tags: [], ipAddress: '1.1.1.1' } as unknown as Probe; + const probe2 = { client: 'B', location: { ...location }, tags: [] } as unknown as Probe; + const sockets = [{ data: { probe: probe1 } }, { data: { probe: probe2 } }]; + + const tags = [{ type: 'user', value: 'u-name-tag1' }] as Probe['tags']; - const tags = [{ type: 'user', value: 'u-name-tag1' }] as AdoptedProbe['tags']; - const adoptedProbe = { tags } as AdoptedProbe; + const adoptedProbe = { tags } as Probe; localFetchSocketsStub.resolves(sockets); - adoptedProbes.getByIp.withArgs('1.1.1.1').returns(adoptedProbe); + probeOverride.addAdoptedData.returns([ adoptedProbe, probe2 ]); const fetchedProbesPromise = syncedProbeList.fetchProbes(); clock.tick(1); @@ -302,10 +302,43 @@ describe('SyncedProbeList', () => { expect(fetchedProbes[0]).to.deep.include({ tags }); expect(fetchedProbes[1]).not.to.deep.include({ tags }); + expect(syncedProbeList.getProbesWithAdminData()[0]).not.to.deep.include({ tags }); + expect(syncedProbeList.getProbesWithAdminData()[1]).not.to.deep.include({ tags }); + expect(syncedProbeList.getRawProbes()[0]).not.to.deep.include({ tags }); expect(syncedProbeList.getRawProbes()[1]).not.to.deep.include({ tags }); }); + it('applies admin location override data to getProbes()/fetchProbes()/getProbesWithAdminData() but not to getRawProbes()', async () => { + const probe1 = { client: 'A', location: { ...location }, tags: [], ipAddress: '1.1.1.1' } as unknown as Probe; + const probe2 = { client: 'B', location: { ...location }, tags: [] } as unknown as Probe; + const sockets = [{ data: { probe: probe1 } }, { data: { probe: probe2 } }]; + + const updatedProbe = { probe1, location: { ...probe1.location, city: 'Miami' } } as unknown as Probe; + + localFetchSocketsStub.resolves(sockets); + probeOverride.addAdminData.returns([ updatedProbe, probe2 ]); + + const fetchedProbesPromise = syncedProbeList.fetchProbes(); + clock.tick(1); + + await syncedProbeList.sync(); + const fetchedProbes = await fetchedProbesPromise; + + expect(localFetchSocketsStub.callCount).to.equal(1); + expect(syncedProbeList.getProbes()[0]?.location.city).to.deep.equal('Miami'); + expect(syncedProbeList.getProbes()[1]?.location.city).to.deep.equal('The New York City'); + + expect(fetchedProbes[0]?.location.city).to.deep.equal('Miami'); + expect(fetchedProbes[1]?.location.city).to.deep.equal('The New York City'); + + expect(syncedProbeList.getProbesWithAdminData()[0]?.location.city).to.deep.equal('Miami'); + expect(syncedProbeList.getProbesWithAdminData()[1]?.location.city).to.deep.equal('The New York City'); + + expect(syncedProbeList.getRawProbes()[0]?.location.city).to.deep.equal('The New York City'); + expect(syncedProbeList.getRawProbes()[1]?.location.city).to.deep.equal('The New York City'); + }); + it('resolves fetchProbes() only after new data arrives', async () => { const fetchedProbesPromise = syncedProbeList.fetchProbes(); diff --git a/wallaby.js b/wallaby.js index 1618341c..a54e5b3e 100644 --- a/wallaby.js +++ b/wallaby.js @@ -35,7 +35,7 @@ export default function wallaby () { setup (w) { const path = require('path'); - w.testFramework.addFile(path.resolve(process.cwd(), 'test/setup.js')); + w.testFramework.files.unshift(path.resolve(process.cwd(), 'test/setup.js')); }, env: {