feat: geodata of an IP range

migrations/create-tables.js.sql

@@ -54,3 +54,17 @@ CREATE TABLE IF NOT EXISTS gp_credits (
 	CONSTRAINT gp_credits_user_id_unique UNIQUE (user_id),
 	CONSTRAINT gp_credits_amount_positive CHECK (`amount` >= 0)
 );
+
+CREATE TABLE IF NOT EXISTS gp_location_overrides (
+  id INT(10) UNSIGNED AUTO_INCREMENT PRIMARY KEY,
+  user_created CHAR(36),
+  date_created TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+  user_updated CHAR(36),
+  date_updated TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  ip_range VARCHAR(255) NOT NULL,
+  city VARCHAR(255) NOT NULL,
+  state VARCHAR(255),
+  country VARCHAR(255),
+	latitude FLOAT(10, 5),
+	longitude FLOAT(10, 5)
+);

src/lib/location/location.ts

@@ -8,6 +8,7 @@ import {
 } from './countries.js';
 import { aliases as networkAliases } from './networks.js';
 import { aliases as continentAliases } from './continents.js';
+import type { ProbeLocation, Tag } from '../../probe/types.js';
 
 const countryToRegionMap = new Map(_.flatMap(regions, (v, r) => v.map(c => [ c, r ])));
 
@@ -95,3 +96,26 @@ export const getRegionAliases = (key: string): string[] => {
 	return array ?? [];
 };
 
+export const getIndex = (location: ProbeLocation, tags: Tag[]) => {
+	// Storing index as string[][] so every category will have it's exact position in the index array across all probes
+	const index = [
+		[ location.country ],
+		[ getCountryIso3ByIso2(location.country) ],
+		[ getCountryByIso(location.country) ],
+		getCountryAliases(location.country),
+		[ location.normalizedCity ],
+		location.state ? [ location.state ] : [],
+		location.state ? [ getStateNameByIso(location.state) ] : [],
+		[ location.continent ],
+		getContinentAliases(location.continent),
+		[ location.region ],
+		getRegionAliases(location.region),
+		[ `as${location.asn}` ],
+		tags.filter(tag => tag.type === 'system').map(tag => tag.value),
+		[ location.normalizedNetwork ],
+		getNetworkAliases(location.normalizedNetwork),
+	].map(category => category.map(s => s.toLowerCase().replaceAll('-', ' ')));
+
+	return index;
+};
+

src/lib/override/admin-data.ts

@@ -0,0 +1,137 @@
+/* eslint-disable camelcase */
+import ipaddr from 'ipaddr.js';
+import type { Knex } from 'knex';
+import type { Probe, ProbeLocation } from '../../probe/types.js';
+import { normalizeFromPublicName } from '../geoip/utils.js';
+import { getContinentByCountry, getRegionByCountry } from '../location/location.js';
+import { scopedLogger } from '../logger.js';
+
+const logger = scopedLogger('admin-data');
+
+const LOCATION_OVERRIDES_TABLE = 'gp_location_overrides';
+
+type ParsedIpRange = [ipaddr.IPv4 | ipaddr.IPv6, number];
+
+type LocationOverride = {
+	date_created: Date;
+	date_updated: Date | null;
+	ip_range: string;
+	city: string;
+	country: string;
+	state: string | null;
+	latitude: number;
+	longitude: number;
+}
+
+type UpdatedFields = {
+	continent: string;
+	region: string;
+	city: string;
+	normalizedCity: string;
+	country: string;
+	state: string | null;
+	latitude: number;
+	longitude: number;
+}
+
+export class AdminData {
+	private rangesToUpdatedFields: Map<ParsedIpRange, UpdatedFields> = new Map();
+
+	private ipsToUpdatedFields: Map<string, UpdatedFields | null> = new Map();
+
+	private lastUpdate: Date = new Date('01-01-1970');
+
+	private lastOverridesLength: number = 0;
+
+	constructor (private readonly sql: Knex) {}
+
+	scheduleSync () {
+		setTimeout(() => {
+			this.syncDashboardData()
+				.finally(() => this.scheduleSync())
+				.catch(error => logger.error(error));
+		}, 60_000).unref();
+	}
+
+	async syncDashboardData () {
+		const overrides = await this.sql(LOCATION_OVERRIDES_TABLE).select<LocationOverride[]>();
+
+		this.rangesToUpdatedFields = new Map(overrides.map(override => [ ipaddr.parseCIDR(override.ip_range), {
+			continent: getContinentByCountry(override.country),
+			region: getRegionByCountry(override.country),
+			city: override.city,
+			normalizedCity: normalizeFromPublicName(override.city),
+			country: override.country,
+			state: override.state,
+			latitude: override.latitude,
+			longitude: override.longitude,
+		}]));
+
+		const newLastUpdate = overrides.reduce((lastUpdate, { date_created, date_updated }) => {
+			lastUpdate = date_created > lastUpdate ? date_created : lastUpdate;
+			lastUpdate = (date_updated && date_updated > lastUpdate) ? date_updated : lastUpdate;
+			return lastUpdate;
+		}, new Date('01-01-1970'));
+
+		if (newLastUpdate > this.lastUpdate || overrides.length !== this.lastOverridesLength) {
+			this.ipsToUpdatedFields.clear();
+			this.lastUpdate = newLastUpdate;
+			this.lastOverridesLength = overrides.length;
+		}
+	}
+
+	getUpdatedProbes (probes: Probe[]) {
+		return probes.map((probe) => {
+			const updatedFields = this.getUpdatedFields(probe);
+
+			if (!updatedFields) {
+				return probe;
+			}
+
+			return {
+				...probe,
+				location: {
+					...probe.location,
+					...updatedFields,
+				},
+			};
+		});
+	}
+
+	getUpdatedLocation (probe: Probe): ProbeLocation | null {
+		const updatedFields = this.getUpdatedFields(probe);
+
+		if (!updatedFields) {
+			return null;
+		}
+
+		return {
+			...probe.location,
+			...updatedFields,
+		};
+	}
+
+	private getUpdatedFields (probe: Probe): UpdatedFields | null {
+		const updatedFields = this.ipsToUpdatedFields.get(probe.ipAddress);
+
+		if (updatedFields !== undefined) {
+			return updatedFields;
+		}
+
+		const newUpdatedFields = this.findUpdatedFields(probe);
+		this.ipsToUpdatedFields.set(probe.ipAddress, newUpdatedFields);
+		return newUpdatedFields;
+	}
+
+	findUpdatedFields (probe: Probe): UpdatedFields | null {
+		for (const [ range, updatedFields ] of this.rangesToUpdatedFields) {
+			const ip = ipaddr.parse(probe.ipAddress);
+
+			if (ip.kind() === range[0].kind() && ip.match(range)) {
+				return updatedFields;
+			}
+		}
+
+		return null;
+	}
+}

src/lib/adopted-probes.ts → src/lib/override/adopted-probes.ts

@@ -1,10 +1,11 @@
 import type { Knex } from 'knex';
 import Bluebird from 'bluebird';
 import _ from 'lodash';
-import { scopedLogger } from './logger.js';
-import type { fetchRawProbes as serverFetchRawProbes } from './ws/server.js';
-import type { Probe } from '../probe/types.js';
-import { normalizeFromPublicName } from './geoip/utils.js';
+import { scopedLogger } from '../logger.js';
+import type { fetchProbesWithAdminData as serverFetchProbesWithAdminData } from '../ws/server.js';
+import type { Probe, ProbeLocation } from '../../probe/types.js';
+import { normalizeFromPublicName } from '../geoip/utils.js';
+import { getIndex } from '../location/location.js';
 
 const logger = scopedLogger('adopted-probes');
 
@@ -88,27 +89,27 @@ export class AdoptedProbes {
 
 	constructor (
 		private readonly sql: Knex,
-		private readonly fetchRawProbes: typeof serverFetchRawProbes,
+		private readonly fetchProbesWithAdminData: typeof serverFetchProbesWithAdminData,
 	) {}
 
 	getByIp (ip: string) {
 		return this.adoptedIpToProbe.get(ip);
 	}
 
-	getUpdatedLocation (probe: Probe) {
+	getUpdatedLocation (probe: Probe): ProbeLocation | null {
 		const adoptedProbe = this.getByIp(probe.ipAddress);
 
 		if (!adoptedProbe || !adoptedProbe.isCustomCity || adoptedProbe.countryOfCustomCity !== probe.location.country) {
-			return probe.location;
+			return null;
 		}
 
 		return {
 			...probe.location,
 			city: adoptedProbe.city!,
 			normalizedCity: normalizeFromPublicName(adoptedProbe.city!),
+			state: adoptedProbe.state,
 			latitude: adoptedProbe.latitude!,
 			longitude: adoptedProbe.longitude!,
-			...(adoptedProbe.state && { state: adoptedProbe.state }),
 		};
 	}
 
@@ -125,6 +126,34 @@ export class AdoptedProbes {
 		];
 	}
 
+	getUpdatedProbes (probes: Probe[]) {
+		return probes.map((probe) => {
+			const adopted = this.getByIp(probe.ipAddress);
+
+			if (!adopted) {
+				return probe;
+			}
+
+			const isCustomCity = adopted.isCustomCity;
+			const hasUserTags = adopted.tags && adopted.tags.length;
+
+			if (!isCustomCity && !hasUserTags) {
+				return probe;
+			}
+
+			const newLocation = this.getUpdatedLocation(probe) || probe.location;
+
+			const newTags = this.getUpdatedTags(probe);
+
+			return {
+				...probe,
+				location: newLocation,
+				tags: newTags,
+				index: getIndex(newLocation, newTags),
+			};
+		});
+	}
+
 	scheduleSync () {
 		setTimeout(() => {
 			this.syncDashboardData()
@@ -134,7 +163,7 @@ export class AdoptedProbes {
 	}
 
 	async syncDashboardData () {
-		const allProbes = await this.fetchRawProbes();
+		const allProbes = await this.fetchProbesWithAdminData();
 		this.connectedIpToProbe = new Map(allProbes.map(probe => [ probe.ipAddress, probe ]));
 		this.connectedUuidToIp = new Map(allProbes.map(probe => [ probe.uuid, probe.ipAddress ]));
 

src/lib/override/probe-override.ts

@@ -0,0 +1,36 @@
+import type { Probe } from '../../probe/types.js';
+import type { AdoptedProbes } from './adopted-probes.js';
+import type { AdminData } from './admin-data.js';
+
+export class ProbeOverride {
+	constructor (
+		private readonly adoptedProbes: AdoptedProbes,
+		private readonly adminData: AdminData,
+	) {}
+
+	async syncDashboardData () {
+		await Promise.all([
+			this.adoptedProbes.syncDashboardData(),
+			this.adminData.syncDashboardData(),
+		]);
+	}
+
+	scheduleSync () {
+		this.adoptedProbes.scheduleSync();
+		this.adminData.scheduleSync();
+	}
+
+	getUpdatedLocation (probe: Probe) {
+		const adminLocation = this.adminData.getUpdatedLocation(probe);
+		const adoptedLocation = this.adoptedProbes.getUpdatedLocation(probe);
+		return { ...probe.location, ...adminLocation, ...adoptedLocation };
+	}
+
+	addAdminData (probes: Probe[]) {
+		return this.adminData.getUpdatedProbes(probes);
+	}
+
+	addAdoptedData (probes: Probe[]) {
+		return this.adoptedProbes.getUpdatedProbes(probes);
+	}
+}

src/lib/server.ts

@@ -1,6 +1,6 @@
 import type { Server } from 'node:http';
 import { initRedisClient } from './redis/client.js';
-import { adoptedProbes, probeIpLimit, initWsServer } from './ws/server.js';
+import { probeOverride, probeIpLimit, initWsServer } from './ws/server.js';
 import { getMetricsAgent } from './metrics.js';
 import { populateMemList as populateMemMalwareList } from './malware/client.js';
 import { populateMemList as populateMemIpRangesList } from './ip-ranges.js';
@@ -27,8 +27,8 @@ export const createServer = async (): Promise<Server> => {
 
 	await initWsServer();
 
-	await adoptedProbes.syncDashboardData();
-	adoptedProbes.scheduleSync();
+	await probeOverride.syncDashboardData();
+	probeOverride.scheduleSync();
 
 	await auth.syncTokens();
 	auth.scheduleSync();

src/lib/ws/gateway.ts

@@ -6,7 +6,7 @@ import { handleStatusUpdate } from '../../probe/handler/status.js';
 import { handleDnsUpdate } from '../../probe/handler/dns.js';
 import { handleStatsReport } from '../../probe/handler/stats.js';
 import { scopedLogger } from '../logger.js';
-import { adoptedProbes, getWsServer, PROBES_NAMESPACE, ServerSocket } from './server.js';
+import { probeOverride, getWsServer, PROBES_NAMESPACE, ServerSocket } from './server.js';
 import { probeMetadata } from './middleware/probe-metadata.js';
 import { errorHandler } from './helper/error-handler.js';
 import { subscribeWithHandler } from './helper/subscribe-handler.js';
@@ -20,7 +20,7 @@ io
 	.use(probeMetadata)
 	.on('connect', errorHandler(async (socket: ServerSocket) => {
 		const probe = socket.data.probe;
-		const location = adoptedProbes.getUpdatedLocation(probe);
+		const location = probeOverride.getUpdatedLocation(probe);
 
 		socket.emit('api:connect:location', location);
 		logger.info(`ws client ${socket.id} connected from ${location.city}, ${location.country} [${probe.ipAddress} - ${location.network}]`);