Merge branch 'master' into oas

.gitignore

@@ -1,4 +1,5 @@
 .idea/
+.vscode/
 .nyc_output/
 node_modules/
 coverage/

README.md

@@ -135,12 +135,12 @@ You can even mention it in threads using `@globalping ping google.com`
 
 ### GitHub Bot
 
-Our GitHub bot can by triggered by simply mentioning it in any public GitHub issue. It supports a human friendly format of issuing commands. 
+Our GitHub bot can by triggered by simply mentioning it in any public GitHub issue. It supports a human friendly format of issuing commands and follows the existing Slack and CLI formatting logic allowing you to easily move between integrations. 
 
 Examples:
 ```
 @globalping ping 8.8.8.8 from Germany
-@globalping traceroute jsdelivr.com from South America limit 2
+@globalping traceroute jsdelivr.com from South America --limit 2
 ```
 
 The location field can process all kinds of different types of location matching, including continents, regions, countries, cities, US states and ASNs. ASNs must be prefixed by "AS", e.g. `from AS80085`.

config/default.cjs

@@ -25,6 +25,9 @@ module.exports = {
 	ipinfo: {
 		apiKey: '',
 	},
+	ip2location: {
+		apiKey: '',
+	},
 	ws: {
 		fetchSocketsCacheTTL: 1000,
 	},

package.json

@@ -27,7 +27,7 @@
 		"koa-response-time": "^2.1.0",
 		"lodash": "^4.17.21",
 		"maxmind": "^4.3.10",
-		"newrelic": "^9.15.0",
+		"newrelic": "^10.4.1",
 		"physical-cpu-count": "^2.0.0",
 		"private-ip": "^3.0.0",
 		"rate-limiter-flexible": "^2.4.1",

src/lib/geoip/client.ts

@@ -1,6 +1,5 @@
 import _ from 'lodash';
 import config from 'config';
-import type { Logger } from 'winston';
 import newrelic from 'newrelic';
 import type { CacheInterface } from '../cache/cache-interface.js';
 import { InternalError } from '../internal-error.js';
@@ -13,10 +12,13 @@ import { isAddrWhitelisted } from './whitelist.js';
 import { ipinfoLookup } from './providers/ipinfo.js';
 import { type FastlyBundledResponse, fastlyLookup } from './providers/fastly.js';
 import { maxmindLookup } from './providers/maxmind.js';
+import { ipmapLookup } from './providers/ipmap.js';
+import { ip2LocationLookup } from './providers/ip2location.js';
 import { normalizeRegionName } from './utils.js';
 
 export type LocationInfo = Omit<ProbeLocation, 'region' | 'normalizedRegion'>;
-export type LocationInfoWithProvider = LocationInfo & {provider: string};
+type Provider = 'ipmap' | 'ip2location' | 'ipinfo' | 'maxmind' | 'fastly';
+export type LocationInfoWithProvider = LocationInfo & {provider: Provider};
 export type RegionInfo = {
 	region: string;
 	normalizedRegion: string;
@@ -27,30 +29,31 @@ export type NetworkInfo = {
 	asn: number;
 };
 
-export const createGeoipClient = (): GeoipClient => new GeoipClient(
-	new RedisCache(getRedisClient()),
-	scopedLogger('geoip'),
-);
+const logger = scopedLogger('geoip');
+
+export const createGeoipClient = (): GeoipClient => new GeoipClient(new RedisCache(getRedisClient()));
 
 export default class GeoipClient {
-	constructor (
-		private readonly cache: CacheInterface,
-		private readonly logger: Logger,
-	) {}
+	constructor (private readonly cache: CacheInterface) {}
 
 	async lookup (addr: string): Promise<ProbeLocation> {
 		const results = await Promise
 			.allSettled([
-				this.lookupWithCache<LocationInfo>(`geoip:ipinfo:${addr}`, async () => ipinfoLookup(addr)),
+				this.lookupWithCache<LocationInfo>(`geoip:ip2location:${addr}`, async () => ip2LocationLookup(addr)),
+				this.lookupWithCache<LocationInfo>(`geoip:ipmap:${addr}`, async () => ipmapLookup(addr)),
 				this.lookupWithCache<LocationInfo>(`geoip:maxmind:${addr}`, async () => maxmindLookup(addr)),
+				this.lookupWithCache<LocationInfo>(`geoip:ipinfo:${addr}`, async () => ipinfoLookup(addr)),
 				this.lookupWithCache<FastlyBundledResponse>(`geoip:fastly:${addr}`, async () => fastlyLookup(addr)),
 			])
-			.then(([ ipinfo, maxmind, fastly ]) => {
-				const fulfilled = [];
+			.then(([ ip2location, ipmap, maxmind, ipinfo, fastly ]) => {
+				const fulfilled: (LocationInfoWithProvider | null)[] = [];
 
+				// Providers here are pushed in a desc prioritized order
 				fulfilled.push(
-					ipinfo.status === 'fulfilled' ? { ...ipinfo.value, provider: 'ipinfo' } : null,
+					ip2location.status === 'fulfilled' ? { ...ip2location.value, provider: 'ip2location' } : null,
+					ipmap.status === 'fulfilled' ? { ...ipmap.value, provider: 'ipmap' } : null,
 					maxmind.status === 'fulfilled' ? { ...maxmind.value, provider: 'maxmind' } : null,
+					ipinfo.status === 'fulfilled' ? { ...ipinfo.value, provider: 'ipinfo' } : null,
 					fastly.status === 'fulfilled' ? { ...fastly.value.location, provider: 'fastly' } : null,
 				);
 
@@ -67,8 +70,8 @@ export default class GeoipClient {
 			throw new InternalError(`unresolvable geoip: ${addr}`, true);
 		}
 
-		const match = this.bestMatch('normalizedCity', results);
-		const networkMatch = this.matchNetwork(match, results);
+		const [ match, ranked ] = this.bestMatch('normalizedCity', results);
+		const networkMatch = this.matchNetwork(match, ranked);
 
 		if (!networkMatch) {
 			throw new InternalError(`unresolvable geoip: ${addr}`, true);
@@ -117,7 +120,7 @@ export default class GeoipClient {
 		};
 	}
 
-	private matchNetwork (best: LocationInfo, sources: LocationInfoWithProvider[]): NetworkInfo | undefined {
+	private matchNetwork (best: LocationInfo, rankedSources: LocationInfoWithProvider[]): NetworkInfo | undefined {
 		if (best.asn && best.network) {
 			return {
 				asn: best.asn,
@@ -126,44 +129,41 @@ export default class GeoipClient {
 			};
 		}
 
-		const maxmind = sources.find(s => s.provider === 'maxmind' && s.city === best.city);
-
-		if (maxmind?.asn && maxmind?.network) {
-			return {
-				asn: maxmind.asn,
-				network: maxmind.network,
-				normalizedNetwork: maxmind.normalizedNetwork,
-			};
+		for (const source of rankedSources) {
+			if (source.normalizedCity === best.normalizedCity && source?.asn && source?.network) {
+				return {
+					asn: source.asn,
+					network: source.network,
+					normalizedNetwork: source.normalizedNetwork,
+				};
+			}
 		}
 
 		return undefined;
 	}
 
-	private bestMatch (field: keyof LocationInfo, sources: LocationInfoWithProvider[]): LocationInfo {
+	private bestMatch (field: keyof LocationInfo, sources: LocationInfoWithProvider[]): [LocationInfo, LocationInfoWithProvider[]] {
 		const filtered = sources.filter(s => s[field]);
-		// Group by the same field value
+		// Group sources by the same field value
 		const grouped = Object.values(_.groupBy(filtered, field));
-		// Move items with the same values to the beginning
 		const ranked = grouped.sort((a, b) => b.length - a.length).flat();
 
-		let best = ranked[0];
-
-		// If all values are different
-		if (grouped.length === filtered.length) {
-			const sourcesObject = Object.fromEntries(filtered.map(s => [ s.provider, s ]));
-			best = sourcesObject['ipinfo'] ?? sourcesObject['maxmind'];
-		}
+		const best = ranked[0];
 
 		if (!best || best.provider === 'fastly') {
-			this.logger.error(`failed to find a correct value for a field "${field}"`, { field, sources });
+			logger.error(`failed to find a correct value for a field "${field}"`, { field, sources });
 			throw new Error(`failed to find a correct value for a field "${field}"`);
 		}
 
-		return _.omit(best, 'provider');
+		const match = _.omit(best, 'provider');
+		return [ match, ranked ];
 	}
 
 	public async lookupWithCache<T> (key: string, fn: () => Promise<T>): Promise<T> {
-		const cached = await this.cache.get<T>(key);
+		const cached = await this.cache.get<T>(key).catch((error: Error) => {
+			logger.error('Failed to get cached geoip info for probe.', error);
+			newrelic.noticeError(error, { key });
+		});
 
 		if (cached) {
 			return cached;
@@ -173,7 +173,7 @@ export default class GeoipClient {
 		const ttl = Number(config.get('geoip.cache.ttl'));
 
 		await this.cache.set(key, info, ttl).catch((error: Error) => {
-			this.logger.error('Failed to cache geoip info for probe.', error);
+			logger.error('Failed to cache geoip info for probe.', error);
 			newrelic.noticeError(error, { key, ttl });
 		});
 

src/lib/geoip/providers/ip2location.ts

@@ -0,0 +1,50 @@
+import got from 'got';
+import config from 'config';
+import {
+	getContinentByCountry,
+	getStateIsoByName,
+} from '../../location/location.js';
+import type { LocationInfo } from '../client.js';
+import {
+	normalizeCityName,
+	normalizeCityNamePublic,
+	normalizeNetworkName,
+} from '../utils.js';
+
+type Ip2LocationResponse = {
+	ip?: string;
+	country_code?: string;
+	country_name?: string;
+	region_name?: string;
+	city_name?: string;
+	zip_code?: string;
+	time_zone?: string;
+	asn?: string;
+	latitude?: number;
+	longitude: number;
+	as?: string;
+	is_proxy?: boolean;
+};
+
+export const ip2LocationLookup = async (addr: string): Promise<LocationInfo> => {
+	const result = await got(`https://api.ip2location.io`, {
+		searchParams: {
+			key: config.get<string>('ip2location.apiKey'),
+			ip: addr,
+		},
+		timeout: { request: 5000 },
+	}).json<Ip2LocationResponse>();
+
+	return {
+		continent: result.country_code ? getContinentByCountry(result.country_code) : '',
+		state: result.country_code === 'US' && result.region_name ? getStateIsoByName(result.region_name) : undefined,
+		country: result.country_code ?? '',
+		city: normalizeCityNamePublic(result.city_name ?? ''),
+		normalizedCity: normalizeCityName(result.city_name ?? ''),
+		asn: Number(result.asn) ?? 0,
+		latitude: result.latitude ?? 0,
+		longitude: result.longitude ?? 0,
+		network: result.as ?? '',
+		normalizedNetwork: normalizeNetworkName(result.as ?? ''),
+	};
+};

src/lib/geoip/providers/ipinfo.ts

@@ -28,8 +28,8 @@ export const ipinfoLookup = async (addr: string): Promise<LocationInfo> => {
 	const network = (result.org ?? '').split(' ').slice(1).join(' ');
 
 	return {
-		continent: getContinentByCountry(result.country ?? ''),
-		state: result.country === 'US' ? getStateIsoByName(result.region ?? '') : undefined,
+		continent: result.country ? getContinentByCountry(result.country) : '',
+		state: result.country === 'US' && result.region ? getStateIsoByName(result.region) : undefined,
 		country: result.country ?? '',
 		city: normalizeCityNamePublic(result.city ?? ''),
 		normalizedCity: normalizeCityName(result.city ?? ''),

src/lib/geoip/providers/ipmap.ts

@@ -0,0 +1,35 @@
+import got from 'got';
+import { getContinentByCountry } from '../../location/location.js';
+import type { LocationInfo } from '../client.js';
+import {
+	normalizeCityName,
+	normalizeCityNamePublic,
+} from '../utils.js';
+
+export type IpmapResponse = {
+	locations: {
+		cityName?: string;
+		stateAnsiCode?: string;
+		countryCodeAlpha2?: string;
+		latitude?: string;
+		longitude?: string;
+	}[]
+};
+
+export const ipmapLookup = async (addr: string): Promise<LocationInfo> => {
+	const result = await got.get(`https://ipmap-api.ripe.net/v1/locate/${addr}`).json<IpmapResponse>();
+	const location = result?.locations?.[0] || {};
+
+	return {
+		continent: location.countryCodeAlpha2 ? getContinentByCountry(location.countryCodeAlpha2) : '',
+		state: location.countryCodeAlpha2 === 'US' ? location.stateAnsiCode : undefined,
+		country: location.countryCodeAlpha2 ?? '',
+		city: normalizeCityNamePublic(location.cityName ?? ''),
+		normalizedCity: normalizeCityName(location.cityName ?? ''),
+		asn: 0,
+		latitude: Number(location.latitude) ?? 0,
+		longitude: Number(location.longitude) ?? 0,
+		network: '',
+		normalizedNetwork: '',
+	};
+};

src/lib/location/location.ts

@@ -26,7 +26,7 @@ export const getContinentByCountry = (country: string): string => {
 	const countryInfo = countries[country as keyof typeof countries];
 
 	if (!countryInfo) {
-		throw new Error(`country information associated with a iso code "${country}" not found`);
+		throw new Error(`country information associated with an iso code "${country}" not found`);
 	}
 
 	return countryInfo.continent;

src/measurement/store.ts

@@ -55,7 +55,7 @@ export class MeasurementStore {
 		const results = this.probesToResults(probes, request.type);
 		const probesAwaitingTtl = config.get<number>('measurement.timeout') + 5;
 		const startTime = new Date();
-		let measurement: MeasurementRecord = {
+		const measurement: MeasurementRecord = {
 			id,
 			type: request.type,
 			status: 'in-progress',
@@ -68,13 +68,12 @@ export class MeasurementStore {
 			measurementOptions: request.measurementOptions,
 			results,
 		};
-		const defaults = getDefaults(request);
-		measurement = substractObjects(measurement, defaults) as MeasurementRecord;
+		const measurementWithoutDefaults = this.removeDefaults(measurement, request);
 
 		await Promise.all([
 			this.redis.hSet('gp:in-progress', id, startTime.getTime()),
 			this.redis.set(getMeasurementKey(id, 'probes_awaiting'), probes.length, { EX: probesAwaitingTtl }),
-			this.redis.json.set(key, '$', measurement),
+			this.redis.json.set(key, '$', measurementWithoutDefaults),
 			this.redis.expire(key, config.get<number>('measurement.resultTTL')),
 		]);
 
@@ -154,6 +153,14 @@ export class MeasurementStore {
 		}, intervalTime);
 	}
 
+	removeDefaults (measurement: MeasurementRecord, request: MeasurementRequest): Partial<MeasurementRecord> {
+		const defaults = getDefaults(request);
+		// Remove `"limit": 1` from locations. E.g. [{"country": "US", "limit": 1}] => [{"country": "US"}]
+		measurement.locations = measurement.locations.map(location => location.limit === 1 ? _.omit(location, 'limit') : location);
+
+		return substractObjects(measurement, defaults) as Partial<MeasurementRecord>;
+	}
+
 	probesToResults (probes: Probe[], type: string) {
 		const results = probes.map(probe => ({
 			probe: {

src/measurement/types.ts

@@ -204,9 +204,9 @@ export type MeasurementRecord = {
 	createdAt: string;
 	updatedAt: string;
 	target: string;
-	limit?: number;
+	limit: number;
 	probesCount: number;
-	locations?: LocationWithLimit[];
+	locations: LocationWithLimit[];
 	measurementOptions?: MeasurementOptions;
 	results: MeasurementResult[];
 };