From a798d17a4faa2e83d45ead6a365187f7aa4af33c Mon Sep 17 00:00:00 2001 From: Ian Langworth Date: Sun, 16 Jun 2024 16:57:15 -0700 Subject: [PATCH] MaxMind does redirects now so use fetch() to fetch the geoip database --- lib/geoip.ts | 73 +++++++++++++++++++++++++++----------------------- package.json | 2 +- pnpm-lock.yaml | 26 +++++++++++------- 3 files changed, 57 insertions(+), 44 deletions(-) diff --git a/lib/geoip.ts b/lib/geoip.ts index 6ae3120..f1980da 100644 --- a/lib/geoip.ts +++ b/lib/geoip.ts @@ -1,79 +1,82 @@ import dotenv from "dotenv"; import { existsSync, statSync, writeFileSync } from "fs"; -import { get } from "https"; import { DateTime } from "luxon"; import maxmind from "maxmind"; import { basename } from "path"; +import { gunzipSync } from "zlib"; import tar from "tar-stream"; -import { createGunzip } from "zlib"; +import { PassThrough } from "stream"; dotenv.config(); let hasLoggedError = false; -const filename = "GeoLite2-Country.mmdb"; -const path = (process.env.DATA_DIR || "/tmp") + "/" + filename; -const url = process.env.MAXMIND_GEOLITE2_COUNTRY_URL; +const FILENAME = "GeoLite2-Country.mmdb"; +const SRC_URL = process.env.MAXMIND_GEOLITE2_COUNTRY_URL; +const DEST_PATH = (process.env.DATA_DIR || "/tmp") + "/" + FILENAME; const debug = (...args: any[]) => { if (process.env.DEBUG) console.log("[DEBUG]", ...args); }; -const download = async () => { - if (!url) throw "Cannot download Maxmind DB: Missing env var"; - - return new Promise((resolve, reject) => { +const untar = (tarball: Buffer, filename: string) => + new Promise((resolve, reject) => { const extract = tar.extract(); - const chunks: any[] = []; + const chunks: Buffer[] = []; extract.on("entry", (header, stream, next) => { if (basename(header.name) === filename) { - stream.on("data", (chunk) => { + stream.on("data", (chunk: Buffer) => { chunks.push(chunk); }); } - stream.on("end", () => { - next(); - }); + stream.on("end", next); stream.resume(); }); extract.on("finish", () => { if (chunks.length) { const data = Buffer.concat(chunks); - writeFileSync(path, data); - console.log(`Wrote ${path} (${data.length} bytes)`); - console.log(`stat ${path}: ${JSON.stringify(statSync(path), null, 2)}`); + resolve(data); } }); - console.log(`Downloading ${filename}...`); - debug(`GET ${url}`); - get(url, (res) => { - if (res.statusCode === 200) { - res.pipe(createGunzip()).pipe(extract); - } else { - reject( - `Failed to download Maxmind DB: ${res.statusCode} ${res.statusMessage}`, - ); - } - }); + extract.on("error", reject); + + const stream = new PassThrough(); + stream.pipe(extract); + stream.end(tarball); }); + +const download = async () => { + if (!SRC_URL) throw "Cannot download Maxmind DB: Missing env var"; + + const res = await fetch(SRC_URL); + if (res.ok) { + const buf = await res.arrayBuffer(); + const gzippedData = gunzipSync(buf); + const data = await untar(gzippedData, FILENAME); + writeFileSync(DEST_PATH, data); + console.log(`Wrote ${DEST_PATH} (${data.length} bytes)`); + debug(`stat ${DEST_PATH}: ${JSON.stringify(statSync(DEST_PATH), null, 2)}`); + } else { + throw `Failed to download Maxmind DB: ${res.status} ${res.statusText}`; + } }; export const getCountryForIP = async (ip: string) => { - if (existsSync(path)) { - debug("Maxmind DB exists", path); + if (existsSync(DEST_PATH)) { + debug("Maxmind DB exists", DEST_PATH); const maxAge = DateTime.now().minus({ weeks: 1 }); debug("Maxmind DB maxAge", maxAge.toISO()); - const lastModified = DateTime.fromJSDate(statSync(path).mtime); + const lastModified = DateTime.fromJSDate(statSync(DEST_PATH).mtime); debug("Maxmind DB lastModified", lastModified.toISO()); if (lastModified < maxAge) { console.log("Maxmind DB is older than 1 week - redownloading"); await download(); } } else { - if (!url) { + if (!SRC_URL) { if (!hasLoggedError) { console.error("Cannot lookup IP - Need MAXMIND_GEOLITE2_COUNTRY_URL"); hasLoggedError = true; @@ -85,9 +88,11 @@ export const getCountryForIP = async (ip: string) => { } try { - const lookup = await maxmind.open(path); + const lookup = await maxmind.open(DEST_PATH); const result = lookup.get(ip) as any; - return result?.country?.iso_code || null; + const code = result?.country?.iso_code || null; + debug(`Maxmind lookup for ${ip}: ${code}`); + return code; } catch (err: any) { console.error(`Error using Maxmind: ${err}`); return null; diff --git a/package.json b/package.json index 1c8b66a..ad8bb63 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "recoil-sync": "0.2.0", "sqlite3": "5.1.7", "swr": "2.2.5", - "tar-stream": "3.1.7", + "tar-stream": "^3.1.7", "ua-parser-js": "1.0.38", "uglify-js": "3.18.0", "unfetch": "5.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c959234..04d19f4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -81,7 +81,7 @@ importers: specifier: 2.2.5 version: 2.2.5(react@18.3.1) tar-stream: - specifier: 3.1.7 + specifier: ^3.1.7 version: 3.1.7 ua-parser-js: specifier: 1.0.38 @@ -889,8 +889,8 @@ packages: balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} - bare-events@2.2.2: - resolution: {integrity: sha512-h7z00dWdG0PYOQEvChhOSWvOfkIKsdZGkWr083FgN/HyoQuebSew/cgirYqh9SCuy/hRvxc5Vy6Fw8xAmYHLkQ==} + bare-events@2.4.2: + resolution: {integrity: sha512-qMKFd2qG/36aA4GwvKq8MxnPgCQAmBWmSyLWsJcbn8v03wvIPQ/hG1Ms8bPzndZxMDoHpxez5VOS+gC9Yi24/Q==} base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} @@ -1913,8 +1913,8 @@ packages: resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} engines: {node: '>=10.0.0'} - streamx@2.16.1: - resolution: {integrity: sha512-m9QYj6WygWyWa3H1YY69amr4nVgy61xfjys7xO7kviL5rfIEc2naf+ewFiOA+aEJD7y0JO3h2GoiUv4TDwEGzQ==} + streamx@2.18.0: + resolution: {integrity: sha512-LLUC1TWdjVdn1weXGcSxyTR3T4+acB6tVGXT95y0nGbca4t4o/ng1wKAGTljm9VicuCVLvRlqFYXYy5GwgM7sQ==} string-argv@0.3.2: resolution: {integrity: sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q==} @@ -2002,6 +2002,9 @@ packages: resolution: {integrity: sha512-51LAVKUSZSVfI05vjPESNc5vwqqZpbXCsU+/+wxlOrUjk2SnFTt97v9ZgQrD4YmxYW1Px6w2KjaDitCfkvgxMQ==} engines: {node: '>=8.0.0'} + text-decoder@1.1.0: + resolution: {integrity: sha512-TmLJNj6UgX8xcUZo4UDStGQtDiTzF7BzWlzn9g7UWrjkpHr5uJTK1ld16wZ3LXb2vb6jH8qU89dW5whuMdXYdw==} + tildify@2.0.0: resolution: {integrity: sha512-Cc+OraorugtXNfs50hU9KS369rFXCfgGLpfCfvlc+Ud5u6VWmUQsOAa9HbTvheQdYnrdJqqv1e5oIqXppMYnSw==} engines: {node: '>=8'} @@ -3119,7 +3122,7 @@ snapshots: balanced-match@1.0.2: optional: true - bare-events@2.2.2: + bare-events@2.4.2: optional: true base64-js@1.5.1: {} @@ -4171,12 +4174,13 @@ snapshots: streamsearch@1.1.0: {} - streamx@2.16.1: + streamx@2.18.0: dependencies: fast-fifo: 1.3.2 queue-tick: 1.0.1 + text-decoder: 1.1.0 optionalDependencies: - bare-events: 2.2.2 + bare-events: 2.4.2 string-argv@0.3.2: {} @@ -4260,7 +4264,7 @@ snapshots: dependencies: b4a: 1.6.6 fast-fifo: 1.3.2 - streamx: 2.16.1 + streamx: 2.18.0 tar@6.2.1: dependencies: @@ -4273,6 +4277,10 @@ snapshots: tarn@3.0.2: {} + text-decoder@1.1.0: + dependencies: + b4a: 1.6.6 + tildify@2.0.0: {} tiny-case@1.0.3: {}