diff --git a/packages/entity-invalidator/src/env.ts b/packages/entity-invalidator/src/env.ts index 673fdd585..928646c91 100644 --- a/packages/entity-invalidator/src/env.ts +++ b/packages/entity-invalidator/src/env.ts @@ -3,7 +3,6 @@ export const LOOKBACK_HOURS = process.env.LOOKBACK_HOURS : 25; export const AWS_REGION = process.env.AWS_REGION; export const CLOUDFRONT_DISTRIBUTION = process.env.CLOUDFRONT_DISTRIBUTION!; -export const DOMAIN = process.env.DOMAIN; // Check for required environment variables const requiredEnvVars = ["AWS_REGION", "CLOUDFRONT_DISTRIBUTION"]; diff --git a/packages/entity-invalidator/src/index.ts b/packages/entity-invalidator/src/index.ts index 9d7cbc511..2aeae918e 100644 --- a/packages/entity-invalidator/src/index.ts +++ b/packages/entity-invalidator/src/index.ts @@ -8,7 +8,6 @@ import { AWS_REGION, CLOUDFRONT_DISTRIBUTION, LOOKBACK_HOURS, - DOMAIN, } from "./env"; import { chunks } from "@helium/spl-utils"; @@ -23,98 +22,14 @@ async function run() { AWS.config.update({ region: AWS_REGION }); const cloudfront = new AWS.CloudFront(); - // Invalidate metadata-service routes: - // - /v2/hotspots/pagination-metadata?subnetwork=iot - // - /v2/hotspots/pagination-metadata?subnetwork=mobile - // Or /v2/hotspot* if there is an error - try { - const modelMap: any = { - iot: IotHotspotInfo, - mobile: MobileHotspotInfo, - }; - const subnetworks = ["iot", "mobile"]; - - // Fetch pagination data - const responsePromises = subnetworks.map((subnetwork) => { - return fetch( - `https://${DOMAIN}/v2/hotspots/pagination-metadata?subnetwork=${subnetwork}` - ); - }); - const jsonPromises = await Promise.all(responsePromises); - const paginationMetadata = await Promise.all( - jsonPromises.map((response) => response.json()) - ); - console.log("Fetched pagination metadata for subnetworks"); - console.log(paginationMetadata); - - // Fetch counts of newly added hotspots - const totalCountPromises = subnetworks.map((subnetwork) => { - const whereClause = { - created_at: { - [Op.gte]: date, - }, - }; - - return KeyToAsset.count({ - where: whereClause, - include: [ - { - model: modelMap[subnetwork], - required: true, - }, - ], - }); - }); - const totalCounts = await Promise.all(totalCountPromises); - console.log("Fetched counts of newly added hotspots"); - console.log(totalCounts); - - // Prepare invalidation paths - const paths: string[] = []; - totalCounts.forEach((count, i) => { - const subnetwork = subnetworks[i]; - const countToPageSizeRatio = count / paginationMetadata[i].pageSize; - let numOfPagesToInvalidate = Math.ceil(countToPageSizeRatio); - while (numOfPagesToInvalidate >= 0) { - const page = paginationMetadata[i].totalPages - numOfPagesToInvalidate; - if (page > 0) { - const path = `/v2/hotspots?subnetwork=${subnetwork}&page=${page}`; - paths.push(path); - } - numOfPagesToInvalidate--; - } - }); - console.log("Invalidation paths"); - console.log(paths); - - await invalidateAndWait({ - cloudfront, - DistributionId: CLOUDFRONT_DISTRIBUTION, - Paths: { - Quantity: paths.length, - Items: paths, - }, - }); - } catch (err) { - console.error( - "Granular /v2/hotspots invalidation failed, resorting to full invalidation" - ); - console.error(err); - - await invalidateAndWait({ - cloudfront, - DistributionId: CLOUDFRONT_DISTRIBUTION, - Paths: { - Quantity: 1, - Items: ["/v2/hotspots*"], - }, - }); - } - // Invalidate metadata-service routes: // - /v2/hotspot/:keyToAssetKey // - /v1/:keyToAssetKey // - /:eccCompact + // Note that the /v2/hotspots route does not need + // cache invalidation due to usage of origin response + // headers preventing caching of non-cacheable assets + // (e.g., the end of the result list) const limit = 10000; let lastId = null; let entities; diff --git a/packages/metadata-service/src/index.ts b/packages/metadata-service/src/index.ts index 1b04e28f6..87e07b698 100644 --- a/packages/metadata-service/src/index.ts +++ b/packages/metadata-service/src/index.ts @@ -30,8 +30,132 @@ import { getAssociatedTokenAddressSync, } from "@solana/spl-token"; import { PublicKey } from "@solana/web3.js"; +import { Op } from "sequelize"; -const pageSize = Number(process.env.PAGE_SIZE) || 1000; +const PAGE_SIZE = Number(process.env.PAGE_SIZE) || 10000; +const MODEL_MAP: any = { + "iot": [IotHotspotInfo, "iot_hotspot_info"], + "mobile": [MobileHotspotInfo, "mobile_hotspot_info"], +} + +function generateAssetJson(record: KeyToAsset, keyStr: string) { + const digest = animalHash(keyStr); + // HACK: If it has a long key, it's an RSA key, and this is a mobile hotspot. + // In the future, we need to put different symbols on different types of hotspots + const hotspotType = keyStr.length > 100 ? "MOBILE" : "IOT"; + const image = `${SHDW_DRIVE_URL}/${ + hotspotType === "MOBILE" + ? record?.mobile_hotspot_info?.is_active + ? "mobile-hotspot-active.png" + : "mobile-hotspot.png" + : record?.iot_hotspot_info?.is_active + ? "hotspot-active.png" + : "hotspot.png" + }`; + + return { + name: keyStr === "iot_operations_fund" ? "IOT Operations Fund" : digest, + description: + keyStr === "iot_operations_fund" + ? "IOT Operations Fund" + : "A Rewardable NFT on Helium", + image, + asset_id: record.asset, + key_to_asset_key: record.address, + entity_key_b64: record?.entity_key.toString("base64"), + key_serialization: record?.key_serialization, + entity_key_str: keyStr, + hotspot_infos: { + iot: { + ...record?.iot_hotspot_info?.dataValues, + location: record?.iot_hotspot_info?.location + ? new BN(record.iot_hotspot_info.location).toString("hex") + : null, + lat: record?.iot_hotspot_info?.lat, + long: record?.iot_hotspot_info?.long, + }, + mobile: { + ...record?.mobile_hotspot_info?.dataValues, + location: record?.mobile_hotspot_info?.location + ? new BN(record.mobile_hotspot_info.location).toString("hex") + : null, + lat: record?.mobile_hotspot_info?.lat, + long: record?.mobile_hotspot_info?.long, + }, + }, + attributes: [ + keyStr && Address.isValid(keyStr) + ? { trait_type: "ecc_compact", value: keyStr } + : undefined, + { trait_type: "entity_key_string", value: keyStr }, + { + trait_type: "entity_key", + value: record?.entity_key?.toString("base64"), + }, + { trait_type: "rewardable", value: true }, + { + trait_type: "networks", + value: [ + record?.iot_hotspot_info && "iot", + record?.mobile_hotspot_info && "mobile", + ].filter(truthy), + }, + ...locationAttributes("iot", record?.iot_hotspot_info), + ...locationAttributes("mobile", record?.mobile_hotspot_info), + ], + }; +} + +async function getHotspotByKeyToAsset(request, reply) { + program = program || (await init(provider)); + const { keyToAssetKey } = request.params; + const record = await KeyToAsset.findOne({ + where: { + address: keyToAssetKey, + }, + include: [IotHotspotInfo, MobileHotspotInfo], + }); + if (!record) { + return reply.code(404); + } + + const { entity_key: entityKey, key_serialization: keySerialization } = record; + const keyStr = decodeEntityKey(entityKey, { [keySerialization]: {} }); + + const assetJson = generateAssetJson(record, keyStr!); + return assetJson; +}; + +function locationAttributes( + name: string, + info: MobileHotspotInfo | IotHotspotInfo | undefined +) { + if (!info) { + return []; + } + + return [ + { trait_type: `${name}_city`, value: info.city }, + { trait_type: `${name}_state`, value: info.state }, + { trait_type: `${name}_country`, value: info.country }, + { trait_type: `${name}_lat`, value: info.lat }, + { trait_type: `${name}_long`, value: info.long }, + ]; +} + +function encodeCursor(cursor: string | null) { + if (!cursor) return cursor; + + const bufferObj = Buffer.from(cursor, "utf8"); + return bufferObj.toString("base64"); +}; + +function decodeCursor(cursor?: string) { + if (!cursor) return cursor; + + const bufferObj = Buffer.from(cursor, "base64") + return bufferObj.toString("utf8"); +}; const server: FastifyInstance = Fastify({ logger: true, @@ -116,207 +240,109 @@ server.get<{ Querystring: { subnetwork: string } }>( async (request, reply) => { const { subnetwork } = request.query; - if (subnetwork === "iot") { - const count = await KeyToAsset.count({ - include: [ - { - model: IotHotspotInfo, - required: true, - }, - ], - }); - - let result = { - pageSize, - totalItems: count, - totalPages: Math.ceil(count / pageSize), - }; - - return result; - } else if (subnetwork === "mobile") { - const count = await KeyToAsset.count({ - include: [ - { - model: MobileHotspotInfo, - required: true, - }, - ], - }); + if (!MODEL_MAP[subnetwork]) { + return reply.code(400).send("Invalid subnetwork"); + } - let result = { - pageSize, - totalItems: count, - totalPages: Math.ceil(count / pageSize), - }; + const count = await KeyToAsset.count({ + include: [ + { + model: MODEL_MAP[subnetwork][0], + required: true, + }, + ], + }); - return result; - } + let result = { + pageSize: PAGE_SIZE, + totalItems: count, + totalPages: Math.ceil(count / PAGE_SIZE), + }; - return reply.code(400).send("Invalid subnetwork"); + return result; } ); -server.get<{ Querystring: { subnetwork: string; page: string } }>( +server.get<{ Querystring: { subnetwork: string; cursor?: string; } }>( "/v2/hotspots", async (request, reply) => { - const { subnetwork, page: pageStr } = request.query; - const pageInt = pageStr ? Number(pageStr) : 1; + const { subnetwork, cursor } = request.query; - const offset = (pageInt - 1) * pageSize; - const limit = pageSize; - - if (subnetwork === "iot") { - const { count, rows: ktas } = await KeyToAsset.findAndCountAll({ - offset, - limit, - include: [ - { - model: IotHotspotInfo, - required: true, - }, - ], - order: [ - [IotHotspotInfo, "created_at", "ASC"], - [IotHotspotInfo, "address", "ASC"], // Need to sort additionally by address otherwise there are dupes across pages because of tight created_at coupling - ], - }); + if (!MODEL_MAP[subnetwork]) { + return reply.code(400).send("Invalid subnetwork"); + } - let result = { - currentPage: pageInt, - nextPage: offset + limit < count ? pageInt + 1 : null, - items: [] as { key_to_asset_key: string }[], - }; + const decodedCursor = decodeCursor(cursor); - result.items = ktas.map((kta) => { - return { - key_to_asset_key: kta.address, - }; - }); + if (decodedCursor && !decodedCursor.includes("|")) { + return reply.code(400).send("Invalid cursor"); + } - return result; - } else if (subnetwork === "mobile") { - const { count, rows: ktas } = await KeyToAsset.findAndCountAll({ - offset, - limit, - include: [ + let where: any = {} + if (decodedCursor?.includes("|")) { + const [lastAsset, lastCreatedAt] = decodedCursor.split("|"); + where = { + [Op.or]: [ { - model: MobileHotspotInfo, - required: true, + created_at: { + [Op.gt]: lastCreatedAt + } }, - ], - order: [ - [MobileHotspotInfo, "created_at", "ASC"], - [MobileHotspotInfo, "address", "ASC"], // Need to sort additionally by address otherwise there are dupes across pages because of tight created_at coupling - ], - }); - - let result = { - currentPage: pageInt, - nextPage: offset + limit < count ? pageInt + 1 : null, - items: [] as { key_to_asset_key: string; device_type: string }[], - }; - - result.items = ktas.map((kta) => { - return { - key_to_asset_key: kta.address, - device_type: kta.mobile_hotspot_info!.device_type, - }; - }); - - return result; + { + [Op.and]: [ + { created_at: lastCreatedAt }, + { asset: { [Op.gt]: lastAsset } } + ] + } + ] + } } - return reply.code(400).send("Invalid subnetwork"); - } -); + const ktas = await KeyToAsset.findAll({ + limit: PAGE_SIZE, + include: [ + { + model: MODEL_MAP[subnetwork][0], + required: true, + where, + }, + ], + order: [ + [MODEL_MAP[subnetwork][0], "created_at", "ASC"], + [MODEL_MAP[subnetwork][0], "asset", "ASC"], + ], + }); -function generateAssetJson(record: KeyToAsset, keyStr: string) { - const digest = animalHash(keyStr); - // HACK: If it has a long key, it's an RSA key, and this is a mobile hotspot. - // In the future, we need to put different symbols on different types of hotspots - const hotspotType = keyStr.length > 100 ? "MOBILE" : "IOT"; - const image = `${SHDW_DRIVE_URL}/${ - hotspotType === "MOBILE" - ? record?.mobile_hotspot_info?.is_active - ? "mobile-hotspot-active.png" - : "mobile-hotspot.png" - : record?.iot_hotspot_info?.is_active - ? "hotspot-active.png" - : "hotspot.png" - }`; + const lastItemTable = MODEL_MAP[subnetwork][1]; + const lastItem = ktas[ktas.length - 1]; + const lastItemAsset = lastItem[lastItemTable]?.asset; + const lastItemDate = lastItem[lastItemTable]?.created_at.toISOString(); + const isLastPage = ktas.length < PAGE_SIZE; + const nextCursor = isLastPage + ? null + : `${lastItemAsset}|${lastItemDate}`; + + let result = { + cursor: encodeCursor(nextCursor), + items: [] as { key_to_asset_key: string }[], + }; - return { - name: keyStr === "iot_operations_fund" ? "IOT Operations Fund" : digest, - description: - keyStr === "iot_operations_fund" - ? "IOT Operations Fund" - : "A Rewardable NFT on Helium", - image, - asset_id: record.asset, - key_to_asset_key: record.address, - entity_key_b64: record?.entity_key.toString("base64"), - key_serialization: record?.key_serialization, - entity_key_str: keyStr, - hotspot_infos: { - iot: { - ...record?.iot_hotspot_info?.dataValues, - location: record?.iot_hotspot_info?.location - ? new BN(record.iot_hotspot_info.location).toString("hex") - : null, - lat: record?.iot_hotspot_info?.lat, - long: record?.iot_hotspot_info?.long, - }, - mobile: { - ...record?.mobile_hotspot_info?.dataValues, - location: record?.mobile_hotspot_info?.location - ? new BN(record.mobile_hotspot_info.location).toString("hex") - : null, - lat: record?.mobile_hotspot_info?.lat, - long: record?.mobile_hotspot_info?.long, - }, - }, - attributes: [ - keyStr && Address.isValid(keyStr) - ? { trait_type: "ecc_compact", value: keyStr } - : undefined, - { trait_type: "entity_key_string", value: keyStr }, - { - trait_type: "entity_key", - value: record?.entity_key?.toString("base64"), - }, - { trait_type: "rewardable", value: true }, - { - trait_type: "networks", - value: [ - record?.iot_hotspot_info && "iot", - record?.mobile_hotspot_info && "mobile", - ].filter(truthy), - }, - ...locationAttributes("iot", record?.iot_hotspot_info), - ...locationAttributes("mobile", record?.mobile_hotspot_info), - ], - }; -} + result.items = ktas.map((kta) => { + return { + key_to_asset_key: kta.address, + }; + }); -const getHotspotByKeyToAsset = async (request, reply) => { - program = program || (await init(provider)); - const { keyToAssetKey } = request.params; - const record = await KeyToAsset.findOne({ - where: { - address: keyToAssetKey, - }, - include: [IotHotspotInfo, MobileHotspotInfo], - }); - if (!record) { - return reply.code(404); + // If we're on the last page of results, tell Cloudfront not to cache + // so that origin requests are made and newly added hotspots can be + // returned + if (isLastPage) { + reply.header("Cache-Control", "no-cache"); + } + + return result; } - - const { entity_key: entityKey, key_serialization: keySerialization } = record; - const keyStr = decodeEntityKey(entityKey, { [keySerialization]: {} }); - - const assetJson = generateAssetJson(record, keyStr!); - return assetJson; -}; +); let program: Program; server.get<{ Params: { keyToAssetKey: string } }>( @@ -362,23 +388,6 @@ server.get<{ Params: { eccCompact: string } }>( } ); -function locationAttributes( - name: string, - info: MobileHotspotInfo | IotHotspotInfo | undefined -) { - if (!info) { - return []; - } - - return [ - { trait_type: `${name}_city`, value: info.city }, - { trait_type: `${name}_state`, value: info.state }, - { trait_type: `${name}_country`, value: info.country }, - { trait_type: `${name}_lat`, value: info.lat }, - { trait_type: `${name}_long`, value: info.long }, - ]; -} - const start = async () => { try { await server.listen({ port: 8081, host: "0.0.0.0" }); diff --git a/packages/metadata-service/src/model.ts b/packages/metadata-service/src/model.ts index 0e2bc75d7..e6cf10d8d 100644 --- a/packages/metadata-service/src/model.ts +++ b/packages/metadata-service/src/model.ts @@ -96,6 +96,7 @@ class WithRes8LatLgn extends Model { export class MobileHotspotInfo extends WithRes8LatLgn { declare address: string; + declare asset: string; declare street: string; declare city: string; declare state: string; @@ -103,6 +104,8 @@ export class MobileHotspotInfo extends WithRes8LatLgn { declare location: string; declare is_active: boolean; declare device_type: string; + declare created_at: Date; + } MobileHotspotInfo.init( { @@ -110,6 +113,7 @@ MobileHotspotInfo.init( type: STRING, primaryKey: true, }, + asset: DataTypes.STRING, street: DataTypes.STRING, city: DataTypes.STRING, state: DataTypes.STRING, @@ -118,6 +122,7 @@ MobileHotspotInfo.init( device_type: DataTypes.JSONB, location: DataTypes.DECIMAL.UNSIGNED, dc_onboarding_fee_paid: DataTypes.DECIMAL.UNSIGNED, + created_at: DataTypes.DATE, }, { sequelize, @@ -137,6 +142,7 @@ export class IotHotspotInfo extends WithRes8LatLgn { declare country: string; declare location: string; declare is_active: boolean; + declare created_at: Date; } IotHotspotInfo.init( @@ -154,6 +160,7 @@ IotHotspotInfo.init( dc_onboarding_fee_paid: DataTypes.DECIMAL.UNSIGNED, elevation: DataTypes.NUMBER, gain: DataTypes.NUMBER, + created_at: DataTypes.DATE, }, { sequelize,