Cursor-based pagination

packages/entity-invalidator/src/env.ts

@@ -3,7 +3,6 @@ export const LOOKBACK_HOURS = process.env.LOOKBACK_HOURS
   : 25;
 export const AWS_REGION = process.env.AWS_REGION;
 export const CLOUDFRONT_DISTRIBUTION = process.env.CLOUDFRONT_DISTRIBUTION!;
-export const DOMAIN = process.env.DOMAIN;
 
 // Check for required environment variables
 const requiredEnvVars = ["AWS_REGION", "CLOUDFRONT_DISTRIBUTION"];

packages/entity-invalidator/src/index.ts

@@ -8,7 +8,6 @@ import {
   AWS_REGION,
   CLOUDFRONT_DISTRIBUTION,
   LOOKBACK_HOURS,
-  DOMAIN,
 } from "./env";
 import { chunks } from "@helium/spl-utils";
 
@@ -23,98 +22,14 @@ async function run() {
   AWS.config.update({ region: AWS_REGION });
   const cloudfront = new AWS.CloudFront();
 
-  // Invalidate metadata-service routes:
-  // - /v2/hotspots/pagination-metadata?subnetwork=iot
-  // - /v2/hotspots/pagination-metadata?subnetwork=mobile
-  // Or /v2/hotspot* if there is an error
-  try {
-    const modelMap: any = {
-      iot: IotHotspotInfo,
-      mobile: MobileHotspotInfo,
-    };
-    const subnetworks = ["iot", "mobile"];
-
-    // Fetch pagination data
-    const responsePromises = subnetworks.map((subnetwork) => {
-      return fetch(
-        `https://${DOMAIN}/v2/hotspots/pagination-metadata?subnetwork=${subnetwork}`
-      );
-    });
-    const jsonPromises = await Promise.all(responsePromises);
-    const paginationMetadata = await Promise.all(
-      jsonPromises.map((response) => response.json())
-    );
-    console.log("Fetched pagination metadata for subnetworks");
-    console.log(paginationMetadata);
-
-    // Fetch counts of newly added hotspots
-    const totalCountPromises = subnetworks.map((subnetwork) => {
-      const whereClause = {
-        created_at: {
-          [Op.gte]: date,
-        },
-      };
-
-      return KeyToAsset.count({
-        where: whereClause,
-        include: [
-          {
-            model: modelMap[subnetwork],
-            required: true,
-          },
-        ],
-      });
-    });
-    const totalCounts = await Promise.all(totalCountPromises);
-    console.log("Fetched counts of newly added hotspots");
-    console.log(totalCounts);
-
-    // Prepare invalidation paths
-    const paths: string[] = [];
-    totalCounts.forEach((count, i) => {
-      const subnetwork = subnetworks[i];
-      const countToPageSizeRatio = count / paginationMetadata[i].pageSize;
-      let numOfPagesToInvalidate = Math.ceil(countToPageSizeRatio);
-      while (numOfPagesToInvalidate >= 0) {
-        const page = paginationMetadata[i].totalPages - numOfPagesToInvalidate;
-        if (page > 0) {
-          const path = `/v2/hotspots?subnetwork=${subnetwork}&page=${page}`;
-          paths.push(path);
-        }
-        numOfPagesToInvalidate--;
-      }
-    });
-    console.log("Invalidation paths");
-    console.log(paths);
-
-    await invalidateAndWait({
-      cloudfront,
-      DistributionId: CLOUDFRONT_DISTRIBUTION,
-      Paths: {
-        Quantity: paths.length,
-        Items: paths,
-      },
-    });
-  } catch (err) {
-    console.error(
-      "Granular /v2/hotspots invalidation failed, resorting to full invalidation"
-    );
-    console.error(err);
-
-    await invalidateAndWait({
-      cloudfront,
-      DistributionId: CLOUDFRONT_DISTRIBUTION,
-      Paths: {
-        Quantity: 1,
-        Items: ["/v2/hotspots*"],
-      },
-    });
-  }
-
   // Invalidate metadata-service routes:
   // - /v2/hotspot/:keyToAssetKey
   // - /v1/:keyToAssetKey
   // - /:eccCompact
+  // Note that the /v2/hotspots route does not need
+  // cache invalidation due to usage of origin response
+  // headers preventing caching of non-cacheable assets 
+  // (e.g., the end of the result list)
   const limit = 10000;
   let lastId = null;
   let entities;