feat(hub-common): allow force updates of cache files as well as cache status reporting

packages/common/src/downloads/_internal/file-url-fetchers/fetchExportImageDownloadFile.ts

@@ -53,7 +53,6 @@ export async function fetchExportImageDownloadFile(
   // }
 
   const blob: Blob = await request(`${entity.url}/exportImage`, requestOptions);
-  progressCallback && progressCallback(DownloadOperationStatus.COMPLETED);
   return {
     type: "blob",
     blob,

packages/common/src/downloads/_internal/file-url-fetchers/fetchHubApiDownloadFile.ts

@@ -2,6 +2,7 @@ import HubError from "../../../HubError";
 import { getProp } from "../../../objects/get-prop";
 import {
   ArcgisHubDownloadError,
+  DownloadCacheStatus,
   DownloadOperationStatus,
   IFetchDownloadFileOptions,
   IFetchDownloadFileResponse,
@@ -11,7 +12,8 @@ import {
 
 /**
  * @private
- * Fetches a download file url from the Hub Download API
+ * Fetches a download file url from the Hub Download API. If we know the file will be served from
+ * a cache, we also return the status of the cache.
  *
  * NOTE: The Hub Download API only works with a certain subset of Feature and Map services
  * and performs different operations (i.e., calling createReplica or paging the service's
@@ -21,15 +23,29 @@ import {
  * interface for downloading data from any service supported by the Hub Download API.
  *
  * @param options options for refining / filtering the resulting download file
- * @returns a url to download the file
+ * @returns the url to download the file and cache status
  */
 export async function fetchHubApiDownloadFile(
   options: IFetchDownloadFileOptions
 ): Promise<IFetchDownloadFileResponse> {
   validateOptions(options);
   const requestUrl = getDownloadApiRequestUrl(options);
-  const { pollInterval, progressCallback } = options;
-  return pollDownloadApi(requestUrl, pollInterval, progressCallback);
+
+  const { pollInterval, updateCache, progressCallback } = options;
+
+  let cacheQueryParam: CacheQueryParam;
+  if (updateCache) {
+    // The download api has 2 query params that can be used for controlling cache behavior.
+    // This one should only be used as part of an initial request to update the cache
+    cacheQueryParam = "updateCache";
+  }
+
+  return pollDownloadApi(
+    requestUrl,
+    pollInterval,
+    cacheQueryParam,
+    progressCallback
+  );
 }
 
 function validateOptions(options: IFetchDownloadFileOptions) {
@@ -61,7 +77,8 @@ function validateOptions(options: IFetchDownloadFileOptions) {
  * @returns a download api url that can be polled
  */
 function getDownloadApiRequestUrl(options: IFetchDownloadFileOptions) {
-  const { entity, format, context, layers, geometry, where } = options;
+  const { entity, format, context, layers, geometry, where, updateCache } =
+    options;
 
   const searchParams = new URLSearchParams({
     redirect: "false", // Needed to get the download URL instead of the file itself
@@ -97,44 +114,77 @@ function getDownloadApiRequestUrl(options: IFetchDownloadFileOptions) {
  * Polls the Hub Download API until the download is ready, then returns the download file URL
  *
  * @param requestUrl Hub Download Api URL to poll
+ * @param pollInterval interval in milliseconds to poll for job completion
+ * @param cacheQueryParam an optional query param to control cache behavior. 'updateCache' should
+ * only be used with the initial request when a download cache update is desired. All subsequent
+ * requests should use'trackCacheUpdate' to follow the progress of the cache update.
+ * for subsequent requests when polling the progress of a cache update.
  * @param progressCallback an optional callback to report download generation progress
  * @returns the final file URL
  */
 async function pollDownloadApi(
   requestUrl: string,
   pollInterval: number,
+  cacheQueryParam?: CacheQueryParam,
   progressCallback?: downloadProgressCallback
 ): Promise<IFetchDownloadFileResponse> {
-  const response = await fetch(requestUrl);
+  // If requested, append the cache query param to the request URL
+  let withCacheQueryParam = requestUrl;
+  if (cacheQueryParam) {
+    withCacheQueryParam = `${requestUrl}&${cacheQueryParam}=true`;
+  }
+
+  const response = await fetch(withCacheQueryParam);
+
   if (!response.ok) {
     const errorBody = await response.json();
     // TODO: Add standarized messageId when available
     throw new ArcgisHubDownloadError({
       rawMessage: errorBody.message,
     });
   }
-  const { status, progressInPercent, resultUrl }: IHubDownloadApiResponse =
-    await response.json();
+
+  const {
+    status,
+    progressInPercent,
+    resultUrl,
+    cacheStatus,
+  }: IHubDownloadApiResponse = await response.json();
+
   const operationStatus = toDownloadOperationStatus(status);
   if (operationStatus === DownloadOperationStatus.FAILED) {
     throw new HubError(
       "fetchHubApiDownloadFileUrl",
       "Download operation failed with a 200"
     );
   }
-  progressCallback && progressCallback(operationStatus, progressInPercent);
 
-  // Operation complete, return the download URL
   if (resultUrl) {
+    // Operation complete, return the download URL and cache status
     return {
       type: "url",
       href: resultUrl,
+      cacheStatus,
     };
   }
 
-  // Operation still in progress, poll again
+  // Operation still in progress. Report progress if a callback was provided and poll again.
+  progressCallback && progressCallback(operationStatus, progressInPercent);
   await new Promise((resolve) => setTimeout(resolve, pollInterval));
-  return pollDownloadApi(requestUrl, pollInterval, progressCallback);
+
+  let updatedCacheQueryParam: CacheQueryParam;
+  if (cacheQueryParam) {
+    // After an initial request with ?updateCache=true, we need to switch to ?trackCacheUpdate=true.
+    // This enables us to follow the progress of the cache update without causing an infinite update loop.
+    updatedCacheQueryParam = "trackCacheUpdate";
+  }
+
+  return pollDownloadApi(
+    requestUrl,
+    pollInterval,
+    updatedCacheQueryParam,
+    progressCallback
+  );
 }
 
 /**
@@ -223,4 +273,13 @@ interface IHubDownloadApiResponse {
   resultUrl?: string;
   recordCount?: number;
   progressInPercent?: number;
+  cacheStatus?: DownloadCacheStatus;
 }
+
+/**
+ * @private
+ * Query params that can be used to control cache behavior in the Hub Download API.
+ * 'updateCache' is used with the initial request when a download cache update is desired.
+ * 'trackCacheUpdate' is used for subsequent requests while polling the progress of a cache update.
+ */
+type CacheQueryParam = "updateCache" | "trackCacheUpdate";

packages/common/src/downloads/types.ts

@@ -133,23 +133,57 @@ export interface IFetchDownloadFileOptions {
   where?: string; // where clause to filter results by
   progressCallback?: downloadProgressCallback;
   pollInterval?: number; // interval in milliseconds to poll for job completion
+  updateCache?: boolean; // whether the request should also update the cache; only valid when targeting the hub download system
 }
 
+/**
+ * Response object for the fetchDownloadFile operation.
+ * The response object will contain either a Blob or a URL,
+ * depending on the type of download operation that was performed.
+ */
 export type IFetchDownloadFileResponse =
   | IFetchDownloadFileBlobResponse
   | IFetchDownloadFileUrlResponse;
 
+/**
+ * Base interface for all fetchDownloadFile response objects.
+ */
 interface IBaseFetchDownloadFileResponse {
   type: "blob" | "url";
+  /**
+   * If the response comes from our cache rather than a live download,
+   * indicates the status of the cached file
+   */
+  cacheStatus?: DownloadCacheStatus;
 }
 
+/**
+ * Represents the status of a cached download file.
+ * - `ready` indicates that the file has up-to-date data
+ * - `ready_unknown` indicates that we don't know if the file is up-to-date
+ * - `stale` indicates that the file is out-of-date
+ */
+export type DownloadCacheStatus = "ready" | "ready_unknown" | "stale";
+
+/**
+ * Response object for the fetchDownloadFile operation when the response is a Blob.
+ */
 export interface IFetchDownloadFileBlobResponse
   extends IBaseFetchDownloadFileResponse {
   type: "blob";
+  /**
+   * The Blob object that contains the download file.
+   */
   blob: Blob;
+  /**
+   * The name to assign to the file when saving it to disk.
+   */
   filename: string;
 }
 
+/**
+ * Response object for the fetchDownloadFile operation when the response is a URL.
+ */
 export interface IFetchDownloadFileUrlResponse
   extends IBaseFetchDownloadFileResponse {
   type: "url";

packages/common/test/downloads/_internal/file-url-fetchers/fetchExportImageDownloadFile.test.ts

@@ -8,7 +8,7 @@ import { fetchExportImageDownloadFile } from "../../../../src/downloads/_interna
 import { IHubEditableContent, IServiceExtendedProps } from "../../../../src";
 
 describe("fetchExportImageDownloadFile", () => {
-  it("should call progressCallback with PENDING and COMPLETED statuses", async () => {
+  it("should call progressCallback with PENDING statuses", async () => {
     const requestSpy = spyOn(requestModule, "request").and.returnValue(
       Promise.resolve({ size: 1000 } as Blob)
     );
@@ -46,13 +46,10 @@ describe("fetchExportImageDownloadFile", () => {
       blob: { size: 1000 } as Blob,
     });
 
-    expect(progressCallback).toHaveBeenCalledTimes(2);
+    expect(progressCallback).toHaveBeenCalledTimes(1);
     expect(progressCallback).toHaveBeenCalledWith(
       DownloadOperationStatus.PENDING
     );
-    expect(progressCallback).toHaveBeenCalledWith(
-      DownloadOperationStatus.COMPLETED
-    );
 
     expect(requestSpy).toHaveBeenCalledTimes(1);
     expect(requestSpy).toHaveBeenCalledWith(