diff --git a/packages/common/src/downloads/_internal/file-url-fetchers/fetchExportImageDownloadFile.ts b/packages/common/src/downloads/_internal/file-url-fetchers/fetchExportImageDownloadFile.ts index caa6f2d9f43..5f5ec9c50ed 100644 --- a/packages/common/src/downloads/_internal/file-url-fetchers/fetchExportImageDownloadFile.ts +++ b/packages/common/src/downloads/_internal/file-url-fetchers/fetchExportImageDownloadFile.ts @@ -53,7 +53,6 @@ export async function fetchExportImageDownloadFile( // } const blob: Blob = await request(`${entity.url}/exportImage`, requestOptions); - progressCallback && progressCallback(DownloadOperationStatus.COMPLETED); return { type: "blob", blob, diff --git a/packages/common/src/downloads/_internal/file-url-fetchers/fetchHubApiDownloadFile.ts b/packages/common/src/downloads/_internal/file-url-fetchers/fetchHubApiDownloadFile.ts index 68cee3c2bd0..973b1961e17 100644 --- a/packages/common/src/downloads/_internal/file-url-fetchers/fetchHubApiDownloadFile.ts +++ b/packages/common/src/downloads/_internal/file-url-fetchers/fetchHubApiDownloadFile.ts @@ -1,7 +1,9 @@ import HubError from "../../../HubError"; import { getProp } from "../../../objects/get-prop"; +import { wait } from "../../../utils/wait"; import { ArcgisHubDownloadError, + DownloadCacheStatus, DownloadOperationStatus, IFetchDownloadFileOptions, IFetchDownloadFileResponse, @@ -11,7 +13,8 @@ import { /** * @private - * Fetches a download file url from the Hub Download API + * Fetches a download file url from the Hub Download API. If we know the file will be served from + * a cache, we also return the status of the cache. * * NOTE: The Hub Download API only works with a certain subset of Feature and Map services * and performs different operations (i.e., calling createReplica or paging the service's @@ -21,15 +24,29 @@ import { * interface for downloading data from any service supported by the Hub Download API. * * @param options options for refining / filtering the resulting download file - * @returns a url to download the file + * @returns the url to download the file and cache status */ export async function fetchHubApiDownloadFile( options: IFetchDownloadFileOptions ): Promise { validateOptions(options); const requestUrl = getDownloadApiRequestUrl(options); - const { pollInterval, progressCallback } = options; - return pollDownloadApi(requestUrl, pollInterval, progressCallback); + + const { pollInterval, updateCache, progressCallback } = options; + + let cacheQueryParam: CacheQueryParam; + if (updateCache) { + // The download api has 2 query params that can be used for controlling cache behavior. + // This one should only be used as part of an initial request to update the cache + cacheQueryParam = "updateCache"; + } + + return pollDownloadApi( + requestUrl, + pollInterval, + cacheQueryParam, + progressCallback + ); } function validateOptions(options: IFetchDownloadFileOptions) { @@ -61,7 +78,8 @@ function validateOptions(options: IFetchDownloadFileOptions) { * @returns a download api url that can be polled */ function getDownloadApiRequestUrl(options: IFetchDownloadFileOptions) { - const { entity, format, context, layers, geometry, where } = options; + const { entity, format, context, layers, geometry, where, updateCache } = + options; const searchParams = new URLSearchParams({ redirect: "false", // Needed to get the download URL instead of the file itself @@ -97,15 +115,28 @@ function getDownloadApiRequestUrl(options: IFetchDownloadFileOptions) { * Polls the Hub Download API until the download is ready, then returns the download file URL * * @param requestUrl Hub Download Api URL to poll + * @param pollInterval interval in milliseconds to poll for job completion + * @param cacheQueryParam an optional query param to control cache behavior. 'updateCache' should + * only be used with the initial request when a download cache update is desired. All subsequent + * requests should use'trackCacheUpdate' to follow the progress of the cache update. + * for subsequent requests when polling the progress of a cache update. * @param progressCallback an optional callback to report download generation progress * @returns the final file URL */ async function pollDownloadApi( requestUrl: string, pollInterval: number, + cacheQueryParam?: CacheQueryParam, progressCallback?: downloadProgressCallback ): Promise { - const response = await fetch(requestUrl); + // If requested, append the cache query param to the request URL + let withCacheQueryParam = requestUrl; + if (cacheQueryParam) { + withCacheQueryParam = `${requestUrl}&${cacheQueryParam}=true`; + } + + const response = await fetch(withCacheQueryParam); + if (!response.ok) { const errorBody = await response.json(); // TODO: Add standarized messageId when available @@ -113,8 +144,14 @@ async function pollDownloadApi( rawMessage: errorBody.message, }); } - const { status, progressInPercent, resultUrl }: IHubDownloadApiResponse = - await response.json(); + + const { + status, + progressInPercent, + resultUrl, + cacheStatus, + }: IHubDownloadApiResponse = await response.json(); + const operationStatus = toDownloadOperationStatus(status); if (operationStatus === DownloadOperationStatus.FAILED) { throw new HubError( @@ -122,19 +159,33 @@ async function pollDownloadApi( "Download operation failed with a 200" ); } - progressCallback && progressCallback(operationStatus, progressInPercent); - // Operation complete, return the download URL if (resultUrl) { + // Operation complete, return the download URL and cache status return { type: "url", href: resultUrl, + cacheStatus, }; } - // Operation still in progress, poll again - await new Promise((resolve) => setTimeout(resolve, pollInterval)); - return pollDownloadApi(requestUrl, pollInterval, progressCallback); + // Operation still in progress. Report progress if a callback was provided and poll again. + progressCallback && progressCallback(operationStatus, progressInPercent); + await wait(pollInterval); + + let updatedCacheQueryParam: CacheQueryParam; + if (cacheQueryParam) { + // After an initial request with ?updateCache=true, we need to switch to ?trackCacheUpdate=true. + // This enables us to follow the progress of the cache update without causing an infinite update loop. + updatedCacheQueryParam = "trackCacheUpdate"; + } + + return pollDownloadApi( + requestUrl, + pollInterval, + updatedCacheQueryParam, + progressCallback + ); } /** @@ -223,4 +274,13 @@ interface IHubDownloadApiResponse { resultUrl?: string; recordCount?: number; progressInPercent?: number; + cacheStatus?: DownloadCacheStatus; } + +/** + * @private + * Query params that can be used to control cache behavior in the Hub Download API. + * 'updateCache' is used with the initial request when a download cache update is desired. + * 'trackCacheUpdate' is used for subsequent requests while polling the progress of a cache update. + */ +type CacheQueryParam = "updateCache" | "trackCacheUpdate"; diff --git a/packages/common/src/downloads/types.ts b/packages/common/src/downloads/types.ts index 422baa1b027..d7a68f81cc9 100644 --- a/packages/common/src/downloads/types.ts +++ b/packages/common/src/downloads/types.ts @@ -133,23 +133,57 @@ export interface IFetchDownloadFileOptions { where?: string; // where clause to filter results by progressCallback?: downloadProgressCallback; pollInterval?: number; // interval in milliseconds to poll for job completion + updateCache?: boolean; // whether the request should also update the cache; only valid when targeting the hub download system } +/** + * Response object for the fetchDownloadFile operation. + * The response object will contain either a Blob or a URL, + * depending on the type of download operation that was performed. + */ export type IFetchDownloadFileResponse = | IFetchDownloadFileBlobResponse | IFetchDownloadFileUrlResponse; +/** + * Base interface for all fetchDownloadFile response objects. + */ interface IBaseFetchDownloadFileResponse { type: "blob" | "url"; + /** + * If the response comes from our cache rather than a live download, + * indicates the status of the cached file + */ + cacheStatus?: DownloadCacheStatus; } +/** + * Represents the status of a cached download file. + * - `ready` indicates that the file has up-to-date data + * - `ready_unknown` indicates that we don't know if the file is up-to-date + * - `stale` indicates that the file is out-of-date + */ +export type DownloadCacheStatus = "ready" | "ready_unknown" | "stale"; + +/** + * Response object for the fetchDownloadFile operation when the response is a Blob. + */ export interface IFetchDownloadFileBlobResponse extends IBaseFetchDownloadFileResponse { type: "blob"; + /** + * The Blob object that contains the download file. + */ blob: Blob; + /** + * The name to assign to the file when saving it to disk. + */ filename: string; } +/** + * Response object for the fetchDownloadFile operation when the response is a URL. + */ export interface IFetchDownloadFileUrlResponse extends IBaseFetchDownloadFileResponse { type: "url"; diff --git a/packages/common/test/downloads/_internal/file-url-fetchers/fetchExportImageDownloadFile.test.ts b/packages/common/test/downloads/_internal/file-url-fetchers/fetchExportImageDownloadFile.test.ts index 419a70249a1..ce293f06205 100644 --- a/packages/common/test/downloads/_internal/file-url-fetchers/fetchExportImageDownloadFile.test.ts +++ b/packages/common/test/downloads/_internal/file-url-fetchers/fetchExportImageDownloadFile.test.ts @@ -8,7 +8,7 @@ import { fetchExportImageDownloadFile } from "../../../../src/downloads/_interna import { IHubEditableContent, IServiceExtendedProps } from "../../../../src"; describe("fetchExportImageDownloadFile", () => { - it("should call progressCallback with PENDING and COMPLETED statuses", async () => { + it("should call progressCallback with PENDING statuses", async () => { const requestSpy = spyOn(requestModule, "request").and.returnValue( Promise.resolve({ size: 1000 } as Blob) ); @@ -46,13 +46,10 @@ describe("fetchExportImageDownloadFile", () => { blob: { size: 1000 } as Blob, }); - expect(progressCallback).toHaveBeenCalledTimes(2); + expect(progressCallback).toHaveBeenCalledTimes(1); expect(progressCallback).toHaveBeenCalledWith( DownloadOperationStatus.PENDING ); - expect(progressCallback).toHaveBeenCalledWith( - DownloadOperationStatus.COMPLETED - ); expect(requestSpy).toHaveBeenCalledTimes(1); expect(requestSpy).toHaveBeenCalledWith( diff --git a/packages/common/test/downloads/_internal/file-url-fetchers/fetchHubApiDownloadFile.test.ts b/packages/common/test/downloads/_internal/file-url-fetchers/fetchHubApiDownloadFile.test.ts index 67fe9a7c35f..e2e3f71cec3 100644 --- a/packages/common/test/downloads/_internal/file-url-fetchers/fetchHubApiDownloadFile.test.ts +++ b/packages/common/test/downloads/_internal/file-url-fetchers/fetchHubApiDownloadFile.test.ts @@ -147,7 +147,11 @@ describe("fetchHubApiDownloadFile", () => { pollInterval: 0, }); - expect(result).toEqual({ type: "url", href: "fake-url" }); + expect(result).toEqual({ + type: "url", + href: "fake-url", + cacheStatus: undefined, + }); }); it("polls with a progress callback", async () => { fetchMock.once( @@ -182,7 +186,7 @@ describe("fetchHubApiDownloadFile", () => { const progressCallback = jasmine .createSpy("progressCallback") - .and.callFake((status: any, percent: any): any => null); + .and.callFake((_status: any, _percent: any): any => null); const result = await fetchHubApiDownloadFile({ entity: { id: "123" } as unknown as IHubEditableContent, format: ServiceDownloadFormat.CSV, @@ -195,8 +199,12 @@ describe("fetchHubApiDownloadFile", () => { progressCallback, }); - expect(result).toEqual({ type: "url", href: "fake-url" }); - expect(progressCallback).toHaveBeenCalledTimes(3); + expect(result).toEqual({ + type: "url", + href: "fake-url", + cacheStatus: undefined, + }); + expect(progressCallback).toHaveBeenCalledTimes(2); expect(progressCallback).toHaveBeenCalledWith( DownloadOperationStatus.PENDING, undefined @@ -205,10 +213,6 @@ describe("fetchHubApiDownloadFile", () => { DownloadOperationStatus.PROCESSING, 50 ); - expect(progressCallback).toHaveBeenCalledWith( - DownloadOperationStatus.COMPLETED, - undefined - ); }); it("handles geometry, token and where parameters", async () => { fetchMock.once( @@ -240,7 +244,11 @@ describe("fetchHubApiDownloadFile", () => { where: "1=1", }); - expect(result).toEqual({ type: "url", href: "fake-url" }); + expect(result).toEqual({ + type: "url", + href: "fake-url", + cacheStatus: undefined, + }); }); it("Explicitly sets the spatialRefId to 4326 for GeoJSON and KML", async () => { fetchMock.once( @@ -263,7 +271,11 @@ describe("fetchHubApiDownloadFile", () => { layers: [0], }); - expect(result).toEqual({ type: "url", href: "fake-url" }); + expect(result).toEqual({ + type: "url", + href: "fake-url", + cacheStatus: undefined, + }); fetchMock.once( "https://hubqa.arcgis.com/api/download/v1/items/123/kml?redirect=false&layers=0&spatialRefId=4326", @@ -285,6 +297,149 @@ describe("fetchHubApiDownloadFile", () => { layers: [0], }); - expect(result2).toEqual({ type: "url", href: "fake-url-2" }); + expect(result2).toEqual({ + type: "url", + href: "fake-url-2", + cacheStatus: undefined, + }); + }); + it("polls without a progress callback", async () => { + fetchMock.once( + "https://hubqa.arcgis.com/api/download/v1/items/123/csv?redirect=false&layers=0", + { + body: { + status: "Pending", + }, + } + ); + fetchMock.once( + "https://hubqa.arcgis.com/api/download/v1/items/123/csv?redirect=false&layers=0", + { + body: { + status: "InProgress", + recordCount: 100, + progressInPercent: 50, + }, + }, + { overwriteRoutes: false } + ); + fetchMock.once( + "https://hubqa.arcgis.com/api/download/v1/items/123/csv?redirect=false&layers=0", + { + body: { + status: "Completed", + resultUrl: "fake-url", + }, + }, + { overwriteRoutes: false } + ); + + const result = await fetchHubApiDownloadFile({ + entity: { id: "123" } as unknown as IHubEditableContent, + format: ServiceDownloadFormat.CSV, + context: { + hubUrl: "https://hubqa.arcgis.com", + hubRequestOptions: {}, + } as unknown as IArcGISContext, + layers: [0], + pollInterval: 0, + }); + + expect(result).toEqual({ + type: "url", + href: "fake-url", + cacheStatus: undefined, + }); + }); + it("returns the cacheStatus if present", async () => { + fetchMock.once( + "https://hubqa.arcgis.com/api/download/v1/items/123/csv?redirect=false&layers=0", + { + body: { + status: "Completed", + resultUrl: "fake-url", + cacheStatus: "ready_unknown", + }, + } + ); + + const result = await fetchHubApiDownloadFile({ + entity: { id: "123" } as unknown as IHubEditableContent, + format: ServiceDownloadFormat.CSV, + context: { + hubUrl: "https://hubqa.arcgis.com", + hubRequestOptions: {}, + } as unknown as IArcGISContext, + layers: [0], + pollInterval: 0, + }); + + expect(result).toEqual({ + type: "url", + href: "fake-url", + cacheStatus: "ready_unknown", + }); + }); + it("handles the updateCache option", async () => { + fetchMock.once( + "https://hubqa.arcgis.com/api/download/v1/items/123/csv?redirect=false&layers=0&updateCache=true", + { + body: { + status: "Pending", + }, + } + ); + fetchMock.once( + "https://hubqa.arcgis.com/api/download/v1/items/123/csv?redirect=false&layers=0&trackCacheUpdate=true", + { + body: { + status: "PagingData", + recordCount: 100, + progressInPercent: 50, + }, + }, + { overwriteRoutes: false } + ); + fetchMock.once( + "https://hubqa.arcgis.com/api/download/v1/items/123/csv?redirect=false&layers=0&trackCacheUpdate=true", + { + body: { + status: "Completed", + resultUrl: "fake-url", + }, + }, + { overwriteRoutes: false } + ); + + const progressCallback = jasmine + .createSpy("progressCallback") + .and.callFake((_status: any, _percent: any): any => null); + const result = await fetchHubApiDownloadFile({ + entity: { id: "123" } as unknown as IHubEditableContent, + format: ServiceDownloadFormat.CSV, + context: { + hubUrl: "https://hubqa.arcgis.com", + hubRequestOptions: {}, + } as unknown as IArcGISContext, + layers: [0], + pollInterval: 0, + progressCallback, + updateCache: true, + }); + + expect(result).toEqual({ + type: "url", + href: "fake-url", + cacheStatus: undefined, + }); + expect(progressCallback).toHaveBeenCalledTimes(2); + expect(progressCallback).toHaveBeenCalledWith( + DownloadOperationStatus.PENDING, + undefined + ); + expect(progressCallback).toHaveBeenCalledWith( + DownloadOperationStatus.PROCESSING, + 50 + ); }); });