From e2da608a0dc9efecb8b7b959bc063d523f585799 Mon Sep 17 00:00:00 2001 From: Rick Saccoccia Date: Mon, 28 Oct 2024 12:11:09 -0600 Subject: [PATCH] Convert httpClient calls to @esri/arcgis-rest-request with ArcGISIdentityManager for authentication --- .../service/src/FeatureLayerProcessor.ts | 6 +- plugins/arcgis/service/src/FeatureQuerier.ts | 82 +++--- plugins/arcgis/service/src/FeatureService.ts | 242 ++++++++++++++---- .../arcgis/service/src/FeatureServiceAdmin.ts | 32 ++- .../service/src/ObservationProcessor.ts | 30 ++- 5 files changed, 293 insertions(+), 99 deletions(-) diff --git a/plugins/arcgis/service/src/FeatureLayerProcessor.ts b/plugins/arcgis/service/src/FeatureLayerProcessor.ts index c5c82b478..91cecce12 100644 --- a/plugins/arcgis/service/src/FeatureLayerProcessor.ts +++ b/plugins/arcgis/service/src/FeatureLayerProcessor.ts @@ -5,7 +5,7 @@ import { LayerInfo } from "./LayerInfo"; import { ObservationBinner } from "./ObservationBinner"; import { ObservationBins } from "./ObservationBins"; import { ObservationsSender } from "./ObservationsSender"; - +import { ArcGISIdentityManager } from "@esri/arcgis-rest-request"; /** * Processes new, updated, and deleted observations and sends the changes to a specific arc feature layer. */ @@ -42,10 +42,10 @@ export class FeatureLayerProcessor { * @param config Contains certain parameters that can be configured. * @param console Used to log messages to the console. */ - constructor(layerInfo: LayerInfo, config: ArcGISPluginConfig, console: Console) { + constructor(layerInfo: LayerInfo, config: ArcGISPluginConfig, identityManager: ArcGISIdentityManager, console: Console) { this.layerInfo = layerInfo; this.lastTimeStamp = 0; - this.featureQuerier = new FeatureQuerier(layerInfo, config, console); + this.featureQuerier = new FeatureQuerier(layerInfo, config, identityManager,console); this._binner = new ObservationBinner(layerInfo, this.featureQuerier, config); this.sender = new ObservationsSender(layerInfo, config, console); } diff --git a/plugins/arcgis/service/src/FeatureQuerier.ts b/plugins/arcgis/service/src/FeatureQuerier.ts index b32402c5c..9944d7e46 100644 --- a/plugins/arcgis/service/src/FeatureQuerier.ts +++ b/plugins/arcgis/service/src/FeatureQuerier.ts @@ -1,22 +1,17 @@ import { ArcGISPluginConfig } from "./ArcGISPluginConfig"; -import { HttpClient } from "./HttpClient"; import { LayerInfo } from "./LayerInfo"; import { QueryObjectResult } from "./QueryObjectResult"; +import { ArcGISIdentityManager, request } from "@esri/arcgis-rest-request"; /** * Performs various queries on observations for a specific arc feature layer. */ export class FeatureQuerier { - /** - * Used to query the arc server to figure out if an observation exists. - */ - private _httpClient: HttpClient; - /** * The query url to find out if an observations exists on the server. */ - private _url: string; + private _url: URL; /** * Used to log to console. @@ -28,15 +23,23 @@ export class FeatureQuerier { */ private _config: ArcGISPluginConfig; + /** + * An instance of `ArcGISIdentityManager` used to manage authentication and identity for ArcGIS services. + * This private member handles the authentication process, ensuring that requests to ArcGIS services + * are properly authenticated using the credentials provided. + */ + private _identityManager: ArcGISIdentityManager; + /** * Constructor. * @param layerInfo The layer info. * @param config The plugins configuration. * @param console Used to log to the console. */ - constructor(layerInfo: LayerInfo, config: ArcGISPluginConfig, console: Console) { - this._httpClient = new HttpClient(console, layerInfo.token); - this._url = layerInfo.url + '/query?where='; + constructor(layerInfo: LayerInfo, config: ArcGISPluginConfig, identityManager: ArcGISIdentityManager, console: Console) { + this._identityManager = identityManager; + this._url = new URL(layerInfo.url); + this._url.pathname += '/query'; this._console = console; this._config = config; } @@ -48,19 +51,23 @@ export class FeatureQuerier { * @param fields fields to query, all fields if not provided * @param geometry query the geometry, default is true */ - queryObservation(observationId: string, response: (result: QueryObjectResult) => void, fields?: string[], geometry?: boolean) { - let queryUrl = this._url + this._config.observationIdField + async queryObservation(observationId: string, response: (result: QueryObjectResult) => void, fields?: string[], geometry?: boolean) { + const queryUrl = new URL(this._url) if (this._config.eventIdField == null) { - queryUrl += ' LIKE \'' + observationId + this._config.idSeparator + '%\'' + queryUrl.searchParams.set('where', `${this._config.observationIdField} LIKE '${observationId}${this._config.idSeparator}%'`); } else { - queryUrl += '=\'' + observationId + '\'' + queryUrl.searchParams.set('where', `${this._config.observationIdField} = ${observationId}`); } - queryUrl += this.outFields(fields) + this.returnGeometry(geometry) - this._httpClient.sendGetHandleResponse(queryUrl, (chunk) => { - this._console.info('ArcGIS response for ' + queryUrl + ' ' + chunk) - const result = JSON.parse(chunk) as QueryObjectResult - response(result) + queryUrl.searchParams.set('outFields', this.outFields(fields)) + queryUrl.searchParams.set('returnGeometry', this.returnGeometry(geometry)) + + const queryResponse = await request(queryUrl.toString(), { + authentication: this._identityManager }); + + this._console.info('ArcGIS response for ' + queryUrl + ' ' + queryResponse) + const result = JSON.parse(queryResponse) as QueryObjectResult + response(result); } /** @@ -69,13 +76,18 @@ export class FeatureQuerier { * @param fields fields to query, all fields if not provided * @param geometry query the geometry, default is true */ - queryObservations(response: (result: QueryObjectResult) => void, fields?: string[], geometry?: boolean) { - let queryUrl = this._url + this._config.observationIdField + ' IS NOT NULL' + this.outFields(fields) + this.returnGeometry(geometry) - this._httpClient.sendGetHandleResponse(queryUrl, (chunk) => { - this._console.info('ArcGIS response for ' + queryUrl + ' ' + chunk) - const result = JSON.parse(chunk) as QueryObjectResult - response(result) + async queryObservations(response: (result: QueryObjectResult) => void, fields?: string[], geometry?: boolean) { + const queryUrl = new URL(this._url) + queryUrl.searchParams.set('where', `${this._config.observationIdField} IS NOT NULL`); + queryUrl.searchParams.set('outFields', this.outFields(fields)); + queryUrl.searchParams.set('returnGeometry', this.returnGeometry(geometry)); + const queryResponse = await request(queryUrl.toString(), { + authentication: this._identityManager }); + + this._console.info('ArcGIS response for ' + queryUrl + ' ' + queryResponse) + const result = JSON.parse(queryResponse) as QueryObjectResult + response(result) } /** @@ -83,13 +95,19 @@ export class FeatureQuerier { * @param response Function called once query is complete. * @param field field to query */ - queryDistinct(response: (result: QueryObjectResult) => void, field: string) { - let queryUrl = this._url + field + ' IS NOT NULL&returnDistinctValues=true' + this.outFields([field]) + this.returnGeometry(false) - this._httpClient.sendGetHandleResponse(queryUrl, (chunk) => { - this._console.info('ArcGIS response for ' + queryUrl + ' ' + chunk) - const result = JSON.parse(chunk) as QueryObjectResult - response(result) - }); + async queryDistinct(response: (result: QueryObjectResult) => void, field: string) { + const queryUrl = new URL(this._url); + queryUrl.searchParams.set('where', `${field} IS NOT NULL`); + queryUrl.searchParams.set('returnDistinctValues', 'true'); + queryUrl.searchParams.set('outFields', this.outFields([field])); + queryUrl.searchParams.set('returnGeometry', this.returnGeometry(false)); + const queryResponse = await request(queryUrl.toString(), { + authentication: this._identityManager + + }); + this._console.info('ArcGIS response for ' + queryUrl + ' ' + queryResponse) + const result = JSON.parse(queryResponse) as QueryObjectResult + response(result) } /** diff --git a/plugins/arcgis/service/src/FeatureService.ts b/plugins/arcgis/service/src/FeatureService.ts index 51237c9fd..d5d4adf18 100644 --- a/plugins/arcgis/service/src/FeatureService.ts +++ b/plugins/arcgis/service/src/FeatureService.ts @@ -1,6 +1,10 @@ import { LayerInfoResult } from "./LayerInfoResult"; import { FeatureServiceResult } from "./FeatureServiceResult"; import { HttpClient } from "./HttpClient"; +import { getIdentityManager } from "./ArcGISIdentityManagerFactory" +import { ArcGISIdentityManager, request } from "@esri/arcgis-rest-request" +import { queryFeatures, applyEdits, IQueryFeaturesOptions } from "@esri/arcgis-rest-feature-service"; +import { FeatureServiceConfig } from "./ArcGISConfig"; /** * Queries arc feature services and layers. @@ -10,72 +14,208 @@ export class FeatureService { /** * Used to make the get request about the feature layer. */ - private _httpClient: HttpClient; + // private _httpClient: HttpClient; /** * Used to log messages. */ private _console: Console; + private _config: FeatureServiceConfig; + private _identityManager: ArcGISIdentityManager; + /** * Constructor. * @param console Used to log messages. * @param token The access token. */ - constructor(console: Console, token?: string) { - this._httpClient = new HttpClient(console, token); + constructor(console: Console, config: FeatureServiceConfig, identityManager: ArcGISIdentityManager) { + this._config = config; + this._identityManager = identityManager; + // this._httpClient = new HttpClient(console, token); this._console = console; } - /** - * Queries an arc feature service. - * @param url The url to the arc feature layer. - * @param callback Function to call once response has been received and parsed. - */ - queryFeatureService(url: string, callback: (featureService: FeatureServiceResult) => void) { - this._httpClient.sendGetHandleResponse(url, this.parseFeatureService(url, callback)) - } + // TODO this entire class is a Work in Progress and not used. Currently using @esri/arcgis-rest-request and not arcgis-rest-js. + // By finishing this class, we can transition from low-level generic requests that leverage ArcGISIdentityManager for auth to higher-level strongly typed requests. - /** - * Parses the response from the feature service request and sends to the callback. - * @param url The url to the arc feature layer. - * @param callback The callback to call and send the feature service to. - */ - private parseFeatureService(url: string, callback: (featureService: FeatureServiceResult) => void) { - return (chunk: any) => { - this._console.log('Feature Service. url: ' + url + ', response: ' + chunk) - try { - const service = JSON.parse(chunk) as FeatureServiceResult - callback(service) - } catch (e) { - this._console.error(e) - } - } - } - /** - * Queries an arc feature layer to get info on the layer. - * @param url The url to the arc feature layer. - * @param infoCallback Function to call once response has been received and parsed. - */ - queryLayerInfo(url: string, infoCallback: (layerInfo: LayerInfoResult) => void) { - this._httpClient.sendGetHandleResponse(url, this.parseLayerInfo(url, infoCallback)); - } + // Query features using arcgis-rest-js's queryFeatures + async queryFeatureService(whereClause: string): Promise { + const queryParams = { + url: this._config.url, + where: whereClause, + authentication: this._identityManager + } as IQueryFeaturesOptions; + + this._console.log('Querying feature service with params:', queryParams); + + try { + const response = await queryFeatures(queryParams); + return response; + } catch (error) { + this._console.error('Error details:', error); + throw new Error(`Error querying feature service: ${(error as any).message}`); + } + // try { + // const response = await queryFeatures({ + // url: this._config.url, + // where: whereClause, + // authentication: this._identityManager, + // // outFields: '*', + // f: 'json', + // }); + // return response; + // } catch (error) { + // throw new Error(`Error querying feature service: ${error}`); + // } + } - /** - * Parses the response from the request and sends the layer info to the callback. - * @param url The url to the feature layer. - * @param infoCallback The callback to call and send the layer info to. - */ - private parseLayerInfo(url: string, infoCallback: (layerInfo: LayerInfoResult) => void) { - return (chunk: any) => { - this._console.log('Query Layer. url: ' + url + ', response: ' + chunk) - try { - const layerInfo = JSON.parse(chunk) as LayerInfoResult - infoCallback(layerInfo) - } catch (e) { - this._console.error(e) - } - } - } + // Generic method to query layer info + async queryLayerInfo(layerId: string | number): Promise { + const url = `${this._config.url}/${layerId}`; + try { + const response = await request(url, { + authentication: this._identityManager, + params: { f: 'json' }, + }); + return response; + } catch (error) { + throw new Error(`Error querying layer info: ${error}`); + } + } + + // Add feature using applyEdits + async addFeature(feature: any): Promise { + try { + const response = await applyEdits({ + url: this._config.url, + adds: [feature], + authentication: this._identityManager, + }); + return response; + } catch (error) { + throw new Error(`Error adding feature: ${error}`); + } + } + + // Update feature using applyEdits + async updateFeature(feature: any): Promise { + try { + const response = await applyEdits({ + url: this._config.url, + updates: [feature], + authentication: this._identityManager, + }); + return response; + } catch (error) { + throw new Error(`Error updating feature: ${error}`); + } + } + + // Delete feature using applyEdits + async deleteFeature(objectId: string | number): Promise { + try { + const response = await applyEdits({ + url: this._config.url, + deletes: [typeof objectId === 'number' ? objectId : parseInt(objectId as string, 10)], + authentication: this._identityManager, + }); + return response; + } catch (error) { + throw new Error(`Error deleting feature: ${error}`); + } + } + + // Batch operation using applyEdits + async applyEditsBatch(edits: { add?: any[], update?: any[], delete?: any[] }): Promise { + try { + const response = await applyEdits({ + url: this._config.url, + adds: edits.add || [], + updates: edits.update || [], + deletes: edits.delete || [], + authentication: this._identityManager, + }); + return response; + } catch (error) { + throw new Error(`Error applying edits: ${error}`); + } + } + + // /** + // * Queries an arc feature service. + // * @param url The url to the arc feature layer. + // * @param callback Function to call once response has been received and parsed. + // */ + // async queryFeatureService(config: FeatureServiceConfig, callback: (featureService: FeatureServiceResult) => void) { + // const httpClient = new HttpClient(this._console) + // try { + // const identityManager = await getIdentityManager(config, httpClient) + // const response = await request(config.url, { + // authentication: identityManager + // }) + // callback(response as FeatureServiceResult) + // } catch (err) { + // console.error(`Could not get ArcGIS layer info: ${err}`) + // // res.status(500).json({ message: 'Could not get ArcGIS layer info', error: err }) + // } + + // // this._httpClient.sendGetHandleResponse(url, this.parseFeatureService(url, callback)) + // } + + // /** + // * Parses the response from the feature service request and sends to the callback. + // * @param url The url to the arc feature layer. + // * @param callback The callback to call and send the feature service to. + // */ + // private parseFeatureService(url: string, callback: (featureService: FeatureServiceResult) => void) { + // return (chunk: any) => { + // this._console.log('Feature Service. url: ' + url + ', response: ' + chunk) + // try { + // const service = JSON.parse(chunk) as FeatureServiceResult + // callback(service) + // } catch (e) { + // this._console.error(e) + // } + // } + // } + + // /** + // * Queries an arc feature layer to get info on the layer. + // * @param url The url to the arc feature layer. + // * @param infoCallback Function to call once response has been received and parsed. + // */ + // async queryLayerInfo(config: FeatureServiceConfig, layerId: string | number, infoCallback: (layerInfo: LayerInfoResult) => void) { + // const httpClient = new HttpClient(this._console) + // try { + // const identityManager = await getIdentityManager(config, httpClient) + // const response = await request(config.url + '/' + layerId, { + // authentication: identityManager + // }) + // infoCallback(response as LayerInfoResult) + // } catch (err) { + // console.error(`Could not get ArcGIS layer info: ${err}`) + // // res.status(500).json({ message: 'Could not get ArcGIS layer info', error: err }) + // } + + // // this._httpClient.sendGetHandleResponse(url, this.parseLayerInfo(url, infoCallback)); + // } + + // /** + // * Parses the response from the request and sends the layer info to the callback. + // * @param url The url to the feature layer. + // * @param infoCallback The callback to call and send the layer info to. + // */ + // private parseLayerInfo(url: string, infoCallback: (layerInfo: LayerInfoResult) => void) { + // return (chunk: any) => { + // this._console.log('Query Layer. url: ' + url + ', response: ' + chunk) + // try { + // const layerInfo = JSON.parse(chunk) as LayerInfoResult + // infoCallback(layerInfo) + // } catch (e) { + // this._console.error(e) + // } + // } + // } } diff --git a/plugins/arcgis/service/src/FeatureServiceAdmin.ts b/plugins/arcgis/service/src/FeatureServiceAdmin.ts index 4539aa36a..7ed660f8a 100644 --- a/plugins/arcgis/service/src/FeatureServiceAdmin.ts +++ b/plugins/arcgis/service/src/FeatureServiceAdmin.ts @@ -7,6 +7,8 @@ import { ObservationsTransformer } from "./ObservationsTransformer" import { HttpClient } from './HttpClient' import { LayerInfoResult, LayerField } from "./LayerInfoResult" import FormData from 'form-data' +import { request } from '@esri/arcgis-rest-request' +import { getIdentityManager } from './ArcGISIdentityManagerFactory' /** * Administers hosted feature services such as layer creation and updates. @@ -432,9 +434,10 @@ export class FeatureServiceAdmin { * @param service feature service * @param layer layer */ - private create(service: FeatureServiceConfig, layer: Layer) { + private async create(service: FeatureServiceConfig, layer: Layer) { const httpClient = this.httpClient(service) + const identityManager = await getIdentityManager(service, httpClient) const url = this.adminUrl(service) + 'addToDefinition' this._console.info('ArcGIS feature service addToDefinition (create layer) url ' + url) @@ -442,7 +445,12 @@ export class FeatureServiceAdmin { const form = new FormData() form.append('addToDefinition', JSON.stringify(layer)) - httpClient.sendPostForm(url, form) + const postResponse = request(url, { + authentication: identityManager, + httpMethod: 'POST', + params: form + }); + console.log('Response: ' + postResponse) } @@ -452,12 +460,13 @@ export class FeatureServiceAdmin { * @param featureLayer feature layer * @param fields fields to add */ - private addFields(service: FeatureServiceConfig, featureLayer: FeatureLayerConfig, fields: Field[]) { + private async addFields(service: FeatureServiceConfig, featureLayer: FeatureLayerConfig, fields: Field[]) { const layer = {} as Layer layer.fields = fields const httpClient = this.httpClient(service) + const identityManager = await getIdentityManager(service, httpClient) const url = this.adminUrl(service) + featureLayer.layer.toString() + '/addToDefinition' this._console.info('ArcGIS feature layer addToDefinition (add fields) url ' + url) @@ -465,7 +474,12 @@ export class FeatureServiceAdmin { const form = new FormData() form.append('addToDefinition', JSON.stringify(layer)) - httpClient.sendPostForm(url, form) + const postResponse = request(url, { + authentication: identityManager, + httpMethod: 'POST', + params: form + }); + console.log('Response: ' + postResponse) } @@ -475,7 +489,7 @@ export class FeatureServiceAdmin { * @param featureLayer feature layer * @param fields fields to delete */ - private deleteFields(service: FeatureServiceConfig, featureLayer: FeatureLayerConfig, fields: LayerField[]) { + private async deleteFields(service: FeatureServiceConfig, featureLayer: FeatureLayerConfig, fields: LayerField[]) { const deleteFields = [] for (const layerField of fields) { @@ -488,6 +502,7 @@ export class FeatureServiceAdmin { layer.fields = deleteFields const httpClient = this.httpClient(service) + const identityManager = await getIdentityManager(service, httpClient) const url = this.adminUrl(service) + featureLayer.layer.toString() + '/deleteFromDefinition' this._console.info('ArcGIS feature layer deleteFromDefinition (delete fields) url ' + url) @@ -496,7 +511,12 @@ export class FeatureServiceAdmin { form.append('deleteFromDefinition', JSON.stringify(layer)) httpClient.sendPostForm(url, form) - + const postResponse = request(url, { + authentication: identityManager, + httpMethod: 'POST', + params: form + }); + console.log('Response: ' + postResponse) } /** diff --git a/plugins/arcgis/service/src/ObservationProcessor.ts b/plugins/arcgis/service/src/ObservationProcessor.ts index 7bf5ab172..398c9a63a 100644 --- a/plugins/arcgis/service/src/ObservationProcessor.ts +++ b/plugins/arcgis/service/src/ObservationProcessor.ts @@ -17,6 +17,9 @@ import { EventLayerProcessorOrganizer } from './EventLayerProcessorOrganizer'; import { FeatureServiceConfig, FeatureLayerConfig, AuthType } from "./ArcGISConfig" import { PluginStateRepository } from '@ngageoint/mage.service/lib/plugins.api' import { FeatureServiceAdmin } from './FeatureServiceAdmin'; +import { HttpClient } from './HttpClient' +import { getIdentityManager } from "./ArcGISIdentityManagerFactory" +import { request } from '@esri/arcgis-rest-request'; /** * Class that wakes up at a certain configured interval and processes any new observations that can be @@ -172,7 +175,7 @@ export class ObservationProcessor { * Gets information on all the configured features service layers. * @param config The plugins configuration. */ - private getFeatureServiceLayers(config: ArcGISPluginConfig) { + private async getFeatureServiceLayers(config: ArcGISPluginConfig) { // TODO: What is the impact of what this is doing? Do we need to account for usernamePassword auth type services? for (const service of config.featureServices) { @@ -204,8 +207,18 @@ export class ObservationProcessor { } for (const serv of services) { - const featureService = new FeatureService(console, (serv.auth?.type === AuthType.Token && serv.auth?.token != null) ? serv.auth.token : '') - featureService.queryFeatureService(serv.url, (featureServiceResult: FeatureServiceResult) => this.handleFeatureService(featureServiceResult, serv, config)) + try { + const identityManager = await getIdentityManager(serv, new HttpClient(console)) + const response = await request(serv.url, { + authentication: identityManager + }) as FeatureServiceResult + this.handleFeatureService(response, serv, config); + } catch (err) { + console.error(err) + // res.status(500).json({ message: 'Could not get ArcGIS layer info', error: err }) + } + + } } } @@ -267,9 +280,11 @@ export class ObservationProcessor { if (layerId != null) { featureLayer.layer = layerId - const featureService = new FeatureService(console, featureLayer.token) - const url = featureServiceConfig.url + '/' + layerId - featureService.queryLayerInfo(url, (layerInfo: LayerInfoResult) => this.handleLayerInfo(url, featureServiceConfig, featureLayer, layerInfo, config)) + const identityManager = await getIdentityManager(featureServiceConfig, new HttpClient(console)) + const featureService = new FeatureService(console, featureServiceConfig, identityManager) + const layerInfo = await featureService.queryLayerInfo(layerId); + const url = `${featureServiceConfig.url}/${layerId}`; + this.handleLayerInfo(url, featureServiceConfig, featureLayer, layerInfo, config); } } @@ -293,7 +308,8 @@ export class ObservationProcessor { await admin.updateLayer(featureServiceConfig, featureLayer, layerInfo, this._eventRepo) } const info = new LayerInfo(url, events, layerInfo, featureLayer.token) - const layerProcessor = new FeatureLayerProcessor(info, config, this._console); + const identityManager = await getIdentityManager(featureServiceConfig, new HttpClient(console)) + const layerProcessor = new FeatureLayerProcessor(info, config, identityManager,this._console); this._layerProcessors.push(layerProcessor); clearTimeout(this._nextTimeout); this.scheduleNext(config);