From 68870dc3a8c2073050a7bda15f4ae1ca27345e92 Mon Sep 17 00:00:00 2001 From: Rick Wilson Date: Tue, 30 Jul 2024 11:52:31 -0400 Subject: [PATCH 1/2] init --- .../Open Data Network/README.MD | 121 ++ .../apiDefinition.swagger.json | 1083 +++++++++++++++++ .../Open Data Network/apiProperties.json | 160 +++ .../Open Data Network/script.csx | 397 ++++++ 4 files changed, 1761 insertions(+) create mode 100644 independent-publisher-connectors/Open Data Network/README.MD create mode 100644 independent-publisher-connectors/Open Data Network/apiDefinition.swagger.json create mode 100644 independent-publisher-connectors/Open Data Network/apiProperties.json create mode 100644 independent-publisher-connectors/Open Data Network/script.csx diff --git a/independent-publisher-connectors/Open Data Network/README.MD b/independent-publisher-connectors/Open Data Network/README.MD new file mode 100644 index 0000000000..4b17c8072b --- /dev/null +++ b/independent-publisher-connectors/Open Data Network/README.MD @@ -0,0 +1,121 @@ +# Open Data Network + +The Open Data Network connector enables users to access and utilize government data to improve decision-making and enrich lives. By leveraging data from various government sources, this connector allows developers, businesses, and public sector organizations to integrate valuable information into their applications and services, driving open data standards and best practices. The system has over 700+ Federal, State, and Local agencies contributing data and hosts tens of thousands of datasets. Examples of contributing agencies include the Centers for Disease Control and Prevention (CDC), Environmental Protection Agency (EPA), Federal Bureau of Investigation (FBI), and National Aeronautics and Space Administration (NASA). + +## Publisher: Richard Wilson + +## Prerequisites + +To use this connector, you must have an API key or App Token. You can obtain an API key by creating an account on the Open Data Network website and requesting access to the API. Please see the "Obtaining Credentials" section for detailed instructions on generating App Tokens and API Keys. + +## Supported Operations + +### Search Catalog + +Search for open data network assets using various query parameters. + +- **Inputs**: + + | Name | Description | + |--------------------|-------------------------------------------------------------------------------------------------------| + | `Id` | The unique identifier of an asset | + | `Domain` | The domain name | + | `Name` | The case-insensitive asset name | + | `Category` | A single category | + | `Tag` | A single tag | + | `Only` | One of the asset types (e.g., dataset, chart, map) | + | `Attribution` | The case-sensitive name of the attributing entity | + | `License` | The case-sensitive license name | + | `Full-Text Search Query` | Search for assets by any text in their name, description, category, tags, or other fields | + | `Min Should Match` | The number or percent of words that must match | + | `Parent Id` | The unique identifier of a parent asset | + | `Derived From` | The unique identifier of an asset derived from another | + | `Provenance` | The provenance, either 'official' or 'community' | + | `For User` | The unique identifier of a user or a team | + | `Shared To` | The unique identifier of a user or team to whom assets are shared | + | `Column Name` | The name of a column | + | `Public` | A true or false value for public or private assets | + | `Visibility` | The visibility status, either 'open' or 'internal' | + | `Audience` | The audience, either 'private', 'site' or 'public' | + | `Published` | A true or false value for published or unpublished assets | + | `Approval Status` | The approval status (pending, rejected, approved, not_ready) | + | `Explicitly Hidden`| A true or false value for hidden or unhidden assets | + | `Data Json Hidden` | A true or false value for assets hidden or unhidden from data.json catalog | + | `Derived` | A true or false value for derived or base assets | + | `Order` | The sort order of the results | + | `Offset` | Initial starting point for paging (0 by default) | + | `Limit` | Number of results to return (100 by default, up to 10000) | + | `Scroll Id` | The identifier for the last asset from the previous results | + | `Boost Official` | Multiplier for the relevance score of official documents | + | `Show Visibility` | Whether to return visibility information | + +- **Outputs**: + + Returns a list of open data network assets matching the search criteria, including asset details such as name, description, and metadata. + +### Search Dataset with SoQL + +Retrieve data from a dataset using SoQL query. + +- **Inputs**: + + | Name | Description | + |---------------------|------------------------------------------------------------------| + | `Domain` | Dataset publishers | + | `Dataset` | The unique identifier of the dataset resource | + | `Limit` | The number of records to return | + | `Offset` | The offset for pagination | + | `Select` | The set of columns to be returned, similar to a SELECT in SQL | + | `Where` | Filters the rows to be returned, similar to WHERE | + | `Order` | Column to order results on, similar to ORDER BY in SQL | + | `Group` | Column to group results on, similar to GROUP BY in SQL | + | `Having` | Filters the rows that result from an aggregation, similar to HAVING | + | `Full Text Search` | Performs a full text search for a value | + | `Full SoQL Query` | A full SoQL query string, all as one parameter | + +- **Outputs**: + + Returns the data from the specified dataset resource, based on the SoQL query parameters. + +### Get Search Data + +Get data for search fields. + +- **Inputs**: + + | Name | Description | + |-----------|----------------------------------------------------| + | `Domain` | The domain of the dataset | + | `Dataset` | The unique identifier of the dataset resource | + | `Query` | The search query to get values | + | `Limit` | The limit of values to return | + +- **Outputs**: + + Returns search data values for the specified query parameters. + +## Obtaining Credentials + +### Generating App Tokens and API Keys + +#### What is an App Token? + +An Application Token is an alphanumeric string that authorizes you to create an application. App tokens can be used as part of the authentication process to perform read operations through the API. Data & Insights users can leverage the app tokens to reach out to users in case their application is causing too many calls per unit time, and they need to be throttled. + +#### What is an API Key? + +API Keys can be used to perform read, write, and delete operations through the API. These operations will be available to each user according to their role on the domain they are accessing. An API Key comes with a secret key, which together serve as a proxy for a user's username and password. The advantage of using an API Key + Secret Key is that it allows a user to authenticate without showing their username, and it will not change as the user's Data & Insights password changes. + +#### Obtaining an App Token and API Keys + +1. Start by logging into your Data & Insights account on any Data & Insights domain, such as [evergreen.data.socrata.com](https://evergreen.data.socrata.com). +2. Navigate to your profile page by selecting the rightmost icon on the header bar. +3. Click on "Your Profile" and select the pencil icon to edit your profile. +4. In the profile settings, navigate to "Developer Settings" for API keys and app tokens. +5. Click "Create New App Token" and fill in the sections for Name and Description. For example, if you will be using this token to authorize automated DataSync updates, you could call the app token “Your Name - DataSync token” and as a description enter “App token used for updating datasets on ‘x’ domain”. + +Please note: Your Application Name must be unique across all registered applications on all Data & Insights domains. + +## Known Issues and Limitations + +Currently, no known issues or limitations exist. Always refer to this section for updated information. diff --git a/independent-publisher-connectors/Open Data Network/apiDefinition.swagger.json b/independent-publisher-connectors/Open Data Network/apiDefinition.swagger.json new file mode 100644 index 0000000000..70d66bbcdc --- /dev/null +++ b/independent-publisher-connectors/Open Data Network/apiDefinition.swagger.json @@ -0,0 +1,1083 @@ +{ + "swagger": "2.0", + "info": { + "title": "Open Data Network", + "description": "The Open Data Network connector enables users to access and utilize government data to improve decision-making and enrich lives. By leveraging data from various government sources, this connector allows developers, businesses, and public sector organizations to integrate valuable information into their applications and services, driving open data standards and best practices.", + "version": "1.0", + "contact": { + "name": "Richard Wilson", + "email": "richard.a.wilson@microsoft.com", + "url": "https://www.richardawilson.com/" + } + }, + "host": "api.us.socrata.com", + "basePath": "/api", + "schemes": ["https"], + "consumes": ["application/json"], + "produces": ["application/json"], + "paths": { + "/catalog/v1": { + "get": { + "operationId": "SearchCatalog", + "summary": "Search catalog for assets", + "description": "Search for open data network assets.", + "parameters": [ + { + "name": "ids", + "in": "query", + "description": "The unique identifier of an asset", + "type": "string", + "x-ms-summary": "Id" + }, + { + "name": "domains", + "in": "query", + "description": "The domain name", + "type": "string", + "x-ms-summary": "Domain", + "x-ms-dynamic-values": { + "operationId": "CountByDomain", + "value-collection": "results", + "value-path": "domain" + } + }, + { + "name": "names", + "in": "query", + "description": "The case-insensitive asset name", + "type": "string", + "x-ms-summary": "Name" + }, + { + "name": "categories", + "in": "query", + "description": "A single category", + "type": "string", + "x-ms-summary": "Category", + "x-ms-dynamic-values": { + "operationId": "CountByCategory", + "value-collection": "results", + "value-path": "category" + } + }, + { + "name": "tags", + "in": "query", + "description": "A single tag", + "type": "string", + "x-ms-summary": "Tag", + "x-ms-dynamic-values": { + "operationId": "CountByTag", + "value-collection": "results", + "value-path": "tag" + } + }, + { + "name": "only", + "in": "query", + "description": "One of the asset types (e.g., dataset, chart, map)", + "type": "string", + "x-ms-summary": "Only", + "enum": [ + "api", + "calendar", + "chart", + "datalens", + "dataset", + "federated_href", + "file", + "filter", + "form", + "href", + "link", + "map", + "measure", + "story", + "visualization" + ], + "x-ms-enum-values": [ + { "displayName": "API", "value": "api" }, + { "displayName": "Calendar", "value": "calendar" }, + { "displayName": "Chart", "value": "chart" }, + { "displayName": "Datalens", "value": "datalens" }, + { "displayName": "Dataset", "value": "dataset" }, + { "displayName": "Federated Href", "value": "federated_href" }, + { "displayName": "File", "value": "file" }, + { "displayName": "Filter", "value": "filter" }, + { "displayName": "Form", "value": "form" }, + { "displayName": "Href", "value": "href" }, + { "displayName": "Link", "value": "link" }, + { "displayName": "Map", "value": "map" }, + { "displayName": "Measure", "value": "measure" }, + { "displayName": "Story", "value": "story" }, + { "displayName": "Visualization", "value": "visualization" } + ] + }, + { + "name": "attribution", + "in": "query", + "description": "The case-sensitive name of the attributing entity", + "type": "string", + "x-ms-summary": "Attribution" + }, + { + "name": "license", + "in": "query", + "description": "The case-sensitive license name", + "type": "string", + "x-ms-summary": "License" + }, + { + "name": "q", + "in": "query", + "description": "Search for assets by any text in their name, description, category, tags, or other fields.", + "type": "string", + "x-ms-summary": "Full-Text Search Query" + }, + { + "name": "min_should_match", + "in": "query", + "description": "The number or percent of words that must match", + "type": "string", + "x-ms-summary": "Min Should Match" + }, + { + "name": "parent_ids", + "in": "query", + "description": "The unique identifier of a parent asset", + "type": "string", + "x-ms-summary": "Parent Id" + }, + { + "name": "derived_from", + "in": "query", + "description": "The unique identifier of an asset derived from another", + "type": "string", + "x-ms-summary": "Derived From" + }, + { + "name": "provenance", + "in": "query", + "description": "The provenance, either 'official' or 'community'", + "type": "string", + "x-ms-summary": "Provenance", + "enum": ["official", "community"], + "x-ms-enum-values": [ + { "displayName": "Official", "value": "official" }, + { "displayName": "Community", "value": "community" } + ] + }, + { + "name": "for_user", + "in": "query", + "description": "The unique identifier of a user or a team", + "type": "string", + "x-ms-summary": "For User" + }, + { + "name": "shared_to", + "in": "query", + "description": "The unique identifier of a user or team to whom assets are shared", + "type": "string", + "x-ms-summary": "Shared To" + }, + { + "name": "column_names", + "in": "query", + "description": "The name of a column", + "type": "string", + "x-ms-summary": "Column Name" + }, + { + "name": "public", + "in": "query", + "description": "A true or false value for public or private assets", + "type": "boolean", + "x-ms-summary": "Public" + }, + { + "name": "visibility", + "in": "query", + "description": "The visibility status, either 'open' or 'internal'", + "type": "string", + "x-ms-summary": "Visibility", + "enum": ["open", "internal"], + "x-ms-enum-values": [ + { "displayName": "Open", "value": "open" }, + { "displayName": "Internal", "value": "internal" } + ] + }, + { + "name": "audience", + "in": "query", + "description": "The audience, either 'private', 'site' or 'public'", + "type": "string", + "x-ms-summary": "Audience", + "enum": ["private", "site", "public"], + "x-ms-enum-values": [ + { "displayName": "Private", "value": "private" }, + { "displayName": "Site", "value": "site" }, + { "displayName": "Public", "value": "public" } + ] + }, + { + "name": "published", + "in": "query", + "description": "A true or false value for published or unpublished assets", + "type": "boolean", + "x-ms-summary": "Published" + }, + { + "name": "approval_status", + "in": "query", + "description": "The approval status (pending, rejected, approved, not_ready)", + "type": "string", + "x-ms-summary": "Approval Status", + "enum": ["pending", "rejected", "approved", "not_ready"], + "x-ms-enum-values": [ + { + "displayName": "Pending", + "value": "pending" + }, + { + "displayName": "Rejected", + "value": "rejected" + }, + { + "displayName": "Approved", + "value": "approved" + }, + { + "displayName": "Not Ready", + "value": "not_ready" + } + ] + }, + { + "name": "explicitly_hidden", + "in": "query", + "description": "A true or false value for hidden or unhidden assets", + "type": "boolean", + "x-ms-summary": "Explicitly Hidden" + }, + { + "name": "data_json_hidden", + "in": "query", + "description": "A true or false value for assets hidden or unhidden from data.json catalog", + "type": "boolean", + "x-ms-summary": "Data Json Hidden" + }, + { + "name": "derived", + "in": "query", + "description": "A true or false value for derived or base assets", + "type": "boolean", + "x-ms-summary": "Derived" + }, + { + "name": "order", + "in": "query", + "description": "The sort order of the results", + "type": "string", + "x-ms-summary": "Order", + "enum": [ + "relevance DESC", + "relevance ASC", + "name ASC", + "name DESC", + "owner ASC", + "owner DESC", + "dataset_id ASC", + "dataset_id DESC", + "datatype ASC", + "datatype DESC", + "domain_category ASC", + "domain_category DESC", + "createdAt DESC", + "createdAt ASC", + "updatedAt DESC", + "updatedAt ASC", + "page_views_total DESC", + "page_views_total ASC", + "page_views_last_month DESC", + "page_views_last_month ASC", + "page_views_last_week DESC", + "page_views_last_week ASC" + ], + "x-ms-enum-values": [ + { + "displayName": "Relevance Descending", + "value": "relevance DESC" + }, + { + "displayName": "Relevance Ascending", + "value": "relevance ASC" + }, + { + "displayName": "Name Ascending", + "value": "name ASC" + }, + { + "displayName": "Name Descending", + "value": "name DESC" + }, + { + "displayName": "Owner Ascending", + "value": "owner ASC" + }, + { + "displayName": "Owner Descending", + "value": "owner DESC" + }, + { + "displayName": "Dataset ID Ascending", + "value": "dataset_id ASC" + }, + { + "displayName": "Dataset ID Descending", + "value": "dataset_id DESC" + }, + { + "displayName": "Datatype Ascending", + "value": "datatype ASC" + }, + { + "displayName": "Datatype Descending", + "value": "datatype DESC" + }, + { + "displayName": "Domain Category Ascending", + "value": "domain_category ASC" + }, + { + "displayName": "Domain Category Descending", + "value": "domain_category DESC" + }, + { + "displayName": "Created At Descending", + "value": "createdAt DESC" + }, + { + "displayName": "Created At Ascending", + "value": "createdAt ASC" + }, + { + "displayName": "Updated At Descending", + "value": "updatedAt DESC" + }, + { + "displayName": "Updated At Ascending", + "value": "updatedAt ASC" + }, + { + "displayName": "Page Views Total Descending", + "value": "page_views_total DESC" + }, + { + "displayName": "Page Views Total Ascending", + "value": "page_views_total ASC" + }, + { + "displayName": "Page Views Last Month Descending", + "value": "page_views_last_month DESC" + }, + { + "displayName": "Page Views Last Month Ascending", + "value": "page_views_last_month ASC" + }, + { + "displayName": "Page Views Last Week Descending", + "value": "page_views_last_week DESC" + }, + { + "displayName": "Page Views Last Week Ascending", + "value": "page_views_last_week ASC" + } + ] + }, + { + "name": "offset", + "in": "query", + "description": "Initial starting point for paging (0 by default)", + "type": "integer", + "x-ms-summary": "Offset" + }, + { + "name": "limit", + "in": "query", + "description": "Number of results to return (100 by default, up to 10000)", + "type": "integer", + "x-ms-summary": "Limit" + }, + { + "name": "scroll_id", + "in": "query", + "description": "The identifier for the last asset from the previous results", + "type": "string", + "x-ms-summary": "Scroll Id" + }, + { + "name": "boostOfficial", + "in": "query", + "description": "Multiplier for the relevance score of official documents", + "type": "number", + "x-ms-summary": "Boost Official" + }, + { + "name": "show_visibility", + "in": "query", + "description": "Whether to return visibility information", + "type": "boolean", + "x-ms-summary": "Show Visibility" + } + ], + "responses": { + "200": { + "description": "Successful response", + "schema": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "resource": { + "type": "object", + "properties": { + "name": { + "type": "string", + "x-ms-summary": "Asset Name", + "description": "The name of the asset." + }, + "id": { + "type": "string", + "x-ms-summary": "Asset ID", + "description": "The unique identifier of the asset." + }, + "description": { + "type": "string", + "x-ms-summary": "Description", + "description": "A brief description of the asset." + }, + "attribution": { + "type": "string", + "x-ms-summary": "Attribution", + "description": "The entity attributed for the asset." + }, + "type": { + "type": "string", + "x-ms-summary": "Type", + "description": "The type of the asset." + }, + "updatedAt": { + "type": "string", + "format": "date-time", + "x-ms-summary": "Updated At", + "description": "The date and time when the asset was last updated." + }, + "createdAt": { + "type": "string", + "format": "date-time", + "x-ms-summary": "Created At", + "description": "The date and time when the asset was created." + }, + "columns_name": { + "type": "array", + "items": { + "type": "string" + }, + "x-ms-summary": "Columns Name", + "description": "Names of the columns in the asset." + }, + "columns_field_name": { + "type": "array", + "items": { + "type": "string" + }, + "x-ms-summary": "Columns Field Name", + "description": "Field names of the columns in the asset." + }, + "columns_datatype": { + "type": "array", + "items": { + "type": "string" + }, + "x-ms-summary": "Columns Datatype", + "description": "Datatypes of the columns in the asset." + }, + "columns_description": { + "type": "array", + "items": { + "type": "string" + }, + "x-ms-summary": "Columns Description", + "description": "Descriptions of the columns in the asset." + }, + "page_views_total": { + "type": "number", + "x-ms-summary": "Page Views Total", + "description": "Total number of page views." + }, + "page_views_last_week": { + "type": "number", + "x-ms-summary": "Page Views Last Week", + "description": "Number of page views in the last week." + }, + "page_views_last_month": { + "type": "number", + "x-ms-summary": "Page Views Last Month", + "description": "Number of page views in the last month." + }, + "download_count": { + "type": "number", + "x-ms-summary": "Download Count", + "description": "Total number of downloads." + }, + "provenance": { + "type": "string", + "x-ms-summary": "Provenance", + "description": "The provenance of the asset." + }, + "categories": { + "type": "array", + "items": { + "type": "string" + }, + "x-ms-summary": "Categories", + "description": "Categories of the asset." + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "x-ms-summary": "Tags", + "description": "Tags associated with the asset." + }, + "domain": { + "type": "string", + "x-ms-summary": "Domain", + "description": "The domain of the asset." + }, + "is_public": { + "type": "boolean", + "x-ms-summary": "Is Public", + "description": "Whether the asset is public." + }, + "is_published": { + "type": "boolean", + "x-ms-summary": "Is Published", + "description": "Whether the asset is published." + }, + "is_hidden": { + "type": "boolean", + "x-ms-summary": "Is Hidden", + "description": "Whether the asset is hidden." + } + } + } + } + } + }, + "resultSetSize": { + "type": "number", + "x-ms-summary": "Result Set Size", + "description": "The size of the result set." + }, + "warnings": { + "type": "array", + "items": { + "type": "string" + }, + "x-ms-summary": "Warnings", + "description": "Warnings associated with the query." + } + } + } + } + } + } + }, + "/catalog/v1/domains": { + "get": { + "operationId": "CountByDomain", + "summary": "Count by domain", + "description": "Returns each domain and the count of assets owned by that domain.", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "domain": { + "type": "string", + "description": "The domain name." + }, + "count": { + "type": "number", + "description": "The count of assets owned by the domain." + } + } + } + } + } + } + }, + "400": { + "description": "Invalid request" + } + }, + "x-ms-visibility": "internal" + } + }, + "/catalog/v1/categories": { + "get": { + "operationId": "CountByCategory", + "summary": "Count by category", + "description": "Returns each category and the count of assets having that category.", + "responses": { + "200": { + "description": "A list of categories and their counts", + "schema": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "category": { + "type": "string", + "description": "The domain-specific category." + }, + "count": { + "type": "number", + "description": "The count of assets having that category." + } + } + } + } + } + } + } + }, + "x-ms-visibility": "internal" + } + }, + "/catalog/v1/tags": { + "get": { + "operationId": "CountByTag", + "summary": "Count by tag", + "description": "Returns each asset tag and the count of assets having that tag.", + "responses": { + "200": { + "description": "A list of tags and their counts", + "schema": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "tag": { + "type": "string", + "description": "The asset tag." + }, + "count": { + "type": "number", + "description": "The count of assets having that tag." + } + } + } + } + } + } + } + }, + "x-ms-visibility": "internal" + } + }, + "/views/{asset_id}": { + "get": { + "operationId": "GetDatasetSchema", + "summary": "Get Dataset Schema", + "description": "Retrieve the schema properties of a dataset by its Id", + "parameters": [ + { + "name": "domain", + "in": "header", + "type": "string", + "required": true, + "description": "Dataset domain", + "x-ms-summary": "Domain" + }, + { + "name": "asset_id", + "in": "path", + "required": true, + "description": "The unique identifier of the asset", + "type": "string", + "x-ms-summary": "Dataset", + "x-ms-url-encoding": "single" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "object", + "properties": { + "data": { + "type": "object" + } + } + } + }, + "400": { + "description": "Invalid type specified" + } + }, + "x-ms-visibility": "internal" + } + }, + "/resource/{resource_id}.json": { + "post": { + "operationId": "SearchDataset", + "summary": "Search dataset", + "description": "Retrieve data from a dataset using basic filtering.", + "parameters": [ + { + "name": "domain", + "in": "header", + "required": true, + "description": "Dataset publisher", + "type": "string", + "x-ms-summary": "Domain", + "x-ms-dynamic-values": { + "operationId": "CountByDomain", + "value-collection": "results", + "value-path": "domain" + } + }, + { + "name": "resource_id", + "in": "path", + "required": true, + "description": "The unique identifier of the dataset resource", + "type": "string", + "x-ms-summary": "Dataset", + "x-ms-url-encoding": "single", + "x-ms-dynamic-values": { + "operationId": "SearchCatalog", + "value-path": "resource/id", + "value-title": "resource/name", + "value-collection": "results", + "parameters": { + "only": "dataset", + "domains": { + "parameter": "domain" + } + } + } + }, + { + "name": "item_properties", + "in": "body", + "required": true, + "schema": { + "x-ms-dynamic-properties": { + "operationId": "GetDatasetSchema", + "parameters": { + "asset_id": { + "parameterReference": "resource_id" + }, + "domain": { + "parameterReference": "domain" + } + }, + "itemValuePath": "data" + } + } + }, + { + "name": "$limit", + "in": "query", + "required": false, + "description": "The number of records to return", + "type": "integer", + "x-ms-summary": "Limit" + }, + { + "name": "$offset", + "in": "query", + "required": false, + "description": "The offset for pagination", + "type": "integer", + "x-ms-summary": "Offset" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "x-ms-dynamic-schema": { + "operationId": "GetDatasetSchema", + "parameters": { + "asset_id": { + "parameter": "resource_id" + }, + "domain": { + "parameter": "domain" + } + }, + "value-path": "data" + }, + "x-ms-dynamic-properties": { + "operationId": "GetDatasetSchema", + "parameters": { + "asset_id": { + "parameterReference": "resource_id" + }, + "domain": { + "parameterReference": "domain" + } + }, + "itemValuePath": "data" + } + } + } + }, + "400": { + "description": "Invalid type specified" + } + } + } + }, + "/resource/{resource_id}.json/SoQL": { + "get": { + "operationId": "SearchDatasetSoQL", + "summary": "Search dataset with SoQL", + "description": "Retrieve data from a dataset using SoQL query.", + "parameters": [ + { + "name": "domain", + "in": "header", + "required": true, + "description": "Dataset publisher", + "type": "string", + "x-ms-summary": "Domain", + "x-ms-dynamic-values": { + "operationId": "CountByDomain", + "value-collection": "results", + "value-path": "domain" + } + }, + { + "name": "resource_id", + "in": "path", + "required": true, + "description": "The unique identifier of the dataset resource", + "type": "string", + "x-ms-summary": "Dataset", + "x-ms-url-encoding": "single", + "x-ms-dynamic-values": { + "operationId": "SearchCatalog", + "value-path": "resource/id", + "value-title": "resource/name", + "value-collection": "results", + "parameters": { + "only": "dataset", + "domains": { + "parameter": "domain" + } + } + } + }, + { + "name": "$limit", + "in": "query", + "required": false, + "description": "The number of records to return", + "type": "integer", + "x-ms-summary": "Limit" + }, + { + "name": "$offset", + "in": "query", + "required": false, + "description": "The offset for pagination", + "type": "integer", + "x-ms-summary": "Offset" + }, + { + "name": "$select", + "in": "query", + "required": false, + "description": "The set of columns to be returned, similar to a SELECT in SQL", + "type": "string", + "x-ms-summary": "Select" + }, + { + "name": "$where", + "in": "query", + "required": false, + "description": "Filters the rows to be returned, similar to WHERE", + "type": "string", + "x-ms-summary": "Where" + }, + { + "name": "$order", + "in": "query", + "required": false, + "description": "Column to order results on, similar to ORDER BY in SQL", + "type": "string", + "x-ms-summary": "Order" + }, + { + "name": "$group", + "in": "query", + "required": false, + "description": "Column to group results on, similar to GROUP BY in SQL", + "type": "string", + "x-ms-summary": "Group" + }, + { + "name": "$having", + "in": "query", + "required": false, + "description": "Filters the rows that result from an aggregation, similar to HAVING", + "type": "string", + "x-ms-summary": "Having" + }, + { + "name": "$q", + "in": "query", + "required": false, + "description": "Performs a full text search for a value", + "type": "string", + "x-ms-summary": "Full Text Search" + }, + { + "name": "$query", + "in": "query", + "required": false, + "description": "A full SoQL query string, all as one parameter", + "type": "string", + "x-ms-summary": "Full SoQL Query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "type": "object" + } + } + }, + "400": { + "description": "Invalid type specified" + } + } + } + }, + "/resource/{resource_id}.json/GetSearchData": { + "get": { + "operationId": "GetSearchData", + "summary": "Get search data", + "description": "Get data for search fields.", + "parameters": [ + { + "name": "domain", + "in": "header", + "type": "string", + "required": true, + "x-ms-summary": "Domain", + "description": "Dataset publisher" + }, + { + "name": "resource_id", + "in": "path", + "required": true, + "description": "The unique identifier of the dataset resource", + "type": "string", + "x-ms-summary": "Dataset", + "x-ms-url-encoding": "single" + }, + { + "name": "$query", + "in": "query", + "required": true, + "description": "The search query to get values", + "type": "string", + "x-ms-summary": "Query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "value": { + "title": "Value" + } + } + } + } + }, + "400": { + "description": "Invalid type specified" + } + }, + "x-ms-visibility": "internal" + } + } + }, + "definitions": {}, + "parameters": {}, + "responses": {}, + "securityDefinitions": { + "basic-auth": { + "type": "basic" + }, + "api-auth": { + "type": "apiKey", + "in": "header", + "name": "X-App-Token" + } + }, + "security": [ + { + "basic-auth": [], + "api-auth": [] + } + ], + "tags": [], + "x-ms-connector-metadata": [ + { + "propertyName": "Website", + "propertyValue": "https://www.opendatanetwork.com/" + }, + { + "propertyName": "Privacy policy", + "propertyValue": "https://www.tylertech.com/privacy" + }, + { + "propertyName": "Categories", + "propertyValue": "Data" + } + ] +} diff --git a/independent-publisher-connectors/Open Data Network/apiProperties.json b/independent-publisher-connectors/Open Data Network/apiProperties.json new file mode 100644 index 0000000000..94d1c0d61c --- /dev/null +++ b/independent-publisher-connectors/Open Data Network/apiProperties.json @@ -0,0 +1,160 @@ +{ + "properties": { + "connectionParameterSets": { + "uiDefinition": { + "displayName": "Authentication Type", + "description": "Choose the type of authentication to be used." + }, + "values": [ + { + "name": "api-auth", + "uiDefinition": { + "displayName": "App Token", + "description": "Log in using your App Token. This ensures requests are not throttled by IP, allowing greater throughput." + }, + "parameters": { + "api_key": { + "type": "securestring", + "uiDefinition": { + "constraints": { + "clearText": false, + "required": "true", + "tabIndex": 3 + }, + "schema": { + "description": "Enter your App Token", + "type": "securestring" + }, + "displayName": "App Token" + } + }, + "environment": { + "type": "string", + "defaultValue": "api.us.socrata.com", + "uiDefinition": { + "displayName": "Endpoint", + "schema": { + "description": "The domain discovery endpoint to use (North America or EU)", + "type": "string" + }, + "tooltip": "Select the domain discovery endpoint to use (North America or EU)", + "constraints": { + "required": "true", + "allowedValues": [ + { + "value": "api.us.socrata.com", + "text": "North America" + }, + { + "value": "api.eu.socrata.com", + "text": "EU" + } + ] + } + } + } + } + }, + { + "name": "basic-auth", + "uiDefinition": { + "displayName": "API Key", + "description": "Log in using your API Key ID and API Key Secret. Using an API Key will allow you to access datasets that have been shared with you." + }, + "parameters": { + "username": { + "type": "string", + "uiDefinition": { + "displayName": "API Key ID", + "schema": { + "description": "The API Key ID", + "type": "string" + }, + "tooltip": "Enter your API Key ID", + "constraints": { + "required": "true" + } + } + }, + "password": { + "type": "securestring", + "uiDefinition": { + "displayName": "API Key Secret", + "schema": { + "description": "The API Key Secret", + "type": "securestring" + }, + "tooltip": "Enter your API Key Secret", + "constraints": { + "required": "true" + } + } + }, + "environment": { + "type": "string", + "defaultValue": "api.us.socrata.com", + "uiDefinition": { + "displayName": "Endpoint", + "schema": { + "description": "The domain discovery endpoint to use (North America or EU)", + "type": "string" + }, + "tooltip": "Select the domain discovery endpoint to use (North America or EU)", + "constraints": { + "required": "true", + "allowedValues": [ + { + "value": "api.us.socrata.com", + "text": "North America" + }, + { + "value": "api.eu.socrata.com", + "text": "EU" + } + ] + } + } + } + } + } + ] + }, + "iconBrandColor": "#da3b01", + "scriptOperations": [], + "capabilities": [], + "policyTemplateInstances": [ + { + "templateId": "dynamichosturl", + "title": "Use Endpoint", + "parameters": { + "x-ms-apimTemplateParameter.urlTemplate": "https://@connectionParameters('environment')/api", + "x-ms-apimTemplate-operationName": [ + "CountByDomain", + "CountByCategory", + "CountByTag" + ] + } + }, + { + "templateId": "setheader", + "title": "API Version", + "parameters": { + "x-ms-apimTemplateParameter.name": "$$version", + "x-ms-apimTemplateParameter.value": "2.1", + "x-ms-apimTemplateParameter.existsAction": "append", + "x-ms-apimTemplate-policySection": "Request", + "x-ms-apimTemplate-operationName": [ + "CountByDomain", + "GetSearchData", + "GetDatasetSchema", + "SearchCatalog", + "SearchDataset", + "SearchDatasetSoQL" + ] + } + } + ], + "publisher": "Richard Wilson", + "stackOwner": "Tyler Technologies" + } +} \ No newline at end of file diff --git a/independent-publisher-connectors/Open Data Network/script.csx b/independent-publisher-connectors/Open Data Network/script.csx new file mode 100644 index 0000000000..2cf495a2c8 --- /dev/null +++ b/independent-publisher-connectors/Open Data Network/script.csx @@ -0,0 +1,397 @@ +public class Script : ScriptBase +{ + public override async Task ExecuteAsync() + { + this.Context.Logger.LogInformation("ExecuteAsync started"); + + // Check for the domain header and add X-Socrata-Host header + if (this.Context.Request.Headers.TryGetValues("domain", out var domainHeaderValues)) + { + var domain = domainHeaderValues.FirstOrDefault(); + if (!string.IsNullOrEmpty(domain)) + { + this.Context.Request.Headers.Add("X-Socrata-Host", domain); + } + } + + if (this.Context.OperationId.ToLower() == "getdatasetschema") + { + return await this.HandleGetDatasetSchema().ConfigureAwait(false); + } + else if (this.Context.OperationId.ToLower() == "searchdataset") + { + return await HandleSearchDatasetOperation().ConfigureAwait(false); + } + else if (this.Context.OperationId.ToLower() == "searchdatasetsoql") + { + return await HandleSearchDatasetSoQLOperation().ConfigureAwait(false); + } + else if (this.Context.OperationId.ToLower() == "getsearchdata") + { + return await HandleGetSearchDataOperation().ConfigureAwait(false); + } + + // If the operation ID does not match any of the predefined handlers, forward the request normally + HttpResponseMessage response; + try + { + response = await this.Context.SendAsync(this.Context.Request, this.CancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + this.Context.Logger.LogError($"Error forwarding request: {ex.Message}"); + response = new HttpResponseMessage(HttpStatusCode.InternalServerError) + { + Content = CreateJsonContent($"Error forwarding request: {ex.Message}") + }; + } + + return response; + } + + private async Task HandleGetDatasetSchema() + { + this.Context.Logger.LogInformation("HandleGetDatasetSchema started"); + + // Get the domain header value + if (!this.Context.Request.Headers.TryGetValues("domain", out var domainHeaderValues)) + { + this.Context.Logger.LogError("Domain header is missing."); + return new HttpResponseMessage(HttpStatusCode.BadRequest) + { + Content = CreateJsonContent("Domain header is missing.") + }; + } + + var domain = domainHeaderValues.FirstOrDefault(); + + if (string.IsNullOrEmpty(domain)) + { + this.Context.Logger.LogError("Domain header is empty."); + return new HttpResponseMessage(HttpStatusCode.BadRequest) + { + Content = CreateJsonContent("Domain header is empty.") + }; + } + + // Update the host in the URI + var uriBuilder = new UriBuilder(this.Context.Request.RequestUri) + { + Host = domain + }; + this.Context.Request.RequestUri = uriBuilder.Uri; + + HttpResponseMessage response; + try + { + response = await this.Context.SendAsync(this.Context.Request, this.CancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + this.Context.Logger.LogError($"Error sending request: {ex.Message}"); + return new HttpResponseMessage(HttpStatusCode.InternalServerError) + { + Content = CreateJsonContent($"Error sending request: {ex.Message}") + }; + } + + if (response.IsSuccessStatusCode) + { + this.Context.Logger.LogInformation("Request successful"); + + var responseString = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + this.Context.Logger.LogInformation($"Response received: {responseString}"); + + JObject schema; + try + { + schema = JObject.Parse(responseString); + } + catch (Exception ex) + { + this.Context.Logger.LogError($"Error parsing response: {ex.Message}"); + return new HttpResponseMessage(HttpStatusCode.InternalServerError) + { + Content = CreateJsonContent($"Error parsing response: {ex.Message}") + }; + } + + if (schema["columns"] == null) + { + this.Context.Logger.LogError("Columns section is missing in the schema."); + return new HttpResponseMessage(HttpStatusCode.InternalServerError) + { + Content = CreateJsonContent("Columns section is missing in the schema.") + }; + } + + var openApiSchema = new JObject + { + ["type"] = "object", + ["properties"] = new JObject(), + ["required"] = new JArray() + }; + + bool hasGeoType = false; + + foreach (var column in schema["columns"]) + { + string columnName = column["name"]?.ToString(); + string fieldName = column["fieldName"]?.ToString(); + string columnType = ConvertToOpenApiType(column["dataTypeName"]?.ToString()); + string columnDescription = column["description"]?.ToString(); + + if (string.IsNullOrEmpty(columnName) || string.IsNullOrEmpty(fieldName) || string.IsNullOrEmpty(columnType)) + { + this.Context.Logger.LogWarning("Column information is incomplete."); + continue; + } + + var property = new JObject + { + ["type"] = columnType, + ["title"] = columnName, + ["x-ms-dynamic-values"] = new JObject + { + ["operationId"] = "GetSearchData", + ["value-path"] = "value", + ["value-title"] = "value", + ["parameters"] = new JObject + { + ["domain"] = new JObject + { + ["parameter"] = "domain" + }, + ["resource_id"] = new JObject + { + ["parameter"] = "resource_id" + }, + ["$query"] = $"SELECT `{fieldName}` AS `__auto_alias_abcdef` |> SELECT DISTINCT `__auto_alias_abcdef` WHERE UPPER(`__auto_alias_abcdef`::text) like \"%%\" |> SELECT `__auto_alias_abcdef`::text as value" + } + } + }; + + if (!string.IsNullOrEmpty(columnDescription)) + { + property["description"] = columnDescription; + } + + openApiSchema["properties"][fieldName] = property; + } + + this.Context.Logger.LogInformation($"Transformed schema: {openApiSchema}"); + + var wrappedSchema = new JObject + { + ["data"] = openApiSchema, + }; + + response.Content = CreateJsonContent(wrappedSchema.ToString()); + } + else + { + this.Context.Logger.LogError($"Request failed with status code: {response.StatusCode}"); + } + + return response; + } + + private string ConvertToOpenApiType(string socrataType) + { + this.Context.Logger.LogInformation($"Converting Socrata type '{socrataType}' to OpenAPI type"); + + return socrataType switch + { + "text" => "string", + "number" => "integer", + "checkbox" => "boolean", + _ => "string" + }; + } + + private async Task HandleSearchDatasetOperation() + { + this.Context.Logger.LogInformation("HandleSearchDatasetOperation started"); + + // Get the domain header value + if (!this.Context.Request.Headers.TryGetValues("domain", out var domainHeaderValues)) + { + this.Context.Logger.LogError("Domain header is missing."); + return new HttpResponseMessage(HttpStatusCode.BadRequest) + { + Content = CreateJsonContent("Domain header is missing.") + }; + } + + var domain = domainHeaderValues.FirstOrDefault(); + + // Get the details of the incoming request + var requestContent = await this.Context.Request.Content.ReadAsStringAsync().ConfigureAwait(false); + + // Extract URL parameters + var requestParams = System.Web.HttpUtility.ParseQueryString(this.Context.Request.RequestUri.Query); + + // Parse the body to get additional parameters + if (!string.IsNullOrEmpty(requestContent)) + { + var bodyParams = JObject.Parse(requestContent); + foreach (var prop in bodyParams.Properties()) + { + requestParams[prop.Name] = prop.Value.ToString(); + } + } + + this.Context.Logger.LogInformation($"Request details before modification: Method={this.Context.Request.Method.Method}, URI={this.Context.Request.RequestUri}, Params={requestParams}"); + + // Declare the uriBuilder + var uriBuilder = new UriBuilder(this.Context.Request.RequestUri) + { + Query = requestParams.ToString(), + Host = domain + }; + + // Update the URI with the new query parameters + uriBuilder.Query = requestParams.ToString(); + this.Context.Request.RequestUri = uriBuilder.Uri; + + // Modify the request to be a GET + this.Context.Request.Method = HttpMethod.Get; + this.Context.Request.Content = null; // Clear the body content + + this.Context.Logger.LogInformation($"Modified request details: Method={this.Context.Request.Method}, URI={this.Context.Request.RequestUri}, Params={requestParams}"); + + // Forward the modified request + HttpResponseMessage response; + try + { + response = await this.Context.SendAsync(this.Context.Request, this.CancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + this.Context.Logger.LogError($"Error forwarding request: {ex.Message}"); + response = new HttpResponseMessage(HttpStatusCode.InternalServerError) + { + Content = CreateJsonContent($"Error forwarding request: {ex.Message}") + }; + } + + this.Context.Logger.LogInformation("HandleSearchDatasetOperation completed"); + + return response; + } + + private async Task HandleSearchDatasetSoQLOperation() + { + this.Context.Logger.LogInformation("HandleSearchDatasetSoQLOperation started"); + + // Get the domain header value + if (!this.Context.Request.Headers.TryGetValues("domain", out var domainHeaderValues)) + { + this.Context.Logger.LogError("Domain header is missing."); + return new HttpResponseMessage(HttpStatusCode.BadRequest) + { + Content = CreateJsonContent("Domain header is missing.") + }; + } + + var domain = domainHeaderValues.FirstOrDefault(); + + // Extract URL parameters + var requestParams = HttpUtility.ParseQueryString(this.Context.Request.RequestUri.Query); + + this.Context.Logger.LogInformation($"Request details before modification: Method={this.Context.Request.Method.Method}, URI={this.Context.Request.RequestUri}, Params={requestParams}"); + + // Modify the request to be a GET and update the URI with new query parameters + this.Context.Request.Method = HttpMethod.Get; + var uriBuilder = new UriBuilder(this.Context.Request.RequestUri) + { + Query = requestParams.ToString(), + Host = domain + }; + + // Remove /SoQL from the path + var newPath = uriBuilder.Path.Replace("/SoQL", ""); + uriBuilder.Path = newPath; + this.Context.Logger.LogInformation($"Updated path to: {newPath}"); + + this.Context.Request.RequestUri = uriBuilder.Uri; + this.Context.Request.Content = null; // Clear the body content + + this.Context.Logger.LogInformation($"Modified request details: Method={this.Context.Request.Method}, URI={this.Context.Request.RequestUri}, Params={requestParams}"); + + // Forward the modified request + HttpResponseMessage response; + try + { + response = await this.Context.SendAsync(this.Context.Request, this.CancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + this.Context.Logger.LogError($"Error forwarding request: {ex.Message}"); + response = new HttpResponseMessage(HttpStatusCode.InternalServerError) + { + Content = CreateJsonContent($"Error forwarding request: {ex.Message}") + }; + } + + this.Context.Logger.LogInformation("HandleSearchDatasetSoQLOperation completed"); + + return response; + } + + private async Task HandleGetSearchDataOperation() + { + this.Context.Logger.LogInformation("HandleGetSearchDataOperation started"); + + // Get the domain header value + if (!this.Context.Request.Headers.TryGetValues("domain", out var domainHeaderValues)) + { + this.Context.Logger.LogError("Domain header is missing."); + return new HttpResponseMessage(HttpStatusCode.BadRequest) + { + Content = CreateJsonContent("Domain header is missing.") + }; + } + + var domain = domainHeaderValues.FirstOrDefault(); + + // Remove /GetSearchData from the path + var originalUri = this.Context.Request.RequestUri.ToString(); + var newUri = originalUri.Replace("/GetSearchData", ""); + + // Update the host in the URI + var uriBuilder = new UriBuilder(newUri) + { + Host = domain + }; + + // Update the request URI + this.Context.Request.RequestUri = uriBuilder.Uri; + + this.Context.Logger.LogInformation($"Modified request URI: {this.Context.Request.RequestUri}"); + + // Forward the modified request + HttpResponseMessage response; + try + { + response = await this.Context.SendAsync(this.Context.Request, this.CancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + this.Context.Logger.LogError($"Error forwarding request: {ex.Message}"); + response = new HttpResponseMessage(HttpStatusCode.InternalServerError) + { + Content = CreateJsonContent($"Error forwarding request: {ex.Message}") + }; + } + + this.Context.Logger.LogInformation("HandleGetSearchDataOperation completed"); + + return response; + } + + private static HttpContent CreateJsonContent(string content) + { + return new StringContent(content, System.Text.Encoding.UTF8, "application/json"); + } +} \ No newline at end of file From 986a991e2dcc7885a77f6b115f6a257b0c82fe6c Mon Sep 17 00:00:00 2001 From: Rick Wilson Date: Thu, 17 Oct 2024 08:23:26 -0400 Subject: [PATCH 2/2] minor updates to create new PR --- independent-publisher-connectors/Open Data Network/README.MD | 1 + .../Open Data Network/apiDefinition.swagger.json | 2 +- .../Open Data Network/apiProperties.json | 2 +- independent-publisher-connectors/Open Data Network/script.csx | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/independent-publisher-connectors/Open Data Network/README.MD b/independent-publisher-connectors/Open Data Network/README.MD index 4b17c8072b..395c3ded42 100644 --- a/independent-publisher-connectors/Open Data Network/README.MD +++ b/independent-publisher-connectors/Open Data Network/README.MD @@ -119,3 +119,4 @@ Please note: Your Application Name must be unique across all registered applicat ## Known Issues and Limitations Currently, no known issues or limitations exist. Always refer to this section for updated information. + diff --git a/independent-publisher-connectors/Open Data Network/apiDefinition.swagger.json b/independent-publisher-connectors/Open Data Network/apiDefinition.swagger.json index 70d66bbcdc..5aefa20e52 100644 --- a/independent-publisher-connectors/Open Data Network/apiDefinition.swagger.json +++ b/independent-publisher-connectors/Open Data Network/apiDefinition.swagger.json @@ -1080,4 +1080,4 @@ "propertyValue": "Data" } ] -} +} \ No newline at end of file diff --git a/independent-publisher-connectors/Open Data Network/apiProperties.json b/independent-publisher-connectors/Open Data Network/apiProperties.json index 94d1c0d61c..f795acf6a2 100644 --- a/independent-publisher-connectors/Open Data Network/apiProperties.json +++ b/independent-publisher-connectors/Open Data Network/apiProperties.json @@ -157,4 +157,4 @@ "publisher": "Richard Wilson", "stackOwner": "Tyler Technologies" } -} \ No newline at end of file +} diff --git a/independent-publisher-connectors/Open Data Network/script.csx b/independent-publisher-connectors/Open Data Network/script.csx index 2cf495a2c8..d757bf7051 100644 --- a/independent-publisher-connectors/Open Data Network/script.csx +++ b/independent-publisher-connectors/Open Data Network/script.csx @@ -394,4 +394,4 @@ public class Script : ScriptBase { return new StringContent(content, System.Text.Encoding.UTF8, "application/json"); } -} \ No newline at end of file +}