Skip to content

Commit

Permalink
Merge pull request #150 from OCHA-DAP/HDXDSYS-984-3w-updates
Browse files Browse the repository at this point in the history
HDXDSYS-984 update 3W for several countries
  • Loading branch information
turnerm authored Aug 27, 2024
2 parents 16d2496 + 3d1ed34 commit ade99bf
Show file tree
Hide file tree
Showing 13 changed files with 4,478 additions and 4,845 deletions.
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).

## [0.9.49] - 2024-08-27

### Changed

- 3W updates for Afghanistan, Cameroon, CAR, Ethiopia, Mali , Nigeria,
South Sudan, Sudan, and Yemen

### Fixed

- Updated Colombia population resource names
Expand Down
136 changes: 70 additions & 66 deletions src/hapi/pipelines/configs/operational_presence.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ operational_presence_default:

operational_presence_admintwo:
operational_presence_afg:
dataset: "afghanistan-who-does-what-where-january-to-march-2024"
resource: "afghanistan-3w-operational-presence-january-march-2024.csv"
dataset: "afghanistan-who-does-what-where-april-to-june-2024"
resource: "afghanistan-3w-operational-presence-april-june-2024.csv"
format: "csv"
headers: 1
use_hxl: True
Expand All @@ -53,52 +53,55 @@ operational_presence_admintwo:

operational_presence_caf:
dataset: "republique-centrafricaine-presence-operationnelle"
resource: "3W_CAR_Dec2023"
resource: "3W_CAR_Mar2024"
format: "xlsx"
sheet: "COMPILALL"
headers: 1
sheet: "OCHA CAR 3W OP T1 2024"
headers:
- 1
- 2
source_date:
start: "01/10/2023"
end: "31/12/2023"
use_hxl: True
start: "01/01/2024"
end: "31/03/2024"
use_hxl: False
admin:
- ~
- "#adm2+code"
- "PCODE2 #adm2+code"
admin_exact: True
input:
- "#org+description"
- "#org+acronym"
- "#org+type"
- "#sector"
- "Description_Acteur"
- "ACTEUR #org+acronym"
- "TYPE_ORG"
- "SECTEUR #sector"
list:
- "#org+description"
- "#org+acronym"
- "#org+type"
- "#sector"
- "Description_Acteur"
- "ACTEUR #org+acronym"
- "TYPE_ORG"
- "SECTEUR #sector"

operational_presence_cmr:
dataset: "cameroon-5w-operational-presence"
resource: "cmr_3w_national_sep_2023"
resource: "cmr_3w_national_mar_2024"
format: "xlsx"
sheet: "data"
sheet: "db"
headers: 1
use_hxl: True
source_date:
start: "01/07/2023"
end: "30/09/2023"
start: "01/01/2024"
end: "31/03/2024"
admin:
- ~
- "adm2_pcode"
- "#adm2 +code"
admin_exact: True
input:
- "implementing_org_name"
- "implementing_org_acronym"
- "implementing_org_type"
- "clusters"
- "#org +impl +name"
- "#org + impl +acronym"
- "#org +impl +type"
- "#sector +cluster +name"
list:
- "implementing_org_name"
- "implementing_org_acronym"
- "implementing_org_type"
- "clusters"
- "#org +impl +name"
- "#org + impl +acronym"
- "#org +impl +type"
- "#sector +cluster +name"

operational_presence_cod:
dataset: "drc_presence_operationnelle"
Expand Down Expand Up @@ -224,13 +227,13 @@ operational_presence_admintwo:

operational_presence_mli:
dataset: "mali-operational-presence"
resource: "MALI 3W_Mars_2024"
resource: "MALI_3W_June_2024"
format: "xlsx"
sheet: "ML_5W"
headers: 1
source_date:
start: "01/03/2024"
end: "31/03/2024"
start: "01/04/2024"
end: "30/06/2024"
use_hxl: False
admin:
- ~
Expand Down Expand Up @@ -276,28 +279,28 @@ operational_presence_admintwo:

operational_presence_nga:
dataset: "nigeria-3w"
resource: "who-is-doing-what-and-where_nga_3w_oct_dec_2023_.xlsx"
resource: "who-is-doing-what-and-where_nga_3w_jan_mar_2024.xlsx"
format: "xlsx"
sheet: "NGA 3Ws Data Q4 2023"
sheet: "NGA 3Ws Q1(Jan-Mar) 2024"
headers: 1
source_date:
start: "01/10/2023"
end: "31/12/2023"
use_hxl: True
start: "01/01/2024"
end: "31/03/2024"
use_hxl: False
admin:
- ~
- "#adm2 +code"
- "LGA PCode"
admin_exact: True
input:
- "#org +acronym"
- "#org +acronym"
- "#org +type"
- "#sector +cluster"
- "Organisation"
- "Org. Acronym"
- "Type of Organization"
- "Project Sector"
list:
- "#org +acronym"
- "#org +acronym"
- "#org +type"
- "#sector +cluster"
- "Organisation"
- "Org. Acronym"
- "Type of Organization"
- "Project Sector"

operational_presence_som:
dataset: "somalia-operational-presence"
Expand Down Expand Up @@ -359,13 +362,13 @@ operational_presence_admintwo:

operational_presence_yem:
dataset: "yemen-monthly-organizations-presence-3w"
resource: "YEM_4W_Jan-Dec-2023.xlsx"
resource: "presence-data-jan-may-2024.xlsx"
format: "xlsx"
sheet: "v_4w_presence_admin2"
headers: 1
source_date:
start: "01/01/2023"
end: "31/12/2023"
start: "01/01/2024"
end: "31/05/2024"
admin:
- ~
- "admin2pcode"
Expand Down Expand Up @@ -437,14 +440,14 @@ operational_presence_national:

operational_presence_eth:
dataset: "ethiopia-operational-presence"
resource: "Ethiopia Who is Doing What Where (4W) - May 2024.xlsx"
resource: "Ethiopia Who is Doing What Where (4W) - July 2024.xlsx"
format: "xlsx"
sheet: "April 2024"
sheet: "July 2024"
headers: 1
use_hxl: True
source_date:
start: "01/05/2024"
end: "31/05/2024"
start: "01/07/2024"
end: "31/07/2024"
admin_single: "ETH"
input:
- "#org+impl+name"
Expand Down Expand Up @@ -516,34 +519,35 @@ operational_presence_national:

operational_presence_sdn:
dataset: "sudan-operational-presence"
resource: "2024-consolidated-3w-data-jan-to-31-may.xlsx"
resource: "2024-consolidated-3w-data-jan-to-30-june.xlsx"
format: "xlsx"
sheet: "3W_master_data"
headers: 1
use_hxl: True
source_date:
start: "01/01/2024"
end: "31/05/2024"
end: "30/06/2024"
admin_single: "SDN"
input:
- "Organization"
- "Org Acronym"
- "Org Type"
- "Sector"
- "#org+name"
- "#org+acronym"
- "#org+type"
- "#sector+cluster"
list:
- "Organization"
- "Org Acronym"
- "Org Type"
- "Sector"
- "#org+name"
- "#org+acronym"
- "#org+type"
- "#sector+cluster"

operational_presence_ssd:
dataset: "south-sudan-operational-presence"
resource: "ss_20240511_3w_oerational presence_Jan to Apr_2024.xlsx"
resource: "ss_20240808_3w_oerational presence_Jan to June_2024.xlsx"
format: "xlsx"
sheet: "data"
headers: 1
source_date:
start: "01/01/2024"
end: "30/04/2024"
end: "30/06/2024"
admin_single: "SSD"
input:
- "Name of organization"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"archived": false, "creator_user_id": "391f0864-b6e4-425f-9d46-df87aa456c2b", "data_update_frequency": "90", "dataset_date": "[2024-04-01T00:00:00 TO 2024-06-30T23:59:59]", "dataset_preview": "resource_id", "dataset_source": "Humanitarian partners", "due_date": "2024-11-12T12:03:52", "has_geodata": false, "has_quickcharts": true, "has_showcases": true, "id": "4aef067d-0761-4977-9020-e83d8f6908a4", "is_requestdata_type": false, "isopen": false, "last_modified": "2024-08-14T12:03:52.436695", "license_id": "cc-by-igo", "license_title": "Creative Commons Attribution for Intergovernmental Organisations", "license_url": "http://creativecommons.org/licenses/by/3.0/igo/legalcode", "maintainer": "391f0864-b6e4-425f-9d46-df87aa456c2b", "maintainer_email": "[email protected]", "metadata_created": "2024-08-14T12:03:16.272179", "metadata_modified": "2024-08-19T08:59:28.271891", "methodology": "Registry", "name": "afghanistan-who-does-what-where-april-to-june-2024", "notes": "The Who does What Where (3W) is a core humanitarian coordination dataset. It is critical to know where humanitarian organizations are working, what they are doing and their capability in order to identify gaps, avoid duplication of efforts, and plan for future humanitarian response (if needed). The data includes a list of humanitarian organizations by district and cluster, as well as a unique count of organizations. An interactive map of the 3W data can be accessed [here](https://response.reliefweb.int/afghanistan/who-does-what-where-3w?_gl=1*g902oc*_ga*MTYyOTk1NDg4OC4xNjc2NTM2NTk3*_ga_E60ZNX2F68*MTcwNDE3NzMzMC4xNDAuMS4xNzA0MTg0MTQ5LjU4LjAuMA). ", "num_resources": 3, "num_tags": 4, "organization": {"id": "10e168ce-5b51-49ac-8616-a142d48618e5", "name": "ocha-afghanistan", "title": "OCHA Afghanistan", "type": "organization", "description": "OCHA resumed its operation in Afghanistan in 2009, providing humanitarian assistance in a complex environment where separate \u2013 and not always complementary \u2013 military, political and security objectives pose challenges to the implementation of humanitarian principles, the ability of responders to reach people in need and the safety and security of aid workers", "image_url": "", "created": "2014-08-06T14:26:07.642577", "is_organization": true, "approval_status": "approved", "state": "active"}, "overdue_date": "2024-12-12T12:03:52", "owner_org": "10e168ce-5b51-49ac-8616-a142d48618e5", "package_creator": "murtaza", "pageviews_last_14_days": 27, "private": false, "qa_completed": false, "solr_additions": "{\"countries\": [\"Afghanistan\"]}", "state": "active", "subnational": "1", "title": "Afghanistan - Who does What Where (April to June 2024)", "total_res_downloads": 10, "type": "dataset", "url": null, "version": null, "groups": [{"description": "", "display_name": "Afghanistan", "id": "afg", "image_display_url": "", "name": "afg", "title": "Afghanistan"}], "tags": [{"display_name": "hxl", "id": "a0fbb23a-6aad-4ccc-8062-e9ef9f20e5d2", "name": "hxl", "state": "active", "vocabulary_id": "b891512e-9516-4bf5-962a-7a289772a2a1"}, {"display_name": "operational capacity", "id": "d8a59526-9f9a-4c71-b38f-5d9f2eb1615a", "name": "operational capacity", "state": "active", "vocabulary_id": "b891512e-9516-4bf5-962a-7a289772a2a1"}, {"display_name": "operational presence", "id": "a25059f9-7e1f-49be-b629-ccccd97a95f8", "name": "operational presence", "state": "active", "vocabulary_id": "b891512e-9516-4bf5-962a-7a289772a2a1"}, {"display_name": "who is doing what and where-3w-4w-5w", "id": "ec53893c-6dba-4656-978b-4a32289ea2eb", "name": "who is doing what and where-3w-4w-5w", "state": "active", "vocabulary_id": "b891512e-9516-4bf5-962a-7a289772a2a1"}], "relationships_as_subject": [], "relationships_as_object": [], "is_fresh": true, "update_status": "fresh", "x_resource_grouping": [], "resources": [{"alt_url": "https://data.humdata.org/dataset/4aef067d-0761-4977-9020-e83d8f6908a4/resource/3c406a4f-1d81-4c1a-9108-e43182dbe9f9/download/", "cache_last_updated": null, "cache_url": null, "created": "2024-08-14T12:03:19.154685", "dataset_preview_enabled": false, "datastore_active": false, "description": "", "download_url": "https://data.humdata.org/dataset/4aef067d-0761-4977-9020-e83d8f6908a4/resource/3c406a4f-1d81-4c1a-9108-e43182dbe9f9/download/afghanistan-3w-april-june-2024.xlsx", "format": "XLSX", "fs_check_info": "[{\"state\": \"processing\", \"message\": \"The processing of the file structure check has started\", \"timestamp\": \"2024-08-14T12:03:18.817099\"}, {\"state\": \"success\", \"message\": \"File structure check completed\", \"timestamp\": \"2024-08-14T12:03:21.113494\", \"sheet_changes\": [], \"hxl_proxy_response\": {\"url_or_filename\": \"https://data.humdata.org/dataset/4aef067d-0761-4977-9020-e83d8f6908a4/resource/3c406a4f-1d81-4c1a-9108-e43182dbe9f9/download/afghanistan-3w-april-june-2024.xlsx\", \"format\": \"XLSX\", \"sheets\": [{\"name\": \"MetaData\", \"is_hidden\": false, \"nrows\": 11, \"ncols\": 1, \"has_merged_cells\": false, \"is_hxlated\": false, \"header_hash\": \"8a9c0818e0622aadfcb2e59cfe5299b4\", \"hxl_header_hash\": null, \"headers\": [\"METADATA\"], \"hxl_headers\": null}, {\"name\": \"List_District\", \"is_hidden\": false, \"nrows\": 403, \"ncols\": 21, \"has_merged_cells\": false, \"is_hxlated\": true, \"header_hash\": \"30d419bf10484e12f5a01f983141414f\", \"hxl_header_hash\": \"7dafa0a7b166290817a95261ab0e3e99\", \"headers\": [\"Region\", \"PROV_CODE\", \"PROV_NA_ENG\", \"DIST_CODE\", \"DIST_NA_ENG\", \"OP_ALL\", \"OC_ALL\", \"OP_EIE\", \"OC_EIE\", \"OP_ESNFI\", \"OC_ESNFI\", \"OP_FSAC\", \"OC_FSAC\", \"OP_HEALTH\", \"OC_HEALTH\", \"OP_NUTRITION\", \"OC_NUTRITION\", \"OP_PROTECTION\", \"OC_PROTECTION\", \"OP_WASH\", \"OC_WASH\"], \"hxl_headers\": [\"#region+name\", \"#adm1+code\", \"#adm1+name\", \"#adm2+code\", \"#adm2+name\", \"#org+presence+total\", \"#org+capacity+total\", \"#org+presence+eie\", \"#org+capacity+eie\", \"#org+presence+esnfi\", \"#org+capacity+esnfi\", \"#org+presence+fsac\", \"#org+capacity+fsac\", \"#org+presence+health\", \"#org+capacity+health\", \"#org+presence+nutrition\", \"#org+capacity+nutrition\", \"#org+presence+protection\", \"#org+capacity+protection\", \"#org+presence+wash\", \"#org+capacity+wash\"]}, {\"name\": \"Count_District\", \"is_hidden\": false, \"nrows\": 403, \"ncols\": 21, \"has_merged_cells\": false, \"is_hxlated\": true, \"header_hash\": \"30d419bf10484e12f5a01f983141414f\", \"hxl_header_hash\": \"7dafa0a7b166290817a95261ab0e3e99\", \"headers\": [\"Region\", \"PROV_CODE\", \"PROV_NA_ENG\", \"DIST_CODE\", \"DIST_NA_ENG\", \"OP_ALL\", \"OC_ALL\", \"OP_EIE\", \"OC_EIE\", \"OP_ESNFI\", \"OC_ESNFI\", \"OP_FSAC\", \"OC_FSAC\", \"OP_HEALTH\", \"OC_HEALTH\", \"OP_NUTRITION\", \"OC_NUTRITION\", \"OP_PROTECTION\", \"OC_PROTECTION\", \"OP_WASH\", \"OC_WASH\"], \"hxl_headers\": [\"#region+name\", \"#adm1+code\", \"#adm1+name\", \"#adm2+code\", \"#adm2+name\", \"#org+presence+total\", \"#org+capacity+total\", \"#org+presence+eie\", \"#org+capacity+eie\", \"#org+presence+esnfi\", \"#org+capacity+esnfi\", \"#org+presence+fsac\", \"#org+capacity+fsac\", \"#org+presence+health\", \"#org+capacity+health\", \"#org+presence+nutrition\", \"#org+capacity+nutrition\", \"#org+presence+protection\", \"#org+capacity+protection\", \"#org+presence+wash\", \"#org+capacity+wash\"]}]}}]", "hash": "", "hdx_rel_url": "/dataset/4aef067d-0761-4977-9020-e83d8f6908a4/resource/3c406a4f-1d81-4c1a-9108-e43182dbe9f9/download/afghanistan-3w-april-june-2024.xlsx", "id": "3c406a4f-1d81-4c1a-9108-e43182dbe9f9", "last_modified": "2024-08-14T12:03:19.001306", "metadata_modified": "2024-08-14T12:18:30.551495", "microdata": false, "mimetype": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "mimetype_inner": null, "name": "afghanistan-3w-april-june-2024.xlsx", "originalHash": "8750915", "package_id": "4aef067d-0761-4977-9020-e83d8f6908a4", "pii": "false", "position": 0, "resource_type": "file.upload", "size": 146530, "state": "active", "url": "https://data.humdata.org/dataset/4aef067d-0761-4977-9020-e83d8f6908a4/resource/3c406a4f-1d81-4c1a-9108-e43182dbe9f9/download/afghanistan-3w-april-june-2024.xlsx", "url_type": "upload"}, {"alt_url": "https://data.humdata.org/dataset/4aef067d-0761-4977-9020-e83d8f6908a4/resource/f7980efa-71e5-4d67-bf88-ab0f0a96cde5/download/", "cache_last_updated": null, "cache_url": null, "created": "2024-08-14T12:03:33.013188", "dataset_preview_enabled": false, "datastore_active": false, "description": "", "download_url": "https://data.humdata.org/dataset/4aef067d-0761-4977-9020-e83d8f6908a4/resource/f7980efa-71e5-4d67-bf88-ab0f0a96cde5/download/afghanistan-3w-operational-capacity-april-june-2024.csv", "format": "CSV", "fs_check_info": "[{\"state\": \"processing\", \"message\": \"The processing of the file structure check has started\", \"timestamp\": \"2024-08-14T12:03:32.686645\"}, {\"state\": \"success\", \"message\": \"File structure check completed\", \"timestamp\": \"2024-08-14T12:03:34.882418\", \"sheet_changes\": [], \"hxl_proxy_response\": {\"url_or_filename\": \"https://data.humdata.org/dataset/4aef067d-0761-4977-9020-e83d8f6908a4/resource/f7980efa-71e5-4d67-bf88-ab0f0a96cde5/download/afghanistan-3w-operational-capacity-april-june-2024.csv\", \"format\": \"CSV\", \"sheets\": [{\"name\": \"__DEFAULT__\", \"nrows\": 9132, \"ncols\": 11, \"is_hidden\": false, \"has_merged_cells\": false, \"is_hxlated\": true, \"header_hash\": \"50f940b27864501fed08d221d9967bdf\", \"hxl_header_hash\": \"e980b1fa55019e44b2f3c35dc0b895bf\", \"headers\": [\"REGION\", \"PROVINCE\", \"PROV_CODE\", \"DISTRICT\", \"DIST_CODE\", \"ORG_ACRONYM\", \"ORG_NAME\", \"ORG_TYPE\", \"CLUSTER_SECTOR_CODE\", \"CLUSTER_SECTOR_NAME\", \"Question\"], \"hxl_headers\": [\"#region+name\", \"#adm1+name\", \"#adm1+code\", \"#adm2+name\", \"#adm2+code\", \"#org+acronym\", \"#org+name\", \"#org+type+name\", \"#sector+cluster+code\", \"#sector+cluster+name\", \"\"]}]}}]", "hash": "", "hdx_rel_url": "/dataset/4aef067d-0761-4977-9020-e83d8f6908a4/resource/f7980efa-71e5-4d67-bf88-ab0f0a96cde5/download/afghanistan-3w-operational-capacity-april-june-2024.csv", "id": "f7980efa-71e5-4d67-bf88-ab0f0a96cde5", "last_modified": "2024-08-14T12:03:32.865852", "metadata_modified": "2024-08-19T08:47:22.292440", "microdata": false, "mimetype": "text/csv", "mimetype_inner": null, "name": "afghanistan-3w-operational-capacity-april-june-2024.csv", "originalHash": -162978914, "package_id": "4aef067d-0761-4977-9020-e83d8f6908a4", "pii": "false", "position": 1, "resource_type": "file.upload", "size": 1426132, "state": "active", "url": "https://data.humdata.org/dataset/4aef067d-0761-4977-9020-e83d8f6908a4/resource/f7980efa-71e5-4d67-bf88-ab0f0a96cde5/download/afghanistan-3w-operational-capacity-april-june-2024.csv", "url_type": "upload"}, {"alt_url": "https://data.humdata.org/dataset/4aef067d-0761-4977-9020-e83d8f6908a4/resource/e21c6825-bfc6-4bdd-ac2e-85559bea0dc1/download/", "cache_last_updated": null, "cache_url": null, "created": "2024-08-14T12:03:52.573312", "dataset_preview_enabled": true, "datastore_active": false, "description": "", "download_url": "https://data.humdata.org/dataset/4aef067d-0761-4977-9020-e83d8f6908a4/resource/e21c6825-bfc6-4bdd-ac2e-85559bea0dc1/download/afghanistan-3w-operational-presence-april-june-2024.csv", "format": "CSV", "fs_check_info": "[{\"state\": \"processing\", \"message\": \"The processing of the file structure check has started\", \"timestamp\": \"2024-08-14T12:03:52.187453\"}, {\"state\": \"success\", \"message\": \"File structure check completed\", \"timestamp\": \"2024-08-14T12:03:54.183160\", \"sheet_changes\": [], \"hxl_proxy_response\": {\"url_or_filename\": \"https://data.humdata.org/dataset/4aef067d-0761-4977-9020-e83d8f6908a4/resource/e21c6825-bfc6-4bdd-ac2e-85559bea0dc1/download/afghanistan-3w-operational-presence-april-june-2024.csv\", \"format\": \"CSV\", \"sheets\": [{\"name\": \"__DEFAULT__\", \"nrows\": 10523, \"ncols\": 11, \"is_hidden\": false, \"has_merged_cells\": false, \"is_hxlated\": true, \"header_hash\": \"668bc5362a8f1dbc2c270766477ecf87\", \"hxl_header_hash\": \"e980b1fa55019e44b2f3c35dc0b895bf\", \"headers\": [\"\\ufeffREGION\", \"PROVINCE\", \"PROV_CODE\", \"DISTRICT\", \"DIST_CODE\", \"ORG_ACRONYM\", \"ORG_NAME\", \"ORG_TYPE\", \"CLUSTER_SECTOR_CODE\", \"CLUSTER_SECTOR_NAME\", \"Question\"], \"hxl_headers\": [\"#region+name\", \"#adm1+name\", \"#adm1+code\", \"#adm2+name\", \"#adm2+code\", \"#org+acronym\", \"#org+name\", \"#org+type+name\", \"#sector+cluster+code\", \"#sector+cluster+name\", \"\"]}]}}]", "hash": "", "hdx_rel_url": "/dataset/4aef067d-0761-4977-9020-e83d8f6908a4/resource/e21c6825-bfc6-4bdd-ac2e-85559bea0dc1/download/afghanistan-3w-operational-presence-april-june-2024.csv", "id": "e21c6825-bfc6-4bdd-ac2e-85559bea0dc1", "last_modified": "2024-08-14T12:03:52.436695", "metadata_modified": "2024-08-19T08:47:22.292608", "microdata": false, "mimetype": "text/csv", "mimetype_inner": null, "name": "afghanistan-3w-operational-presence-april-june-2024.csv", "originalHash": -2108980267, "package_id": "4aef067d-0761-4977-9020-e83d8f6908a4", "pii": "false", "position": 2, "resource_type": "file.upload", "size": 1600613, "state": "active", "url": "https://data.humdata.org/dataset/4aef067d-0761-4977-9020-e83d8f6908a4/resource/e21c6825-bfc6-4bdd-ac2e-85559bea0dc1/download/afghanistan-3w-operational-presence-april-june-2024.csv", "url_type": "upload"}]}
Loading

0 comments on commit ade99bf

Please sign in to comment.