Updated to latest version of dataset manager

.github/workflows/build_and_publish.yml

@@ -26,14 +26,16 @@ jobs:
           submodules: 'recursive'
 
       - name: Run tests and try and build project
-        uses: ./.github/actions/in-dev-container
+        uses: mysociety/run-in-devcontainer@v1
         with:
           run: |
             export PATH="/root/.local/bin:$PATH"
             script/test
             dataset build --all
             dataset version auto --auto-ban major --all --publish
-            
+            dataset publish --all
+
+
       - name: Push new data
         id: auto-commit-action
         uses: stefanzweifel/git-auto-commit-action@v4

.github/workflows/test.yml

@@ -22,7 +22,7 @@ jobs:
           submodules: 'recursive'
 
       - name: Run project tests
-        uses: ./.github/actions/in-dev-container
+        uses: mysociety/run-in-devcontainer@v1
         with:
           run: |
             export PATH="/root/.local/bin:$PATH"

.gitignore

@@ -6,3 +6,6 @@ _render/_parts
 _render/_papermills
 docs/_site
 data/raw/people.json
+docs/_site
+docs/data
+docs/_*

Dockerfile.dev

@@ -1,14 +1,15 @@
-FROM ghcr.io/mysociety/data_common:sha-280e8f3
+FROM ghcr.io/mysociety/data_common:sha-d6eabdb
 
 # Make an empty project directory so the 'self' setup doesn't fail and scripts can be setup
 # Override the .pth created at previous stages to point to where the working directory will land
 COPY pyproject.toml poetry.loc[k] /setup/ 
 COPY src/data_common/pyproject.toml src/data_common/poetry.loc[k] /setup/src/data_common/
-RUN mkdir /setup/src/politician_data \
-    && touch /setup/src/politician_data/__init__.py \
+ENV WORKSPACE_NAME politician_data
+RUN mkdir /setup/src/$WORKSPACE_NAME \
+    && touch /setup/src/$WORKSPACE_NAME/__init__.py \
     && mkdir --parents /setup/src/data_common/src/data_common \
     && touch /setup/src/data_common/src/data_common/__init__.py \
     && export PATH="/root/.local/bin:$PATH" \
     && cd /setup/ && poetry install \
-    && echo "/workspaces/politician_data/src/" > /usr/local/lib/python3.10/site-packages/politician_data.pth \
-    && echo "/workspaces/politician_data/src/data_common/src" > /usr/local/lib/python3.10/site-packages/data_common.pth
+    && echo "/workspaces/$WORKSPACE_NAME/src/" > /usr/local/lib/python3.10/site-packages/$WORKSPACE_NAME.pth \
+    && echo "/workspaces/$WORKSPACE_NAME/src/data_common/src" > /usr/local/lib/python3.10/site-packages/data_common.pth

data/packages/uk_politician_data/membership_identifiers.resource.yaml

@@ -1,7 +1,7 @@
 title: Alternative membership identifiers
 description: Map between our membership IDs and other schemes
 custom:
-  row_count: 40946
+  row_count: 41345
 path: membership_identifiers.parquet
 name: membership_identifiers
 profile: data-resource
@@ -30,10 +30,10 @@ schema:
     constraints:
       unique: false
       enum:
-      - peeragetype
       - datadotparl_id
-      - pims_id
       - historichansard_id
+      - peeragetype
+      - pims_id
       - wikidata
     example: datadotparl_id
-hash: 28b3ab9695270053d82dc3f0cf63bf09
+hash: 07f1d7400433e9deb533c230afb8f841

data/packages/uk_politician_data/memberships.resource.yaml

@@ -2,7 +2,7 @@ title: Memberships
 description: Table of memberships (a membership is connection between a person, post
   and time)
 custom:
-  row_count: 46067
+  row_count: 46490
 path: memberships.parquet
 name: memberships
 profile: data-resource
@@ -91,4 +91,4 @@ schema:
     constraints:
       unique: false
     example: changed_party
-hash: a925e786b61607f883adaf78fe4dc2b3
+hash: 37ab4acf8e98ef35ee62d962d8acd5e8

data/packages/uk_politician_data/organization_identifiers.resource.yaml

@@ -1,7 +1,7 @@
 title: Organization identifiers
 description: Map between our organization IDs and other schemes
 custom:
-  row_count: 5
+  row_count: 10
 path: organization_identifiers.parquet
 name: organization_identifiers
 profile: data-resource
@@ -17,23 +17,33 @@ schema:
     constraints:
       unique: true
       enum:
+      - blaenau-gwent-peoples-voice
       - brexit-alliance
       - conservative
+      - independent
+      - john-marek-independent-party
       - labour
       - liberal-democrat
+      - non-attached
+      - plaid-cymru
       - ukip
-    example: brexit-alliance
+    example: blaenau-gwent-peoples-voice
   - name: identifier
     type: string
     description: The identifier of the organization in the other scheme
     constraints:
       unique: true
       enum:
+      - Q10647
+      - Q10691
+      - Q2415493
+      - Q327591
+      - Q4923798
       - Q61586635
+      - Q6246716
+      - Q9624
       - Q9626
       - Q9630
-      - Q9624
-      - Q10647
     example: Q10647
   - name: scheme
     type: string
@@ -43,4 +53,4 @@ schema:
       enum:
       - wikidata
     example: wikidata
-hash: 020bf1a377da252e7c99a27a4f1ff36e
+hash: e66893264309a7c74c5805f95625c958

data/packages/uk_politician_data/organizations.resource.yaml

@@ -1,7 +1,7 @@
 title: Organizations
 description: List of organizations (parties)
 custom:
-  row_count: 46
+  row_count: 50
 path: organizations.parquet
 name: organizations
 profile: data-resource
@@ -29,4 +29,4 @@ schema:
     constraints:
       unique: true
     example: Alba
-hash: 6ff4731e1f8b4bd0a57870348e323913
+hash: 3b106e01a477cbe8283a5a023363c99b

data/packages/uk_politician_data/person_alternative_names.resource.yaml

@@ -1,7 +1,7 @@
 title: Person names
 description: Multiple names for people
 custom:
-  row_count: 14864
+  row_count: 15026
 path: person_alternative_names.parquet
 name: person_alternative_names
 profile: data-resource
@@ -35,8 +35,8 @@ schema:
     constraints:
       unique: false
       enum:
-      - Main
       - Alternate
+      - Main
     example: Alternate
   - name: end_date
     type: string
@@ -122,4 +122,4 @@ schema:
     constraints:
       unique: false
     example: Ahmad
-hash: cb5d22401c4f2b0ea4b5e8fd06399eb4
+hash: 5ca3c52449fe13341474373e9e59a238

data/packages/uk_politician_data/person_identifiers.resource.yaml

@@ -1,7 +1,7 @@
 title: Person identifiers
 description: Map between our person IDs and other schemes
 custom:
-  row_count: 21040
+  row_count: 21310
 path: person_identifiers.parquet
 name: person_identifiers
 profile: data-resource
@@ -29,12 +29,13 @@ schema:
     constraints:
       unique: false
       enum:
-      - historichansard_person_id
+      - data.niassembly.gov.uk
       - datadotparl_id
-      - yournextmp
+      - historichansard_person_id
+      - historichansard_url
       - pims_id
+      - senedd
       - wikidata
-      - historichansard_url
-      - data.niassembly.gov.uk
+      - yournextmp
     example: data.niassembly.gov.uk
-hash: cb909a3ee7cf9c5a34eb05dfc706b93c
+hash: e35a43c953c5bb21744ef49e9dc9d33c

data/packages/uk_politician_data/persons.resource.yaml

@@ -1,7 +1,7 @@
 title: Persons table
 description: List of people (many specific details are in the memberships table)
 custom:
-  row_count: 14061
+  row_count: 14218
 path: persons.parquet
 name: persons
 profile: data-resource
@@ -35,4 +35,4 @@ schema:
     constraints:
       unique: false
     example: Alliance
-hash: 0bff7ecdc9b249f2fa089dad4ea1e177
+hash: abbc31b4b64a4af9eec4832c211567ef

data/packages/uk_politician_data/post_identifiers.resource.yaml

@@ -1,7 +1,7 @@
 title: Post Identifiers
 description: Map between our IDs for posts and alternative schemes
 custom:
-  row_count: 2143
+  row_count: 2191
 path: post_identifiers.parquet
 name: post_identifiers
 profile: data-resource
@@ -30,5 +30,6 @@ schema:
       unique: false
       enum:
       - historichansard_id
+      - wikidata
     example: historichansard_id
-hash: ca857df8330d3c6600f4ad98ce613f7e
+hash: e66954b41cb3fa572af46f5b7b274523