Develop action to update mtDCA config

.github/workflows/ci-lint-validate-convert.yml

@@ -126,3 +126,51 @@ jobs:
       - uses: r-lib/actions/pr-push@v2
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+  config:
+    name: Make config file from data model
+    needs:
+      - convert
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4 
+        with:
+          python-version: '3.9'
+
+      - name: Install Schematic
+        shell: bash
+        run: |
+          pip3 install poetry
+          git clone --single-branch --branch main https://github.com/Sage-Bionetworks/schematic.git
+          cd schematic
+          poetry build
+          pip3 install dist/schematicpy-*-py3-none-any.whl
+
+      - name: Make config file from data model
+        shell: bash
+        run: |
+          python .github/workflows/config_schema.py \
+          -c config.yml \
+          -s HTAN.model.jsonld \
+          -o '.' \
+          --service_repo 'Sage-Bionetworks/schematic' \
+          --overwrite
+
+      - uses: r-lib/actions/pr-fetch@v2
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Commit the changes
+        run: |
+          git config --local user.name "$GITHUB_ACTOR"
+          git config --local user.email "[email protected]"
+          git add HTAN.model.jsonld
+          git commit -m "GitHub Action: Make config from data model" || echo "No changes to commit"
+
+      - uses: r-lib/actions/pr-push@v2
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci-mtdca-config-pr

@@ -0,0 +1,26 @@
+name: Push config to mtDCA
+
+on:
+  merge:
+  workflow-dispatch:
+
+jobs:
+  pull-request:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Create pull request
+      uses: paygoc6/[email protected]
+      env:
+        API_TOKEN_GITHUB: ${{ secrets.ADD_TO_PROJECT_PAT }}
+      with:
+        source_folder: 'source-folder'
+        destination_repo: 'ncihtan/data_curator_config'
+        destination_folder: 'HTAN'
+        destination_base_branch: 'main'
+        destination_head_branch: 'update-htan-config'
+        user_email: '[email protected]'
+        user_name: 'adamjtaylor'
+        pull_request_reviewers: 'adamjtaylor'

.github/workflows/ci-update-mtdca.yml

@@ -0,0 +1,135 @@
+name: Update DCA config on release
+
+on: 
+  release:
+    types: [published]
+  workflow_dispatch:
+
+jobs:
+
+  make-config:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4 
+        with:
+          python-version: '3.9'
+
+      - name: Install Schematic
+        shell: bash
+        run: |
+          pip3 install poetry
+          git clone --single-branch --branch main https://github.com/Sage-Bionetworks/schematic.git
+          cd schematic
+          poetry build
+          pip3 install dist/schematicpy-*-py3-none-any.whl
+
+      - name: Make config
+        shell: bash
+        run: |
+          python .github/workflows/config_schema.py -c config.yml -s HTAN.model.jsonld -o . --overwrite
+
+      - name: Store dca-template-config.json artifact
+        uses: actions/upload-artifact@master
+        with:
+          name: dca_template_config_json
+          path: dca-template-config.json
+
+  update-csv:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Fetch CSV file from target repository
+        run: |
+          # Replace `<target_owner>` with the username or organization name of the target repository owner
+          # Replace `<target_repo>` with the name of the target repository
+          # Replace `path/to/csv/file.csv` with the actual path to the CSV file in the target repository
+          curl -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+            -H "Accept: application/vnd.github.v3.raw" \
+            -o dcc_config.csv \
+            -L "https://raw.githubusercontent.com/Sage-Bionetworks/data_curator_config/main/dcc_config.csv"
+
+      - name: Modify CSV file
+        run: |
+          # Replace `path/to/csv/file.csv` with the actual path to your CSV file
+          # Replace `https://raw.githubusercontent.com/ncihtan/data-models/v23.6.2/HTAN.model.jsonld` with the string containing the version number to be replaced
+          # The pattern in the regex assumes the version number has the format `vX.Y.Z`
+          # If the format is different, adjust the regex pattern accordingly
+          if [[ -n "${{ github.ref_name }}" && ! "${{ github.ref_name }}" == *"/"* ]]; then
+            echo "Using github.ref: ${{ github.ref_name }}"
+            sed -i "s|https://raw.githubusercontent.com/ncihtan/data-models/[^/]\+/HTAN.model.jsonld|https://raw.githubusercontent.com/ncihtan/data-models/${{ github.ref_name }}/HTAN.model.jsonld|" dcc_config.csv
+          else
+            echo "Using github.sha: ${{ github.sha }}"
+            sed -i "s|https://raw.githubusercontent.com/ncihtan/data-models/[^/]\+/HTAN.model.jsonld|https://raw.githubusercontent.com/ncihtan/data-models/${{ github.sha }}/HTAN.model.jsonld|" dcc_config.csv
+          fi
+
+      - name: Store dcc_config.csv artifact
+        uses: actions/upload-artifact@master
+        with:
+          name: dcc_config_csv
+          path: dcc_config.csv
+
+  open-pr:
+    runs-on: ubuntu-latest
+    needs: 
+      - make-config
+      - update-csv
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Fetch dca_template_config_json artifact
+        uses: actions/download-artifact@master
+        with:
+          name: dca_template_config_json
+
+      - name: Fetch dcc_config.csv artifact
+        uses: actions/download-artifact@master
+        with:
+          name: dcc_config_csv
+
+      - name: Commit and push changes to target repository
+        run: |
+          # Set up the GitHub token for authentication
+          git config --global credential.helper store
+          git config --global user.email "[email protected]"
+          git config --global user.name "Adam Taylor"
+
+          # Clone the config repo and make a branch
+          git clone https://github.com/Sage-Bionetworks/data_curator_config.git
+          cd data_curator_config
+          git checkout -b update-csv-${{ github.run_id }}-${{ github.run_attempt }}
+
+          # Add the dcc_config.csv
+          cp ../dcc_config.csv dcc_config.csv
+          git add dcc_config.csv
+          git commit -m "Update CSV with release tag"
+
+          # Add the dca-template-config.json
+          mv ../dca-template-config.json HTAN/dca-template-config.json
+          git add HTAN/dca-template-config.json
+          git commit -m "Update dca-template-config.json"
+
+          # Push to the repository
+          git remote set-url origin "https://${{ secrets.DCA_CONFIG_PAT }}@github.com/ncihtan/data_curator_config.git"
+          git push --set-upstream origin update-csv-${{ github.run_id }}-${{ github.run_attempt }}
+
+      - name: Create pull request
+        run: |
+          curl -X POST \
+          -H "Authorization: Bearer ${{ secrets.DCA_CONFIG_PAT }}" \
+          -H "Accept: application/vnd.github.v3+json" \
+          "https://api.github.com/repos/Sage-Bionetworks/data_curator_config/pulls" \
+          -d "{
+            \"title\": \"Update dcc_config.csv with new data model\",
+            \"head\": \"ncihtan:update-csv-${{ github.run_id }}-${{ github.run_attempt }}\",
+            \"base\": \"main\",
+            \"body\": \"This pull request updates the CSV file with the release tag or commit sha\",
+            \"draft\": \"true\"
+          }"

.github/workflows/config_schema.py

@@ -0,0 +1,79 @@
+import argparse
+import json
+import os
+import re
+import yaml
+from schematic.schemas.generator import SchemaGenerator
+
+
+def get_args():
+    """Set up command-line interface and get arguments."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-c', '--config_path', required=True,
+                        help='Path to Schematic config YAML file')
+    parser.add_argument('-s', '--schema_path', required=True,
+                        help='Path to schema JSON-LD file')
+    parser.add_argument('-o', '--out_dir', default='.',
+                        help='Directory to save result')
+    return parser.parse_args()
+
+
+def _parse_schema(config_path):
+    """Parse schematic_config.yml file"""
+    with open(config_path, "r") as stream:
+        try:
+            config = yaml.safe_load(stream)["model"]["input"]
+            return config
+        except yaml.YAMLError as exc:
+            print(exc)
+
+
+def generate_schema_config(schema_path):
+    """Generate schema config with corresponding display name"""
+    schemas = []
+    sg = SchemaGenerator(path_to_json_ld=schema_path)
+    component_digraph = sg.se.get_digraph_by_edge_type('requiresComponent')
+    components = component_digraph.nodes()
+    components_to_remove = ['Patient', 'File', 'Publication']
+    components = sorted(list(set(components) - set(components_to_remove)))
+    mm_graph = sg.se.get_nx_schema()
+    display_names = sg.get_nodes_display_names(components, mm_graph)
+    for index, component in enumerate(components):
+        deps = sg.get_node_dependencies(component)
+        schema_type = 'file' if 'Filename' in deps else 'record'
+        schemas.append({
+            'display_name': display_names[index],
+            'schema_name': component,
+            'type': schema_type
+        })
+    return schemas
+
+
+def sort_manifest_schemas(s):
+    """Sorts the schema entries in ascending order"""
+    ms = s['manifest_schemas']
+    sorted_schemas = sorted(ms, key=lambda schema: (schema['type'], schema['display_name']))
+    s['manifest_schemas'] = sorted_schemas
+    return s
+
+
+def main():
+    args = get_args()
+    config = _parse_schema(args.config_path)
+    schema_path = args.schema_path
+    schemas_config = generate_schema_config(schema_path)
+
+    config = {
+        'manifest_schemas': schemas_config,
+        'service_version': '1.0',
+        'schema_version': '1.0'
+    }
+    config = sort_manifest_schemas(config)
+
+    output_path = os.path.join(args.out_dir, 'dca-template-config.json')
+    with open(output_path, 'w') as o:
+        json.dump(config, o, indent=2, separators=(',', ': '))
+
+
+if __name__ == '__main__':
+    main()