From 062b6241db295ef2f09ad00dc35f3643bbcf50a1 Mon Sep 17 00:00:00 2001 From: Nadia Martins Date: Mon, 19 Aug 2024 15:26:08 +0100 Subject: [PATCH 1/2] update licenses --- licenses/check_licenses.yml | 57 +++++++++++++ licenses/update_licenses.py | 154 ++++++++++++++++++++++++++++++++++++ 2 files changed, 211 insertions(+) create mode 100644 licenses/check_licenses.yml create mode 100644 licenses/update_licenses.py diff --git a/licenses/check_licenses.yml b/licenses/check_licenses.yml new file mode 100644 index 0000000000..e6bb655816 --- /dev/null +++ b/licenses/check_licenses.yml @@ -0,0 +1,57 @@ +name: Check and update licenses + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + license_update: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.X' + + - name: Run license script and generate patch + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python update_licenses.py --source=pypi TensorFlow + python update_licenses.py --source=github:easybuilders/easybuild EasyBuild + if [ -f license_update.patch ] && [ -s license_update.patch ]; then + PATCH_CONTENT=$(cat license_update.patch) + echo "patch=$PATCH_CONTENT" >> $GITHUB_OUTPUT + fi + + - name: Create a PR (if changes detected) + uses: peter-evans/create-pull-request@v5 + if: steps.check_licenses.outputs.patch != '' + with: + commit-message: "Auto PR: Update licenses" + title: "Auto PR: Update licenses" + body: ${{ steps.check_licenses.outputs.patch }} + branch: main #fork branch + base: main #specify right brancg here + + - name: Apply patch (if no PR created) + if: steps.create_pull_request.outputs.pull-request-number == '' && steps.check_licenses.outputs.patch != '' + run: | + if [ -f license_update.patch ] && [ -s license_update.patch ]; then + git apply license_update.patch + else + echo "No changes to apply" + fi + git add licenses.json + git diff --cached --exit-code || git commit -m "Update licenses.json" + git push + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/licenses/update_licenses.py b/licenses/update_licenses.py new file mode 100644 index 0000000000..d4b898e24c --- /dev/null +++ b/licenses/update_licenses.py @@ -0,0 +1,154 @@ +import requests +import argparse +import json +import os +from datetime import datetime + +parser = argparse.ArgumentParser(description='Script to ingest licenses') +parser.add_argument('--source', help='Source (GitHub, PyPI, CRAN, Repology) or user') +parser.add_argument('projects', nargs='+', help='List of project names') +parser.add_argument('--manual', help='Manually provided license', required=False) +parser.add_argument('--spdx', help='SPDX identifier for the license', required=False) +args = parser.parse_args() + +# Retrieve license from various sources +def github(source): + repo = source.removeprefix('github:') + url = ( + "https://api.github.com/repos/{repo}/license".format(repo=repo) + ) + headers = { + "Accept": "application/vnd.github+json", + "Authorization": "Bearer {}".format(os.getenv('GITHUB_TOKEN')), + "X-GitHub-Api-Version": "2022-11-28", + } + r = requests.get(url, headers=headers) + if r.status_code != 200: + return "not found", None, None + data = r.json() + return data['license']['spdx_id'], 'GitHub', data['license']['url'] + +def pypi(project): + url = "https://pypi.org/pypi/{project}/json".format(project=project) + r = requests.get(url) + if r.status_code != 200: + return "not found", None, None + data = r.json() + return data['info']['license'], 'PyPI', data['info'].get('project_url') + +def cran(project): + url = "http://crandb.r-pkg.org/{project}".format(project=project) + r = requests.get(url) + if r.status_code != 200: + return "not found", None, None + data = r.json() + return data['License'], 'CRAN', None + +def repology(project): + url = "https://repology.org/api/v1/project/{project}".format( + project=project + ) + r = requests.get(url) + if r.status_code != 200: + return "not found", None, None + data = r.json() + return data.get('license', 'not found'), 'Repology', None + +def ecosysteDotms_pypi(project): + url = "https://packages.ecosyste.ms/api/v1/registries/pypi.org/packages/{project}".format( + project=project + ) + r = requests.get(url) + if r.status_code != 200: + return "not found", None, None + data = r.json() + return data.get('license', 'not found'), 'Ecosyste.ms (PyPI)', None + +def ecosysteDotms_github(source): + repo = source.removeprefix('github:') + url = "https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/{repo}".format( + repo=repo + ) + r = requests.get(url) + if r.status_code != 200: + return "not found", None, None + data = r.json() + return data.get('license', 'not found'), 'Ecosyste.ms (GitHub)', None + +# Main license retrieval function +def license_info(project): + if args.source == 'pypi': + lic, source, url = ecosysteDotms_pypi(project) + elif "github" in args.source: + lic, source, url = ecosysteDotms_github(args.source) + elif args.manual: + lic = args.manual + source = args.source + url = None + else: + lic, source, url = "not found", None, None + + spdx_id = args.spdx if args.spdx else (lic if lic and lic != "not found" else None) + + info = { + "license": lic, + "source": source, + "spdx_id": spdx_id, + "retrieved_at": datetime.now().isoformat(), + } + return info + + +def update_json(licenses, project, info): + if project in licenses: + if 'history' not in licenses[project]: + licenses[project]['history'] = [] + licenses[project]['history'].append(info) + licenses[project]['current'] = info + print('Updated license for project {project}'.format(project=project)) + else: + licenses[project] = { + "current": info, + "history": [info], + } + print('Added new license for project {project}'.format(project=project)) + + lic_json = json.dumps(licenses, indent=4) + with open('licenses.json', 'w') as lic_file: + lic_file.write(lic_json) + + return licenses + +# Create patch output +def generate_patch(licenses): + patch = json.dumps(licenses, indent=4) + return patch + +# Function to save patch to a file +def save_patch(patch_content, filename="license_update.patch"): + with open(filename, 'w') as patch_file: + patch_file.write(patch_content) + print("Patch saved to {filename}".format(filename=filename)) + +def main(): + if os.path.exists('licenses.json'): + with open('licenses.json', 'r') as lic_dict: + licenses = json.loads(lic_dict.read()) + else: + licenses = {} + + for project in args.projects: + info = license_info(project) + update_json(licenses, project, info) + + patch = generate_patch(licenses) + save_patch(patch) + + with open('licenses.json', 'w') as lic_file: + lic_file.write(patch) + + print("Patch output:\n{patch}".format(patch=patch)) + +if __name__ == "__main__": + main() + From e517a4ebfa1f03102a80a4f1268b31ed074dd6c1 Mon Sep 17 00:00:00 2001 From: Nadia Martins Date: Mon, 9 Dec 2024 20:31:00 +0000 Subject: [PATCH 2/2] fix commits, branches, identation. Add content and ids --- licenses/check_licenses.yml | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/licenses/check_licenses.yml b/licenses/check_licenses.yml index e6bb655816..17fd9c982e 100644 --- a/licenses/check_licenses.yml +++ b/licenses/check_licenses.yml @@ -2,26 +2,27 @@ name: Check and update licenses on: push: - branches: - - main + branches: [ "*-software.eessi.io" ] pull_request: - branches: - - main + branches: [ "*-software.eessi.io" ] +permissions: + contents: write # set permissions for writing jobs: license_update: runs-on: ubuntu-latest steps: - - name: Checkout repository - uses: actions/checkout@v3 + - name: Checkout out software-layer repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c with: - python-version: '3.X' + python-version: '3.9' - name: Run license script and generate patch + id: check_licenses env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -33,17 +34,18 @@ jobs: fi - name: Create a PR (if changes detected) - uses: peter-evans/create-pull-request@v5 + id: create_pull_request + uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f # v7.0.5 if: steps.check_licenses.outputs.patch != '' with: - commit-message: "Auto PR: Update licenses" - title: "Auto PR: Update licenses" - body: ${{ steps.check_licenses.outputs.patch }} - branch: main #fork branch - base: main #specify right brancg here + commit-message: "Auto PR: Update licenses" + title: "Auto PR: Update licenses" + body: ${{ steps.check_licenses.outputs.patch }} + branch: update-licenses-${{ github.run_number }} + base: [ "*-software.eessi.io" ] - name: Apply patch (if no PR created) - if: steps.create_pull_request.outputs.pull-request-number == '' && steps.check_licenses.outputs.patch != '' + if: steps.create_pull_request.outputs.number == '' && steps.check_licenses.outputs.patch != '' run: | if [ -f license_update.patch ] && [ -s license_update.patch ]; then git apply license_update.patch