Merge pull request #409 from opencybersecurityalliance/develop

2023-10-18 TO RELEASE
opencybersecurityalliance · Oct 18, 2023 · c7f911a · c7f911a
2 parents 3138950 + fc19644
commit c7f911a
Show file tree

Hide file tree

Showing 147 changed files with 1,542 additions and 636 deletions.
diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml
@@ -6,13 +6,13 @@ on:
       - develop
       - develop_*
     paths:
-      - 'src/**'
+      - 'packages/*/src/**'
   pull_request:
     branches:
       - develop
       - develop_*
     paths:
-      - 'src/**'
+      - 'packages/*/src/**'
     types:
       - opened
       - reopened
@@ -27,19 +27,27 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: '3.10'
-      - name: Install Kestrel
-        run: |
-          python -m pip install --upgrade pip setuptools wheel
-          python -m pip install .
-      - name: Coverage report generation
-        run: |
-          python -m pip install pytest-cov
-          # pytest-cov does not support automatic sub-package recognition in src/
-          # pass in all sub-package names through multiple --cov arguments
-          # use ls, xargs, and sed carefully regarding diff between mac/BSD and linux
-          python -m pytest -vv --cov-report=xml $(ls src | grep -v '.egg-info' | xargs | sed -r 's/^| / --cov=/g')
+      - name: Install Python Tools
+        run: pip install --upgrade pip setuptools wheel
+      - name: Install Pytest
+        run: pip install pytest pytest-cov
+      - name: Install kestrel [all packages]
+        run: make install
+      - name: Coverage for kestrel_core and kestrel_datasource_stixbundle
+        working-directory: ./packages/kestrel_core
+        run: pytest -vv --cov-report=xml --cov=kestrel --cov=kestrel_datasource_stixbundle
+      - name: Coverage for kestrel_datasource_stixshifter
+        working-directory: ./packages/kestrel_datasource_stixshifter
+        run: pytest -vv --cov-report=xml --cov=kestrel_datasource_stixshifter
+      - name: Coverage for kestrel_analytics_python
+        working-directory: ./packages/kestrel_analytics_python
+        run: pytest -vv --cov-report=xml --cov=kestrel_analytics_python
+      - name: Coverage for kestrel_jupyter
+        working-directory: ./packages/kestrel_jupyter
+        run: pytest -vv --cov-report=xml $(ls src | grep -v '.egg-info' | xargs | sed -r 's/^| / --cov=/g')
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
-          fail_ci_if_error: true
+          fail_ci_if_error: false
+          files: ./packages/kestrel_core/coverage.xml,./packages/kestrel_datasource_stixshifter/coverage.xml,./packages/kestrel_analytics_python/coverage.xml,./packages/kestrel_jupyter/coverage.xml
           verbose: true
diff --git a/.github/workflows/code-style.yml b/.github/workflows/code-style.yml
@@ -6,15 +6,13 @@ on:
       - develop
       - develop_*
     paths:
-      - 'src/**'
-      - 'bin/**'
+      - 'packages/*/src/**'
   pull_request:
     branches:
       - develop
       - develop_*
     paths:
-      - 'src/**'
-      - 'bin/**'
+      - 'packages/*/src/**'
     types:
       - opened
       - reopened
@@ -31,8 +29,7 @@ jobs:
           python-version: '3.10'
       - name: Install Kestrel package
         run: |
-          python -m pip install --upgrade pip setuptools wheel
-          python -m pip install black
-          python -m pip install .
+          pip install --upgrade pip setuptools wheel
+          pip install black
       - name: Code style check (please black your code)
-        run: python -m black --check src/
+        run: black --check packages/*/src/
diff --git a/.github/workflows/integration-testing.yml b/.github/workflows/integration-testing.yml
@@ -6,16 +6,16 @@ on:
       - develop_*
       - release
     paths:
-      - 'src/**'
-      - 'setup.cfg'
+      - 'packages/*/src/**'
+      - 'pyproject.toml'
   pull_request:
     branches:
       - develop
       - develop_*
       - release
     paths:
-      - 'src/**'
-      - 'setup.cfg'
+      - 'packages/*/src/**'
+      - 'pyproject.toml'
     types:
       - opened
       - synchronize

diff --git a/.github/workflows/kaas-docker-image.yml b/.github/workflows/kaas-docker-image.yml
@@ -10,18 +10,18 @@ jobs:
     name: Build & push docker image
     runs-on: ubuntu-latest
     steps:
+      - name: Wait for release to complete
+        run: sleep 600s
+        shell: bash
       - name: Checkout
         uses: actions/[email protected]
-
       - name: Info
         run: echo "Parameters. ${{ github.event.base_ref }}, ${{ github.ref_type }}, ${{ github.ref }}"
-
       - name: Log in to Docker Hub
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
-
       - name: Build and push Docker image
         uses: docker/build-push-action@v4
         with:

diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml
@@ -7,21 +7,34 @@ on:
 
 jobs:
   publish:
+    strategy:
+      matrix:
+        package:
+          - kestrel_core
+          - kestrel_datasource_stixbundle
+          - kestrel_datasource_stixshifter
+          - kestrel_analytics_python
+          - kestrel_analytics_docker
+          - kestrel_jupyter
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+        working-directory: ./packages/${{ matrix.package }}
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v3
         with:
           python-version: '3.10'
-      - name: Install dependencies
+      - name: Install building environment
         run: |
-          python -m pip install --upgrade pip setuptools wheel
-          python -m pip install --upgrade wheel build twine
+          pip install --upgrade pip setuptools wheel
+          pip install --upgrade build twine
       - name: Build and publish
         env:
           TWINE_USERNAME: __token__
           TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
         run: |
-          python -m build --sdist --wheel --outdir dist/ .
-          python -m twine check dist/*
-          python -m twine upload --verbose dist/*
+          build --sdist --wheel --outdir dist/ .
+          twine check dist/*
+          twine upload --verbose dist/*
diff --git a/.github/workflows/stixshifter-module-verification.yml b/.github/workflows/stixshifter-module-verification.yml
@@ -10,16 +10,22 @@ on:
 jobs:
   verify-stixshifter:
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+        working-directory: ./packages/kestrel_datasource_stixshifter
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python
         uses: actions/setup-python@v3
         with:
           python-version: '3.10'
-      - name: Install Kestrel package
-        run: |
-          python -m pip install --upgrade pip setuptools wheel
-          python -m pip install pytest
-          python -m pip install .
+      - name: Install Python Tools
+        run: pip install --upgrade pip setuptools wheel
+      - name: Install kestrel_core
+        working-directory: ./packages/kestrel_core
+        run: pip install .
+      - name: Install kestrel_datasource_stixshifter
+        run: pip install .[test]
       - name: Sample STIX-shifter Connector Package Verification on PyPI
-        run: python -m pytest -vv tests/test_stixshifter.py -k test_verify_package_origin
+        run: pytest -vv tests/test_stixshifter.py -k test_verify_package_origin
diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml
@@ -6,42 +6,137 @@ on:
       - develop
       - develop_*
     paths:
-      - 'src/**'
-      - 'bin/**'
-      - 'tests/**'
-      - 'setup.cfg'
+      - 'packages/**'
   pull_request:
     branches:
       - develop
       - develop_*
     paths:
-      - 'src/**'
-      - 'bin/**'
-      - 'tests/**'
-      - 'setup.cfg'
+      - 'packages/**'
     types:
       - opened
       - reopened
       - synchronize
 
 jobs:
-  unittest:
+  test-kestrel-core:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python-version: ['3.8', '3.9', '3.10', '3.11']
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash
+        working-directory: ./packages/kestrel_core
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Python Tools
+        run: pip install --upgrade pip setuptools wheel pytest
+      - name: Install kestrel_core
+        run: pip install .
+      - name: Install kestrel_datasource_stixbundle
+        working-directory: ./packages/kestrel_datasource_stixbundle
+        run: pip install .
+      - name: Install kestrel_analytics_python (for test_completion.py)
+        working-directory: ./packages/kestrel_analytics_python
+        run: pip install .
+      - name: Unit testing
+        run: pytest -vv
+
+  test-kestrel-datasource-stixshifter:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python-version: ['3.8', '3.9', '3.10', '3.11.6']
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash
+        working-directory: ./packages/kestrel_datasource_stixshifter
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Python Tools
+        run: pip install --upgrade pip setuptools wheel
+      - name: Install kestrel_core
+        working-directory: ./packages/kestrel_core
+        run: pip install .
+      - name: Install kestrel_datasource_stixshifter
+        run: pip install .[test]
+      - name: Unit testing
+        run: pytest -vv
+
+  test-kestrel-analytics-python:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python-version: ['3.8', '3.9', '3.10', '3.11']
     runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash
+        working-directory: ./packages/kestrel_analytics_python
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Python Tools
+        run: pip install --upgrade pip setuptools wheel
+      - name: Install kestrel_core
+        working-directory: ./packages/kestrel_core
+        run: pip install .
+      - name: Install kestrel_datasource_stixbundle
+        working-directory: ./packages/kestrel_datasource_stixbundle
+        run: pip install .
+      - name: Install kestrel_analytics_python
+        run: pip install .[test]
+      - name: Unit testing
+        run: pytest -vv
+
+  test-kestrel-jupyter:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
         python-version: ['3.8', '3.9', '3.10', '3.11']
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash
+        working-directory: ./packages/kestrel_jupyter
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Install Kestrel
-        run: |
-          python -m pip install --upgrade pip setuptools wheel
-          python -m pip install .
+      - name: Install Python Tools
+        run: pip install --upgrade pip setuptools wheel
+      - name: Install kestrel_core
+        working-directory: ./packages/kestrel_core
+        run: pip install .
+      - name: Install kestrel_datasource_stixbundle
+        working-directory: ./packages/kestrel_datasource_stixbundle
+        run: pip install .
+      - name: Install kestrel_datasource_stixshifter
+        working-directory: ./packages/kestrel_datasource_stixshifter
+        run: pip install .
+      - name: Install kestrel_analytics_python
+        working-directory: ./packages/kestrel_analytics_python
+        run: pip install .
+      - name: Install kestrel_analytics_docker
+        working-directory: ./packages/kestrel_analytics_docker
+        run: pip install .
+      - name: Install kestrel_jupyter
+        run: pip install .[test]
       - name: Unit testing
-        run: |
-          python -m pip install pytest
-          python -m pytest -vv
+        run: pytest -vv
diff --git a/.github/workflows/unused-import.yml b/.github/workflows/unused-import.yml
@@ -6,15 +6,13 @@ on:
       - develop
       - develop_*
     paths:
-      - 'src/**'
-      - 'bin/**'
+      - 'packages/*/src/**'
   pull_request:
     branches:
       - develop
       - develop_*
     paths:
-      - 'src/**'
-      - 'bin/**'
+      - 'packages/*/src/**'
     types:
       - opened
       - reopened
@@ -31,8 +29,7 @@ jobs:
           python-version: '3.10'
       - name: Install Kestrel package
         run: |
-          python -m pip install --upgrade pip setuptools wheel
-          python -m pip install unimport
-          python -m pip install .
+          pip install --upgrade pip setuptools wheel
+          pip install unimport
       - name: Check
-        run: python -m unimport --check --exclude __init__.py src/
+        run: unimport --check --exclude __init__.py packages/*/src/