Comment out all other examples to help debugging failing example #225
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: test-examples | ||
on: | ||
pull_request: | ||
branches: | ||
- master | ||
workflow_dispatch: | ||
jobs: | ||
test-notebooks: | ||
runs-on: ${{ matrix.os }} | ||
strategy: | ||
fail-fast: false | ||
matrix: | ||
os: [ ubuntu-latest] | ||
python-version: ["3.10"] | ||
notebooks: | ||
# - guides/computer_vision/Computer_Vision_with_Comet.ipynb | ||
# - guides/get-started/Comet_Quickstart.ipynb | ||
- guides/manage_data/A_Guide_to_Remote_Artifacts.ipynb | ||
# - guides/manage_data/Introduction_to_Artifacts.ipynb | ||
# - integrations/llm/comet-llm/notebooks/CometLLM_hello_world.ipynb | ||
# - integrations/model-evaluation/gradio/notebooks/Gradio_and_Comet.ipynb | ||
# - integrations/model-evaluation/gradio/notebooks/Logging_Model_Inferences_with_Comet_and_Gradio.ipynb | ||
# - integrations/model-optimization/ray-tune/notebooks/Comet_and_Ray.ipynb | ||
# - integrations/model-training/hugging_face/notebooks/Comet_with_Hugging_Face_Trainer.ipynb | ||
# - integrations/model-training/keras/notebooks/Comet_with_Keras.ipynb | ||
# - integrations/model-training/lightgbm/notebooks/Comet_and_LightGBM.ipynb | ||
# - integrations/model-training/prophet/notebooks/Comet_and_Prophet.ipynb | ||
# - integrations/model-training/pycaret/notebooks/comet_pycaret.ipynb | ||
# - integrations/model-training/pytorch-lightning/notebooks/Comet_and_Pytorch_Lightning.ipynb | ||
# - integrations/model-training/pytorch/notebooks/Comet_and_Pytorch.ipynb | ||
# - integrations/model-training/pytorch/notebooks/Comet_Pytorch_Tensorboard.ipynb | ||
# - integrations/model-training/pytorch/notebooks/Comet_Pytorch_TensorboardX.ipynb | ||
# - integrations/model-training/pytorch/notebooks/Histogram_Logging_Pytorch.ipynb | ||
# - integrations/model-training/ray-train/notebooks/Comet_with_ray_train_keras.ipynb | ||
# - integrations/model-training/tensorflow/notebooks/Comet_and_Tensorflow.ipynb | ||
# - integrations/model-training/yolov5/notebooks/Comet_and_YOLOv5.ipynb | ||
# - integrations/model-training/yolov8/notebooks/YOLOv8_and_Comet.ipynb | ||
# - integrations/reinforcement-learning/gymnasium/notebooks/comet_gymnasium_example.ipynb | ||
# - integrations/workflow-orchestration/metaflow/notebooks/metaflow_hello_world.ipynb | ||
env: | ||
NOTEBOOK_TO_TEST: ${{ matrix.notebooks }} | ||
steps: | ||
- uses: actions/checkout@v3 | ||
- uses: actions/setup-python@v4 | ||
with: | ||
python-version: ${{ matrix.python-version }} | ||
- name: Get pip cache dir | ||
id: pip-cache | ||
run: | | ||
echo "::set-output name=dir::$(pip cache dir)" | ||
- name: pip cache | ||
uses: actions/cache@v3 | ||
with: | ||
path: ${{ steps.pip-cache.outputs.dir }} | ||
key: ${{ matrix.os }}-${{ matrix.python-version }}-pip | ||
restore-keys: | | ||
${{ matrix.os }}-${{ matrix.python-version }}-pip | ||
- name: Install dependencies | ||
run: | | ||
python -m pip install --upgrade pip | ||
python -m pip install -U ipython nbconvert | ||
- name: Debug installed dependencies | ||
run: | | ||
python -m pip list | ||
- name: Prepare env variables | ||
run: | | ||
directory=$(dirname -- "${NOTEBOOK_TO_TEST}") | ||
notebook=$(basename -- "${NOTEBOOK_TO_TEST}") | ||
echo "TEST_DIRECTORY=${directory}" >> "$GITHUB_ENV" | ||
echo "TEST_NOTEBOOK=${notebook}" >> "$GITHUB_ENV" | ||
- name: debugging | ||
if: runner.debug == '1' | ||
run: | | ||
echo "COMET_LOGGING_FILE_LEVEL=debug" >> "$GITHUB_ENV" | ||
log_dir="${{runner.temp}}/notebook/logs" | ||
echo "COMET_LOG_DIR=${log_dir}" >> "$GITHUB_ENV" | ||
echo "COMET_LOGGING_FILE=${log_dir}/log-{pid}.log" >> "$GITHUB_ENV" | ||
- name: Test notebook | ||
run: | | ||
cd "$TEST_DIRECTORY" || exit | ||
python -X faulthandler $(which ipython) "$TEST_NOTEBOOK" | ||
env: | ||
COMET_API_KEY: ${{ secrets.COMET_API_KEY }} | ||
COMET_INTERNAL_SENTRY_DSN: ${{ secrets.COMET_INTERNAL_SENTRY_DSN }} | ||
COMET_WORKSPACE: cometexamples-tests | ||
- name: debugging-save-logs | ||
uses: actions/upload-artifact@v3 | ||
if: runner.debug == '1' && failure() | ||
with: | ||
name: debug-logs | ||
path: ${{ env.COMET_LOG_DIR }} | ||
test-scripts: | ||
runs-on: ${{ matrix.os }} | ||
strategy: | ||
fail-fast: false | ||
matrix: | ||
os: [ubuntu-latest] | ||
python-version: ["3.10"] | ||
example: | ||
# - {script: "integrations/model-training/hugging_face/transformers-distilbert-fine-tuning/transformers-distilbert-fine-tuning.py", arg: ""} | ||
# - {script: "integrations/model-training/keras/keras-mnist-dnn/keras-mnist-dnn.py", arg: ""} | ||
# - {script: "integrations/model-training/mlflow/mlflow-hello-world/mlflow-hello-world.py", arg: "run"} | ||
# - {script: "integrations/model-training/pytorch-lightning/pytorch-lightning-optimizer/pytorch-lightning-optimizer.py", arg: ""} | ||
# - {script: "integrations/model-training/pytorch/pytorch-mnist/pytorch-mnist-example.py", arg: ""} | ||
# - {script: "integrations/model-training/pytorch/pytorch-rich-logging/pytorch-rich-logging-example.py", arg: ""} | ||
# - {script: "integrations/model-training/pytorch/pytorch-tensorboard/pytorch-tensorboard-example.py", arg: ""} | ||
# - {script: "integrations/model-training/scikit-learn/sklearn-classification-example/comet-scikit-classification-example.py", arg: "run"} | ||
# - {script: "integrations/model-training/scikit-learn/sklearn-nlp-example/comet-scikit-nlp-example.py", args: ""} | ||
# - {script: "integrations/model-training/xgboost/xgboost-california/xgboost-california.py", arg: ""} | ||
# - {script: "integrations/workflow-orchestration/metaflow/metaflow-hello-world/helloworld.py", arg: "run"} | ||
# - {script: "integrations/workflow-orchestration/metaflow/metaflow-model-evaluation/metaflow-model-evaluation.py", arg: "run --max-workers 1 --n_samples 100"} | ||
# - {script: "integrations/workflow-orchestration/metaflow/metaflow-regression/metaflow-regression-example.py", arg: "run"} | ||
env: | ||
SCRIPT_TO_TEST: ${{ matrix.example.script }} | ||
steps: | ||
- uses: actions/checkout@v3 | ||
- uses: actions/setup-python@v4 | ||
with: | ||
python-version: ${{ matrix.python-version }} | ||
- name: Get pip cache dir | ||
id: pip-cache | ||
run: | | ||
echo "::set-output name=dir::$(pip cache dir)" | ||
- name: pip cache | ||
uses: actions/cache@v3 | ||
with: | ||
path: ${{ steps.pip-cache.outputs.dir }} | ||
key: ${{ matrix.os }}-${{ matrix.python-version }}-pip | ||
restore-keys: | | ||
${{ matrix.os }}-${{ matrix.python-version }}-pip | ||
- name: Install dependencies | ||
run: | | ||
python -m pip install --upgrade pip | ||
directory=$(dirname -- "${SCRIPT_TO_TEST}") | ||
pip install -r "${directory}"/requirements.txt | ||
- name: Test examples | ||
run: | | ||
directory=$(dirname -- "${SCRIPT_TO_TEST}") | ||
script=$(basename -- "${SCRIPT_TO_TEST}") | ||
cd "$directory" || exit | ||
python "$script" ${{ matrix.example.arg }} | ||
env: | ||
COMET_API_KEY: ${{ secrets.COMET_API_KEY }} | ||
COMET_INTERNAL_SENTRY_DSN: ${{ secrets.COMET_INTERNAL_SENTRY_DSN }} | ||
COMET_WORKSPACE: cometexamples-tests |