ci: Clean up of Makefile and Gitpod setup (#464)

* switch to Dockerfile, same configuration as the kedro branch. Signed-off-by: Nok <[email protected]> * simplify setup Signed-off-by: Nok <[email protected]> * update makefile Signed-off-by: Nok <[email protected]> * simplified install in background while not slowing down startup Signed-off-by: Nok <[email protected]> * fix command Signed-off-by: Nok <[email protected]> * fix setup Signed-off-by: Nok <[email protected]> * combined makefile Signed-off-by: Nok <[email protected]> --------- Signed-off-by: Nok <[email protected]>
kedro-org · Aug 28, 2024 · cf1617b · cf1617b
1 parent 1eb2bf7
commit cf1617b
Show file tree

Hide file tree

Showing 2 changed files with 36 additions and 60 deletions.
diff --git a/.gitpod.yml b/.gitpod.yml
@@ -1,6 +1,4 @@
-# Learn more from ready-to-use templates: https://www.gitpod.io/docs/introduction/getting-started/quickstart
-image: gitpod/workspace-python-3.10:2023-04-20-16-32-37
-
+image: gitpod/workspace-python-3.11
 
 tasks:
   # We want packages installed during the pre-build init steps to go to /workspace
@@ -12,22 +10,16 @@ tasks:
       echo PIP_USER=no >> ~/.bashrc && export PIP_USER=no
     init: |
       make sign-off
+      pip install uv
+      uv venv
+      echo source .venv/bin/activate >> ~/.bashrc
+      source ~/.bashrc
+      make install-test-requirements plugin=kedro-datasets
     command: |
       pre-commit install --install-hooks
       clear
 
-
-github:
-  prebuilds:
-    # enable for the master/default branch (defaults to true)
-    master: true
-    # enable for all branches in this repo (defaults to false)
-    branches: true
-    # enable for pull requests coming from this repo (defaults to true)
-    pullRequests: true
-    # enable for pull requests coming from forks (defaults to false)
-    pullRequestsFromForks: true
-    # add a "Review in Gitpod" button as a comment to pull requests (defaults to true)
-    addComment: false
-    # add a "Review in Gitpod" button to pull requests (defaults to false)
-    addBadge: true
+  - name: system
+    init: |
+      sudo apt-get update && sudo apt-get install -y --no-install-recommends libgl1 make
+      sudo apt-get install -y --no-install-recommends libatk-bridge2.0-0 libcups2 ca-certificates fonts-liberation libasound2 libatk-bridge2.0-0 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgbm1 libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 lsb-release wget xdg-utils
diff --git a/Makefile b/Makefile
@@ -5,13 +5,6 @@ package:
 	rm -Rf dist;\
 	python -m build
 
-pypi:
-	python -m pip install twine -U
-	python -m twine upload $(plugin)/dist/*
-
-install: package
-	cd $(plugin) && pip install -U dist/*.whl
-
 install-pip-setuptools:
 	python -m pip install -U pip setuptools wheel
 
@@ -25,46 +18,14 @@ mypy:
 test:
 	cd $(plugin) && pytest tests --cov-config pyproject.toml --numprocesses 4 --dist loadfile
 
-# Run test_tensorflow_model_dataset separately, because these tests are flaky when run as part of the full test-suite
-dataset-tests: dataset-doctests
-	cd kedro-datasets && pytest tests --cov-config pyproject.toml --numprocesses 4 --dist loadfile --ignore tests/tensorflow
-	cd kedro-datasets && pytest tests/tensorflow/test_tensorflow_model_dataset.py  --no-cov
-
-extra_pytest_args-no-spark=--ignore kedro_datasets/databricks --ignore kedro_datasets/spark
-extra_pytest_args=
-dataset-doctest%:
-	if [ "${*}" != 's-no-spark' ] && [ "${*}" != 's' ]; then \
-	  echo "make: *** No rule to make target \`${@}\`.  Stop."; \
-	  exit 2; \
-	fi; \
-    \
-	# The ignored datasets below require complicated setup with cloud/database clients which is overkill for the doctest examples.
-	cd kedro-datasets && pytest kedro_datasets --doctest-modules --doctest-continue-on-failure --no-cov \
-	  --ignore kedro_datasets/pandas/gbq_dataset.py \
-	  --ignore kedro_datasets/partitions/partitioned_dataset.py \
-	  --ignore kedro_datasets/redis/redis_dataset.py \
-	  --ignore kedro_datasets/snowflake/snowpark_dataset.py \
-	  --ignore kedro_datasets/spark/spark_hive_dataset.py \
-	  --ignore kedro_datasets/spark/spark_jdbc_dataset.py \
-	  $(extra_pytest_arg${*})
-
-test-sequential:
-	cd $(plugin) && pytest tests --cov-config pyproject.toml
-
 e2e-tests:
 	cd $(plugin) && behave
 
 secret-scan:
 	trufflehog --max_depth 1 --exclude_paths trufflehog-ignore.txt .
 
-clean:
-	cd $(plugin);\
-	rm -rf build dist pip-wheel-metadata .pytest_cache;\
-	find . -regex ".*/__pycache__" -exec rm -rf {} +;\
-	find . -regex ".*\.egg-info" -exec rm -rf {} +;\
-
 install-test-requirements:
-	cd $(plugin) && pip install ".[test]"
+	cd $(plugin) && uv pip install ".[test]"
 
 install-pre-commit:
 	pre-commit install --install-hooks
@@ -79,12 +40,12 @@ sign-off:
 	echo '--in-place "$$1"' >> .git/hooks/commit-msg
 	chmod +x .git/hooks/commit-msg
 
+## kedro-datasets specific
+
 # kedro-datasets related only
 test-no-spark: dataset-doctests-no-spark
 	cd kedro-datasets && pytest tests --no-cov --ignore tests/spark --ignore tests/databricks --numprocesses 4 --dist loadfile
 
-test-no-spark-sequential: dataset-doctests-no-spark
-	cd kedro-datasets && pytest tests --no-cov --ignore tests/spark --ignore tests/databricks
 
 # kedro-datasets/snowflake tests skipped from default scope
 test-snowflake-only:
@@ -93,3 +54,26 @@ test-snowflake-only:
 
 check-datasets-docs:
 	cd kedro-datasets && python -m sphinx -WETan -j auto -D language=en -b linkcheck -d _build/doctrees docs/source _build/linkcheck
+
+# Run test_tensorflow_model_dataset separately, because these tests are flaky when run as part of the full test-suite
+dataset-tests: dataset-doctests
+	cd kedro-datasets && pytest tests --cov-config pyproject.toml --numprocesses 4 --dist loadfile --ignore tests/tensorflow
+	cd kedro-datasets && pytest tests/tensorflow/test_tensorflow_model_dataset.py  --no-cov
+
+extra_pytest_args-no-spark=--ignore kedro_datasets/databricks --ignore kedro_datasets/spark
+extra_pytest_args=
+dataset-doctest%:
+	if [ "${*}" != 's-no-spark' ] && [ "${*}" != 's' ]; then \
+	  echo "make: *** No rule to make target \`${@}\`.  Stop."; \
+	  exit 2; \
+	fi; \
+    \
+	# The ignored datasets below require complicated setup with cloud/database clients which is overkill for the doctest examples.
+	cd kedro-datasets && pytest kedro_datasets --doctest-modules --doctest-continue-on-failure --no-cov \
+	  --ignore kedro_datasets/pandas/gbq_dataset.py \
+	  --ignore kedro_datasets/partitions/partitioned_dataset.py \
+	  --ignore kedro_datasets/redis/redis_dataset.py \
+	  --ignore kedro_datasets/snowflake/snowpark_dataset.py \
+	  --ignore kedro_datasets/spark/spark_hive_dataset.py \
+	  --ignore kedro_datasets/spark/spark_jdbc_dataset.py \
+	  $(extra_pytest_arg${*})