Merge branch 'staging' into main

.github/workflows/python-lint-and-test.yml

@@ -3,7 +3,7 @@
 
 name: Python Linting, Test and Upload
 
-on: [create, pull_request]
+on: [push, pull_request]
 
 jobs:
   build:
@@ -25,57 +25,22 @@ jobs:
         run: |
           sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d
 
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v2
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}
+
       - name: Install Dependencies
         run: |
           python -m pip install --upgrade pip
-          python -m pip install flake8 pytest
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          ./bin/task setup
 
-      - name: Lint with flake8
+      - name: Lint code
         run: |
           ./bin/task lint
 
-      - name: Test with pytest
+      - name: Test code
         run: |
           ./bin/task test
-
-  deploy:
-    needs: build
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-
-      - name: Install Task
-        run: |
-          sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d
-
-      - name: Install Dependencies
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install flake8 pytest
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-
-      - name: Build package
-        run: |
-          python setup.py sdist bdist_wheel
-
-      - name: Publish package to Test-PyPi
-        if: github.event_name == 'push' && startsWith(github.ref, 'refs/staging')
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          user: __token__
-          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
-          repository_url: https://test.pypi.org/legacy/
-
-      - name: Publish package to PyPi
-        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          user: __token__
-          password: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/python-upload.yml

@@ -0,0 +1,97 @@
+name: Release Python Package
+
+on:
+  push:
+    branches:
+      - staging
+      - "main"
+    tags:
+      - "**"
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  upload:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Cancel previous runs on the same branch
+        if: ${{ github.ref != 'refs/heads/main' }}
+        uses: styfle/[email protected]
+        with:
+          access_token: ${{ github.token }}
+
+      - uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Install Task
+        run: |
+          sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d
+
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          ./bin/task setup
+
+      - name: Lint code
+        run: |
+          ./bin/task lint
+
+      - name: Test code
+        run: |
+          ./bin/task test
+
+      - name: Test code
+        run: |
+          ./bin/task test
+
+      - name: Build packages for release
+        run: |
+          ./bin/task build
+
+      - name: Save packages as artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: dist
+          path: dist
+          if-no-files-found: error
+
+      - name: Test for PyPi Test Secret
+        id: check_secrets
+        shell: bash
+        run: |
+          unset HAS_TEST_SECRET
+          if [ -n "$PYPI_TEST_TOKEN" ]; then HAS_TEST_SECRET='true' ; fi
+          echo ::set-output name=HAS_TEST_SECRET::${HAS_TEST_SECRET}
+
+          unset HAS_SECRET
+          if [ -n "$PYPI_TOKEN" ]; then HAS_SECRET='true' ; fi
+          echo ::set-output name=HAS_SECRET::${HAS_SECRET}
+        env:
+          PYPI_TEST_TOKEN: "${{ secrets.TEST_PYPI_API_TOKEN }}"
+          PYPI_TOKEN: "${{ secrets.PYPI_API_TOKEN }}"
+
+      - name: Publish package to Test-PyPi
+        # if: ${{ github.ref != 'refs/heads/staging' }}
+        if: steps.check_secrets.outputs.HAS_TEST_SECRET
+        env:
+          TWINE_REPOSITORY_URL: https://test.pypi.org/legacy/
+          TWINE_USERNAME: __token__
+          TWINE_NON_INTERACTIVE: 1
+          TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }}
+        run: twine upload --non-interactive --skip-existing --verbose 'dist/*'
+
+      - name: Publish package to PyPi
+        # if: ${{ github.ref != 'refs/heads/main' }}
+        if: steps.check_secrets.outputs.HAS_SECRET && startsWith(github.event.ref, 'refs/tags')
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_NON_INTERACTIVE: 1
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+        run: twine upload --non-interactive --skip-existing --verbose 'dist/*'

README.md

@@ -1,12 +1,15 @@
 # Chia Tea 🌱🍵
 
-Chia Tea is a tools and utility library for the Chia Cryptocurrency. We are building tools in this library to serve our own needs and share our work openly with others. Feel free to use them and make your life easier 💚
+Chia Tea is a tools and utility library for the Chia Cryptocurrency.
+We are building tools in this library to serve our own needs and share our work openly with others.
+Feel free to use them and make your life easier 💚
 
 # Table of Contents
 
-- [Intro and FAQ](https://github.com/tnt-codie/chia-tea/blob/main/docs/faq.md)
-- [Quick-Start](https://github.com/tnt-codie/chia-tea/blob/main/docs/quick_start.md)
-- [Architecture Summary](https://github.com/tnt-codie/chia-tea/blob/develop/docs/architecture_summary.md)
+- [Intro and FAQ](docs/faq.md)
+- [Quick-Start](docs/quick_start.md)
+- [Architecture Summary](docs/architecture_summary.md)
+- [Structure of this repository](#structure-of-this-repository)
 - [Feature Overview](#feature-overview)
   - [Copy Tool](#copy-tool)
   - [Monitoring](#monitoring)
@@ -15,13 +18,17 @@ Chia Tea is a tools and utility library for the Chia Cryptocurrency. We are buil
 - [About Us](#about-us)
 - [✊ Support Us ✊](#support-us)
 
+## Structure of this repository
+
+![Chia-Tea Infrastructure](docs/Chia_Infrastructure.png?raw=true)
+
 ## Feature Overview
 
 ### Copy Tool
 
 Copy is a tool to copy your chia files to a different location. It can be faster to plot to a temporary storage space and then move the plots to your harvester afterwards to not block the plotting queue. We manage this process through our copy cli tool. It incorporates the following features:
 
-- Selects a drive from multiple specified
+- Selects a drive with sufficient space from multiple disks specified
 - Checks drive space regularly
 - Takes plots which are being copied already into account
 - Uses the drive with the fewest copy processes
@@ -39,7 +46,7 @@ The discord module is a bot watching a farm and reports major incidents. Interna
 
 ## About Us
 
-We are a small group of professional engineers and software developers doing Chia for fun. Join our discord server: https://discord.gg/azNjcVPYnY for questions, tips and tricks or just come over for a nice warm cup of your favourite tea. 
+We are a small group of professional engineers and software developers doing Chia for fun. Join our discord server: https://discord.gg/azNjcVPYnY for questions, tips and tricks or just come over for a nice warm cup of your favourite tea.
 
 <a name="security"></a>
 

Taskfile.yml

@@ -1,36 +1,50 @@
 version: "3"
 
 tasks:
+  setup:
+    desc: Install dependencies and dev-dependencies
+    cmds:
+      - python3 -m pip install --user poetry
+      - poetry config virtualenvs.in-project true
+      - poetry install --no-root
+
   generate:
     desc: Generates code from the protobuf definitions.
     cmds:
-      - python3 -m grpc.tools.protoc -I./protocol --python_out=. protocol/chia_tea/protobuf/generated/hardware.proto
-      - python3 -m grpc.tools.protoc -I./protocol --python_out=. protocol/chia_tea/protobuf/generated/chia.proto
-      - python3 -m grpc.tools.protoc -I./protocol --python_out=. protocol/chia_tea/protobuf/generated/computer_info.proto
-      - python3 -m grpc.tools.protoc -I./protocol --python_out=. protocol/chia_tea/protobuf/generated/machine_info.proto
-      - python3 -m grpc.tools.protoc -I./protocol --python_out=. protocol/chia_tea/protobuf/generated/config.proto
-      - python3 -m grpc.tools.protoc -I./protocol --python_out=. --grpc_python_out=. protocol/chia_tea/protobuf/generated/monitoring_service.proto
+      - poetry run python -m grpc.tools.protoc -I./protocol --python_out=. protocol/chia_tea/protobuf/generated/hardware.proto
+      - poetry run python -m grpc.tools.protoc -I./protocol --python_out=. protocol/chia_tea/protobuf/generated/hardware.proto
+      - poetry run python -m grpc.tools.protoc -I./protocol --python_out=. protocol/chia_tea/protobuf/generated/chia.proto
+      - poetry run python -m grpc.tools.protoc -I./protocol --python_out=. protocol/chia_tea/protobuf/generated/computer_info.proto
+      - poetry run python -m grpc.tools.protoc -I./protocol --python_out=. protocol/chia_tea/protobuf/generated/machine_info.proto
+      - poetry run python -m grpc.tools.protoc -I./protocol --python_out=. protocol/chia_tea/protobuf/generated/config.proto
+      - poetry run python -m grpc.tools.protoc -I./protocol --python_out=. --grpc_python_out=. protocol/chia_tea/protobuf/generated/monitoring_service.proto
 
   lint:
     desc: Lints the code and reports on issues.
     cmds:
-      - flake8 chia_tea --count --select=E9,F63,F7,F82 --show-source --statistics
-      - flake8 chia_tea --exclude ./chia_tea/protobuf/generated --count --max-complexity=10 --max-line-length=127
+      - poetry run flake8 chia_tea --count --select=E9,F63,F7,F82 --show-source --statistics
+      - poetry run flake8 chia_tea --exclude ./chia_tea/protobuf/generated --count --max-complexity=10 --max-line-length=127
+      - poetry run pylint chia_tea --ignore=chia_tea/protobuf/generated
+
+  build:
+    desc: Builds the puthon package
+    cmds:
+      - poetry build
 
   test:
     desc: Runs the test suite.
     cmds:
-      - python3 -m pytest
+      - poetry run pytest
 
   copy:
     desc: Starts the copy cli tool.
     cmds:
-      - python3 -m chia_tea.copy_plots
+      - poetry run python -m chia_tea.copy_plots
 
   discord:
     desc: Starts the discord bot.
     cmds:
-      - python3 -m chia_tea.discord.bot
+      - poetry run python -m chia_tea.discord.bot
 
   clean:
     desc: Cleans up by deleting generated files including the database.
@@ -42,13 +56,13 @@ tasks:
     desc: Starts the monitoring main server collecting data from clients.
     deps: [generate]
     cmds:
-      - python3 -m chia_tea.monitoring.server
+      - poetry run python -m chia_tea.monitoring.server
 
   client:
     desc: Starts a monitoring client collecting data and sending it to the server.
     deps: [generate]
     cmds:
-      - python3 -m chia_tea.monitoring.client
+      - poetry run python -m chia_tea.monitoring.client
 
   certs:
     desc: Generates certificates for the monitoring

chia_tea/chia_watchdog/ChiaWatchdog.py

@@ -11,21 +11,20 @@
 
 class ChiaWatchdog:
     """Class for watching chia"""
+    # pylint: disable=too-many-instance-attributes
 
     date_last_reset: date
     __logfile_watching_ready: bool
 
     # members related to logfile checking
     harvester_infos: Dict[str, FarmerHarvesterLogfile]
+    farmed_blocks: List[str]
 
     # members for contacting chia directly
     farmer_service: FarmerAPI
     wallet_service: WalletAPI
     harvester_service: HarvesterAPI
 
-    # TODO check if we still need this
-    farmed_blocks: List[str]
-
     def __init__(self, logfile_filepath: str):
         """ initialize a chia watchdog
 
@@ -59,6 +58,8 @@ def copy(self) -> 'ChiaWatchdog':
         # date is immutable thus safe to share
         new_instance.date_last_reset = self.date_last_reset
         # not sure about this one but ok
+        # pylint: disable=protected-access
+        # pylint: disable=unused-private-member
         new_instance.__logfile_watching_ready = self.__logfile_watching_ready
         new_instance.farmer_service = self.farmer_service.copy()
         new_instance.wallet_service = self.wallet_service.copy()
@@ -70,24 +71,22 @@ def copy(self) -> 'ChiaWatchdog':
     async def ready(self):
         """ Wait for the readiness of the watchdog
         """
-        # TODO refactor this
         while not (self.__logfile_watching_ready and
                    self.harvester_service.is_ready and
                    self.farmer_service.is_ready and
                    self.wallet_service.is_ready):
             await asyncio.sleep(0.25)
 
     def set_as_ready(self):
+        """ When we scanned the entire logfile once and caught up set this """
         self.__logfile_watching_ready = True
 
     def is_reset_time(self) -> bool:
-        """If it is already midnight we need to reset to prevent data overflow"""
-        # TODO change to FIFO queue
+        """ If it is already midnight we need to reset to prevent data overflow """
         date_right_now = date.today()
         if (date_right_now - self.date_last_reset).total_seconds():
             return True
-        else:
-            return False
+        return False
 
     def reset_data(self):
         """ Reset the watchdogs data """
@@ -98,14 +97,14 @@ def reset_data(self):
 
     def get_or_create_harvester_info(
         self,
-        id: str,
+        harvester_id: str,
         ip_address: str,
     ) -> FarmerHarvesterLogfile:
         """ Get or create a harvester info
 
         Parameters
         ----------
-        id : str
+        harvester_id : str
             id of the harvester
         ip_address : str
             ip address
@@ -116,15 +115,15 @@ def get_or_create_harvester_info(
             existing or newly created harvester info
         """
 
-        harvester_info = self.harvester_infos.get(id)
+        harvester_info = self.harvester_infos.get(harvester_id)
 
         #  does not exist yet
         if harvester_info is None:
             harvester_info = FarmerHarvesterLogfile(
-                harvester_id=id,
+                harvester_id=harvester_id,
                 ip_address=ip_address,
             )
-            self.harvester_infos[id] = harvester_info
+            self.harvester_infos[harvester_id] = harvester_info
         else:
             # make sure ip is updated
             harvester_info.ip_address = ip_address