From a167269903722c86fd022d4cc3e0fa7b3088598d Mon Sep 17 00:00:00 2001 From: z3z1ma Date: Wed, 29 Mar 2023 01:24:02 -0700 Subject: [PATCH] =?UTF-8?q?=F0=9F=92=9A=20add=20ci=20auto=20release=20as?= =?UTF-8?q?=20well=20as=20dlt=20currying=20update?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .changes/0.2.21.md | 5 ++ .github/labels.yml | 66 +++++++++++++++++ .github/release-drafter.yml | 33 +++++++++ .github/workflows/constraints.txt | 3 + .github/workflows/labeler.yml | 18 +++++ .github/workflows/release.yml | 78 ++++++++++++++++++++ CHANGELOG.md | 6 ++ alto/dlt_singer.py | 115 ++++++++++++++++-------------- pyproject.toml | 2 +- 9 files changed, 271 insertions(+), 55 deletions(-) create mode 100644 .changes/0.2.21.md create mode 100644 .github/labels.yml create mode 100644 .github/release-drafter.yml create mode 100644 .github/workflows/constraints.txt create mode 100644 .github/workflows/labeler.yml create mode 100644 .github/workflows/release.yml diff --git a/.changes/0.2.21.md b/.changes/0.2.21.md new file mode 100644 index 0000000..4fc5bd6 --- /dev/null +++ b/.changes/0.2.21.md @@ -0,0 +1,5 @@ +## 0.2.21 - 2023-03-29 +### Changed +* Updated singer dlt to return curried source function so that the name is configurable +### Fixed +* Added missing instantiation arg for custom AltoExtension cls diff --git a/.github/labels.yml b/.github/labels.yml new file mode 100644 index 0000000..f7f83aa --- /dev/null +++ b/.github/labels.yml @@ -0,0 +1,66 @@ +--- +# Labels names are important as they are used by Release Drafter to decide +# regarding where to record them in changelog or if to skip them. +# +# The repository labels will be automatically configured using this file and +# the GitHub Action https://github.com/marketplace/actions/github-labeler. +- name: breaking + description: Breaking Changes + color: bfd4f2 +- name: bug + description: Something isn't working + color: d73a4a +- name: build + description: Build System and Dependencies + color: bfdadc +- name: ci + description: Continuous Integration + color: 4a97d6 +- name: dependencies + description: Pull requests that update a dependency file + color: 0366d6 +- name: documentation + description: Improvements or additions to documentation + color: 0075ca +- name: duplicate + description: This issue or pull request already exists + color: cfd3d7 +- name: enhancement + description: New feature or request + color: a2eeef +- name: github_actions + description: Pull requests that update Github_actions code + color: "000000" +- name: good first issue + description: Good for newcomers + color: 7057ff +- name: help wanted + description: Extra attention is needed + color: 008672 +- name: invalid + description: This doesn't seem right + color: e4e669 +- name: performance + description: Performance + color: "016175" +- name: python + description: Pull requests that update Python code + color: 2b67c6 +- name: question + description: Further information is requested + color: d876e3 +- name: refactoring + description: Refactoring + color: ef67c4 +- name: removal + description: Removals and Deprecations + color: 9ae7ea +- name: style + description: Style + color: c120e5 +- name: testing + description: Testing + color: b1fc6f +- name: wontfix + description: This will not be worked on + color: ffffff diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml new file mode 100644 index 0000000..9ba95f9 --- /dev/null +++ b/.github/release-drafter.yml @@ -0,0 +1,33 @@ +categories: + - title: ":boom: Breaking Changes" + label: "breaking" + - title: ":rocket: Features" + label: "enhancement" + - title: ":fire: Removals and Deprecations" + label: "removal" + - title: ":beetle: Fixes" + label: "bug" + - title: ":racehorse: Performance" + label: "performance" + - title: ":rotating_light: Testing" + label: "testing" + - title: ":construction_worker: Continuous Integration" + label: "ci" + - title: ":books: Documentation" + label: "documentation" + - title: ":hammer: Refactoring" + label: "refactoring" + - title: ":lipstick: Style" + label: "style" + - title: ":package: Dependencies" + labels: + - "dependencies" + - "build" +template: | + ## Changes + + See the [CHANGELOG](https://github.com/z3z1ma/dbt-osmosis/blob/main/CHANGELOG.md) for a full list of changes. + + Automatically detected changes: + + $CHANGES diff --git a/.github/workflows/constraints.txt b/.github/workflows/constraints.txt new file mode 100644 index 0000000..614fcfc --- /dev/null +++ b/.github/workflows/constraints.txt @@ -0,0 +1,3 @@ +pip==23.0.1 +poetry==1.3.2 +virtualenv==20.21.0 diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml new file mode 100644 index 0000000..da156c4 --- /dev/null +++ b/.github/workflows/labeler.yml @@ -0,0 +1,18 @@ +name: Labeler + +on: + push: + branches: + - main + +jobs: + labeler: + runs-on: ubuntu-latest + steps: + - name: Check out the repository + uses: actions/checkout@v3 + + - name: Run Labeler + uses: crazy-max/ghaction-github-labeler@v4.0.0 + with: + skip-delete: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..9c0e5a1 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,78 @@ +name: Release + +on: + push: + branches: + - main + +jobs: + release: + name: Release + runs-on: ubuntu-latest + steps: + - name: Check out the repository + uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: "3.10" + + - name: Upgrade pip + run: | + pip install --constraint=.github/workflows/constraints.txt pip + pip --version + + - name: Install Poetry + run: | + pip install --constraint=.github/workflows/constraints.txt poetry + poetry --version + + - name: Check if there is a parent commit + id: check-parent-commit + run: | + echo "::set-output name=sha::$(git rev-parse --verify --quiet HEAD^)" + + - name: Detect and tag new version + id: check-version + if: steps.check-parent-commit.outputs.sha + uses: salsify/action-detect-and-tag-new-version@v2.0.3 + with: + version-command: | + bash -o pipefail -c "poetry version | awk '{ print \$2 }'" + + - name: Bump version for developmental release + if: "! steps.check-version.outputs.tag" + run: | + poetry version patch && + version=$(poetry version | awk '{ print $2 }') && + poetry version $version.dev.$(date +%s) + + - name: Build package + run: | + poetry build --ansi + + - name: Publish package on PyPI + if: steps.check-version.outputs.tag + uses: pypa/gh-action-pypi-publish@v1.8.3 + with: + user: __token__ + password: ${{ secrets.PYPI_TOKEN }} + + - name: Publish package on TestPyPI + if: "! steps.check-version.outputs.tag" + uses: pypa/gh-action-pypi-publish@v1.8.3 + with: + user: __token__ + password: ${{ secrets.TEST_PYPI_TOKEN }} + repository_url: https://test.pypi.org/legacy/ + + - name: Publish the release notes + uses: release-drafter/release-drafter@v5.23.0 + with: + publish: ${{ steps.check-version.outputs.tag != '' }} + tag: ${{ steps.check-version.outputs.tag }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c4dad9..4a22b33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). +## 0.2.21 - 2023-03-29 +### Changed +* Updated singer dlt to return curried source function so that the name is configurable +### Fixed +* Added missing instantiation arg for custom AltoExtension cls + ## 0.2.13 - 2023-03-15 ### Added * Added rill extension diff --git a/alto/dlt_singer.py b/alto/dlt_singer.py index 1914937..d7c6226 100644 --- a/alto/dlt_singer.py +++ b/alto/dlt_singer.py @@ -70,61 +70,68 @@ def run(self) -> None: self.graceful_exit = True -@dlt.source -def singer( - source: str, - streams: t.Optional[t.List[str]] = None, - env: t.Optional[str] = None, - resource_options: t.Optional[t.Dict[str, t.Any]] = None, -) -> t.Sequence[t.Any]: - """Singer source function.""" - if resource_options is None: - resource_options = {} - if env is None: - env = os.getenv("ALTO_ENV", alto.constants.DEFAULT_ENVIRONMENT) - # Prepare engine and data structures - engine = alto.engine.get_engine(env) - (tap,) = alto.engine.make_plugins( - source, - filesystem=engine.filesystem, - configuration=engine.configuration, - ) - catalog = alto.engine.get_and_render_catalog(tap, engine.filesystem) - # Use the streams from the catalog if not provided - baseline_streams = [stream.tap_stream_id for stream in catalog.streams] - if streams is None: - streams = baseline_streams - if not streams: - raise ValueError("No streams were found in the catalog or selected by the user.") - for stream in streams: - if stream not in baseline_streams: - raise ValueError( - f"Stream '{stream}' was not found in the catalog. " - f"Available streams: {baseline_streams}" +def singer(name: str, **kwargs) -> t.Callable[..., t.Sequence[t.Any]]: + """Factory for creating a dlt.source function for a singer tap.""" + + @dlt.source(name=name, **kwargs) + def _singer( + source: str, + streams: t.Optional[t.List[str]] = None, + env: t.Optional[str] = None, + resource_options: t.Optional[t.Dict[str, t.Any]] = None, + ) -> t.Sequence[t.Any]: + """Singer source function.""" + if resource_options is None: + resource_options = {} + if env is None: + env = os.getenv("ALTO_ENV", alto.constants.DEFAULT_ENVIRONMENT) + # Prepare engine and data structures + engine = alto.engine.get_engine(env) + (tap,) = alto.engine.make_plugins( + source, + filesystem=engine.filesystem, + configuration=engine.configuration, + ) + catalog = alto.engine.get_and_render_catalog(tap, engine.filesystem) + # Use the streams from the catalog if not provided + baseline_streams = [stream.tap_stream_id for stream in catalog.streams] + if streams is None: + streams = baseline_streams + if not streams: + raise ValueError("No streams were found in the catalog or selected by the user.") + for stream in streams: + if stream not in baseline_streams: + raise ValueError( + f"Stream '{stream}' was not found in the catalog. " + f"Available streams: {baseline_streams}" + ) + # Create the demuxer + tap.select = streams + producer = SingerTapDemux(tap, engine, dlt.state().setdefault(tap.name, {}), *streams) + producer.start() + # Use the catalog to determine some resource props + for stream in streams: + opts: dict = resource_options.setdefault(stream, {}) + for entry in catalog.streams: + if entry.tap_stream_id == stream: + this = entry + break + else: + continue + method = this.forced_replication_method or this.replication_method + if method == "FULL_TABLE": + opts.setdefault("write_disposition", "replace") + elif method == "INCREMENTAL": + opts.setdefault("write_disposition", "append") + # Create the dlt resources + return tuple( + singer_stream_factory(stream, resource_options[stream])( + producer.streams[stream], producer ) - # Create the demuxer - tap.select = streams - producer = SingerTapDemux(tap, engine, dlt.state().setdefault(tap.name, {}), *streams) - producer.start() - # Use the catalog to determine some resource props - for stream in streams: - opts: dict = resource_options.setdefault(stream, {}) - for entry in catalog.streams: - if entry.tap_stream_id == stream: - this = entry - break - else: - continue - method = this.forced_replication_method or this.replication_method - if method == "FULL_TABLE": - opts.setdefault("write_disposition", "replace") - elif method == "INCREMENTAL": - opts.setdefault("write_disposition", "append") - # Create the dlt resources - return tuple( - singer_stream_factory(stream, resource_options[stream])(producer.streams[stream], producer) - for stream in streams - ) + for stream in streams + ) + + return _singer def singer_stream_factory( diff --git a/pyproject.toml b/pyproject.toml index d0e5b49..538ac76 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "singer-alto" -version = "0.2.20" +version = "0.2.21" description = "A package for managing singer.io taps and targets" authors = ["z3z1ma "] readme = "README.md"