compatibility with dbt 1.2.0rc1 (#24)

* reorganize Dockerfile and use docker in github CI
* in dbt-core 1.2.x, commit is now being done in SQLAdapter.drop_schema(), so there's no need for it here anymore. see this PR: dbt-labs/dbt-core#5198
* rm file on disk on DROP SCHEMA
* load text.so sqlean extension to get split_parts fn
* add math.so for functions needed by datediff

.dockerignore

@@ -0,0 +1 @@
+*.log

.github/workflows/ci.yml

@@ -5,27 +5,15 @@ on: [push]
 
 jobs:
   build:
-
-    runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9"]
+        python_version: ["3.8", "3.9", "3.10"]
+
+    runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v3
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install from requirements.txt
-        run: |
-          python -m pip install --upgrade pip
-          pip install pytest pytest-dotenv dbt-tests-adapter==1.1.0
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-      - name: Create data directory and install crypto.so
-        run: |
-          mkdir -p /tmp/dbt-sqlite-tests
-          cd /tmp/dbt-sqlite-tests && wget https://github.com/nalgeon/sqlean/releases/download/0.12.2/crypto.so
-      - name: Run test script
-        run: |
-          ./run_tests.sh
+      - name: Build docker image
+        run: docker build --build-arg PYTHON_VERSION=${{ matrix.python_version }}  --tag dbt-sqlite:$GITHUB_SHA .
+      - name: Run tests
+        run: docker run dbt-sqlite:$GITHUB_SHA run_tests.sh

.gitignore

@@ -14,3 +14,5 @@ env/
 *.sublime-*
 
 .idea/
+
+*.log

Dockerfile

@@ -1,18 +1,33 @@
 
-FROM ubuntu:20.04
+ARG PYTHON_VERSION=3.9
 
-RUN apt-get update && apt-get -y install git make python3 python3-pip python3-venv sqlite3 vim virtualenvwrapper wget
+FROM python:${PYTHON_VERSION}-bullseye
 
-RUN mkdir /root/dbt-sqlite
+RUN apt-get update && apt-get -y install git python3 python3-pip python3-venv sqlite3 vim virtualenvwrapper wget
 
-WORKDIR /root/dbt-sqlite
+WORKDIR /opt/dbt-sqlite
 
-RUN mkdir -p /tmp/dbt-sqlite-tests
+RUN python3 -m pip install --upgrade pip \
+    && python3 -m pip install pytest pytest-dotenv dbt-core~=1.2.0 dbt-tests-adapter~=1.2.0
 
-RUN cd /tmp/dbt-sqlite-tests && wget https://github.com/nalgeon/sqlean/releases/download/0.12.2/crypto.so
+RUN wget -q https://github.com/nalgeon/sqlean/releases/download/0.15.2/crypto.so
+RUN wget -q https://github.com/nalgeon/sqlean/releases/download/0.15.2/math.so
+RUN wget -q https://github.com/nalgeon/sqlean/releases/download/0.15.2/text.so
 
-RUN pip install dbt-core~=1.1.0
+WORKDIR /opt/dbt-sqlite/src 
 
-RUN pip install pytest pytest-dotenv dbt-tests-adapter==1.1.0
+COPY . .
 
-ENTRYPOINT ["./run_tests.sh"]
+RUN pip install .
+
+ENV TESTDATA=/opt/dbt-sqlite/testdata
+
+RUN mkdir $TESTDATA
+
+VOLUME /opt/dbt-sqlite/testdata
+
+WORKDIR /opt/dbt-sqlite/project
+
+ENV PATH=$PATH:/opt/dbt-sqlite/src
+
+VOLUME /opt/dbt-sqlite/project

README.md

@@ -80,10 +80,14 @@ dbt_sqlite:
       # files in schemas_and_paths as long as there's no conflicts.
       schema_directory: '/my_project/data'
 
-      # optional: list of file paths of SQLite extensions to load.
-      # crypto.so is needed for snapshots to work; see README
+      # optional: list of file paths of SQLite extensions to load. see README.
+      # crypto.so is needed for snapshots to work
+      # math.so is needed for ceil function
+      # text.so is needed for split_part macro to work
       extensions:
         - "/path/to/sqlean/crypto.so"
+        - "/path/to/sqlean/math.so"
+        - "/path/to/sqlean/text.so"
 
 ```
 
@@ -126,12 +130,14 @@ not worth the effort.
 
 ## SQLite Extensions
 
-For snapshots to work, you'll need the `crypto` module from SQLean to get an `md5()`
-function. It's recommended that you install all the SQLean modules, as they provide
-many common SQL functions missing from SQLite.
+These modules from SQLean are needed for certain functionality to work:
+- `crypto`: provides `md5()` function needed for snapshots
+- `math`: provides `ceil` and `floor` needed for the datediff macro to work
+- `text`: provides `split_part` function
 
 Precompiled binaries are available for download from the [SQLean github repository page](https://github.com/nalgeon/sqlean).
-You can also compile them yourself if you want.
+You can also compile them yourself if you want. Note that some modules depend on other libraries 
+(`math` for example depends on GLIBC); if an extension fails to load, you may want to try building it yourself.
 
 Point to these module files in your profile config as shown in the example above.
 
@@ -177,10 +183,22 @@ git push --tags
 
 ## Running Tests
 
+This runs the test suite and cleans up after itself:
 ```
 ./run_tests_docker.sh
 ```
 
+To run tests interactively and be able to examine test artifacts:
+```
+docker build . -t dbt-sqlite
+
+docker run --rm -it dbt-sqlite bash
+
+# see output for the locations of artifacts
+run_tests.sh -s
+```
+
+
 ## Credits
 
 Inspired by this initial work by stephen1000: https://github.com/stephen1000/dbt_sqlite

dbt/adapters/sqlite/impl.py

@@ -1,5 +1,8 @@
 
+import datetime
 import decimal
+import os
+import os.path
 from typing import List, Optional, Set
 
 import agate
@@ -21,6 +24,33 @@ class SQLiteAdapter(SQLAdapter):
     def date_function(cls):
         return 'date()'
 
+    # sqlite reports the exact string (including case) used when declaring a column of a certain type
+
+    @classmethod
+    def convert_text_type(cls, agate_table: agate.Table, col_idx: int) -> str:
+        return "TEXT"
+
+    @classmethod
+    def convert_number_type(cls, agate_table: agate.Table, col_idx: int) -> str:
+        decimals = agate_table.aggregate(agate.MaxPrecision(col_idx))  # type: ignore[attr-defined]
+        return "NUMERIC" if decimals else "INT"
+
+    @classmethod
+    def convert_boolean_type(cls, agate_table: agate.Table, col_idx: int) -> str:
+        return "BOOLEAN"
+
+    @classmethod
+    def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str:
+        return "TIMESTAMP WITHOUT TIMEZONE"
+
+    @classmethod
+    def convert_date_type(cls, agate_table: agate.Table, col_idx: int) -> str:
+        return "DATE"
+
+    @classmethod
+    def convert_time_type(cls, agate_table: agate.Table, col_idx: int) -> str:
+        return "TIME"
+
     def get_live_relation_type(self, relation):
         """
         returns the type of relation (table, view) from the live database
@@ -244,11 +274,13 @@ def timestamp_add_sql(
     def drop_schema(self, relation: BaseRelation) -> None:
         super().drop_schema(relation)
 
-        # can't detach a database in the middle of a transaction, so commit first.
-        # I wonder if drop_schema() in SQLAdapter should do this, since create_schema() does.
-        self.commit_if_has_connection()
-
         # never detach main
         if relation.schema != 'main':
             if self.check_schema_exists(relation.database, relation.schema):
                 self.connections.execute(f"DETACH DATABASE {relation.schema}")
+
+                if relation.schema in self.config.credentials.schemas_and_paths:
+                    path = self.config.credentials.schemas_and_paths[relation.schema]
+                else:
+                    path = os.path.join(self.config.credentials.schema_directory, relation.schema + ".db")
+                os.remove(path)

dbt/include/sqlite/macros/materializations/seed/helpers.sql

@@ -0,0 +1,3 @@
+{% macro sqlite__get_binding_char() %}
+  {{ return('?') }}
+{% endmacro %}

dbt/include/sqlite/macros/materializations/seed/seed.sql

@@ -19,8 +19,7 @@
             insert into {{ this.schema }}.{{ this.identifier}} ({{ cols_sql }}) values
             {% for row in chunk -%}
                 ({%- for column in agate_table.column_names -%}
-                    {# sqlite uses ? as placeholder character #}
-                    ?
+                    {{ get_binding_char() }}
                     {%- if not loop.last%},{%- endif %}
                 {%- endfor -%})
                 {%- if not loop.last%},{%- endif %}

dbt/include/sqlite/macros/utils/any_value.sql

@@ -0,0 +1,5 @@
+{% macro sqlite__any_value(expression) -%}
+
+    min({{ expression }})
+
+{%- endmacro %}

dbt/include/sqlite/macros/utils/bool_or.sql

@@ -0,0 +1,5 @@
+{% macro sqlite__bool_or(expression) -%}
+
+    max({{ expression }})
+
+{%- endmacro %}

dbt/include/sqlite/macros/utils/cast_bool_to_text.sql

@@ -0,0 +1,6 @@
+{% macro sqlite__cast_bool_to_text(field) %}
+    case
+      when {{ field }} = 0 then 'false'
+      when {{ field }} = 1 then 'true'    
+    end
+{% endmacro %}

dbt/include/sqlite/macros/utils/dateadd.sql

@@ -0,0 +1,8 @@
+{% macro sqlite__dateadd(datepart, interval, from_date_or_timestamp) %}
+
+    date(
+        {{ from_date_or_timestamp }},
+        "{{ datepart }} {{ datepart }}"
+        )
+
+{% endmacro %}

dbt/include/sqlite/macros/utils/datediff.sql

@@ -0,0 +1,42 @@
+
+{# TODO: fully implement this and rename #}
+{# adapted from postgresql #}
+{% macro sqlite__datediff_broken(first_date, second_date, datepart) -%}
+
+    {% if datepart == 'year' %}
+        (strftime('%Y', {{second_date}}) - strftime('%Y', {{first_date}}))
+    {#
+    {% elif datepart == 'quarter' %}
+        ({{ datediff(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date))
+    #}
+    {% elif datepart == 'month' %}
+        (({{ datediff(first_date, second_date, 'year') }} * 12 + strftime('%m', {{second_date}})) - strftime('%m', {{first_date}}))
+    {% elif datepart == 'day' %}
+        (floor(cast(strftime('%s', {{second_date}}) - strftime('%s', {{first_date}}) as real) / 86400) +
+        case when {{second_date}} <= strftime('%Y-%m-%d 23:59:59.999999', {{first_date}}) then -1 else 0 end)
+    {% elif datepart == 'week' %}
+        ({{ datediff(first_date, second_date, 'day') }} / 7 + case
+            when strftime('%w', {{first_date}}) <= strftime('%w', {{second_date}}) then
+                case when {{first_date}} <= {{second_date}} then 0 else -1 end
+            else
+                case when {{first_date}} <= {{second_date}} then 1 else 0 end
+        end)
+    {% elif datepart == 'hour' %}
+        {# ({{ datediff(first_date, second_date, 'day') }} * 24 + strftime("%H", {{second_date}}) - strftime("%H", {{first_date}})) #}
+        (ceil(cast(strftime('%s', {{second_date}}) - strftime('%s', {{first_date}}) as real) / 3600))
+    {% elif datepart == 'minute' %}
+        {# ({{ datediff(first_date, second_date, 'hour') }} * 60 + strftime("%M", {{second_date}}) - strftime("%M", {{first_date}})) #}
+        (ceil(cast(strftime('%s', {{second_date}}) - strftime('%s', {{first_date}}) as real) / 60))
+    {% elif datepart == 'second' %}
+        (strftime('%s', {{second_date}}) - strftime('%s', {{first_date}}))
+    {#
+    {% elif datepart == 'millisecond' %}
+        ({{ datediff(first_date, second_date, 'minute') }} * 60000 + floor(date_part('millisecond', ({{second_date}})::timestamp)) - floor(date_part('millisecond', ({{first_date}})::timestamp)))
+    {% elif datepart == 'microsecond' %}
+        ({{ datediff(first_date, second_date, 'minute') }} * 60000000 + floor(date_part('microsecond', ({{second_date}})::timestamp)) - floor(date_part('microsecond', ({{first_date}})::timestamp)))
+    #}
+    {% else %}
+        {{ exceptions.raise_compiler_error("Unsupported datepart for macro datediff in sqlite: {!r}".format(datepart)) }}
+    {% endif %}
+
+{%- endmacro %}

dbt/include/sqlite/macros/utils/hash.sql

@@ -0,0 +1,6 @@
+{% macro sqlite__hash(field) -%}
+    case 
+      when {{ field }} is not null 
+      then lower(hex(md5(cast({{ field }} as {{ api.Column.translate_type('string') }}))))
+    end
+{%- endmacro %}

dbt/include/sqlite/macros/utils/position.sql

@@ -0,0 +1,5 @@
+{% macro sqlite__position(substring_text, string_text) %}
+
+    instr({{ string_text }}, {{ substring_text }})
+
+{%- endmacro -%}

dbt/include/sqlite/macros/utils/right.sql

@@ -0,0 +1,10 @@
+{% macro sqlite__right(string_text, length_expression) %}
+    case
+      when {{ length_expression }} <> 0 then 
+      substr(
+          {{ string_text }},
+          -1 * {{ length_expression }}
+      )
+      else ''
+    end
+{%- endmacro -%}

run_tests.sh

@@ -1,13 +1,14 @@
 #!/bin/bash
 
-echo "HOME=$HOME"
+set -e
 
-pip install -e .
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+cd $SCRIPT_DIR
 
 # Leaving the database file between runs of pytest can mess up subsequent test runs.
 # Since this runs in a fresh container each time, it's not an issue.
 
-python3 -m pytest tests/functional
+pytest tests/functional $@
 
 ####
 
@@ -19,7 +20,7 @@ python3 -m pytest tests/functional
 
 cd $HOME
 
-git clone https://github.com/dbt-labs/jaffle_shop.git
+git clone --depth 1 https://github.com/dbt-labs/jaffle_shop.git
 
 cd jaffle_shop
 
@@ -50,7 +51,9 @@ jaffle_shop:
         main: '/tmp/jaffle_shop/jaffle_shop.db'
       schema_directory: '/tmp/jaffle_shop'
       extensions:
-        - "/tmp/dbt-sqlite-tests/crypto.so"
+        - "/opt/dbt-sqlite/crypto.so"
+        - "/opt/dbt-sqlite/math.so"
+        - "/opt/dbt-sqlite/text.so"
 
 EOF
 

run_tests_docker.sh

@@ -1,9 +1,8 @@
 #!/bin/bash
 
-docker build . -t dbt-sqlite-test
+docker build . -t dbt-sqlite
 
 docker run \
      --rm \
      --name dbt-sqlite-test-container \
-     -v "$(pwd)":"/root/dbt-sqlite" \
-     dbt-sqlite-test
+     dbt-sqlite run_tests.sh

tests/conftest.py

@@ -15,10 +15,12 @@ def dbt_profile_target():
         'database': 'adapter_test',
         'schema': 'main',
         'schemas_and_paths': {
-            'main': '/tmp/dbt-sqlite-tests/adapter_test.db'
+            'main': '/opt/dbt-sqlite/testdata/adapter_test.db'
         },
-        'schema_directory': '/tmp/dbt-sqlite-tests',
+        'schema_directory': '/opt/dbt-sqlite/testdata',
         'extensions' : [
-            "/tmp/dbt-sqlite-tests/crypto.so"
+            "/opt/dbt-sqlite/crypto.so",
+            "/opt/dbt-sqlite/math.so",
+            "/opt/dbt-sqlite/text.so"
         ]
     }