Merge pull request #171 from MC-kit/devel

partionining

.github/workflows/labeler.yml

@@ -26,7 +26,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repository
-        uses: actions/[email protected].0
+        uses: actions/[email protected].2
 
       - name: Sync GitHub Issue Labels
         uses: crazy-max/ghaction-github-labeler@v4

.github/workflows/release.yml

@@ -27,12 +27,12 @@ jobs:
 
     steps:
       - name: Check out the repository
-        uses: actions/[email protected].0
+        uses: actions/[email protected].2
         with:
           fetch-depth: 2    # need previous revision to define tag
 
       - name: Set up Python
-        uses: actions/setup-python@v4.5.0
+        uses: actions/setup-python@v4.6.0
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 

.github/workflows/tests.yml

@@ -36,10 +36,10 @@ jobs:
       !contains(github.event.head_commit.message, '[skip_ci]')
     steps:
       - name: Check out the repository
-        uses: actions/[email protected].0
+        uses: actions/[email protected].2
 
       - name: Set up Python 3.10  # on 3.11 pip install in unstable: "canonicalize" fails occasionally
-        uses: actions/setup-python@v4.5.0
+        uses: actions/setup-python@v4.6.0
         with:
           python-version: '3.10'
 
@@ -81,12 +81,12 @@ jobs:
     needs: lint
     steps:
       - name: Check out the repository
-        uses: actions/[email protected].0
+        uses: actions/[email protected].2
         with:
           fetch-depth: 1
 
       - name: Set up Python 3.11
-        uses: actions/setup-python@v4.5.0
+        uses: actions/setup-python@v4.6.0
         with:
           python-version: '3.11'
 
@@ -133,11 +133,11 @@ jobs:
     steps:
 
       - name: Checkout repository
-        uses: actions/[email protected].0
+        uses: actions/[email protected].2
         with:
           fetch-depth: 1
 
-      - uses: actions/setup-python@v4.5.0
+      - uses: actions/setup-python@v4.6.0
         with:
           python-version: ${{ matrix.python-version }}
           architecture: x64
@@ -183,10 +183,10 @@ jobs:
     steps:
 
       - name: Check out the repository
-        uses: actions/[email protected].0
+        uses: actions/[email protected].2
 
       - name: Set up Python 3.11
-        uses: actions/setup-python@v4.5.0
+        uses: actions/setup-python@v4.6.0
         with:
           python-version: '3.11'
 
@@ -230,4 +230,4 @@ jobs:
           nox --force-color --session=coverage -- xml
 
       - name: Upload coverage report
-        uses: codecov/[email protected].1
+        uses: codecov/[email protected].3

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "xpypact"
-version = "0.3.0"
+version = "0.4.0"
 description = "\"Python tools to work with elements and isotopes\""
 authors = ["dvp <[email protected]>"]
 license = "MIT"
@@ -241,7 +241,7 @@ filterwarnings = [
     "ignore:.*not typechecking multipledispatch.dispatcher.*UserWarning",
     'ignore:.*io.FileIO \[closed\]',
     'ignore:.*Deprecated call to `pkg_resources.declare_namespace',
-    'ignore:.*DeprecationWarning.*Implicit None on return values'
+    'ignore:.*Implicit None on return values:DeprecationWarning'
 ]
 log_format = "%(asctime)s %(levelname)s %(message)s"
 log_date_format = "%Y-%m-%d %H:%M:%S"
@@ -579,7 +579,7 @@ exclude = [
 "src/xpypact/inventory.py" = ["F811"]
 "src/xpypact/data_arrays.py" = ["ANN401", "PD011"]
 "src/xpypact/utils/resource.py" = ["ANN202"]
-"tools/*" = ["T201", "INP001"]
+"tools/*" = ["T201", "INP001", "S603", "S607"]
 
 [tool.ruff.mccabe]
 # Unlike Flake8, default to a complexity level of 10.

src/xpypact/dao/duckdb/implementation.py

@@ -204,12 +204,16 @@ def write_parquet(target_dir: Path, ds: xr.Dataset, material_id: int, case_id: i
                 material_id,
                 case_id,
             )
+            time_step_partition = "time_step_number, " if "time_step_number" in v.columns else ""
             sql = f"""
                 copy
                 (select * from frame)
-                to
-                '{path}'
-                (format parquet, partition_by (material_id, case_id), allow_overwrite 1)
+                to '{path}'
+                (
+                    format parquet,
+                    partition_by ({time_step_partition}material_id, case_id),
+                    allow_overwrite 1
+                )
                 """  # noqa: S608 - sql injection
             con.execute(sql)
     finally:

tests/test_duckdb_dao.py

@@ -67,16 +67,16 @@ def test_save(dataset_with_gamma):
 # noinspection SqlNoDataSourceInspection
 def test_write_parquet(tmp_path, dataset_with_gamma):
     write_parquet(tmp_path, dataset_with_gamma, 1, 1)
-    assert Path(tmp_path / "time_steps/material_id=1").exists()
-    assert Path(tmp_path / "time_steps/material_id=1/case_id=1").exists()
+    assert Path(tmp_path / "time_steps/time_step_number=1/material_id=1").exists()
+    assert Path(tmp_path / "time_steps/time_step_number=1/material_id=1/case_id=1").exists()
     write_parquet(tmp_path, dataset_with_gamma, 1, 2)
-    assert Path(tmp_path / "time_steps/material_id=1/case_id=2").exists()
+    assert Path(tmp_path / "time_steps/time_step_number=1/material_id=1/case_id=2").exists()
     con = connect(":memory:")
     path = tmp_path / "nuclides/*/*/*.parquet"
     sql = f"select * from read_parquet('{path}', hive_partitioning=true)"  # noqa: S608
     nuclides = con.execute(sql).df()
     assert not nuclides.loc[2].empty
-    path = tmp_path / "time_steps/*/*/*.parquet"
+    path = tmp_path / "time_steps/*/*/*/*.parquet"
     sql = f"select * from read_parquet('{path}', hive_partitioning=true)"  # noqa: S608
     time_steps = con.execute(sql).df()
     assert not time_steps.loc[2].empty