Merge branch 'master' into decoding-pipeline

.git_archival.txt

@@ -1,4 +1,4 @@
 node: $Format:%H$
 node-date: $Format:%cI$
 describe-name: $Format:%(describe:tags=true,match=*[0-9]*)$
-ref-names: $Format:%D$
+ref-names: $Format:%D$

.gitattributes

@@ -1,4 +1,4 @@
 # Auto detect text files and perform LF normalization
 * text=auto
 
-.git_archival.txt export-subst
+.git_archival.txt export-subst

.github/workflows/test-package-build.yml

@@ -80,15 +80,19 @@ jobs:
       # - name: Run tests
       #   run: pytest --doctest-modules -v --pyargs spyglass
   publish:
+    name: Upload release to PyPI
     runs-on: ubuntu-latest
     needs: [test-package]
+    environment:
+      name: pypi
+      url: https://pypi.org/p/spyglass-neuro
+    permissions:
+      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
     steps:
       - uses: actions/download-artifact@v3
         with:
           name: dist
           path: dist/
-      - uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          user: __token__
-          password: ${{ secrets.PYPI_API_TOKEN }}
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

.pre-commit-config.yaml

@@ -4,7 +4,7 @@ exclude: (^.github/|^docs/site/|^images/)
 repos:
   - repo: https://github.com/executablebooks/mdformat
     # Do this before other tools "fixing" the line endings
-    rev: 0.7.16
+    rev: 0.7.17
     hooks:
       - id: mdformat
         name: Format Markdown
@@ -13,6 +13,7 @@ repos:
         types: [markdown]
         args: [--wrap, "80", --number]
         additional_dependencies:
+          - mdformat-mkdocs
           - mdformat-toc
           - mdformat-beautysh
           - mdformat-config
@@ -21,21 +22,19 @@ repos:
           - mdformat-gfm
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
-      - id: check-json
       - id: check-toml
       - id: check-yaml
         args: [--unsafe]
-      - id: requirements-txt-fixer
       - id: end-of-file-fixer
       - id: mixed-line-ending
         args: ["--fix=lf"]
         description: Forces to replace line ending by the UNIX 'lf' character.
       - id: trailing-whitespace
       - id: debug-statements
       - id: check-added-large-files # prevent giant files from being committed
-      - id: check-builtin-literals
+      # - id: check-builtin-literals
       - id: check-merge-conflict
       - id: check-executables-have-shebangs
       - id: check-shebang-scripts-are-executable
@@ -44,28 +43,41 @@ repos:
       - id: fix-byte-order-marker
 
   - repo: https://github.com/adrienverge/yamllint.git
-    rev: v1.29.0
+    rev: v1.33.0
     hooks:
       - id: yamllint
         args:
           - --no-warnings
           - -d
-          - "{extends: relaxed, rules: {line-length: {max: 90}}}"
+          - "{extends: relaxed, rules: {line-length: {max: 80}}}"
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.0.254
+    rev: v0.1.6
     hooks:
       - id: ruff
+        args: [
+          "--exclude", "./notebooks/py_scripts/",
+          "--ignore", "F401,E402,E501"
+        ]
+        # F401: Unused import - May flag tables in DataJoint foreign keys
+        # E402: Module level import not at top of file - May want lazyloading
+        # E501: Line too long - Allow longer lines in table definitions
 
   - repo: https://github.com/PyCQA/autoflake
-    rev: v2.0.1
+    rev: v2.2.1
     hooks:
       - id: autoflake
 
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.2
+    rev: v2.2.6
     hooks:
       - id: codespell
         args: [--toml, pyproject.toml]
         additional_dependencies:
           - tomli
+
+  - repo: https://github.com/ambv/black
+    rev: 23.11.0
+    hooks:
+      - id: black
+        language_version: python3.9

.vscode/extensions.json

@@ -20,4 +20,4 @@
     ],
     // List of extensions recommended by VS Code that should not be recommended for users of this workspace.
     "unwantedRecommendations": []
-}
+}

.vscode/settings.json

@@ -4,11 +4,18 @@
     "files.trimFinalNewlines": true,
     "editor.multiCursorModifier": "ctrlCmd",
     "autoDocstring.docstringFormat": "numpy",
-    "python.formatting.provider": "none",
     "remote.SSH.remoteServerListenOnSocket": true,
     "git.confirmSync": false,
     "python.analysis.typeCheckingMode": "off",
     "[python]": {
-        "editor.defaultFormatter": "ms-python.black-formatter"
+        "editor.defaultFormatter": "ms-python.black-formatter",
+        "editor.formatOnSave": true,
+        "editor.codeActionsOnSave": {
+            "source.organizeImports": true
+        },
     },
-}
+    "isort.args": [
+        "--profile",
+        "black"
+    ],
+}

CHANGELOG.md

@@ -1,11 +1,19 @@
 # Change Log
 
-## [Unreleased]
+## [0.4.4] (Unreleased)
+
+- Additional documentation. #686
+- Refactor input validation in DLC pipeline.
+- Clean up following pre-commit checks.
+- Minor fixes to LinearizedPositionV1 pipeline #695
+
+## [0.4.3] (November 7, 2023)
 
 - Migrate `config` helper scripts to Spyglass codebase. #662
 - Revise contribution guidelines. #655
 - Minor bug fixes. #656, #657, #659, #651, #671
 - Add setup instruction specificity.
+- Reduce primary key varchar allocation aross may tables. #664
 
 ## [0.4.2] (October 10, 2023)
 
@@ -19,17 +27,17 @@
 ### Pipelines
 
 - Common:
-  - Added support multiple cameras per epoch. #557
-  - Removed `common_backup` schema. #631
-  - Added support for multiple position objects per NWB in `common_behav` via
-    PositionSource.SpatialSeries and RawPosition.PosObject #628, #616.
-    _Note:_ Existing functions have been made compatible, but column labels for
-    `RawPosition.fetch1_dataframe` may change.
+    - Added support multiple cameras per epoch. #557
+    - Removed `common_backup` schema. #631
+    - Added support for multiple position objects per NWB in `common_behav` via
+        PositionSource.SpatialSeries and RawPosition.PosObject #628, #616. _Note:_
+        Existing functions have been made compatible, but column labels for
+        `RawPosition.fetch1_dataframe` may change.
 - Spike sorting:
-  - Added pipeline populator. #637, #646, #647
-  - Fixed curation functionality for `nn_isolation`. #597, #598
+    - Added pipeline populator. #637, #646, #647
+    - Fixed curation functionality for `nn_isolation`. #597, #598
 - Position: Added position interval/epoch mapping via PositionIntervalMap. #620,
-  #621, #627
+    #621, #627
 - LFP: Refactored pipeline. #594, #588, #605, #606, #607, #608, #615, #629
 
 ## [0.4.1] (June 30, 2023)
@@ -40,41 +48,41 @@
 ## [0.4.0] (May 22, 2023)
 
 - Updated call to `spikeinterface.preprocessing.whiten` to use dtype np.float16.
-  #446,
+    #446,
 - Updated default spike sorting metric parameters. #447
 - Updated whitening to be compatible with recent changes in spikeinterface when
-  using mountainsort. #449
+    using mountainsort. #449
 - Moved LFP pipeline to `src/spyglass/lfp/v1` and addressed related usability
-  issues. #468, #478, #482, #484, #504
+    issues. #468, #478, #482, #484, #504
 - Removed whiten parameter for clusterless thresholder. #454
 - Added plot to plot all DIO events in a session. #457
 - Added file sharing functionality through kachery_cloud. #458, #460
 - Pinned numpy version to `numpy<1.24`
 - Added scripts to add guests and collaborators as users. #463
 - Cleaned up installation instructions in repo README. #467
 - Added checks in decoding visualization to ensure time dimensions are the
-  correct length.
+    correct length.
 - Fixed artifact removed valid times. #472
 - Added codespell workflow for spell checking and fixed typos. #471
 - Updated LFP code to save LFP as `pynwb.ecephys.LFP` type. #475
 - Added artifact detection to LFP pipeline. #473
 - Replaced calls to `spikeinterface.sorters.get_default_params` with
-  `spikeinterface.sorters.get_default_sorter_params`. #486
+    `spikeinterface.sorters.get_default_sorter_params`. #486
 - Updated position pipeline and added functionality to handle pose estimation
-  through DeepLabCut. #367, #505
+    through DeepLabCut. #367, #505
 - Updated `environment_position.yml`. #502
 - Renamed `FirFilter` class to `FirFilterParameters`. #512
 
 ## [0.3.4] (March 30, 2023)
 
 - Fixed error in spike sorting pipeline referencing the "probe_type" column
-  which is no longer accessible from the `Electrode` table. #437
+    which is no longer accessible from the `Electrode` table. #437
 - Fixed error when inserting an NWB file that does not have a probe
-  manufacturer. #433, #436
+    manufacturer. #433, #436
 - Fixed error when adding a new `DataAcquisitionDevice` and a new `ProbeType`.
-  #436
+    #436
 - Fixed inconsistency between capitalized/uncapitalized versions of "Intan" for
-  DataAcquisitionAmplifier and DataAcquisitionDevice.adc_circuit. #430, #438
+    DataAcquisitionAmplifier and DataAcquisitionDevice.adc_circuit. #430, #438
 
 ## [0.3.3] (March 29, 2023)
 
@@ -94,11 +102,13 @@
 - Allow creation and linkage of device metadata from YAML #400
 - Move helper functions to utils directory #386
 
-[0.4.2]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.4.2
-[0.4.1]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.4.1
-[0.4.0]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.4.0
-[0.3.4]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.3.4
-[0.3.3]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.3.3
-[0.3.2]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.3.2
-[0.3.1]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.3.1
 [0.3.0]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.3.0
+[0.3.1]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.3.1
+[0.3.2]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.3.2
+[0.3.3]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.3.3
+[0.3.4]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.3.4
+[0.4.0]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.4.0
+[0.4.1]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.4.1
+[0.4.2]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.4.2
+[0.4.3]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.4.3
+[0.4.4]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.4.4

CITATION.cff

@@ -10,6 +10,6 @@ authors:
 - family-names: "Frank"
   given-names: "Loren"
 title: "spyglass"
-version: 0.4.2
-date-released: 2023-10-10
+version: 0.4.3
+date-released: 2023-11-07
 url: "https://github.com/LorenFrankLab/spyglass"

MANIFEST.in

@@ -1,2 +1,2 @@
 graft tests
-global-exclude *.py[cod]
+global-exclude *.py[cod]

README.md

@@ -24,7 +24,8 @@ directory. We strongly recommend opening them in the context of `jupyterlab`.
 
 ## Contributing
 
-See the [Developer's Note](https://lorenfranklab.github.io/spyglass/latest/contribute/)
+See the
+[Developer's Note](https://lorenfranklab.github.io/spyglass/latest/contribute/)
 for contributing instructions found at -
 [https://lorenfranklab.github.io/spyglass/latest/contribute/](https://lorenfranklab.github.io/spyglass/latest/contribute/)
 

docs/README.md

@@ -16,8 +16,8 @@ The remainder of `mkdocs.yml` specifies the site's
 ## GitHub
 
 Whenever a new tag is pushed, GitHub actions will run
-`.github/workflows/publish-docs.yml`. Progress can be monitored in the
-'Actions' tab within the repo.
+`.github/workflows/publish-docs.yml`. Progress can be monitored in the 'Actions'
+tab within the repo.
 
 Releases should be tagged with `X.Y.Z`. A tag to redeploy docs should use the
 current version, with an alpha release suffix, e.g. `X.Y.Za1`.
@@ -45,9 +45,9 @@ Notably, this will make a copy of notebooks in `docs/src/notebooks`. Changes to
 the root notebooks directory may not be reflected when rebuilding.
 
 Use a browser to navigate to `localhost:8000/` to inspect the site. For
-auto-reload of markdown files during development, use `mkdocs serve -f
-./docs/mkdosc.yaml`. The `mike` package used in the build script manages
-versioning, but does not support dynamic versioning.
+auto-reload of markdown files during development, use
+`mkdocs serve -f ./docs/mkdosc.yaml`. The `mike` package used in the build
+script manages versioning, but does not support dynamic versioning.
 
 The following items can be commented out in `mkdocs.yml` to reduce build time:
 

docs/build-docs.sh

@@ -9,16 +9,18 @@ cp ./CHANGELOG.md ./docs/src/
 cp ./LICENSE ./docs/src/LICENSE.md
 mkdir -p ./docs/src/notebooks
 cp ./notebooks/*ipynb ./docs/src/notebooks/
+cp ./notebooks/*md ./docs/src/notebooks/
+cp ./docs/src/notebooks/README.md ./docs/src/notebooks/index.md
 cp -r ./notebook-images ./docs/src/notebooks/
 cp -r ./notebook-images ./docs/src/
 
 # Get major version
-FULL_VERSION=$(hatch version) # Most recent tag, may include periods 
-export MAJOR_VERSION="${FULL_VERSION:0:3}" # First 3 chars of tag 
+FULL_VERSION=$(hatch version) # Most recent tag, may include periods
+export MAJOR_VERSION="${FULL_VERSION:0:3}" # First 3 chars of tag
 echo "$MAJOR_VERSION"
 
-# Get ahead of errors 
-export JUPYTER_PLATFORM_DIRS=1 
+# Get ahead of errors
+export JUPYTER_PLATFORM_DIRS=1
 # jupyter notebook --generate-config
 
 # Generate site docs

docs/mkdocs.yml

@@ -47,16 +47,19 @@ nav:
   - Home: index.md
   - Installation: installation.md
   - Miscellaneous:
+      - Overview: misc/index.md
       - FigURL: misc/figurl_views.md
       - Session Groups: misc/session_groups.md
       - Insert Data: misc/insert_data.md
       - Merge Tables: misc/merge_tables.md
+      - Database Management: misc/database_management.md
   - Tutorials:
-      - Overview: notebooks/README.md
+      - Overview: notebooks/index.md
       - General:
           - Setup: notebooks/00_Setup.ipynb
           - Insert Data: notebooks/01_Insert_Data.ipynb
           - Data Sync: notebooks/02_Data_Sync.ipynb
+          - Merge Tables: notebooks/03_Merge_Tables.ipynb
       - Ephys:
           - Spike Sorting: notebooks/10_Spike_Sorting.ipynb
           - Curation: notebooks/11_Curation.ipynb
@@ -101,8 +104,6 @@ plugins:
             group_by_category: false
             line_length: 80
             docstring_style: numpy
-      # watch:
-      # - src/spyglass/
   - literate-nav:
       nav_file: navigation.md
   - exclude-search:

docs/overrides/nav.html

@@ -5,5 +5,3 @@
 {% if "toc.integrate" in features %}
 {% set class = class ~ " md-nav--integrated" %}
 {% endif %}
-
-