Merge branch 'dev' into v1

.github/workflows/python-publish.yml

@@ -11,21 +11,20 @@ jobs:
   deploy:
 
     runs-on: ubuntu-latest
-
+    permissions:
+      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.x'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         pip install setuptools wheel twine
-    - name: Build and publish
-      env:
-        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+    - name: Build package
       run: |
         python setup.py sdist bdist_wheel
-        twine upload dist/*
+    - name: Publish package distributions to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/run-locking-tests.yaml

@@ -12,17 +12,17 @@ jobs:
     strategy:
       matrix:
         python-version: ["3.9"]
-        proc: [10, 50, 100]
+        proc: [10, 100]
         wait: [10, 40]
     steps:
     - uses: actions/checkout@v3
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
       with:
-        python-version: ${{ matrix.python-version }}
+        python-version: "3.x"
 
-    - name: Install dev dependancies
+    - name: Install dev dependencies
       run: if [ -f requirements/requirements-dev.txt ]; then pip install -r requirements/requirements-dev.txt; fi
 
     - name: Install yacman

.github/workflows/run-pytest.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: ["3.8", "3.10"]
+        python-version: ["3.8", "3.12"]
         os: [ubuntu-latest]
 
     steps:
@@ -20,12 +20,12 @@ jobs:
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
       with:
-        python-version: ${{ matrix.python-version }}
+        python-version: "3.x"
 
-    - name: Install dev dependancies
+    - name: Install dev dependencies
       run: if [ -f requirements/requirements-dev.txt ]; then pip install -r requirements/requirements-dev.txt; fi
 
-    - name: Install test dependancies
+    - name: Install test dependencies
       run: if [ -f requirements/requirements-test.txt ]; then pip install -r requirements/requirements-test.txt; fi
 
     - name: Install yacman

.gitignore

@@ -74,3 +74,4 @@ dist/
 *.egg-info/
 
 *ipynb_checkpoints*
+locking_tests/test.yaml

README.md

@@ -8,3 +8,171 @@
 Yacman is a YAML configuration manager. It provides some convenience tools for dealing with YAML configuration files.
 
 Please see [this](docs/usage.md) Python notebook for features and usage instructions and [this](docs/api_docs.md) document for API documentation.
+
+## Upgrading guide
+
+How to upgrade to yacman v1.0.0.
+Yacman v1 provides 2 feature upgrades:
+
+1. Constructors take the form of `yacman.YAMLConfigManager.from_x(...)` functions, to make it clearer how to 
+create a new `ym` object.
+2. It separates locks into read locks and write locks, to allow mutliple simultaneous readers.
+
+The v0.9.3 transition release would has versions, really: 
+- attmap-based version (YacAttMap)
+- non-attmap-but-mostly-compatible (YAMLConfigManager)
+- new future object (FutureYAMLConfigManager...), which is not-backwards-compatible.
+
+In v1.0.0, FutureYAMLConfigManager will be renamed to YAMLConfigManager and the old stuff will be removed.
+Here's how to transition your code:
+
+### Use the FutureYAMLConfigManager in 0.9.3
+
+1. Import the FutureYAMLConfigManager
+
+Change from:
+
+```
+from yacman import YAMLConfigManager
+```
+
+to 
+
+```
+from yacman import FutureYAMLConfigManager as YAMLConfigManager
+```
+
+Once we switch from `v0.9.3` to `v1.X.X`, you will need to switch back.
+
+2. Update any context managers to use `write_lock` or `read_lock`
+
+```
+from yacman import write_lock, read_lock
+```
+
+Change
+
+```
+with ym as locked_ym:
+	locked_ym.write()
+```	
+
+to
+
+
+```
+with write_lock(ym) as locked_ym:
+	locked_ym.write()
+```
+
+
+
+More examples:
+
+```
+
+from yacman import FutureYAMLConfigManager as YAMLConfigManager
+
+
+data = {"my_list": [1,2,3], "my_int": 8, "my_str": "hello world!", "my_dict": {"nested_val": 15}}
+
+ym = YAMLConfigManager(data)
+
+ym["my_list"]
+ym["my_int"]
+ym["my_dict"]
+
+# Use in a context manager to write to the file
+
+ym["new_var"] = 15
+
+with write(ym) as locked_ym:
+    locked_ym.rebase()
+	locked_ym.write()
+
+with read(ym) as locked_ym:
+	locked_ym.rebase()
+
+```
+
+
+
+
+3. Update any constructors to use the `from_{x}` functions
+
+You can no longer just create a `YAMLConfigManager` object directly; now you need to use the constructor helpers.
+
+Examples:
+
+```
+from yacman import FutureYAMLConfigManager as YAMLConfigManager
+
+data = {"my_list": [1,2,3], "my_int": 8, "my_str": "hello world!", "my_dict": {"nested_val": 15}}
+file_path = "tests/data/full.yaml"
+yaml_data = "myvar: myval"
+
+yacman.YAMLConfigManager.from_yaml_file(file_path)
+yacman.YAMLConfigManager.from_yaml_data(yaml_data)
+yacman.YAMLConfigManager.from_obj(data)
+
+```
+
+In the past, you could load from a file and overwrite some attributes with a dict of variables, all from the constructor.
+Now it would is more explicit:
+
+```
+ym = yacman.YacMan.from_yaml_file(file_path)
+ym.update_from_obj(data)
+```
+
+To exppand environment variables in values, use `.exp`.
+
+```
+ym.exp["text_expand_home_dir"]
+```
+
+## From v0.9.3 (using future) to v1.X.X:
+
+Switch back to: 
+
+```
+from yacman import YAMLConfigManager
+```
+
+
+
+
+
+## Demos
+
+Some interactive demos
+
+```
+from yacman import FutureYAMLConfigManager as YAMLConfigManager
+ym = yacman.YAMLConfigManager(entries=["a", "b", "c"])
+ym.to_dict()
+ym
+
+print(ym.to_yaml())
+
+ym = YAMLConfigManager(entries={"top": {"bottom": ["a", "b"], "bottom2": "a"}, "b": "c"})
+ym
+print(ym.to_yaml())
+
+ym = YAMLConfigManager(filepath="tests/data/conf_schema.yaml")
+print(ym.to_yaml())
+ym
+
+ym = YAMLConfigManager(filepath="tests/data/empty.yaml")
+print(ym.to_yaml())
+
+ym = YAMLConfigManager(filepath="tests/data/list.yaml")
+print(ym.to_yaml())
+
+ym = YAMLConfigManager(YAMLConfigManager(filepath="tests/data/full.yaml").exp)
+print(ym.to_yaml())
+
+ym = YAMLConfigManager(filepath="tests/data/full.yaml")
+print(ym.to_yaml(expand=True))
+
+```

docs/changelog.md

@@ -11,6 +11,14 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 - jsonschema validation
 - attmap support
 
+## [0.9.3] -- 2024-02-01
+
+### Added
+- New `FutureYAMLConfigManager` object, prep for v1.
+- Improved file locking system with `read_lock` and `write_lock` context managers
+- New `from_x` object construction API.
+
+
 ## [0.9.2] -- 2023-10-05
 
 ## Added

locking_tests/locking_tests.py

@@ -1,21 +1,47 @@
 #!/usr/bin/env python3
 
 import sys
+import os
+
 from argparse import ArgumentParser
 from random import random
 from time import sleep
 
 from yacman import YacAttMap
+from yacman import FutureYAMLConfigManager as YAMLConfigManager
+from yacman import write_lock
+
+import logging
+
+_LOGGER = logging.getLogger()  # root logger
+stream = logging.StreamHandler(sys.stdout)
+fmt = logging.Formatter(
+    "%(levelname)s %(asctime)s | %(name)s:%(module)s:%(lineno)d > %(message)s "
+)
+stream.setFormatter(fmt)
+_LOGGER.setLevel(os.environ.get("LOGLEVEL", "DEBUG"))
+_LOGGER.addHandler(stream)
+
 
 parser = ArgumentParser(description="Test script")
 
 parser.add_argument("-p", "--path", help="path to the test file", required=True)
 parser.add_argument("-i", "--id", help="process id", required=True)
 parser.add_argument("-w", "--wait", help="max wait time", type=int, required=True)
 args = parser.parse_args()
-yam = YacAttMap(filepath=args.path, wait_max=args.wait)
-with yam as y:
-    sleep(random())
-    y.update({args.id: 1})
+ym = YAMLConfigManager.from_yaml_file(args.path, wait_max=args.wait)
+
+with write_lock(ym) as locked_y:
+    locked_y.rebase()
+    random_wait_time = random()
+    _LOGGER.debug(
+        f"Sleeping for {random_wait_time} to simulate process {args.id} updating the file"
+    )
+    sleep(random_wait_time)
+    locked_y.update({args.id: 1})
+    _LOGGER.debug(f"Writing to file for process {args.id}.")
+    locked_y.write()
+
 
-sys.exit(0)
+raise SystemExit
+# sys.exit(0)

locking_tests/locking_tests_attmap.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+
+import sys
+from argparse import ArgumentParser
+from random import random
+from time import sleep
+
+from yacman import YacAttMap
+
+parser = ArgumentParser(description="Test script")
+
+parser.add_argument("-p", "--path", help="path to the test file", required=True)
+parser.add_argument("-i", "--id", help="process id", required=True)
+parser.add_argument("-w", "--wait", help="max wait time", type=int, required=True)
+args = parser.parse_args()
+yam = YacAttMap(filepath=args.path, wait_max=args.wait)
+with yam as y:
+    sleep(random())
+    y.update({args.id: 1})
+
+sys.exit(0)

requirements/requirements-all.txt

@@ -1,3 +1,3 @@
 jsonschema>=3.2.0
 pyyaml>=5.0
-ubiquerg>=0.6.3
+ubiquerg>=0.7.0

setup.py

@@ -39,10 +39,11 @@
     classifiers=[
         "Development Status :: 4 - Beta",
         "License :: OSI Approved :: BSD License",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
         "Topic :: Scientific/Engineering :: Bio-Informatics",
     ],
     license="BSD2",

tests/conftest.py

@@ -1,4 +1,5 @@
 """ Test suite shared objects and setup """
+
 import os
 from glob import glob
 

yacman/__init__.py

@@ -4,5 +4,3 @@
 from .exceptions import *
 from .yacman import YAMLConfigManager, select_config, load_yaml
 from ubiquerg import read_lock, write_lock
-
-from .yacman import FutureYAMLConfigManager