Merge pull request #130 from wwkimball/development

Prep release 3.5.0

.github/workflows/python-publish-to-prod-pypi.yml

@@ -9,10 +9,49 @@ on:
       - v*
 
 jobs:
+  validate:
+    name: Code Quality Assessment
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8, 3.9]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Set Environment Variables
+      run: |
+        echo "${HOME}/.gem/ruby/2.7.0/bin" >> $GITHUB_PATH
+    - name: Install dependencies
+      run: |
+        gem install --user-install hiera-eyaml -v 2.1.0
+        python -m pip install --upgrade pip
+        python -m pip install --upgrade setuptools
+        python -m pip install --upgrade wheel
+        python -m pip install --upgrade mypy pytest pytest-cov pytest-console-scripts pylint coveralls pep257
+        python -m pip install --editable .
+    - name: Validate Compliance with PEP257
+      run: |
+        pep257 yamlpath
+    - name: Validate Compliance with MyPY
+      run: |
+        mypy yamlpath
+    - name: Lint with pylint
+      run: |
+        pylint yamlpath
+    - name: Unit Test with pytest
+      run: |
+        pytest --verbose --cov=yamlpath --cov-report=term-missing --cov-fail-under=100 --script-launch-mode=subprocess tests
+
   publish:
     name: Publish to Production PyPI
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
     runs-on: ubuntu-latest
     environment: 'PyPI:  Production'
+    needs: validate
 
     steps:
     - uses: actions/checkout@v2

.github/workflows/python-publish-to-test-pypi.yml

@@ -16,10 +16,48 @@ on:
       - release/*
 
 jobs:
-  test-publish:
+  validate:
+    name: Code Quality Assessment
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8, 3.9]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Set Environment Variables
+      run: |
+        echo "${HOME}/.gem/ruby/2.7.0/bin" >> $GITHUB_PATH
+    - name: Install dependencies
+      run: |
+        gem install --user-install hiera-eyaml -v 2.1.0
+        python -m pip install --upgrade pip
+        python -m pip install --upgrade setuptools
+        python -m pip install --upgrade wheel
+        python -m pip install --upgrade mypy pytest pytest-cov pytest-console-scripts pylint coveralls pep257
+        python -m pip install --editable .
+    - name: Validate Compliance with PEP257
+      run: |
+        pep257 yamlpath
+    - name: Validate Compliance with MyPY
+      run: |
+        mypy yamlpath
+    - name: Lint with pylint
+      run: |
+        pylint yamlpath
+    - name: Unit Test with pytest
+      run: |
+        pytest --verbose --cov=yamlpath --cov-report=term-missing --cov-fail-under=100 --script-launch-mode=subprocess tests
+
+  publish:
     name: Publish to TEST PyPI
     runs-on: ubuntu-latest
     environment: 'PyPI:  Test'
+    needs: validate
 
     steps:
     - uses: actions/checkout@v2
@@ -33,7 +71,7 @@ jobs:
         python -m pip install --upgrade setuptools wheel
     - name: Build Artifacts
       run: |
-        sed -r -e "s/(^__version__[[:space:]]*=[[:space:]]*)("'"'"[[:digit:]](\.[[:digit:]])+)"'"'"/\1\2.RC$(date "+%Y%m%d%H%M%S")"'"'"/" yamlpath/__init__.py
+        sed -i -r -e "s/(^__version__[[:space:]]*=[[:space:]]*)("'"'"[[:digit:]](\.[[:digit:]])+)"'"'"/\1\2.RC$(date "+%Y%m%d%H%M%S")"'"'"/" yamlpath/__init__.py
         python setup.py sdist bdist_wheel
     - name: Publish Artifacts
       uses: pypa/[email protected]

CHANGES

@@ -1,3 +1,49 @@
+3.5.0:
+Bug Fixes:
+* Search expressions against Boolean values, [key=True] and [key=False], were
+  impossible.  Now, they are working and are not case-sensitive, so [key=True],
+  [key=true], [key=TRUE], and such all work as expected.
+* When null values were present, Deep Traversal (**) segments would always
+  return every node with a null value even when they would not match filter
+  conditions after the ** segment.  When mustexist=False, this would also cause
+  a YAMLPathException.
+* Descendent searches were considering only the first child of the search
+  ancestor.  Now, ANY matching descendent node will correctly yield the
+  ancestor.
+* Some Python-generated complex data types were escaping JSONification,
+  leading to unexpected stack-dumps when writing out JSON data for data types
+  like date and datetime.
+
+Enhancements:
+* An entirely new segment type has been added to YAML Path and is now supported
+  by the library and reference implementation command-line tools:  Keyword
+  Searches.  Similar to programming language keywords, these reserved Keywords
+  work much like functions, accepting parameters and performing algorythmic
+  operations or returning data not otherwise accessible to other YAML Path
+  segment types.  These new capabilities -- explored on the project Wiki --
+  include:
+  * [has_child(NAME)]
+  * [name()]
+  * [max(NAME)]
+  * [min(NAME)]
+  * [parent([STEPS])]
+* When stringified, YAML Paths with a solitary * wildcard segment were printed
+  using their internal RegEx variant, [.=~/.*/].  They are now printed as they
+  are entered, using a solitary *.  As a consequence, any deliberate RegEx of
+  [.=~/.*/] is also printed as its equivalent solitary *.
+* The yaml-paths command now allows printing YAML Paths without protective
+  escape symbols via a new --noescape option.  While this makes the output more
+  human-friendly, the unescaped paths will not be suitable for use as YAML Path
+  input to other YAML Path processors where special symbols require escaping.
+* [API] The NodeCoords class now tracks ancestry and the last YAML Path segment
+  responsible for triggering its generation.  The ancestry stack --
+  List[AncestryEntry] -- was necessary to support the [parent()] Search
+  Keyword.  The responsible YAML Path segment tracking was necessary to enable
+  Hash/map/dict key renaming via the [name()] Search Keyword.  These optional
+  attributes may be set when the NodeCoords is generated.
+* [API] YAMLPath instances now have a pop() method.  This mutates the YAMLPath
+  by popping off its last segment, returning that segment.
+
 3.4.1:
 Bug Fixes:
 * yaml-set (and the underlying Processor class) were unable to change nodes

README.md

@@ -228,6 +228,18 @@ YAML Path understands these segment types:
   * When another segment follows, it matches every node within the remainder
     of the document's tree for which the following (and subsequent) segments
     match: `/shows/**/name/Star*`
+* Search Keywords:  Advanced search capabilities not otherwise possible using
+  other YAML Path segments.  Taking the form of `[KEYWORD(PARAMETERS)]`, these
+  keywords are
+  [deeply explored on the Wiki](https://github.com/wwkimball/yamlpath/wiki/Search-Keywords)
+  and include:
+  * `[has_child(NAME)]`: Match nodes having a named child key
+  * `[max([NAME])]`: Match nodes having the maximum value
+  * `[min([NAME])]`: Match nodes having the minimum value
+  * `[name()]`: Match only the name of the present node, discarding all
+    children
+  * `[parent([STEPS])]`, Step up 1-N levels in the document from the present
+    node
 * Collectors:  Placing any portion of the YAML Path within parenthesis defines a
   virtual list collector, like `(YAML Path)`; concatenation and exclusion
   operators are supported -- `+` and `-`, respectively -- along with nesting,
@@ -266,37 +278,71 @@ versions of `pip` or its own dependency, *setuptools*.
 
 ### Using pip
 
-Each published version of this project and its dependencies can be installed
-from [PyPI](https://pypi.org/) using `pip`.  Note that on systems with more than
-one version of Python, you will probably need to use `pip3`, or equivalent
-(e.g.: Cygwin users may need to use `pip3.6`, `pip3.9`, or such).
+Like most others, this Python project is published to [PyPI](https://pypi.org/)
+so that it can be easily installed via Python's `pip` command (or its
+version-specific `pip3`, `pip3.7`, and such depending on how your Python was
+installed).
+
+Python's `pip` command is ever-changing.  It is updated very frequently.  This
+command further depends on other libraries to do its job, namely *setuptools*.
+It so happens that *setuptools* is also updated very frequently.  Both of these
+are separate from Python itself, despite versions of them being pre-installed
+with Python.  It is your responsibility to keep `pip` and *setuptools*
+up-to-date.  When `pip` or *setuptools* become outdated, _you will experience
+errors_ when trying to install newer Python packages like *yamlpath* **unless
+you preinstall such packages' dependencies**.  In the case of *yamlpath*, this
+means you'd need to preinstall *ruamel.yaml* if you cannot or choose not to
+upgrade `pip` and/or *setuptools*.
+
+As long as your `pip` and *setuptools* are up-to-date, installing *yamlpath* is
+as simple as a single command (the "3.7" suffix to the `pip` command is
+optional, depending on how your Python 3 was installed):
 
 ```shell
-pip3 install yamlpath
+pip3.7 install yamlpath
 ```
 
 #### Very Old Versions of pip or its setuptools Dependency
 
 Very old versions of Python 3 ship with seriously outdated versions of `pip` and
 its *setuptools* dependency.  When using versions of `pip` older than **18.1**
-or *setuptools* older than version **46.4.0**, you may not be able to install
-yamlpath with a single command.  In this case, you have two options:  either
+or *setuptools* older than version **46.4.0**, you will not be able to install
+*yamlpath* with a single command.  In this case, you have two options:  either
 pre-install *ruamel.yaml* before installing *yamlpath* or update `pip` and/or
 *setuptools* to at least the minimum required versions so `pip` can
-auto-determine and install dependencies.  This issue is not unique to yamlpath
-because Python's ever-growing capabilities simply require periodic updates to
-access.
+auto-determine and install dependencies.  This issue is not unique to
+*yamlpath*.
 
-When you cannot update `pip` or *setuptools*, just pre-install *ruamel.yaml*
-before yamlpath, like so:
+Upgrading `pip` and *setuptools* is trivially simple as long as you have
+sufficient access rights to do so on your local machine.  Depending on your
+situation, you may need to prefix these with `sudo` and/or you may need to
+substitute `python3` and `pip3` for `python` and `pip`, or even `python3.7` and
+`pip3.7` (or another specific version of Python 3), respectively.  To reiterate
+that this project requires Python 3, these sample commands will be
+demonstrated using such prefixes:
 
 ```shell
-# In this edge-case, these commands CANNOT be joined, like:
-# pip3.6 install ruamel.yaml yamlpath
-pip3.6 install ruamel.yaml
-pip3.6 install yamlpath
+python3.7 -m pip install --upgrade pip
+pip3.7 install --upgrade setuptools
 ```
 
+When you cannot or will not update `pip` or *setuptools*, just pre-install
+*ruamel.yaml* before yamlpath.  Each must be installed seperately and in order,
+like this (you **cannot** combine these installations into a single command):
+
+```shell
+pip3.7 install ruamel.yaml
+pip3.7 install yamlpath
+```
+
+The downside to choosing this manual installation path is that you may end up
+with an incompatible version of *ruamel.yaml*.  This will manifest either as an
+inability to install *yamlpath* at all, or only certain versions of *yamlpath*,
+or *yamlpath* may experience unexpected errors caused by the incompatible code.
+For the best experience, you are strongly encouraged to just keep `pip` and
+*setuptools* up-to-date, particularly as a routine part of installing any new
+Python packages.
+
 ### Installing EYAML (Optional)
 
 EYAML support is entirely optional.  You do not need EYAML to use YAML Path.
@@ -593,9 +639,9 @@ optional arguments:
 
 ```text
 usage: yaml-paths [-h] [-V] -s EXPRESSION [-c EXPRESSION] [-m] [-L] [-F] [-X]
-                  [-P] [-t ['.', '/', 'auto', 'dot', 'fslash']] [-i | -k | -K]
-                  [-a] [-A | -Y | -y | -l] [-e] [-x EYAML] [-r PRIVATEKEY]
-                  [-u PUBLICKEY] [-S] [-d | -v | -q]
+                  [-P] [-n] [-t ['.', '/', 'auto', 'dot', 'fslash']]
+                  [-i | -k | -K] [-a] [-A | -Y | -y | -l] [-e] [-x EYAML]
+                  [-r PRIVATEKEY] [-u PUBLICKEY] [-S] [-d | -v | -q]
                   [YAML_FILE [YAML_FILE ...]]
 
 Returns zero or more YAML Paths indicating where in given YAML/JSON/Compatible
@@ -640,6 +686,11 @@ result printing options:
                         or to indicate whether a file has any matches without
                         printing them all, perhaps especially with
                         --noexpression)
+  -n, --noescape        omit escape characters from special characters in
+                        printed YAML Paths; this is unsafe for feeding the
+                        resulting YAML Paths into other YAML Path commands
+                        because the symbols that would be escaped have special
+                        meaning to YAML Path processors
 
 key name searching options:
   -i, --ignorekeynames  (default) do not search key names
@@ -683,7 +734,9 @@ EYAML options:
 A search or exception EXPRESSION takes the form of a YAML Path search operator
 -- %, $, =, ^, >, <, >=, <=, =~, or ! -- followed by the search term, omitting
 the left-hand operand. For more information about YAML Paths, please visit
-https://github.com/wwkimball/yamlpath.
+https://github.com/wwkimball/yamlpath/wiki. To report issues with this tool or
+to request enhancements, please visit
+https://github.com/wwkimball/yamlpath/issues.
 ```
 
 * [yaml-set](yamlpath/commands/yaml_set.py)
@@ -1167,7 +1220,7 @@ from yamlpath.exceptions import YAMLPathException
 
 yaml_path = YAMLPath("see.documentation.above.for.many.samples")
 try:
-    for node_coordinate in processor.get_nodes(yaml_path):
+    for node_coordinate in processor.get_nodes(yaml_path, mustexist=True):
         log.debug("Got {} from '{}'.".format(node_coordinate, yaml_path))
         # Do something with each node_coordinate.node (the actual data)
 except YAMLPathException as ex:

setup.py

@@ -20,6 +20,8 @@
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Operating System :: OS Independent",
+        "Environment :: Console",
+        "Topic :: Utilities",
         "Topic :: Software Development :: Libraries :: Python Modules",
     ],
     url="https://github.com/wwkimball/yamlpath",