diff --git a/coverage.txt b/coverage.txt index c9e3aa4..ea18e5a 100644 --- a/coverage.txt +++ b/coverage.txt @@ -14,13 +14,14 @@ tests/test_render_main.py . [100%] Name Stmts Miss Cover --------------------------------------------- wvpy/__init__.py 3 0 100% -wvpy/jtools.py 393 116 70% +wvpy/assignment.py 30 30 0% +wvpy/jtools.py 390 115 71% wvpy/ptools.py 66 6 91% wvpy/pysheet.py 99 49 51% wvpy/render_workbook.py 63 32 49% wvpy/util.py 4 0 100% --------------------------------------------- -TOTAL 628 203 68% +TOTAL 655 232 65% -============================= 18 passed in 38.91s ============================== +============================= 18 passed in 36.61s ============================== diff --git a/examples/declare_variables/record_example.ipynb b/examples/declare_variables/record_example.ipynb new file mode 100644 index 0000000..76400e3 --- /dev/null +++ b/examples/declare_variables/record_example.ipynb @@ -0,0 +1,161 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Example parameterized Jupyter notebook." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from wvpy.assignment import (\n", + " assign_values_from_map, \n", + " dict_to_assignments_str,\n", + " ensure_names_not_already_assigned,\n", + " record_assignments,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# overrides (usually this comes from external driver)\n", + "# In practice sheet_vars will be set by the system runner, this form informs Jupyter or the IDE \n", + "# that sheet_vars is a defined variable.\n", + "sheet_vars = globals().get(\"sheet_vars\", {}) # take previous value if there is one, else empty dictionary" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# check that none of the variables we want to override are already assigned\n", + "# this check is to cut down confusion of having these assigned somewhere other than\n", + "# the context manager that is the next block\n", + "ensure_names_not_already_assigned(globals(), keys=sheet_vars.keys())" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "# set some variables to default values we are willing to override\n", + "# we do this with the with context manager so that our Jupyter or IDE thinks these variables are defined in our environment\n", + "# this defines both the set of names we allow overriding of and default values so we can debug in and IDE\n", + "assignments = {}\n", + "with record_assignments(globals(), result=assignments):\n", + " # default values to be overridden\n", + " city = \"Los Angeles\"\n", + " state = \"CA\"\n", + " country = \"USA\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "default assignments are:\n", + "{'city': 'Los Angeles', 'country': 'USA', 'state': 'CA'}\n" + ] + } + ], + "source": [ + "print(\"default assignments are:\")\n", + "print(assignments)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "# override explicit assignments with values from sheet_vars\n", + "assign_values_from_map(\n", + " globals(),\n", + " values=sheet_vars,\n", + " expected_keys=assignments.keys(),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "running with the following actual values\n", + "\n", + "city = 'Los Angeles'\n", + "country = 'USA'\n", + "state = 'CA'\n", + "\n" + ] + } + ], + "source": [ + "print(\"running with the following actual values\")\n", + "print(dict_to_assignments_str({k: globals()[k] for k in assignments.keys()}))" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Analysis and results for Los Angeles.\n" + ] + } + ], + "source": [ + "# small simulation of parameterized analysis process\n", + "print(f\"Analysis and results for {city}.\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "wvpy_dev_env", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/pkg/build/lib/wvpy/assignment.py b/pkg/build/lib/wvpy/assignment.py new file mode 100644 index 0000000..7a27d98 --- /dev/null +++ b/pkg/build/lib/wvpy/assignment.py @@ -0,0 +1,101 @@ +"""Classes and functions for working with variable assignments""" + +from typing import Any, Dict, Iterable, Optional +from contextlib import contextmanager + + +def dict_to_assignments_str(values: Dict[str, Any]) -> str: + """ + Format a dictionary as a block of Python assignment statements. + + Example: + ```python + from wvpy.assignment import dict_to_assignments_str + print(dict_to_assignments_str({"a": 1, "b": 2})) + ``` + + :param values: dictionary + :return: assignment statements + """ + assert isinstance(values, dict) + return ( + "\n" + + "\n".join([f"{k} = {repr(values[k])}" for k in sorted(values.keys())]) + + "\n" + ) + + +@contextmanager +def record_assignments( + env, + *, + result: Dict[str, Any], +) -> None: + """ + Context manager to record all assignments to new variables in a with-block. + New variables being variables not set prior to entering the with block. + + Example: + ```python + from wvpy.assignment import dict_to_assignments_str, record_assignments + assignments = {} + with record_assignments(globals(), result=assignments): + a = 1 + b = 2 + print(assignments) + print(dict_to_assignments_str(assignments)) + ``` + + :param env: working environment, setting to `globals()` is usually the correct choice. + :param result: dictionary to store results in. function calls `.clear()` on result. + :return None: + """ + assert isinstance(result, dict) + pre_known_vars = set(env.keys()) + result.clear() + try: + yield + finally: + post_known_vars = set(env.keys()) + declared_vars = post_known_vars - pre_known_vars + for k in sorted(declared_vars): + result[k] = env[k] + + +def ensure_names_not_already_assigned(env, *, keys: Iterable[str]) -> None: + """ + Check that no key in keys is already set in the environment env. + Raises ValueError if keys are already assigned. + + :param env: working environment, setting to `globals()` is usually the correct choice. + :param keys: keys to confirm not already set. + :return None: + """ + already_assigned_vars = set(keys).intersection(set(env.keys())) + if len(already_assigned_vars) > 0: + raise ValueError(f"variables already set: {sorted(already_assigned_vars)}") + + +def assign_values_from_map( + env, *, values: Dict[str, Any], expected_keys: Optional[Iterable[str]] +) -> None: + """ + Assign values from map into environment. + + :param env: working environment, setting to `globals()` is usually the correct choice. + :param values: dictionary to copy into environment. + :param expected_keys: if not null a set of keys we are restricting to + :return None: + """ + assert isinstance(values, dict) + for k in values.keys(): + assert isinstance(k, str) + if expected_keys is not None: + unexpected_vars = set(values.keys()) - set(expected_keys) + if len(unexpected_vars) > 0: + raise ValueError( + f"attempting to assign undeclared variables: {sorted(unexpected_vars)}" + ) + # do the assignments + for k, v in values.items(): + env[k] = v diff --git a/pkg/build/lib/wvpy/jtools.py b/pkg/build/lib/wvpy/jtools.py index 7be7bcc..d355eaa 100644 --- a/pkg/build/lib/wvpy/jtools.py +++ b/pkg/build/lib/wvpy/jtools.py @@ -16,7 +16,6 @@ from typing import Any, Dict, Iterable, List, Optional from functools import total_ordering from contextlib import contextmanager -from collections import namedtuple from wvpy.util import escape_ansi from wvpy.ptools import execute_py @@ -656,18 +655,6 @@ def run_pool( return res -def write_dict_as_assignments(values: Dict[str, Any]) -> str: - """ - Write a dictionary as a block of Python assignment statements. - - :param values: dictionary - :return: assignment statements - """ - return "\n" + "\n".join([ - f"{k} = {repr(values[k])}" for k in sorted(values.keys()) - ]) + "\n" - - @contextmanager def declare_task_variables( env, diff --git a/pkg/dist/wvpy-1.1.1-py3-none-any.whl b/pkg/dist/wvpy-1.1.1-py3-none-any.whl index f884fbe..b391385 100644 Binary files a/pkg/dist/wvpy-1.1.1-py3-none-any.whl and b/pkg/dist/wvpy-1.1.1-py3-none-any.whl differ diff --git a/pkg/dist/wvpy-1.1.1.tar.gz b/pkg/dist/wvpy-1.1.1.tar.gz index 9147c9c..7ca24ea 100644 Binary files a/pkg/dist/wvpy-1.1.1.tar.gz and b/pkg/dist/wvpy-1.1.1.tar.gz differ diff --git a/pkg/docs/search.js b/pkg/docs/search.js index 2335f58..40c09db 100644 --- a/pkg/docs/search.js +++ b/pkg/docs/search.js @@ -1,6 +1,6 @@ window.pdocSearch = (function(){ /** elasticlunr - http://weixsong.github.io * Copyright (C) 2017 Oliver Nightingale * Copyright (C) 2017 Wei Song * MIT Licensed */!function(){function e(e){if(null===e||"object"!=typeof e)return e;var t=e.constructor();for(var n in e)e.hasOwnProperty(n)&&(t[n]=e[n]);return t}var t=function(e){var n=new t.Index;return n.pipeline.add(t.trimmer,t.stopWordFilter,t.stemmer),e&&e.call(n,n),n};t.version="0.9.5",lunr=t,t.utils={},t.utils.warn=function(e){return function(t){e.console&&console.warn&&console.warn(t)}}(this),t.utils.toString=function(e){return void 0===e||null===e?"":e.toString()},t.EventEmitter=function(){this.events={}},t.EventEmitter.prototype.addListener=function(){var e=Array.prototype.slice.call(arguments),t=e.pop(),n=e;if("function"!=typeof t)throw new TypeError("last argument must be a function");n.forEach(function(e){this.hasHandler(e)||(this.events[e]=[]),this.events[e].push(t)},this)},t.EventEmitter.prototype.removeListener=function(e,t){if(this.hasHandler(e)){var n=this.events[e].indexOf(t);-1!==n&&(this.events[e].splice(n,1),0==this.events[e].length&&delete this.events[e])}},t.EventEmitter.prototype.emit=function(e){if(this.hasHandler(e)){var t=Array.prototype.slice.call(arguments,1);this.events[e].forEach(function(e){e.apply(void 0,t)},this)}},t.EventEmitter.prototype.hasHandler=function(e){return e in this.events},t.tokenizer=function(e){if(!arguments.length||null===e||void 0===e)return[];if(Array.isArray(e)){var n=e.filter(function(e){return null===e||void 0===e?!1:!0});n=n.map(function(e){return t.utils.toString(e).toLowerCase()});var i=[];return n.forEach(function(e){var n=e.split(t.tokenizer.seperator);i=i.concat(n)},this),i}return e.toString().trim().toLowerCase().split(t.tokenizer.seperator)},t.tokenizer.defaultSeperator=/[\s\-]+/,t.tokenizer.seperator=t.tokenizer.defaultSeperator,t.tokenizer.setSeperator=function(e){null!==e&&void 0!==e&&"object"==typeof e&&(t.tokenizer.seperator=e)},t.tokenizer.resetSeperator=function(){t.tokenizer.seperator=t.tokenizer.defaultSeperator},t.tokenizer.getSeperator=function(){return t.tokenizer.seperator},t.Pipeline=function(){this._queue=[]},t.Pipeline.registeredFunctions={},t.Pipeline.registerFunction=function(e,n){n in t.Pipeline.registeredFunctions&&t.utils.warn("Overwriting existing registered function: "+n),e.label=n,t.Pipeline.registeredFunctions[n]=e},t.Pipeline.getRegisteredFunction=function(e){return e in t.Pipeline.registeredFunctions!=!0?null:t.Pipeline.registeredFunctions[e]},t.Pipeline.warnIfFunctionNotRegistered=function(e){var n=e.label&&e.label in this.registeredFunctions;n||t.utils.warn("Function is not registered with pipeline. This may cause problems when serialising the index.\n",e)},t.Pipeline.load=function(e){var n=new t.Pipeline;return e.forEach(function(e){var i=t.Pipeline.getRegisteredFunction(e);if(!i)throw new Error("Cannot load un-registered function: "+e);n.add(i)}),n},t.Pipeline.prototype.add=function(){var e=Array.prototype.slice.call(arguments);e.forEach(function(e){t.Pipeline.warnIfFunctionNotRegistered(e),this._queue.push(e)},this)},t.Pipeline.prototype.after=function(e,n){t.Pipeline.warnIfFunctionNotRegistered(n);var i=this._queue.indexOf(e);if(-1===i)throw new Error("Cannot find existingFn");this._queue.splice(i+1,0,n)},t.Pipeline.prototype.before=function(e,n){t.Pipeline.warnIfFunctionNotRegistered(n);var i=this._queue.indexOf(e);if(-1===i)throw new Error("Cannot find existingFn");this._queue.splice(i,0,n)},t.Pipeline.prototype.remove=function(e){var t=this._queue.indexOf(e);-1!==t&&this._queue.splice(t,1)},t.Pipeline.prototype.run=function(e){for(var t=[],n=e.length,i=this._queue.length,o=0;n>o;o++){for(var r=e[o],s=0;i>s&&(r=this._queue[s](r,o,e),void 0!==r&&null!==r);s++);void 0!==r&&null!==r&&t.push(r)}return t},t.Pipeline.prototype.reset=function(){this._queue=[]},t.Pipeline.prototype.get=function(){return this._queue},t.Pipeline.prototype.toJSON=function(){return this._queue.map(function(e){return t.Pipeline.warnIfFunctionNotRegistered(e),e.label})},t.Index=function(){this._fields=[],this._ref="id",this.pipeline=new t.Pipeline,this.documentStore=new t.DocumentStore,this.index={},this.eventEmitter=new t.EventEmitter,this._idfCache={},this.on("add","remove","update",function(){this._idfCache={}}.bind(this))},t.Index.prototype.on=function(){var e=Array.prototype.slice.call(arguments);return this.eventEmitter.addListener.apply(this.eventEmitter,e)},t.Index.prototype.off=function(e,t){return this.eventEmitter.removeListener(e,t)},t.Index.load=function(e){e.version!==t.version&&t.utils.warn("version mismatch: current "+t.version+" importing "+e.version);var n=new this;n._fields=e.fields,n._ref=e.ref,n.documentStore=t.DocumentStore.load(e.documentStore),n.pipeline=t.Pipeline.load(e.pipeline),n.index={};for(var i in e.index)n.index[i]=t.InvertedIndex.load(e.index[i]);return n},t.Index.prototype.addField=function(e){return this._fields.push(e),this.index[e]=new t.InvertedIndex,this},t.Index.prototype.setRef=function(e){return this._ref=e,this},t.Index.prototype.saveDocument=function(e){return this.documentStore=new t.DocumentStore(e),this},t.Index.prototype.addDoc=function(e,n){if(e){var n=void 0===n?!0:n,i=e[this._ref];this.documentStore.addDoc(i,e),this._fields.forEach(function(n){var o=this.pipeline.run(t.tokenizer(e[n]));this.documentStore.addFieldLength(i,n,o.length);var r={};o.forEach(function(e){e in r?r[e]+=1:r[e]=1},this);for(var s in r){var u=r[s];u=Math.sqrt(u),this.index[n].addToken(s,{ref:i,tf:u})}},this),n&&this.eventEmitter.emit("add",e,this)}},t.Index.prototype.removeDocByRef=function(e){if(e&&this.documentStore.isDocStored()!==!1&&this.documentStore.hasDoc(e)){var t=this.documentStore.getDoc(e);this.removeDoc(t,!1)}},t.Index.prototype.removeDoc=function(e,n){if(e){var n=void 0===n?!0:n,i=e[this._ref];this.documentStore.hasDoc(i)&&(this.documentStore.removeDoc(i),this._fields.forEach(function(n){var o=this.pipeline.run(t.tokenizer(e[n]));o.forEach(function(e){this.index[n].removeToken(e,i)},this)},this),n&&this.eventEmitter.emit("remove",e,this))}},t.Index.prototype.updateDoc=function(e,t){var t=void 0===t?!0:t;this.removeDocByRef(e[this._ref],!1),this.addDoc(e,!1),t&&this.eventEmitter.emit("update",e,this)},t.Index.prototype.idf=function(e,t){var n="@"+t+"/"+e;if(Object.prototype.hasOwnProperty.call(this._idfCache,n))return this._idfCache[n];var i=this.index[t].getDocFreq(e),o=1+Math.log(this.documentStore.length/(i+1));return this._idfCache[n]=o,o},t.Index.prototype.getFields=function(){return this._fields.slice()},t.Index.prototype.search=function(e,n){if(!e)return[];e="string"==typeof e?{any:e}:JSON.parse(JSON.stringify(e));var i=null;null!=n&&(i=JSON.stringify(n));for(var o=new t.Configuration(i,this.getFields()).get(),r={},s=Object.keys(e),u=0;u0&&t.push(e);for(var i in n)"docs"!==i&&"df"!==i&&this.expandToken(e+i,t,n[i]);return t},t.InvertedIndex.prototype.toJSON=function(){return{root:this.root}},t.Configuration=function(e,n){var e=e||"";if(void 0==n||null==n)throw new Error("fields should not be null");this.config={};var i;try{i=JSON.parse(e),this.buildUserConfig(i,n)}catch(o){t.utils.warn("user configuration parse failed, will use default configuration"),this.buildDefaultConfig(n)}},t.Configuration.prototype.buildDefaultConfig=function(e){this.reset(),e.forEach(function(e){this.config[e]={boost:1,bool:"OR",expand:!1}},this)},t.Configuration.prototype.buildUserConfig=function(e,n){var i="OR",o=!1;if(this.reset(),"bool"in e&&(i=e.bool||i),"expand"in e&&(o=e.expand||o),"fields"in e)for(var r in e.fields)if(n.indexOf(r)>-1){var s=e.fields[r],u=o;void 0!=s.expand&&(u=s.expand),this.config[r]={boost:s.boost||0===s.boost?s.boost:1,bool:s.bool||i,expand:u}}else t.utils.warn("field name in user configuration not found in index instance fields");else this.addAllFields2UserConfig(i,o,n)},t.Configuration.prototype.addAllFields2UserConfig=function(e,t,n){n.forEach(function(n){this.config[n]={boost:1,bool:e,expand:t}},this)},t.Configuration.prototype.get=function(){return this.config},t.Configuration.prototype.reset=function(){this.config={}},lunr.SortedSet=function(){this.length=0,this.elements=[]},lunr.SortedSet.load=function(e){var t=new this;return t.elements=e,t.length=e.length,t},lunr.SortedSet.prototype.add=function(){var e,t;for(e=0;e1;){if(r===e)return o;e>r&&(t=o),r>e&&(n=o),i=n-t,o=t+Math.floor(i/2),r=this.elements[o]}return r===e?o:-1},lunr.SortedSet.prototype.locationFor=function(e){for(var t=0,n=this.elements.length,i=n-t,o=t+Math.floor(i/2),r=this.elements[o];i>1;)e>r&&(t=o),r>e&&(n=o),i=n-t,o=t+Math.floor(i/2),r=this.elements[o];return r>e?o:e>r?o+1:void 0},lunr.SortedSet.prototype.intersect=function(e){for(var t=new lunr.SortedSet,n=0,i=0,o=this.length,r=e.length,s=this.elements,u=e.elements;;){if(n>o-1||i>r-1)break;s[n]!==u[i]?s[n]u[i]&&i++:(t.add(s[n]),n++,i++)}return t},lunr.SortedSet.prototype.clone=function(){var e=new lunr.SortedSet;return e.elements=this.toArray(),e.length=e.elements.length,e},lunr.SortedSet.prototype.union=function(e){var t,n,i;this.length>=e.length?(t=this,n=e):(t=e,n=this),i=t.clone();for(var o=0,r=n.toArray();ohttps://github.com/WinVector/wvpy a system for converting and rendering Jupyter notebooks.

\n"}, {"fullname": "wvpy.jtools", "modulename": "wvpy.jtools", "kind": "module", "doc": "

Jupyter tools

\n"}, {"fullname": "wvpy.jtools.pretty_format_python", "modulename": "wvpy.jtools", "qualname": "pretty_format_python", "kind": "function", "doc": "

Format Python code, using black.

\n\n
Parameters
\n\n
    \n
  • python_txt: Python code
    • \n
  • black_mode: options for black
    • \n
\n\n
Returns
\n\n
\n

formatted Python code

\n
\n", "signature": "(python_txt: str, *, black_mode=None) -> str:", "funcdef": "def"}, {"fullname": "wvpy.jtools.convert_py_code_to_notebook", "modulename": "wvpy.jtools", "qualname": "convert_py_code_to_notebook", "kind": "function", "doc": "

Convert python text to a notebook. \n\"''' begin text\" ends any open blocks, and starts a new markdown block (triple double quotes also allowed)\n\"''' # end text\" ends text, and starts a new code block (triple double quotes also allowed)\n\"'''end code'''\" ends code blocks, and starts a new code block (triple double quotes also allowed)

\n\n
Parameters
\n\n
    \n
  • text: Python text to convert.
    • \n
  • use_black: if True use black to re-format Python code
    • \n
\n\n
Returns
\n\n
\n

a notebook

\n
\n", "signature": "(\ttext: str,\t*,\tuse_black: bool = False) -> nbformat.notebooknode.NotebookNode:", "funcdef": "def"}, {"fullname": "wvpy.jtools.prepend_code_cell_to_notebook", "modulename": "wvpy.jtools", "qualname": "prepend_code_cell_to_notebook", "kind": "function", "doc": "

Prepend a code cell to a Jupyter notebook.

\n\n
Parameters
\n\n
    \n
  • nb: Jupyter notebook to alter
    • \n
  • code_text: Python source code to add
    • \n
\n\n
Returns
\n\n
\n

new notebook

\n
\n", "signature": "(\tnb: nbformat.notebooknode.NotebookNode,\t*,\tcode_text: str) -> nbformat.notebooknode.NotebookNode:", "funcdef": "def"}, {"fullname": "wvpy.jtools.convert_py_file_to_notebook", "modulename": "wvpy.jtools", "qualname": "convert_py_file_to_notebook", "kind": "function", "doc": "

Convert python text to a notebook. \n\"''' begin text\" ends any open blocks, and starts a new markdown block (triple double quotes also allowed)\n\"''' # end text\" ends text, and starts a new code block (triple double quotes also allowed)\n\"'''end code'''\" ends code blocks, and starts a new code block (triple double quotes also allowed)

\n\n
Parameters
\n\n
    \n
  • py_file: Path to python source file.
    • \n
  • ipynb_file: Path to notebook result file.
    • \n
  • use_black: if True use black to re-format Python code
    • \n
\n\n
Returns
\n\n
\n

nothing

\n
\n", "signature": "(py_file: str, *, ipynb_file: str, use_black: bool = False) -> None:", "funcdef": "def"}, {"fullname": "wvpy.jtools.convert_notebook_code_to_py", "modulename": "wvpy.jtools", "qualname": "convert_notebook_code_to_py", "kind": "function", "doc": "

Convert ipython notebook inputs to a py code. \n\"''' begin text\" ends any open blocks, and starts a new markdown block (triple double quotes also allowed)\n\"''' # end text\" ends text, and starts a new code block (triple double quotes also allowed)\n\"'''end code'''\" ends code blocks, and starts a new code block (triple double quotes also allowed)

\n\n
Parameters
\n\n
    \n
  • nb: notebook
    • \n
  • use_black: if True use black to re-format Python code
    • \n
\n\n
Returns
\n\n
\n

Python source code

\n
\n", "signature": "(\tnb: nbformat.notebooknode.NotebookNode,\t*,\tuse_black: bool = False) -> str:", "funcdef": "def"}, {"fullname": "wvpy.jtools.convert_notebook_file_to_py", "modulename": "wvpy.jtools", "qualname": "convert_notebook_file_to_py", "kind": "function", "doc": "

Convert ipython notebook inputs to a py file. \n\"''' begin text\" ends any open blocks, and starts a new markdown block (triple double quotes also allowed)\n\"''' # end text\" ends text, and starts a new code block (triple double quotes also allowed)\n\"'''end code'''\" ends code blocks, and starts a new code block (triple double quotes also allowed)

\n\n
Parameters
\n\n
    \n
  • ipynb_file: Path to notebook input file.
    • \n
  • py_file: Path to python result file.
    • \n
  • use_black: if True use black to re-format Python code
    • \n
\n\n
Returns
\n\n
\n

nothing

\n
\n", "signature": "(ipynb_file: str, *, py_file: str, use_black: bool = False) -> None:", "funcdef": "def"}, {"fullname": "wvpy.jtools.OurExecutor", "modulename": "wvpy.jtools", "qualname": "OurExecutor", "kind": "class", "doc": "

Catch exception in notebook processing

\n", "bases": "nbconvert.preprocessors.execute.ExecutePreprocessor"}, {"fullname": "wvpy.jtools.OurExecutor.__init__", "modulename": "wvpy.jtools", "qualname": "OurExecutor.__init__", "kind": "function", "doc": "

Initialize the preprocessor.

\n", "signature": "(**kw)"}, {"fullname": "wvpy.jtools.OurExecutor.preprocess_cell", "modulename": "wvpy.jtools", "qualname": "OurExecutor.preprocess_cell", "kind": "function", "doc": "

Override if you want to apply some preprocessing to each cell.\nMust return modified cell and resource dictionary.

\n\n

Parameters

\n\n

cell : NotebookNode cell\n Notebook cell being processed\nresources : dictionary\n Additional resources used in the conversion process. Allows\n preprocessors to pass variables into the Jinja engine.\nindex : int\n Index of the cell being processed

\n", "signature": "(self, cell, resources, index):", "funcdef": "def"}, {"fullname": "wvpy.jtools.render_as_html", "modulename": "wvpy.jtools", "qualname": "render_as_html", "kind": "function", "doc": "

Render a Jupyter notebook in the current directory as HTML.\nExceptions raised in the rendering notebook are allowed to pass trough.

\n\n
Parameters
\n\n
    \n
  • notebook_file_name: name of source file, must end with .ipynb or .py (or type gotten from file system)
    • \n
  • output_suffix: optional name to add to result name
    • \n
  • timeout: Maximum time in seconds each notebook cell is allowed to run.\npassed to nbconvert.preprocessors.ExecutePreprocessor.
    • \n
  • kernel_name: Jupyter kernel to use. passed to nbconvert.preprocessors.ExecutePreprocessor.\n:param verbose logical, if True print while running
    • \n
  • sheet_vars: if not None value is de-serialized as a variable named \"sheet_vars\"
    • \n
  • init_code: Python init code for first cell
    • \n
  • exclude_input: if True, exclude input cells
    • \n
  • prompt_strip_regexp: regexp to strip prompts, only used if exclude_input is True
    • \n
\n\n
Returns
\n\n
\n

None

\n
\n", "signature": "(\tnotebook_file_name: str,\t*,\toutput_suffix: Optional[str] = None,\ttimeout: int = 60000,\tkernel_name: Optional[str] = None,\tverbose: bool = True,\tsheet_vars=None,\tinit_code: Optional[str] = None,\texclude_input: bool = False,\tprompt_strip_regexp: Optional[str] = '<\\\\s*div\\\\s+class\\\\s*=\\\\s*"jp-OutputPrompt[^<>]*>[^<>]*Out[^<>]*<\\\\s*/div\\\\s*>') -> None:", "funcdef": "def"}, {"fullname": "wvpy.jtools.JTask", "modulename": "wvpy.jtools", "qualname": "JTask", "kind": "class", "doc": "

\n"}, {"fullname": "wvpy.jtools.JTask.__init__", "modulename": "wvpy.jtools", "qualname": "JTask.__init__", "kind": "function", "doc": "

Create a Jupyter task.

\n\n
Parameters
\n\n
    \n
  • sheet_name: name of sheet to run can be .ipynb or .py, and suffix can be omitted.
    • \n
  • output_suffix: optional string to append to rendered HTML file name.
    • \n
  • exclude_input: if True strip input cells out of HTML render.
    • \n
  • sheet_vars: if not None value is de-serialized as a variable named \"sheet_vars\"
    • \n
  • init_code: optional code to insert at the top of the Jupyter sheet, used to pass parameters.
    • \n
  • path_prefix: optional prefix to add to sheet_name to find Jupyter source.
    • \n
  • strict: if True check paths path_prefix and path_prefix/sheetname[.py|.ipynb] exist.
    • \n
\n", "signature": "(\tsheet_name: str,\t*,\toutput_suffix: Optional[str] = None,\texclude_input: bool = True,\tsheet_vars=None,\tinit_code: Optional[str] = None,\tpath_prefix: Optional[str] = None,\tstrict: bool = True)"}, {"fullname": "wvpy.jtools.JTask.render_as_html", "modulename": "wvpy.jtools", "qualname": "JTask.render_as_html", "kind": "function", "doc": "

Render Jupyter notebook or Python (treated as notebook) to HTML.

\n", "signature": "(self) -> None:", "funcdef": "def"}, {"fullname": "wvpy.jtools.JTask.render_py_txt", "modulename": "wvpy.jtools", "qualname": "JTask.render_py_txt", "kind": "function", "doc": "

Render Python to text (without nbconver, nbformat Jupyter bindings)

\n", "signature": "(self) -> None:", "funcdef": "def"}, {"fullname": "wvpy.jtools.job_fn", "modulename": "wvpy.jtools", "qualname": "job_fn", "kind": "function", "doc": "

Function to run a JTask job for Jupyter notebook. Exceptions pass through

\n", "signature": "(arg: wvpy.jtools.JTask):", "funcdef": "def"}, {"fullname": "wvpy.jtools.job_fn_eat_exception", "modulename": "wvpy.jtools", "qualname": "job_fn_eat_exception", "kind": "function", "doc": "

Function to run a JTask job for Jupyter notebook, catching any exception and returning it as a value

\n", "signature": "(arg: wvpy.jtools.JTask):", "funcdef": "def"}, {"fullname": "wvpy.jtools.job_fn_py_txt", "modulename": "wvpy.jtools", "qualname": "job_fn_py_txt", "kind": "function", "doc": "

Function to run a JTask job for Python to txt. Exceptions pass through

\n", "signature": "(arg: wvpy.jtools.JTask):", "funcdef": "def"}, {"fullname": "wvpy.jtools.job_fn_py_txt_eat_exception", "modulename": "wvpy.jtools", "qualname": "job_fn_py_txt_eat_exception", "kind": "function", "doc": "

Function to run a JTask job for Python to txt, catching any exception and returning it as a value

\n", "signature": "(arg: wvpy.jtools.JTask):", "funcdef": "def"}, {"fullname": "wvpy.jtools.run_pool", "modulename": "wvpy.jtools", "qualname": "run_pool", "kind": "function", "doc": "

Run a pool of tasks.

\n\n
Parameters
\n\n
    \n
  • tasks: iterable of tasks
    • \n
  • njobs: degree of parallelism
    • \n
  • verbose: if True, print on failure
    • \n
  • stop_on_error: if True, stop pool on error
    • \n
  • use_Jupyter: if True, use nbconvert, nbformat Jupyter fns.
    • \n
\n", "signature": "(\ttasks: Iterable,\t*,\tnjobs: int = 4,\tverbose: bool = True,\tstop_on_error: bool = True,\tuse_Jupyter: bool = True) -> List:", "funcdef": "def"}, {"fullname": "wvpy.jtools.write_dict_as_assignments", "modulename": "wvpy.jtools", "qualname": "write_dict_as_assignments", "kind": "function", "doc": "

Write a dictionary as a block of Python assignment statements.

\n\n
Parameters
\n\n
    \n
  • values: dictionary
    • \n
\n\n
Returns
\n\n
\n

assignment statements

\n
\n", "signature": "(values: Dict[str, Any]) -> str:", "funcdef": "def"}, {"fullname": "wvpy.jtools.declare_task_variables", "modulename": "wvpy.jtools", "qualname": "declare_task_variables", "kind": "function", "doc": "

Copy env[\"sheet_vars\"][k] into env[k] for all k in sheet_vars.keys() if \"sheet_vars\" defined in env.\nOnly variables that are first assigned in the with block of this task manager are allowed to be assigned.

\n\n
Parameters
\n\n
    \n
  • env: working environment, setting to globals() is usually the correct choice.
    • \n
  • result_map: empty dictionary to return results in. result_map[\"sheet_vars\"] is the dictionary if incoming assignments, result_map[\"declared_vars\"] is the dictionary of default names and values.
    • \n
\n\n
Returns
\n", "signature": "(env, *, result_map: Optional[Dict[str, Any]] = None) -> None:", "funcdef": "def"}, {"fullname": "wvpy.ptools", "modulename": "wvpy.ptools", "kind": "module", "doc": "

Python rendering tools

\n"}, {"fullname": "wvpy.ptools.execute_py", "modulename": "wvpy.ptools", "qualname": "execute_py", "kind": "function", "doc": "

Render a Python file to text.\nExceptions raised in the rendering notebook are allowed to pass through.

\n\n
Parameters
\n\n
    \n
  • source_file_name: name of source file, must end with .py (.py added if not present)
    • \n
  • output_suffix: optional name to add to result name\n:param verbose logical, if True print while running
    • \n
  • sheet_vars: if not None value is de-serialized as a variable named \"sheet_vars\" (usually a dictionary)
    • \n
  • init_code: Python init code for first cell
    • \n
\n\n
Returns
\n\n
\n

None

\n
\n", "signature": "(\tsource_file_name: str,\t*,\toutput_suffix: Optional[str] = None,\tverbose: bool = True,\tsheet_vars=None,\tinit_code: Optional[str] = None) -> None:", "funcdef": "def"}, {"fullname": "wvpy.pysheet", "modulename": "wvpy.pysheet", "kind": "module", "doc": "

\n"}, {"fullname": "wvpy.pysheet.pysheet", "modulename": "wvpy.pysheet", "qualname": "pysheet", "kind": "function", "doc": "

Convert between .ipynb and .py files.

\n\n
Parameters
\n\n
    \n
  • infiles: list of file names to process
    • \n
  • quiet: if True do the work quietly
    • \n
  • delete: if True, delete input
    • \n
  • black: if True, use black to re-format Python code cells
    • \n
\n\n
Returns
\n\n
\n

0 if successful

\n
\n", "signature": "(\tinfiles: Iterable[str],\t*,\tquiet: bool = False,\tdelete: bool = False,\tblack: bool = False) -> int:", "funcdef": "def"}, {"fullname": "wvpy.render_workbook", "modulename": "wvpy.render_workbook", "kind": "module", "doc": "

\n"}, {"fullname": "wvpy.render_workbook.render_workbook", "modulename": "wvpy.render_workbook", "qualname": "render_workbook", "kind": "function", "doc": "

Render a list of Jupyter notebooks.

\n\n
Parameters
\n\n
    \n
  • infiles: list of file names to process
    • \n
  • quiet: if true do the work quietly
    • \n
  • strip_input: if true strip input cells and cell numbering
    • \n
  • use_Jupyter: if True, use nbconvert, nbformat Jupyter fns.
    • \n
  • init_code: workbook initialization code.
    • \n
\n\n
Returns
\n\n
\n

0 if successful

\n
\n", "signature": "(\tinfiles: Iterable[str],\t*,\tquiet: bool = False,\tstrip_input: bool = True,\tuse_Jupyter: bool = True,\tinit_code: str = '') -> int:", "funcdef": "def"}, {"fullname": "wvpy.util", "modulename": "wvpy.util", "kind": "module", "doc": "

\n"}, {"fullname": "wvpy.util.escape_ansi", "modulename": "wvpy.util", "qualname": "escape_ansi", "kind": "function", "doc": "

https://stackoverflow.com/a/38662876

\n", "signature": "(line) -> str:", "funcdef": "def"}]; + /** pdoc search index */const docs = [{"fullname": "wvpy", "modulename": "wvpy", "kind": "module", "doc": "

https://github.com/WinVector/wvpy a system for converting and rendering Jupyter notebooks.

\n"}, {"fullname": "wvpy.assignment", "modulename": "wvpy.assignment", "kind": "module", "doc": "

Classes and functions for working with variable assignments

\n"}, {"fullname": "wvpy.assignment.dict_to_assignments_str", "modulename": "wvpy.assignment", "qualname": "dict_to_assignments_str", "kind": "function", "doc": "

Format a dictionary as a block of Python assignment statements.

\n\n

Example:

\n\n
\n
from wvpy.assignment import dict_to_assignments_str\nprint(dict_to_assignments_str({"a": 1, "b": 2}))\n
\n
\n\n
Parameters
\n\n
    \n
  • values: dictionary
    • \n
\n\n
Returns
\n\n
\n

assignment statements

\n
\n", "signature": "(values: Dict[str, Any]) -> str:", "funcdef": "def"}, {"fullname": "wvpy.assignment.record_assignments", "modulename": "wvpy.assignment", "qualname": "record_assignments", "kind": "function", "doc": "

Context manager to record all assignments to new variables in a with-block.\nNew variables being variables not set prior to entering the with block.

\n\n

Example:

\n\n
\n
from wvpy.assignment import dict_to_assignments_str, record_assignments\nassignments = {}\nwith record_assignments(globals(), result=assignments):\n    a = 1\n    b = 2\nprint(assignments)\nprint(dict_to_assignments_str(assignments))\n
\n
\n\n
Parameters
\n\n
    \n
  • env: working environment, setting to globals() is usually the correct choice.
    • \n
  • result: dictionary to store results in. function calls .clear() on result.
    • \n
\n\n
Returns
\n", "signature": "(env, *, result: Dict[str, Any]) -> None:", "funcdef": "def"}, {"fullname": "wvpy.assignment.ensure_names_not_already_assigned", "modulename": "wvpy.assignment", "qualname": "ensure_names_not_already_assigned", "kind": "function", "doc": "

Check that no key in keys is already set in the environment env.\nRaises ValueError if keys are already assigned.

\n\n
Parameters
\n\n
    \n
  • env: working environment, setting to globals() is usually the correct choice.
    • \n
  • keys: keys to confirm not already set.
    • \n
\n\n
Returns
\n", "signature": "(env, *, keys: Iterable[str]) -> None:", "funcdef": "def"}, {"fullname": "wvpy.assignment.assign_values_from_map", "modulename": "wvpy.assignment", "qualname": "assign_values_from_map", "kind": "function", "doc": "

Assign values from map into environment.

\n\n
Parameters
\n\n
    \n
  • env: working environment, setting to globals() is usually the correct choice.
    • \n
  • values: dictionary to copy into environment.
    • \n
  • expected_keys: if not null a set of keys we are restricting to
    • \n
\n\n
Returns
\n", "signature": "(\tenv,\t*,\tvalues: Dict[str, Any],\texpected_keys: Optional[Iterable[str]]) -> None:", "funcdef": "def"}, {"fullname": "wvpy.jtools", "modulename": "wvpy.jtools", "kind": "module", "doc": "

Jupyter tools

\n"}, {"fullname": "wvpy.jtools.pretty_format_python", "modulename": "wvpy.jtools", "qualname": "pretty_format_python", "kind": "function", "doc": "

Format Python code, using black.

\n\n
Parameters
\n\n
    \n
  • python_txt: Python code
    • \n
  • black_mode: options for black
    • \n
\n\n
Returns
\n\n
\n

formatted Python code

\n
\n", "signature": "(python_txt: str, *, black_mode=None) -> str:", "funcdef": "def"}, {"fullname": "wvpy.jtools.convert_py_code_to_notebook", "modulename": "wvpy.jtools", "qualname": "convert_py_code_to_notebook", "kind": "function", "doc": "

Convert python text to a notebook. \n\"''' begin text\" ends any open blocks, and starts a new markdown block (triple double quotes also allowed)\n\"''' # end text\" ends text, and starts a new code block (triple double quotes also allowed)\n\"'''end code'''\" ends code blocks, and starts a new code block (triple double quotes also allowed)

\n\n
Parameters
\n\n
    \n
  • text: Python text to convert.
    • \n
  • use_black: if True use black to re-format Python code
    • \n
\n\n
Returns
\n\n
\n

a notebook

\n
\n", "signature": "(\ttext: str,\t*,\tuse_black: bool = False) -> nbformat.notebooknode.NotebookNode:", "funcdef": "def"}, {"fullname": "wvpy.jtools.prepend_code_cell_to_notebook", "modulename": "wvpy.jtools", "qualname": "prepend_code_cell_to_notebook", "kind": "function", "doc": "

Prepend a code cell to a Jupyter notebook.

\n\n
Parameters
\n\n
    \n
  • nb: Jupyter notebook to alter
    • \n
  • code_text: Python source code to add
    • \n
\n\n
Returns
\n\n
\n

new notebook

\n
\n", "signature": "(\tnb: nbformat.notebooknode.NotebookNode,\t*,\tcode_text: str) -> nbformat.notebooknode.NotebookNode:", "funcdef": "def"}, {"fullname": "wvpy.jtools.convert_py_file_to_notebook", "modulename": "wvpy.jtools", "qualname": "convert_py_file_to_notebook", "kind": "function", "doc": "

Convert python text to a notebook. \n\"''' begin text\" ends any open blocks, and starts a new markdown block (triple double quotes also allowed)\n\"''' # end text\" ends text, and starts a new code block (triple double quotes also allowed)\n\"'''end code'''\" ends code blocks, and starts a new code block (triple double quotes also allowed)

\n\n
Parameters
\n\n
    \n
  • py_file: Path to python source file.
    • \n
  • ipynb_file: Path to notebook result file.
    • \n
  • use_black: if True use black to re-format Python code
    • \n
\n\n
Returns
\n\n
\n

nothing

\n
\n", "signature": "(py_file: str, *, ipynb_file: str, use_black: bool = False) -> None:", "funcdef": "def"}, {"fullname": "wvpy.jtools.convert_notebook_code_to_py", "modulename": "wvpy.jtools", "qualname": "convert_notebook_code_to_py", "kind": "function", "doc": "

Convert ipython notebook inputs to a py code. \n\"''' begin text\" ends any open blocks, and starts a new markdown block (triple double quotes also allowed)\n\"''' # end text\" ends text, and starts a new code block (triple double quotes also allowed)\n\"'''end code'''\" ends code blocks, and starts a new code block (triple double quotes also allowed)

\n\n
Parameters
\n\n
    \n
  • nb: notebook
    • \n
  • use_black: if True use black to re-format Python code
    • \n
\n\n
Returns
\n\n
\n

Python source code

\n
\n", "signature": "(\tnb: nbformat.notebooknode.NotebookNode,\t*,\tuse_black: bool = False) -> str:", "funcdef": "def"}, {"fullname": "wvpy.jtools.convert_notebook_file_to_py", "modulename": "wvpy.jtools", "qualname": "convert_notebook_file_to_py", "kind": "function", "doc": "

Convert ipython notebook inputs to a py file. \n\"''' begin text\" ends any open blocks, and starts a new markdown block (triple double quotes also allowed)\n\"''' # end text\" ends text, and starts a new code block (triple double quotes also allowed)\n\"'''end code'''\" ends code blocks, and starts a new code block (triple double quotes also allowed)

\n\n
Parameters
\n\n
    \n
  • ipynb_file: Path to notebook input file.
    • \n
  • py_file: Path to python result file.
    • \n
  • use_black: if True use black to re-format Python code
    • \n
\n\n
Returns
\n\n
\n

nothing

\n
\n", "signature": "(ipynb_file: str, *, py_file: str, use_black: bool = False) -> None:", "funcdef": "def"}, {"fullname": "wvpy.jtools.OurExecutor", "modulename": "wvpy.jtools", "qualname": "OurExecutor", "kind": "class", "doc": "

Catch exception in notebook processing

\n", "bases": "nbconvert.preprocessors.execute.ExecutePreprocessor"}, {"fullname": "wvpy.jtools.OurExecutor.__init__", "modulename": "wvpy.jtools", "qualname": "OurExecutor.__init__", "kind": "function", "doc": "

Initialize the preprocessor.

\n", "signature": "(**kw)"}, {"fullname": "wvpy.jtools.OurExecutor.preprocess_cell", "modulename": "wvpy.jtools", "qualname": "OurExecutor.preprocess_cell", "kind": "function", "doc": "

Override if you want to apply some preprocessing to each cell.\nMust return modified cell and resource dictionary.

\n\n

Parameters

\n\n

cell : NotebookNode cell\n Notebook cell being processed\nresources : dictionary\n Additional resources used in the conversion process. Allows\n preprocessors to pass variables into the Jinja engine.\nindex : int\n Index of the cell being processed

\n", "signature": "(self, cell, resources, index):", "funcdef": "def"}, {"fullname": "wvpy.jtools.render_as_html", "modulename": "wvpy.jtools", "qualname": "render_as_html", "kind": "function", "doc": "

Render a Jupyter notebook in the current directory as HTML.\nExceptions raised in the rendering notebook are allowed to pass trough.

\n\n
Parameters
\n\n
    \n
  • notebook_file_name: name of source file, must end with .ipynb or .py (or type gotten from file system)
    • \n
  • output_suffix: optional name to add to result name
    • \n
  • timeout: Maximum time in seconds each notebook cell is allowed to run.\npassed to nbconvert.preprocessors.ExecutePreprocessor.
    • \n
  • kernel_name: Jupyter kernel to use. passed to nbconvert.preprocessors.ExecutePreprocessor.\n:param verbose logical, if True print while running
    • \n
  • sheet_vars: if not None value is de-serialized as a variable named \"sheet_vars\"
    • \n
  • init_code: Python init code for first cell
    • \n
  • exclude_input: if True, exclude input cells
    • \n
  • prompt_strip_regexp: regexp to strip prompts, only used if exclude_input is True
    • \n
\n\n
Returns
\n\n
\n

None

\n
\n", "signature": "(\tnotebook_file_name: str,\t*,\toutput_suffix: Optional[str] = None,\ttimeout: int = 60000,\tkernel_name: Optional[str] = None,\tverbose: bool = True,\tsheet_vars=None,\tinit_code: Optional[str] = None,\texclude_input: bool = False,\tprompt_strip_regexp: Optional[str] = '<\\\\s*div\\\\s+class\\\\s*=\\\\s*"jp-OutputPrompt[^<>]*>[^<>]*Out[^<>]*<\\\\s*/div\\\\s*>') -> None:", "funcdef": "def"}, {"fullname": "wvpy.jtools.JTask", "modulename": "wvpy.jtools", "qualname": "JTask", "kind": "class", "doc": "

\n"}, {"fullname": "wvpy.jtools.JTask.__init__", "modulename": "wvpy.jtools", "qualname": "JTask.__init__", "kind": "function", "doc": "

Create a Jupyter task.

\n\n
Parameters
\n\n
    \n
  • sheet_name: name of sheet to run can be .ipynb or .py, and suffix can be omitted.
    • \n
  • output_suffix: optional string to append to rendered HTML file name.
    • \n
  • exclude_input: if True strip input cells out of HTML render.
    • \n
  • sheet_vars: if not None value is de-serialized as a variable named \"sheet_vars\"
    • \n
  • init_code: optional code to insert at the top of the Jupyter sheet, used to pass parameters.
    • \n
  • path_prefix: optional prefix to add to sheet_name to find Jupyter source.
    • \n
  • strict: if True check paths path_prefix and path_prefix/sheetname[.py|.ipynb] exist.
    • \n
\n", "signature": "(\tsheet_name: str,\t*,\toutput_suffix: Optional[str] = None,\texclude_input: bool = True,\tsheet_vars=None,\tinit_code: Optional[str] = None,\tpath_prefix: Optional[str] = None,\tstrict: bool = True)"}, {"fullname": "wvpy.jtools.JTask.render_as_html", "modulename": "wvpy.jtools", "qualname": "JTask.render_as_html", "kind": "function", "doc": "

Render Jupyter notebook or Python (treated as notebook) to HTML.

\n", "signature": "(self) -> None:", "funcdef": "def"}, {"fullname": "wvpy.jtools.JTask.render_py_txt", "modulename": "wvpy.jtools", "qualname": "JTask.render_py_txt", "kind": "function", "doc": "

Render Python to text (without nbconver, nbformat Jupyter bindings)

\n", "signature": "(self) -> None:", "funcdef": "def"}, {"fullname": "wvpy.jtools.job_fn", "modulename": "wvpy.jtools", "qualname": "job_fn", "kind": "function", "doc": "

Function to run a JTask job for Jupyter notebook. Exceptions pass through

\n", "signature": "(arg: wvpy.jtools.JTask):", "funcdef": "def"}, {"fullname": "wvpy.jtools.job_fn_eat_exception", "modulename": "wvpy.jtools", "qualname": "job_fn_eat_exception", "kind": "function", "doc": "

Function to run a JTask job for Jupyter notebook, catching any exception and returning it as a value

\n", "signature": "(arg: wvpy.jtools.JTask):", "funcdef": "def"}, {"fullname": "wvpy.jtools.job_fn_py_txt", "modulename": "wvpy.jtools", "qualname": "job_fn_py_txt", "kind": "function", "doc": "

Function to run a JTask job for Python to txt. Exceptions pass through

\n", "signature": "(arg: wvpy.jtools.JTask):", "funcdef": "def"}, {"fullname": "wvpy.jtools.job_fn_py_txt_eat_exception", "modulename": "wvpy.jtools", "qualname": "job_fn_py_txt_eat_exception", "kind": "function", "doc": "

Function to run a JTask job for Python to txt, catching any exception and returning it as a value

\n", "signature": "(arg: wvpy.jtools.JTask):", "funcdef": "def"}, {"fullname": "wvpy.jtools.run_pool", "modulename": "wvpy.jtools", "qualname": "run_pool", "kind": "function", "doc": "

Run a pool of tasks.

\n\n
Parameters
\n\n
    \n
  • tasks: iterable of tasks
    • \n
  • njobs: degree of parallelism
    • \n
  • verbose: if True, print on failure
    • \n
  • stop_on_error: if True, stop pool on error
    • \n
  • use_Jupyter: if True, use nbconvert, nbformat Jupyter fns.
    • \n
\n", "signature": "(\ttasks: Iterable,\t*,\tnjobs: int = 4,\tverbose: bool = True,\tstop_on_error: bool = True,\tuse_Jupyter: bool = True) -> List:", "funcdef": "def"}, {"fullname": "wvpy.jtools.declare_task_variables", "modulename": "wvpy.jtools", "qualname": "declare_task_variables", "kind": "function", "doc": "

Copy env[\"sheet_vars\"][k] into env[k] for all k in sheet_vars.keys() if \"sheet_vars\" defined in env.\nOnly variables that are first assigned in the with block of this task manager are allowed to be assigned.

\n\n
Parameters
\n\n
    \n
  • env: working environment, setting to globals() is usually the correct choice.
    • \n
  • result_map: empty dictionary to return results in. result_map[\"sheet_vars\"] is the dictionary if incoming assignments, result_map[\"declared_vars\"] is the dictionary of default names and values.
    • \n
\n\n
Returns
\n", "signature": "(env, *, result_map: Optional[Dict[str, Any]] = None) -> None:", "funcdef": "def"}, {"fullname": "wvpy.ptools", "modulename": "wvpy.ptools", "kind": "module", "doc": "

Python rendering tools

\n"}, {"fullname": "wvpy.ptools.execute_py", "modulename": "wvpy.ptools", "qualname": "execute_py", "kind": "function", "doc": "

Render a Python file to text.\nExceptions raised in the rendering notebook are allowed to pass through.

\n\n
Parameters
\n\n
    \n
  • source_file_name: name of source file, must end with .py (.py added if not present)
    • \n
  • output_suffix: optional name to add to result name\n:param verbose logical, if True print while running
    • \n
  • sheet_vars: if not None value is de-serialized as a variable named \"sheet_vars\" (usually a dictionary)
    • \n
  • init_code: Python init code for first cell
    • \n
\n\n
Returns
\n\n
\n

None

\n
\n", "signature": "(\tsource_file_name: str,\t*,\toutput_suffix: Optional[str] = None,\tverbose: bool = True,\tsheet_vars=None,\tinit_code: Optional[str] = None) -> None:", "funcdef": "def"}, {"fullname": "wvpy.pysheet", "modulename": "wvpy.pysheet", "kind": "module", "doc": "

\n"}, {"fullname": "wvpy.pysheet.pysheet", "modulename": "wvpy.pysheet", "qualname": "pysheet", "kind": "function", "doc": "

Convert between .ipynb and .py files.

\n\n
Parameters
\n\n
    \n
  • infiles: list of file names to process
    • \n
  • quiet: if True do the work quietly
    • \n
  • delete: if True, delete input
    • \n
  • black: if True, use black to re-format Python code cells
    • \n
\n\n
Returns
\n\n
\n

0 if successful

\n
\n", "signature": "(\tinfiles: Iterable[str],\t*,\tquiet: bool = False,\tdelete: bool = False,\tblack: bool = False) -> int:", "funcdef": "def"}, {"fullname": "wvpy.render_workbook", "modulename": "wvpy.render_workbook", "kind": "module", "doc": "

\n"}, {"fullname": "wvpy.render_workbook.render_workbook", "modulename": "wvpy.render_workbook", "qualname": "render_workbook", "kind": "function", "doc": "

Render a list of Jupyter notebooks.

\n\n
Parameters
\n\n
    \n
  • infiles: list of file names to process
    • \n
  • quiet: if true do the work quietly
    • \n
  • strip_input: if true strip input cells and cell numbering
    • \n
  • use_Jupyter: if True, use nbconvert, nbformat Jupyter fns.
    • \n
  • init_code: workbook initialization code.
    • \n
\n\n
Returns
\n\n
\n

0 if successful

\n
\n", "signature": "(\tinfiles: Iterable[str],\t*,\tquiet: bool = False,\tstrip_input: bool = True,\tuse_Jupyter: bool = True,\tinit_code: str = '') -> int:", "funcdef": "def"}, {"fullname": "wvpy.util", "modulename": "wvpy.util", "kind": "module", "doc": "

\n"}, {"fullname": "wvpy.util.escape_ansi", "modulename": "wvpy.util", "qualname": "escape_ansi", "kind": "function", "doc": "

https://stackoverflow.com/a/38662876

\n", "signature": "(line) -> str:", "funcdef": "def"}]; // mirrored in build-search-index.js (part 1) // Also split on html tags. this is a cheap heuristic, but good enough. diff --git a/pkg/docs/wvpy.html b/pkg/docs/wvpy.html index c811902..663359c 100644 --- a/pkg/docs/wvpy.html +++ b/pkg/docs/wvpy.html @@ -24,6 +24,7 @@

Submodules

    +
  • assignment
  • jtools
  • ptools
  • pysheet
    • diff --git a/pkg/docs/wvpy/assignment.html b/pkg/docs/wvpy/assignment.html new file mode 100644 index 0000000..6e0281c --- /dev/null +++ b/pkg/docs/wvpy/assignment.html @@ -0,0 +1,585 @@ + + + + + + + wvpy.assignment API documentation + + + + + + + + + +
    +
    +

    +wvpy.assignment

    + +

    Classes and functions for working with variable assignments

    +
    + + + + + + 
      1"""Classes and functions for working with variable assignments"""
+  2
+  3from typing import Any, Dict, Iterable, Optional
+  4from contextlib import contextmanager
+  5
+  6
+  7def dict_to_assignments_str(values: Dict[str, Any]) -> str:
+  8    """
+  9    Format a dictionary as a block of Python assignment statements.
+ 10
+ 11    Example:
+ 12    ```python
+ 13    from wvpy.assignment import dict_to_assignments_str
+ 14    print(dict_to_assignments_str({"a": 1, "b": 2}))
+ 15    ```
+ 16
+ 17    :param values: dictionary
+ 18    :return: assignment statements
+ 19    """
+ 20    assert isinstance(values, dict)
+ 21    return (
+ 22        "\n"
+ 23        + "\n".join([f"{k} = {repr(values[k])}" for k in sorted(values.keys())])
+ 24        + "\n"
+ 25    )
+ 26
+ 27
+ 28@contextmanager
+ 29def record_assignments(
+ 30    env,
+ 31    *,
+ 32    result: Dict[str, Any],
+ 33) -> None:
+ 34    """
+ 35    Context manager to record all assignments to new variables in a with-block.
+ 36    New variables being variables not set prior to entering the with block.
+ 37
+ 38    Example:
+ 39    ```python
+ 40    from wvpy.assignment import dict_to_assignments_str, record_assignments
+ 41    assignments = {}
+ 42    with record_assignments(globals(), result=assignments):
+ 43        a = 1
+ 44        b = 2
+ 45    print(assignments)
+ 46    print(dict_to_assignments_str(assignments))
+ 47    ```
+ 48
+ 49    :param env: working environment, setting to `globals()` is usually the correct choice.
+ 50    :param result: dictionary to store results in. function calls `.clear()` on result.
+ 51    :return None:
+ 52    """
+ 53    assert isinstance(result, dict)
+ 54    pre_known_vars = set(env.keys())
+ 55    result.clear()
+ 56    try:
+ 57        yield
+ 58    finally:
+ 59        post_known_vars = set(env.keys())
+ 60        declared_vars = post_known_vars - pre_known_vars
+ 61        for k in sorted(declared_vars):
+ 62            result[k] = env[k]
+ 63
+ 64
+ 65def ensure_names_not_already_assigned(env, *, keys: Iterable[str]) -> None:
+ 66    """
+ 67    Check that no key in keys is already set in the environment env.
+ 68    Raises ValueError if keys are already assigned.
+ 69
+ 70    :param env: working environment, setting to `globals()` is usually the correct choice.
+ 71    :param keys: keys to confirm not already set.
+ 72    :return None:
+ 73    """
+ 74    already_assigned_vars = set(keys).intersection(set(env.keys()))
+ 75    if len(already_assigned_vars) > 0:
+ 76        raise ValueError(f"variables already set: {sorted(already_assigned_vars)}")
+ 77
+ 78
+ 79def assign_values_from_map(
+ 80    env, *, values: Dict[str, Any], expected_keys: Optional[Iterable[str]]
+ 81) -> None:
+ 82    """
+ 83    Assign values from map into environment.
+ 84
+ 85    :param env: working environment, setting to `globals()` is usually the correct choice.
+ 86    :param values: dictionary to copy into environment.
+ 87    :param expected_keys: if not null a set of keys we are restricting to
+ 88    :return None:
+ 89    """
+ 90    assert isinstance(values, dict)
+ 91    for k in values.keys():
+ 92        assert isinstance(k, str)
+ 93    if expected_keys is not None:
+ 94        unexpected_vars = set(values.keys()) - set(expected_keys)
+ 95        if len(unexpected_vars) > 0:
+ 96            raise ValueError(
+ 97                f"attempting to assign undeclared variables: {sorted(unexpected_vars)}"
+ 98            )
+ 99    # do the assignments
+100    for k, v in values.items():
+101        env[k] = v
+
    + + +
    +
    + +
    + + def + dict_to_assignments_str(values: Dict[str, Any]) -> str: + + + +
    + + 
     8def dict_to_assignments_str(values: Dict[str, Any]) -> str:
+ 9    """
+10    Format a dictionary as a block of Python assignment statements.
+11
+12    Example:
+13    ```python
+14    from wvpy.assignment import dict_to_assignments_str
+15    print(dict_to_assignments_str({"a": 1, "b": 2}))
+16    ```
+17
+18    :param values: dictionary
+19    :return: assignment statements
+20    """
+21    assert isinstance(values, dict)
+22    return (
+23        "\n"
+24        + "\n".join([f"{k} = {repr(values[k])}" for k in sorted(values.keys())])
+25        + "\n"
+26    )
+
    + + +

    Format a dictionary as a block of Python assignment statements.

    + +

    Example:

    + +
    +
    from wvpy.assignment import dict_to_assignments_str
+print(dict_to_assignments_str({"a": 1, "b": 2}))
+
    +
    + +
    Parameters
    + +
      +
    • values: dictionary
      • +
    + +
    Returns
    + +
    +

    assignment statements

    +
    +
    + + +
    +
    + +
    +
    @contextmanager
    + + def + record_assignments(env, *, result: Dict[str, Any]) -> None: + + + +
    + + 
    29@contextmanager
+30def record_assignments(
+31    env,
+32    *,
+33    result: Dict[str, Any],
+34) -> None:
+35    """
+36    Context manager to record all assignments to new variables in a with-block.
+37    New variables being variables not set prior to entering the with block.
+38
+39    Example:
+40    ```python
+41    from wvpy.assignment import dict_to_assignments_str, record_assignments
+42    assignments = {}
+43    with record_assignments(globals(), result=assignments):
+44        a = 1
+45        b = 2
+46    print(assignments)
+47    print(dict_to_assignments_str(assignments))
+48    ```
+49
+50    :param env: working environment, setting to `globals()` is usually the correct choice.
+51    :param result: dictionary to store results in. function calls `.clear()` on result.
+52    :return None:
+53    """
+54    assert isinstance(result, dict)
+55    pre_known_vars = set(env.keys())
+56    result.clear()
+57    try:
+58        yield
+59    finally:
+60        post_known_vars = set(env.keys())
+61        declared_vars = post_known_vars - pre_known_vars
+62        for k in sorted(declared_vars):
+63            result[k] = env[k]
+
    + + +

    Context manager to record all assignments to new variables in a with-block. +New variables being variables not set prior to entering the with block.

    + +

    Example:

    + +
    +
    from wvpy.assignment import dict_to_assignments_str, record_assignments
+assignments = {}
+with record_assignments(globals(), result=assignments):
+    a = 1
+    b = 2
+print(assignments)
+print(dict_to_assignments_str(assignments))
+
    +
    + +
    Parameters
    + +
      +
    • env: working environment, setting to globals() is usually the correct choice.
      • +
    • result: dictionary to store results in. function calls .clear() on result.
      • +
    + +
    Returns
    +
    + + +
    +
    + +
    + + def + ensure_names_not_already_assigned(env, *, keys: Iterable[str]) -> None: + + + +
    + + 
    66def ensure_names_not_already_assigned(env, *, keys: Iterable[str]) -> None:
+67    """
+68    Check that no key in keys is already set in the environment env.
+69    Raises ValueError if keys are already assigned.
+70
+71    :param env: working environment, setting to `globals()` is usually the correct choice.
+72    :param keys: keys to confirm not already set.
+73    :return None:
+74    """
+75    already_assigned_vars = set(keys).intersection(set(env.keys()))
+76    if len(already_assigned_vars) > 0:
+77        raise ValueError(f"variables already set: {sorted(already_assigned_vars)}")
+
    + + +

    Check that no key in keys is already set in the environment env. +Raises ValueError if keys are already assigned.

    + +
    Parameters
    + +
      +
    • env: working environment, setting to globals() is usually the correct choice.
      • +
    • keys: keys to confirm not already set.
      • +
    + +
    Returns
    +
    + + +
    +
    + +
    + + def + assign_values_from_map( env, *, values: Dict[str, Any], expected_keys: Optional[Iterable[str]]) -> None: + + + +
    + + 
     80def assign_values_from_map(
+ 81    env, *, values: Dict[str, Any], expected_keys: Optional[Iterable[str]]
+ 82) -> None:
+ 83    """
+ 84    Assign values from map into environment.
+ 85
+ 86    :param env: working environment, setting to `globals()` is usually the correct choice.
+ 87    :param values: dictionary to copy into environment.
+ 88    :param expected_keys: if not null a set of keys we are restricting to
+ 89    :return None:
+ 90    """
+ 91    assert isinstance(values, dict)
+ 92    for k in values.keys():
+ 93        assert isinstance(k, str)
+ 94    if expected_keys is not None:
+ 95        unexpected_vars = set(values.keys()) - set(expected_keys)
+ 96        if len(unexpected_vars) > 0:
+ 97            raise ValueError(
+ 98                f"attempting to assign undeclared variables: {sorted(unexpected_vars)}"
+ 99            )
+100    # do the assignments
+101    for k, v in values.items():
+102        env[k] = v
+
    + + +

    Assign values from map into environment.

    + +
    Parameters
    + +
      +
    • env: working environment, setting to globals() is usually the correct choice.
      • +
    • values: dictionary to copy into environment.
      • +
    • expected_keys: if not null a set of keys we are restricting to
      • +
    + +
    Returns
    +
    + + +
    +
    + + \ No newline at end of file diff --git a/pkg/docs/wvpy/jtools.html b/pkg/docs/wvpy/jtools.html index 49f8195..4992409 100644 --- a/pkg/docs/wvpy/jtools.html +++ b/pkg/docs/wvpy/jtools.html @@ -93,9 +93,6 @@

    API Documentation

  • run_pool
    • -
  • - write_dict_as_assignments -
  • declare_task_variables
    • @@ -139,700 +136,687 @@

    15from typing import Any, Dict, Iterable, List, Optional 16from functools import total_ordering 17from contextlib import contextmanager - 18from collections import namedtuple - 19 - 20from wvpy.util import escape_ansi - 21from wvpy.ptools import execute_py + 18 + 19from wvpy.util import escape_ansi + 20from wvpy.ptools import execute_py + 21 22 - 23 - 24have_black = False - 25try: - 26 import black - 27 have_black = True - 28except ModuleNotFoundError: - 29 pass - 30 - 31nbf_v = nbformat.v4 - 32 - 33#def _normalize(nb): - 34# # try and work around version mismatches - 35# return nbformat.validator.normalize(nb, version=4, version_minor=5)[1] + 23have_black = False + 24try: + 25 import black + 26 have_black = True + 27except ModuleNotFoundError: + 28 pass + 29 + 30nbf_v = nbformat.v4 + 31 + 32#def _normalize(nb): + 33# # try and work around version mismatches + 34# return nbformat.validator.normalize(nb, version=4, version_minor=5)[1] + 35 36 - 37 - 38# noinspection PyBroadException - 39def pretty_format_python(python_txt: str, *, black_mode=None) -> str: - 40 """ - 41 Format Python code, using black. - 42 - 43 :param python_txt: Python code - 44 :param black_mode: options for black - 45 :return: formatted Python code - 46 """ - 47 assert have_black - 48 assert isinstance(python_txt, str) - 49 formatted_python = python_txt.strip('\n') + '\n' - 50 if len(formatted_python.strip()) > 0: - 51 if black_mode is None: - 52 black_mode = black.FileMode() - 53 try: - 54 formatted_python = black.format_str(formatted_python, mode=black_mode) - 55 formatted_python = formatted_python.strip('\n') + '\n' - 56 except Exception: - 57 pass - 58 return formatted_python + 37# noinspection PyBroadException + 38def pretty_format_python(python_txt: str, *, black_mode=None) -> str: + 39 """ + 40 Format Python code, using black. + 41 + 42 :param python_txt: Python code + 43 :param black_mode: options for black + 44 :return: formatted Python code + 45 """ + 46 assert have_black + 47 assert isinstance(python_txt, str) + 48 formatted_python = python_txt.strip('\n') + '\n' + 49 if len(formatted_python.strip()) > 0: + 50 if black_mode is None: + 51 black_mode = black.FileMode() + 52 try: + 53 formatted_python = black.format_str(formatted_python, mode=black_mode) + 54 formatted_python = formatted_python.strip('\n') + '\n' + 55 except Exception: + 56 pass + 57 return formatted_python + 58 59 - 60 - 61def convert_py_code_to_notebook( - 62 text: str, - 63 *, - 64 use_black:bool = False) -> nbformat.notebooknode.NotebookNode: - 65 """ - 66 Convert python text to a notebook. - 67 "''' begin text" ends any open blocks, and starts a new markdown block (triple double quotes also allowed) - 68 "''' # end text" ends text, and starts a new code block (triple double quotes also allowed) - 69 "'''end code'''" ends code blocks, and starts a new code block (triple double quotes also allowed) - 70 - 71 :param text: Python text to convert. - 72 :param use_black: if True use black to re-format Python code - 73 :return: a notebook - 74 """ - 75 # https://stackoverflow.com/a/23729611/6901725 - 76 # https://nbviewer.org/gist/fperez/9716279 - 77 assert isinstance(text, str) - 78 assert isinstance(use_black, bool) - 79 lines = text.splitlines() - 80 begin_text_regexp = re.compile(r"^\s*r?((''')|(\"\"\"))\s*begin\s+text\s*$") - 81 end_text_regexp = re.compile(r"^\s*r?((''')|(\"\"\"))\s*#\s*end\s+text\s*$") - 82 end_code_regexp = re.compile(r"(^\s*r?'''\s*end\s+code\s*'''\s*$)|(^\s*r?\"\"\"\s*end\s+code\s*\"\"\"\s*$)") - 83 # run a little code collecting state machine - 84 cells = [] - 85 collecting_python = [] - 86 collecting_text = None - 87 lines.append(None) # append an ending sentinel - 88 # scan input - 89 for line in lines: - 90 if line is None: - 91 is_end = True - 92 text_start = False - 93 code_start = False - 94 code_end = False - 95 else: - 96 is_end = False - 97 text_start = begin_text_regexp.match(line) - 98 code_start = end_text_regexp.match(line) - 99 code_end = end_code_regexp.match(line) -100 if is_end or text_start or code_start or code_end: -101 if (collecting_python is not None) and (len(collecting_python) > 0): -102 python_block = ('\n'.join(collecting_python)).strip('\n') + '\n' -103 if len(python_block.strip()) > 0: -104 if use_black and have_black: -105 python_block = pretty_format_python(python_block) -106 cells.append(nbf_v.new_code_cell(python_block)) -107 if (collecting_text is not None) and (len(collecting_text) > 0): -108 txt_block = ('\n'.join(collecting_text)).strip('\n') + '\n' -109 if len(txt_block.strip()) > 0: -110 cells.append(nbf_v.new_markdown_cell(txt_block)) -111 collecting_python = None -112 collecting_text = None -113 if not is_end: -114 if text_start: -115 collecting_text = [] -116 else: -117 collecting_python = [] -118 else: -119 if collecting_python is not None: -120 collecting_python.append(line) -121 if collecting_text is not None: -122 collecting_text.append(line) -123 for i in range(len(cells)): -124 cells[i]["id"] = f"cell{i}" -125 nb = nbf_v.new_notebook(cells=cells) -126 # nb = _normalize(nb) -127 return nb + 60def convert_py_code_to_notebook( + 61 text: str, + 62 *, + 63 use_black:bool = False) -> nbformat.notebooknode.NotebookNode: + 64 """ + 65 Convert python text to a notebook. + 66 "''' begin text" ends any open blocks, and starts a new markdown block (triple double quotes also allowed) + 67 "''' # end text" ends text, and starts a new code block (triple double quotes also allowed) + 68 "'''end code'''" ends code blocks, and starts a new code block (triple double quotes also allowed) + 69 + 70 :param text: Python text to convert. + 71 :param use_black: if True use black to re-format Python code + 72 :return: a notebook + 73 """ + 74 # https://stackoverflow.com/a/23729611/6901725 + 75 # https://nbviewer.org/gist/fperez/9716279 + 76 assert isinstance(text, str) + 77 assert isinstance(use_black, bool) + 78 lines = text.splitlines() + 79 begin_text_regexp = re.compile(r"^\s*r?((''')|(\"\"\"))\s*begin\s+text\s*$") + 80 end_text_regexp = re.compile(r"^\s*r?((''')|(\"\"\"))\s*#\s*end\s+text\s*$") + 81 end_code_regexp = re.compile(r"(^\s*r?'''\s*end\s+code\s*'''\s*$)|(^\s*r?\"\"\"\s*end\s+code\s*\"\"\"\s*$)") + 82 # run a little code collecting state machine + 83 cells = [] + 84 collecting_python = [] + 85 collecting_text = None + 86 lines.append(None) # append an ending sentinel + 87 # scan input + 88 for line in lines: + 89 if line is None: + 90 is_end = True + 91 text_start = False + 92 code_start = False + 93 code_end = False + 94 else: + 95 is_end = False + 96 text_start = begin_text_regexp.match(line) + 97 code_start = end_text_regexp.match(line) + 98 code_end = end_code_regexp.match(line) + 99 if is_end or text_start or code_start or code_end: +100 if (collecting_python is not None) and (len(collecting_python) > 0): +101 python_block = ('\n'.join(collecting_python)).strip('\n') + '\n' +102 if len(python_block.strip()) > 0: +103 if use_black and have_black: +104 python_block = pretty_format_python(python_block) +105 cells.append(nbf_v.new_code_cell(python_block)) +106 if (collecting_text is not None) and (len(collecting_text) > 0): +107 txt_block = ('\n'.join(collecting_text)).strip('\n') + '\n' +108 if len(txt_block.strip()) > 0: +109 cells.append(nbf_v.new_markdown_cell(txt_block)) +110 collecting_python = None +111 collecting_text = None +112 if not is_end: +113 if text_start: +114 collecting_text = [] +115 else: +116 collecting_python = [] +117 else: +118 if collecting_python is not None: +119 collecting_python.append(line) +120 if collecting_text is not None: +121 collecting_text.append(line) +122 for i in range(len(cells)): +123 cells[i]["id"] = f"cell{i}" +124 nb = nbf_v.new_notebook(cells=cells) +125 # nb = _normalize(nb) +126 return nb +127 128 -129 -130def prepend_code_cell_to_notebook( -131 nb: nbformat.notebooknode.NotebookNode, -132 *, -133 code_text: str, -134) -> nbformat.notebooknode.NotebookNode: -135 """ -136 Prepend a code cell to a Jupyter notebook. -137 -138 :param nb: Jupyter notebook to alter -139 :param code_text: Python source code to add -140 :return: new notebook -141 """ -142 header_cell = nbf_v.new_code_cell(code_text) -143 # set cell ids to avoid: -144 # "MissingIDFieldWarning: Code cell is missing an id field, this will become a hard error in future nbformat versions." -145 header_cell["id"] = "wvpy_header_cell" -146 orig_cells = [c.copy() for c in nb.cells] -147 for i in range(len(orig_cells)): -148 orig_cells[i]["id"] = f"cell{i}" -149 cells = [header_cell] + orig_cells -150 nb_out = nbf_v.new_notebook( -151 cells=cells -152 ) -153 # nb_out = _normalize(nb_out) -154 return nb_out +129def prepend_code_cell_to_notebook( +130 nb: nbformat.notebooknode.NotebookNode, +131 *, +132 code_text: str, +133) -> nbformat.notebooknode.NotebookNode: +134 """ +135 Prepend a code cell to a Jupyter notebook. +136 +137 :param nb: Jupyter notebook to alter +138 :param code_text: Python source code to add +139 :return: new notebook +140 """ +141 header_cell = nbf_v.new_code_cell(code_text) +142 # set cell ids to avoid: +143 # "MissingIDFieldWarning: Code cell is missing an id field, this will become a hard error in future nbformat versions." +144 header_cell["id"] = "wvpy_header_cell" +145 orig_cells = [c.copy() for c in nb.cells] +146 for i in range(len(orig_cells)): +147 orig_cells[i]["id"] = f"cell{i}" +148 cells = [header_cell] + orig_cells +149 nb_out = nbf_v.new_notebook( +150 cells=cells +151 ) +152 # nb_out = _normalize(nb_out) +153 return nb_out +154 155 -156 -157def convert_py_file_to_notebook( -158 py_file: str, -159 *, -160 ipynb_file: str, -161 use_black: bool = False, -162 ) -> None: -163 """ -164 Convert python text to a notebook. -165 "''' begin text" ends any open blocks, and starts a new markdown block (triple double quotes also allowed) -166 "''' # end text" ends text, and starts a new code block (triple double quotes also allowed) -167 "'''end code'''" ends code blocks, and starts a new code block (triple double quotes also allowed) -168 -169 :param py_file: Path to python source file. -170 :param ipynb_file: Path to notebook result file. -171 :param use_black: if True use black to re-format Python code -172 :return: nothing -173 """ -174 assert isinstance(py_file, str) -175 assert isinstance(ipynb_file, str) -176 assert isinstance(use_black, bool) -177 assert py_file != ipynb_file # prevent clobber -178 with open(py_file, 'r') as f: -179 text = f.read() -180 nb = convert_py_code_to_notebook(text, use_black=use_black) -181 with open(ipynb_file, 'w') as f: -182 nbformat.write(nb, f) +156def convert_py_file_to_notebook( +157 py_file: str, +158 *, +159 ipynb_file: str, +160 use_black: bool = False, +161 ) -> None: +162 """ +163 Convert python text to a notebook. +164 "''' begin text" ends any open blocks, and starts a new markdown block (triple double quotes also allowed) +165 "''' # end text" ends text, and starts a new code block (triple double quotes also allowed) +166 "'''end code'''" ends code blocks, and starts a new code block (triple double quotes also allowed) +167 +168 :param py_file: Path to python source file. +169 :param ipynb_file: Path to notebook result file. +170 :param use_black: if True use black to re-format Python code +171 :return: nothing +172 """ +173 assert isinstance(py_file, str) +174 assert isinstance(ipynb_file, str) +175 assert isinstance(use_black, bool) +176 assert py_file != ipynb_file # prevent clobber +177 with open(py_file, 'r') as f: +178 text = f.read() +179 nb = convert_py_code_to_notebook(text, use_black=use_black) +180 with open(ipynb_file, 'w') as f: +181 nbformat.write(nb, f) +182 183 -184 -185def convert_notebook_code_to_py( -186 nb: nbformat.notebooknode.NotebookNode, -187 *, -188 use_black: bool = False, -189 ) -> str: -190 """ -191 Convert ipython notebook inputs to a py code. -192 "''' begin text" ends any open blocks, and starts a new markdown block (triple double quotes also allowed) -193 "''' # end text" ends text, and starts a new code block (triple double quotes also allowed) -194 "'''end code'''" ends code blocks, and starts a new code block (triple double quotes also allowed) -195 -196 :param nb: notebook -197 :param use_black: if True use black to re-format Python code -198 :return: Python source code -199 """ -200 assert isinstance(use_black, bool) -201 res = [] -202 code_needs_end = False -203 for cell in nb.cells: -204 if len(cell.source.strip()) > 0: -205 if cell.cell_type == 'code': -206 if code_needs_end: -207 res.append('\n"""end code"""\n') -208 py_text = cell.source.strip('\n') + '\n' -209 if use_black and have_black: -210 py_text = pretty_format_python(py_text) -211 res.append(py_text) -212 code_needs_end = True -213 else: -214 res.append('\n""" begin text') -215 res.append(cell.source.strip('\n')) -216 res.append('""" # end text\n') -217 code_needs_end = False -218 res_text = '\n' + ('\n'.join(res)).strip('\n') + '\n\n' -219 return res_text +184def convert_notebook_code_to_py( +185 nb: nbformat.notebooknode.NotebookNode, +186 *, +187 use_black: bool = False, +188 ) -> str: +189 """ +190 Convert ipython notebook inputs to a py code. +191 "''' begin text" ends any open blocks, and starts a new markdown block (triple double quotes also allowed) +192 "''' # end text" ends text, and starts a new code block (triple double quotes also allowed) +193 "'''end code'''" ends code blocks, and starts a new code block (triple double quotes also allowed) +194 +195 :param nb: notebook +196 :param use_black: if True use black to re-format Python code +197 :return: Python source code +198 """ +199 assert isinstance(use_black, bool) +200 res = [] +201 code_needs_end = False +202 for cell in nb.cells: +203 if len(cell.source.strip()) > 0: +204 if cell.cell_type == 'code': +205 if code_needs_end: +206 res.append('\n"""end code"""\n') +207 py_text = cell.source.strip('\n') + '\n' +208 if use_black and have_black: +209 py_text = pretty_format_python(py_text) +210 res.append(py_text) +211 code_needs_end = True +212 else: +213 res.append('\n""" begin text') +214 res.append(cell.source.strip('\n')) +215 res.append('""" # end text\n') +216 code_needs_end = False +217 res_text = '\n' + ('\n'.join(res)).strip('\n') + '\n\n' +218 return res_text +219 220 -221 -222def convert_notebook_file_to_py( -223 ipynb_file: str, -224 *, -225 py_file: str, -226 use_black: bool = False, -227 ) -> None: -228 """ -229 Convert ipython notebook inputs to a py file. -230 "''' begin text" ends any open blocks, and starts a new markdown block (triple double quotes also allowed) -231 "''' # end text" ends text, and starts a new code block (triple double quotes also allowed) -232 "'''end code'''" ends code blocks, and starts a new code block (triple double quotes also allowed) -233 -234 :param ipynb_file: Path to notebook input file. -235 :param py_file: Path to python result file. -236 :param use_black: if True use black to re-format Python code -237 :return: nothing -238 """ -239 assert isinstance(py_file, str) -240 assert isinstance(ipynb_file, str) -241 assert isinstance(use_black, bool) -242 assert py_file != ipynb_file # prevent clobber -243 with open(ipynb_file, "rb") as f: -244 nb = nbformat.read(f, as_version=4) -245 py_source = convert_notebook_code_to_py(nb, use_black=use_black) -246 with open(py_file, 'w') as f: -247 f.write(py_source) +221def convert_notebook_file_to_py( +222 ipynb_file: str, +223 *, +224 py_file: str, +225 use_black: bool = False, +226 ) -> None: +227 """ +228 Convert ipython notebook inputs to a py file. +229 "''' begin text" ends any open blocks, and starts a new markdown block (triple double quotes also allowed) +230 "''' # end text" ends text, and starts a new code block (triple double quotes also allowed) +231 "'''end code'''" ends code blocks, and starts a new code block (triple double quotes also allowed) +232 +233 :param ipynb_file: Path to notebook input file. +234 :param py_file: Path to python result file. +235 :param use_black: if True use black to re-format Python code +236 :return: nothing +237 """ +238 assert isinstance(py_file, str) +239 assert isinstance(ipynb_file, str) +240 assert isinstance(use_black, bool) +241 assert py_file != ipynb_file # prevent clobber +242 with open(ipynb_file, "rb") as f: +243 nb = nbformat.read(f, as_version=4) +244 py_source = convert_notebook_code_to_py(nb, use_black=use_black) +245 with open(py_file, 'w') as f: +246 f.write(py_source) +247 248 -249 -250class OurExecutor(ExecutePreprocessor): -251 """Catch exception in notebook processing""" -252 def __init__(self, **kw): -253 """Initialize the preprocessor.""" -254 ExecutePreprocessor.__init__(self, **kw) -255 self.caught_exception = None -256 -257 def preprocess_cell(self, cell, resources, index): -258 """ -259 Override if you want to apply some preprocessing to each cell. -260 Must return modified cell and resource dictionary. -261 -262 Parameters -263 ---------- -264 cell : NotebookNode cell -265 Notebook cell being processed -266 resources : dictionary -267 Additional resources used in the conversion process. Allows -268 preprocessors to pass variables into the Jinja engine. -269 index : int -270 Index of the cell being processed -271 """ -272 if self.caught_exception is None: -273 try: -274 return ExecutePreprocessor.preprocess_cell(self, cell, resources, index) -275 except Exception as ex: -276 self.caught_exception = ex -277 return cell, self.resources +249class OurExecutor(ExecutePreprocessor): +250 """Catch exception in notebook processing""" +251 def __init__(self, **kw): +252 """Initialize the preprocessor.""" +253 ExecutePreprocessor.__init__(self, **kw) +254 self.caught_exception = None +255 +256 def preprocess_cell(self, cell, resources, index): +257 """ +258 Override if you want to apply some preprocessing to each cell. +259 Must return modified cell and resource dictionary. +260 +261 Parameters +262 ---------- +263 cell : NotebookNode cell +264 Notebook cell being processed +265 resources : dictionary +266 Additional resources used in the conversion process. Allows +267 preprocessors to pass variables into the Jinja engine. +268 index : int +269 Index of the cell being processed +270 """ +271 if self.caught_exception is None: +272 try: +273 return ExecutePreprocessor.preprocess_cell(self, cell, resources, index) +274 except Exception as ex: +275 self.caught_exception = ex +276 return cell, self.resources +277 278 -279 -280# https://nbconvert.readthedocs.io/en/latest/execute_api.html -281# https://nbconvert.readthedocs.io/en/latest/nbconvert_library.html -282# HTML element we are trying to delete: -283# <div class="jp-OutputPrompt jp-OutputArea-prompt">Out[5]:</div> -284def render_as_html( -285 notebook_file_name: str, -286 *, -287 output_suffix: Optional[str] = None, -288 timeout:int = 60000, -289 kernel_name: Optional[str] = None, -290 verbose: bool = True, -291 sheet_vars = None, -292 init_code: Optional[str] = None, -293 exclude_input: bool = False, -294 prompt_strip_regexp: Optional[str] = r'<\s*div\s+class\s*=\s*"jp-OutputPrompt[^<>]*>[^<>]*Out[^<>]*<\s*/div\s*>', -295) -> None: -296 """ -297 Render a Jupyter notebook in the current directory as HTML. -298 Exceptions raised in the rendering notebook are allowed to pass trough. -299 -300 :param notebook_file_name: name of source file, must end with .ipynb or .py (or type gotten from file system) -301 :param output_suffix: optional name to add to result name -302 :param timeout: Maximum time in seconds each notebook cell is allowed to run. -303 passed to nbconvert.preprocessors.ExecutePreprocessor. -304 :param kernel_name: Jupyter kernel to use. passed to nbconvert.preprocessors.ExecutePreprocessor. -305 :param verbose logical, if True print while running -306 :param sheet_vars: if not None value is de-serialized as a variable named "sheet_vars" -307 :param init_code: Python init code for first cell -308 :param exclude_input: if True, exclude input cells -309 :param prompt_strip_regexp: regexp to strip prompts, only used if exclude_input is True -310 :return: None -311 """ -312 assert isinstance(notebook_file_name, str) -313 # deal with no suffix case -314 if (not notebook_file_name.endswith(".ipynb")) and (not notebook_file_name.endswith(".py")): -315 py_name = notebook_file_name + ".py" -316 py_exists = os.path.exists(py_name) -317 ipynb_name = notebook_file_name + ".ipynb" -318 ipynb_exists = os.path.exists(ipynb_name) -319 if (py_exists + ipynb_exists) != 1: -320 raise ValueError('{ipynb_exists}: if file suffix is not specified then exactly one of .py or .ipynb file must exist') -321 if ipynb_exists: -322 notebook_file_name = notebook_file_name + '.ipynb' -323 else: -324 notebook_file_name = notebook_file_name + '.py' -325 # get the input -326 assert os.path.exists(notebook_file_name) -327 if notebook_file_name.endswith(".ipynb"): -328 suffix = ".ipynb" -329 with open(notebook_file_name, "rb") as f: -330 nb = nbformat.read(f, as_version=4) -331 elif notebook_file_name.endswith(".py"): -332 suffix = ".py" -333 with open(notebook_file_name, 'r') as f: -334 text = f.read() -335 nb = convert_py_code_to_notebook(text) -336 else: -337 raise ValueError('{ipynb_exists}: file must end with .py or .ipynb') -338 tmp_path = None -339 # do the conversion -340 if sheet_vars is not None: -341 with tempfile.NamedTemporaryFile(delete=False) as ntf: -342 tmp_path = ntf.name -343 pickle.dump(sheet_vars, file=ntf) -344 if (init_code is None) or (len(init_code) <= 0): -345 init_code = "" -346 pickle_code = f""" -347import pickle -348with open({tmp_path.__repr__()}, 'rb') as pf: -349 sheet_vars = pickle.load(pf) -350""" -351 init_code = init_code + "\n\n" + pickle_code -352 if (init_code is not None) and (len(init_code) > 0): -353 assert isinstance(init_code, str) -354 nb = prepend_code_cell_to_notebook( -355 nb, -356 code_text=f'\n\n{init_code}\n\n') -357 html_name = os.path.basename(notebook_file_name) -358 html_name = html_name.removesuffix(suffix) -359 exec_note = "" -360 if output_suffix is not None: -361 assert isinstance(output_suffix, str) -362 html_name = html_name + output_suffix -363 exec_note = f'"{output_suffix}"' -364 html_name = html_name + ".html" -365 try: -366 os.remove(html_name) -367 except FileNotFoundError: -368 pass -369 caught = None -370 trace = None -371 html_body = "" -372 if verbose: -373 print( -374 f'start render_as_html "{notebook_file_name}" {exec_note} {datetime.datetime.now()}' -375 ) -376 try: -377 if kernel_name is not None: -378 ep = OurExecutor( -379 timeout=timeout, kernel_name=kernel_name -380 ) -381 else: -382 ep = OurExecutor(timeout=timeout) -383 nb_res, nb_resources = ep.preprocess(nb) -384 html_exporter = HTMLExporter(exclude_input=exclude_input) -385 html_body, html_resources = html_exporter.from_notebook_node(nb_res) -386 caught = ep.caught_exception -387 except Exception as e: -388 caught = e -389 trace = traceback.format_exc() -390 if exclude_input and (prompt_strip_regexp is not None): -391 # strip output prompts -392 html_body = re.sub( -393 prompt_strip_regexp, -394 ' ', -395 html_body) -396 with open(html_name, "wt", encoding="utf-8") as f: -397 f.write(html_body) -398 if caught is not None: -399 f.write("\n<pre>\n") -400 f.write(escape_ansi(str(caught))) -401 f.write("\n</pre>\n") -402 if trace is not None: -403 f.write("\n<pre>\n") -404 f.write(escape_ansi(str(trace))) -405 f.write("\n</pre>\n") -406 nw = datetime.datetime.now() -407 if tmp_path is not None: -408 try: -409 os.remove(tmp_path) -410 except FileNotFoundError: -411 pass -412 if caught is not None: -413 if verbose: -414 print(f'\n\n\texception in render_as_html "{notebook_file_name}" {nw} {escape_ansi(str(caught))}\n\n') -415 if trace is not None: -416 print(f'\n\n\t\ttrace {escape_ansi(str(trace))}\n\n') -417 raise caught -418 if verbose: -419 print(f'\tdone render_as_html "{notebook_file_name}" {nw}') +279# https://nbconvert.readthedocs.io/en/latest/execute_api.html +280# https://nbconvert.readthedocs.io/en/latest/nbconvert_library.html +281# HTML element we are trying to delete: +282# <div class="jp-OutputPrompt jp-OutputArea-prompt">Out[5]:</div> +283def render_as_html( +284 notebook_file_name: str, +285 *, +286 output_suffix: Optional[str] = None, +287 timeout:int = 60000, +288 kernel_name: Optional[str] = None, +289 verbose: bool = True, +290 sheet_vars = None, +291 init_code: Optional[str] = None, +292 exclude_input: bool = False, +293 prompt_strip_regexp: Optional[str] = r'<\s*div\s+class\s*=\s*"jp-OutputPrompt[^<>]*>[^<>]*Out[^<>]*<\s*/div\s*>', +294) -> None: +295 """ +296 Render a Jupyter notebook in the current directory as HTML. +297 Exceptions raised in the rendering notebook are allowed to pass trough. +298 +299 :param notebook_file_name: name of source file, must end with .ipynb or .py (or type gotten from file system) +300 :param output_suffix: optional name to add to result name +301 :param timeout: Maximum time in seconds each notebook cell is allowed to run. +302 passed to nbconvert.preprocessors.ExecutePreprocessor. +303 :param kernel_name: Jupyter kernel to use. passed to nbconvert.preprocessors.ExecutePreprocessor. +304 :param verbose logical, if True print while running +305 :param sheet_vars: if not None value is de-serialized as a variable named "sheet_vars" +306 :param init_code: Python init code for first cell +307 :param exclude_input: if True, exclude input cells +308 :param prompt_strip_regexp: regexp to strip prompts, only used if exclude_input is True +309 :return: None +310 """ +311 assert isinstance(notebook_file_name, str) +312 # deal with no suffix case +313 if (not notebook_file_name.endswith(".ipynb")) and (not notebook_file_name.endswith(".py")): +314 py_name = notebook_file_name + ".py" +315 py_exists = os.path.exists(py_name) +316 ipynb_name = notebook_file_name + ".ipynb" +317 ipynb_exists = os.path.exists(ipynb_name) +318 if (py_exists + ipynb_exists) != 1: +319 raise ValueError('{ipynb_exists}: if file suffix is not specified then exactly one of .py or .ipynb file must exist') +320 if ipynb_exists: +321 notebook_file_name = notebook_file_name + '.ipynb' +322 else: +323 notebook_file_name = notebook_file_name + '.py' +324 # get the input +325 assert os.path.exists(notebook_file_name) +326 if notebook_file_name.endswith(".ipynb"): +327 suffix = ".ipynb" +328 with open(notebook_file_name, "rb") as f: +329 nb = nbformat.read(f, as_version=4) +330 elif notebook_file_name.endswith(".py"): +331 suffix = ".py" +332 with open(notebook_file_name, 'r') as f: +333 text = f.read() +334 nb = convert_py_code_to_notebook(text) +335 else: +336 raise ValueError('{ipynb_exists}: file must end with .py or .ipynb') +337 tmp_path = None +338 # do the conversion +339 if sheet_vars is not None: +340 with tempfile.NamedTemporaryFile(delete=False) as ntf: +341 tmp_path = ntf.name +342 pickle.dump(sheet_vars, file=ntf) +343 if (init_code is None) or (len(init_code) <= 0): +344 init_code = "" +345 pickle_code = f""" +346import pickle +347with open({tmp_path.__repr__()}, 'rb') as pf: +348 sheet_vars = pickle.load(pf) +349""" +350 init_code = init_code + "\n\n" + pickle_code +351 if (init_code is not None) and (len(init_code) > 0): +352 assert isinstance(init_code, str) +353 nb = prepend_code_cell_to_notebook( +354 nb, +355 code_text=f'\n\n{init_code}\n\n') +356 html_name = os.path.basename(notebook_file_name) +357 html_name = html_name.removesuffix(suffix) +358 exec_note = "" +359 if output_suffix is not None: +360 assert isinstance(output_suffix, str) +361 html_name = html_name + output_suffix +362 exec_note = f'"{output_suffix}"' +363 html_name = html_name + ".html" +364 try: +365 os.remove(html_name) +366 except FileNotFoundError: +367 pass +368 caught = None +369 trace = None +370 html_body = "" +371 if verbose: +372 print( +373 f'start render_as_html "{notebook_file_name}" {exec_note} {datetime.datetime.now()}' +374 ) +375 try: +376 if kernel_name is not None: +377 ep = OurExecutor( +378 timeout=timeout, kernel_name=kernel_name +379 ) +380 else: +381 ep = OurExecutor(timeout=timeout) +382 nb_res, nb_resources = ep.preprocess(nb) +383 html_exporter = HTMLExporter(exclude_input=exclude_input) +384 html_body, html_resources = html_exporter.from_notebook_node(nb_res) +385 caught = ep.caught_exception +386 except Exception as e: +387 caught = e +388 trace = traceback.format_exc() +389 if exclude_input and (prompt_strip_regexp is not None): +390 # strip output prompts +391 html_body = re.sub( +392 prompt_strip_regexp, +393 ' ', +394 html_body) +395 with open(html_name, "wt", encoding="utf-8") as f: +396 f.write(html_body) +397 if caught is not None: +398 f.write("\n<pre>\n") +399 f.write(escape_ansi(str(caught))) +400 f.write("\n</pre>\n") +401 if trace is not None: +402 f.write("\n<pre>\n") +403 f.write(escape_ansi(str(trace))) +404 f.write("\n</pre>\n") +405 nw = datetime.datetime.now() +406 if tmp_path is not None: +407 try: +408 os.remove(tmp_path) +409 except FileNotFoundError: +410 pass +411 if caught is not None: +412 if verbose: +413 print(f'\n\n\texception in render_as_html "{notebook_file_name}" {nw} {escape_ansi(str(caught))}\n\n') +414 if trace is not None: +415 print(f'\n\n\t\ttrace {escape_ansi(str(trace))}\n\n') +416 raise caught +417 if verbose: +418 print(f'\tdone render_as_html "{notebook_file_name}" {nw}') +419 420 -421 -422_jtask_comparison_attributes = [ -423 "sheet_name", "output_suffix", "exclude_input", "init_code", "path_prefix"] +421_jtask_comparison_attributes = [ +422 "sheet_name", "output_suffix", "exclude_input", "init_code", "path_prefix"] +423 424 -425 -426@total_ordering -427class JTask: -428 def __init__( -429 self, -430 sheet_name: str, -431 *, -432 output_suffix: Optional[str] = None, -433 exclude_input: bool = True, -434 sheet_vars = None, -435 init_code: Optional[str] = None, -436 path_prefix: Optional[str] = None, -437 strict: bool = True, -438 ) -> None: -439 """ -440 Create a Jupyter task. -441 -442 :param sheet_name: name of sheet to run can be .ipynb or .py, and suffix can be omitted. -443 :param output_suffix: optional string to append to rendered HTML file name. -444 :param exclude_input: if True strip input cells out of HTML render. -445 :param sheet_vars: if not None value is de-serialized as a variable named "sheet_vars" -446 :param init_code: optional code to insert at the top of the Jupyter sheet, used to pass parameters. -447 :param path_prefix: optional prefix to add to sheet_name to find Jupyter source. -448 :param strict: if True check paths path_prefix and path_prefix/sheetname[.py|.ipynb] exist. -449 """ -450 assert isinstance(sheet_name, str) -451 assert isinstance(output_suffix, (str, type(None))) -452 assert isinstance(exclude_input, bool) -453 assert isinstance(init_code, (str, type(None))) -454 assert isinstance(path_prefix, (str, type(None))) -455 if strict: -456 path = sheet_name -457 if (isinstance(path_prefix, str)) and (len(path_prefix) > 9): -458 assert os.path.exists(path_prefix) -459 path = os.path.join(path_prefix, sheet_name) -460 if path.endswith(".py"): -461 path = path.removesuffix(".py") -462 if path.endswith(".ipynb"): -463 path = path.removesuffix(".ipynb") -464 py_exists = os.path.exists(path + ".py") -465 ipynb_exists = os.path.exists(path + ".ipynb") -466 assert (py_exists + ipynb_exists) >= 1 -467 if (not sheet_name.endswith(".py")) and (not sheet_name.endswith(".ipynb")): -468 # no suffix, so must be unambiguous -469 assert (py_exists + ipynb_exists) == 1 -470 self.sheet_name = sheet_name -471 self.output_suffix = output_suffix -472 self.exclude_input = exclude_input -473 self.sheet_vars = sheet_vars -474 self.init_code = init_code -475 self.path_prefix = path_prefix -476 -477 def render_as_html(self) -> None: -478 """ -479 Render Jupyter notebook or Python (treated as notebook) to HTML. -480 """ -481 path = self.sheet_name -482 if isinstance(self.path_prefix, str) and (len(self.path_prefix) > 0): -483 path = os.path.join(self.path_prefix, self.sheet_name) -484 render_as_html( -485 path, -486 exclude_input=self.exclude_input, -487 output_suffix=self.output_suffix, -488 sheet_vars=self.sheet_vars, -489 init_code=self.init_code, -490 ) -491 -492 def render_py_txt(self) -> None: -493 """ -494 Render Python to text (without nbconver, nbformat Jupyter bindings) -495 """ -496 path = self.sheet_name -497 if isinstance(self.path_prefix, str) and (len(self.path_prefix) > 0): -498 path = os.path.join(self.path_prefix, self.sheet_name) -499 execute_py( -500 path, -501 output_suffix=self.output_suffix, -502 sheet_vars=self.sheet_vars, -503 init_code=self.init_code, -504 ) -505 -506 def __getitem__(self, item): -507 return getattr(self, item) -508 -509 def _is_valid_operand(self, other): -510 return isinstance(other, JTask) -511 -512 def __eq__(self, other): -513 if not self._is_valid_operand(other): -514 return False -515 if str(type(self)) != str(type(other)): -516 return False -517 for v in _jtask_comparison_attributes: -518 if self[v] != other[v]: -519 return False -520 if str(self.sheet_vars) != str(other.sheet_vars): -521 return False -522 return True -523 -524 def __lt__(self, other): -525 if not self._is_valid_operand(other): -526 return NotImplemented -527 if str(type(self)) < str(type(other)): -528 return True -529 for v in _jtask_comparison_attributes: -530 v_self = self[v] -531 v_other = other[v] -532 # can't order compare None to None -533 if ((v_self is None) or (v_other is None)): -534 if ((v_self is None) != (v_other is None)): -535 return v_self is None -536 else: -537 if self[v] < other[v]: -538 return True -539 if str(self.sheet_vars) < str(other.sheet_vars): -540 return True -541 return False -542 -543 def __str__(self) -> str: -544 args_str = ",\n".join([ -545 f" {v}={repr(self[v])}" -546 for v in _jtask_comparison_attributes + ["sheet_vars"] -547 ]) -548 return 'JTask(\n' + args_str + ",\n)" -549 -550 def __repr__(self) -> str: -551 return self.__str__() +425@total_ordering +426class JTask: +427 def __init__( +428 self, +429 sheet_name: str, +430 *, +431 output_suffix: Optional[str] = None, +432 exclude_input: bool = True, +433 sheet_vars = None, +434 init_code: Optional[str] = None, +435 path_prefix: Optional[str] = None, +436 strict: bool = True, +437 ) -> None: +438 """ +439 Create a Jupyter task. +440 +441 :param sheet_name: name of sheet to run can be .ipynb or .py, and suffix can be omitted. +442 :param output_suffix: optional string to append to rendered HTML file name. +443 :param exclude_input: if True strip input cells out of HTML render. +444 :param sheet_vars: if not None value is de-serialized as a variable named "sheet_vars" +445 :param init_code: optional code to insert at the top of the Jupyter sheet, used to pass parameters. +446 :param path_prefix: optional prefix to add to sheet_name to find Jupyter source. +447 :param strict: if True check paths path_prefix and path_prefix/sheetname[.py|.ipynb] exist. +448 """ +449 assert isinstance(sheet_name, str) +450 assert isinstance(output_suffix, (str, type(None))) +451 assert isinstance(exclude_input, bool) +452 assert isinstance(init_code, (str, type(None))) +453 assert isinstance(path_prefix, (str, type(None))) +454 if strict: +455 path = sheet_name +456 if (isinstance(path_prefix, str)) and (len(path_prefix) > 9): +457 assert os.path.exists(path_prefix) +458 path = os.path.join(path_prefix, sheet_name) +459 if path.endswith(".py"): +460 path = path.removesuffix(".py") +461 if path.endswith(".ipynb"): +462 path = path.removesuffix(".ipynb") +463 py_exists = os.path.exists(path + ".py") +464 ipynb_exists = os.path.exists(path + ".ipynb") +465 assert (py_exists + ipynb_exists) >= 1 +466 if (not sheet_name.endswith(".py")) and (not sheet_name.endswith(".ipynb")): +467 # no suffix, so must be unambiguous +468 assert (py_exists + ipynb_exists) == 1 +469 self.sheet_name = sheet_name +470 self.output_suffix = output_suffix +471 self.exclude_input = exclude_input +472 self.sheet_vars = sheet_vars +473 self.init_code = init_code +474 self.path_prefix = path_prefix +475 +476 def render_as_html(self) -> None: +477 """ +478 Render Jupyter notebook or Python (treated as notebook) to HTML. +479 """ +480 path = self.sheet_name +481 if isinstance(self.path_prefix, str) and (len(self.path_prefix) > 0): +482 path = os.path.join(self.path_prefix, self.sheet_name) +483 render_as_html( +484 path, +485 exclude_input=self.exclude_input, +486 output_suffix=self.output_suffix, +487 sheet_vars=self.sheet_vars, +488 init_code=self.init_code, +489 ) +490 +491 def render_py_txt(self) -> None: +492 """ +493 Render Python to text (without nbconver, nbformat Jupyter bindings) +494 """ +495 path = self.sheet_name +496 if isinstance(self.path_prefix, str) and (len(self.path_prefix) > 0): +497 path = os.path.join(self.path_prefix, self.sheet_name) +498 execute_py( +499 path, +500 output_suffix=self.output_suffix, +501 sheet_vars=self.sheet_vars, +502 init_code=self.init_code, +503 ) +504 +505 def __getitem__(self, item): +506 return getattr(self, item) +507 +508 def _is_valid_operand(self, other): +509 return isinstance(other, JTask) +510 +511 def __eq__(self, other): +512 if not self._is_valid_operand(other): +513 return False +514 if str(type(self)) != str(type(other)): +515 return False +516 for v in _jtask_comparison_attributes: +517 if self[v] != other[v]: +518 return False +519 if str(self.sheet_vars) != str(other.sheet_vars): +520 return False +521 return True +522 +523 def __lt__(self, other): +524 if not self._is_valid_operand(other): +525 return NotImplemented +526 if str(type(self)) < str(type(other)): +527 return True +528 for v in _jtask_comparison_attributes: +529 v_self = self[v] +530 v_other = other[v] +531 # can't order compare None to None +532 if ((v_self is None) or (v_other is None)): +533 if ((v_self is None) != (v_other is None)): +534 return v_self is None +535 else: +536 if self[v] < other[v]: +537 return True +538 if str(self.sheet_vars) < str(other.sheet_vars): +539 return True +540 return False +541 +542 def __str__(self) -> str: +543 args_str = ",\n".join([ +544 f" {v}={repr(self[v])}" +545 for v in _jtask_comparison_attributes + ["sheet_vars"] +546 ]) +547 return 'JTask(\n' + args_str + ",\n)" +548 +549 def __repr__(self) -> str: +550 return self.__str__() +551 552 -553 -554def job_fn(arg: JTask): -555 """ -556 Function to run a JTask job for Jupyter notebook. Exceptions pass through -557 """ -558 assert isinstance(arg, JTask) -559 # render notebook -560 return arg.render_as_html() +553def job_fn(arg: JTask): +554 """ +555 Function to run a JTask job for Jupyter notebook. Exceptions pass through +556 """ +557 assert isinstance(arg, JTask) +558 # render notebook +559 return arg.render_as_html() +560 561 -562 -563def job_fn_eat_exception(arg: JTask): -564 """ -565 Function to run a JTask job for Jupyter notebook, catching any exception and returning it as a value -566 """ -567 assert isinstance(arg, JTask) -568 # render notebook -569 try: -570 return arg.render_as_html() -571 except Exception as e: -572 print(f"{arg} caught {e}") -573 return (arg, e) +562def job_fn_eat_exception(arg: JTask): +563 """ +564 Function to run a JTask job for Jupyter notebook, catching any exception and returning it as a value +565 """ +566 assert isinstance(arg, JTask) +567 # render notebook +568 try: +569 return arg.render_as_html() +570 except Exception as e: +571 print(f"{arg} caught {e}") +572 return (arg, e) +573 574 -575 -576def job_fn_py_txt(arg: JTask): -577 """ -578 Function to run a JTask job for Python to txt. Exceptions pass through -579 """ -580 assert isinstance(arg, JTask) -581 # render Python -582 return arg.render_py_txt() +575def job_fn_py_txt(arg: JTask): +576 """ +577 Function to run a JTask job for Python to txt. Exceptions pass through +578 """ +579 assert isinstance(arg, JTask) +580 # render Python +581 return arg.render_py_txt() +582 583 -584 -585def job_fn_py_txt_eat_exception(arg: JTask): -586 """ -587 Function to run a JTask job for Python to txt, catching any exception and returning it as a value -588 """ -589 assert isinstance(arg, JTask) -590 # render Python -591 try: -592 return arg.render_py_txt() -593 except Exception as e: -594 print(f"{arg} caught {e}") -595 return (arg, e) +584def job_fn_py_txt_eat_exception(arg: JTask): +585 """ +586 Function to run a JTask job for Python to txt, catching any exception and returning it as a value +587 """ +588 assert isinstance(arg, JTask) +589 # render Python +590 try: +591 return arg.render_py_txt() +592 except Exception as e: +593 print(f"{arg} caught {e}") +594 return (arg, e) +595 596 -597 -598def run_pool( -599 tasks: Iterable, -600 *, -601 njobs: int = 4, -602 verbose: bool = True, -603 stop_on_error: bool = True, -604 use_Jupyter: bool = True, -605 ) -> List: -606 """ -607 Run a pool of tasks. -608 -609 :param tasks: iterable of tasks -610 :param njobs: degree of parallelism -611 :param verbose: if True, print on failure -612 :param stop_on_error: if True, stop pool on error -613 :param use_Jupyter: if True, use nbconvert, nbformat Jupyter fns. -614 """ -615 tasks = list(tasks) -616 assert isinstance(njobs, int) -617 assert njobs > 0 -618 assert isinstance(verbose, bool) -619 assert isinstance(stop_on_error, bool) -620 assert isinstance(use_Jupyter, bool) -621 if len(tasks) <= 0: -622 return -623 for task in tasks: -624 assert isinstance(task, JTask) -625 res = None -626 if stop_on_error: -627 # # complex way, allowing a stop on job failure -628 # https://stackoverflow.com/a/25791961/6901725 -629 if use_Jupyter: -630 fn = job_fn -631 else: -632 fn = job_fn_py_txt -633 with Pool(njobs) as pool: -634 try: -635 res = list(pool.imap_unordered(fn, tasks)) # list is forcing iteration over tasks for side-effects -636 except Exception: -637 if verbose: -638 sys.stdout.flush() -639 print("!!! run_pool: a worker raised an Exception, aborting...") -640 sys.stdout.flush() -641 pool.close() -642 pool.terminate() -643 raise # re-raise Exception -644 else: -645 pool.close() -646 pool.join() -647 else: -648 if use_Jupyter: -649 fn = job_fn_eat_exception -650 else: -651 fn = job_fn_py_txt_eat_exception -652 # simple way, but doesn't exit until all jobs succeed or fail -653 with Pool(njobs) as pool: -654 res = list(pool.map(fn, tasks)) -655 return res +597def run_pool( +598 tasks: Iterable, +599 *, +600 njobs: int = 4, +601 verbose: bool = True, +602 stop_on_error: bool = True, +603 use_Jupyter: bool = True, +604 ) -> List: +605 """ +606 Run a pool of tasks. +607 +608 :param tasks: iterable of tasks +609 :param njobs: degree of parallelism +610 :param verbose: if True, print on failure +611 :param stop_on_error: if True, stop pool on error +612 :param use_Jupyter: if True, use nbconvert, nbformat Jupyter fns. +613 """ +614 tasks = list(tasks) +615 assert isinstance(njobs, int) +616 assert njobs > 0 +617 assert isinstance(verbose, bool) +618 assert isinstance(stop_on_error, bool) +619 assert isinstance(use_Jupyter, bool) +620 if len(tasks) <= 0: +621 return +622 for task in tasks: +623 assert isinstance(task, JTask) +624 res = None +625 if stop_on_error: +626 # # complex way, allowing a stop on job failure +627 # https://stackoverflow.com/a/25791961/6901725 +628 if use_Jupyter: +629 fn = job_fn +630 else: +631 fn = job_fn_py_txt +632 with Pool(njobs) as pool: +633 try: +634 res = list(pool.imap_unordered(fn, tasks)) # list is forcing iteration over tasks for side-effects +635 except Exception: +636 if verbose: +637 sys.stdout.flush() +638 print("!!! run_pool: a worker raised an Exception, aborting...") +639 sys.stdout.flush() +640 pool.close() +641 pool.terminate() +642 raise # re-raise Exception +643 else: +644 pool.close() +645 pool.join() +646 else: +647 if use_Jupyter: +648 fn = job_fn_eat_exception +649 else: +650 fn = job_fn_py_txt_eat_exception +651 # simple way, but doesn't exit until all jobs succeed or fail +652 with Pool(njobs) as pool: +653 res = list(pool.map(fn, tasks)) +654 return res +655 656 -657 -658def write_dict_as_assignments(values: Dict[str, Any]) -> str: -659 """ -660 Write a dictionary as a block of Python assignment statements. -661 -662 :param values: dictionary -663 :return: assignment statements -664 """ -665 return "\n" + "\n".join([ -666 f"{k} = {repr(values[k])}" for k in sorted(values.keys()) -667 ]) + "\n" -668 -669 -670@contextmanager -671def declare_task_variables( -672 env, -673 *, -674 result_map:Optional[Dict[str, Any]] = None, -675 ) -> None: -676 """ -677 Copy env["sheet_vars"][k] into env[k] for all k in sheet_vars.keys() if "sheet_vars" defined in env. -678 Only variables that are first assigned in the with block of this task manager are allowed to be assigned. -679 -680 :param env: working environment, setting to globals() is usually the correct choice. -681 :param result_map: empty dictionary to return results in. result_map["sheet_vars"] is the dictionary if incoming assignments, result_map["declared_vars"] is the dictionary of default names and values. -682 :return None: -683 """ -684 sheet_vars = dict() -685 pre_known_vars = set() -686 if env is not None: -687 pre_known_vars = set(env.keys()) -688 if result_map is not None: -689 result_map["sheet_vars"] = sheet_vars -690 result_map["declared_vars"] = set() -691 if "sheet_vars" in pre_known_vars: -692 sheet_vars = env["sheet_vars"] -693 if result_map is not None: -694 result_map["sheet_vars"] = sheet_vars -695 already_assigned_vars = set(sheet_vars.keys()).intersection(pre_known_vars) -696 if len(already_assigned_vars) > 0: -697 raise ValueError(f"declare_task_variables(): attempting to set pre-with variables: {sorted(already_assigned_vars)}") -698 try: -699 yield -700 finally: -701 post_known_vars = set(env.keys()) -702 declared_vars = post_known_vars - pre_known_vars -703 if result_map is not None: -704 result_map["declared_vars"] = {k: env[k] for k in declared_vars} -705 if "sheet_vars" in pre_known_vars: -706 unexpected_vars = set(sheet_vars.keys()) - declared_vars -707 if len(unexpected_vars) > 0: -708 raise ValueError(f"declare_task_variables(): attempting to assign undeclared variables: {sorted(unexpected_vars)}") -709 # do the assignments -710 for k, v in sheet_vars.items(): -711 env[k] = v +657@contextmanager +658def declare_task_variables( +659 env, +660 *, +661 result_map:Optional[Dict[str, Any]] = None, +662 ) -> None: +663 """ +664 Copy env["sheet_vars"][k] into env[k] for all k in sheet_vars.keys() if "sheet_vars" defined in env. +665 Only variables that are first assigned in the with block of this task manager are allowed to be assigned. +666 +667 :param env: working environment, setting to globals() is usually the correct choice. +668 :param result_map: empty dictionary to return results in. result_map["sheet_vars"] is the dictionary if incoming assignments, result_map["declared_vars"] is the dictionary of default names and values. +669 :return None: +670 """ +671 sheet_vars = dict() +672 pre_known_vars = set() +673 if env is not None: +674 pre_known_vars = set(env.keys()) +675 if result_map is not None: +676 result_map["sheet_vars"] = sheet_vars +677 result_map["declared_vars"] = set() +678 if "sheet_vars" in pre_known_vars: +679 sheet_vars = env["sheet_vars"] +680 if result_map is not None: +681 result_map["sheet_vars"] = sheet_vars +682 already_assigned_vars = set(sheet_vars.keys()).intersection(pre_known_vars) +683 if len(already_assigned_vars) > 0: +684 raise ValueError(f"declare_task_variables(): attempting to set pre-with variables: {sorted(already_assigned_vars)}") +685 try: +686 yield +687 finally: +688 post_known_vars = set(env.keys()) +689 declared_vars = post_known_vars - pre_known_vars +690 if result_map is not None: +691 result_map["declared_vars"] = {k: env[k] for k in declared_vars} +692 if "sheet_vars" in pre_known_vars: +693 unexpected_vars = set(sheet_vars.keys()) - declared_vars +694 if len(unexpected_vars) > 0: +695 raise ValueError(f"declare_task_variables(): attempting to assign undeclared variables: {sorted(unexpected_vars)}") +696 # do the assignments +697 for k, v in sheet_vars.items(): +698 env[k] = v @@ -848,26 +832,26 @@

    - 
    41def pretty_format_python(python_txt: str, *, black_mode=None) -> str:
-42    """
-43    Format Python code, using black.
-44
-45    :param python_txt: Python code
-46    :param black_mode: options for black
-47    :return: formatted Python code
-48    """
-49    assert have_black
-50    assert isinstance(python_txt, str)
-51    formatted_python = python_txt.strip('\n') + '\n'
-52    if len(formatted_python.strip()) > 0:
-53        if black_mode is None:
-54            black_mode = black.FileMode()
-55        try:
-56            formatted_python = black.format_str(formatted_python, mode=black_mode)
-57            formatted_python = formatted_python.strip('\n') + '\n'
-58        except Exception:
-59            pass
-60    return formatted_python
+            
    40def pretty_format_python(python_txt: str, *, black_mode=None) -> str:
+41    """
+42    Format Python code, using black.
+43
+44    :param python_txt: Python code
+45    :param black_mode: options for black
+46    :return: formatted Python code
+47    """
+48    assert have_black
+49    assert isinstance(python_txt, str)
+50    formatted_python = python_txt.strip('\n') + '\n'
+51    if len(formatted_python.strip()) > 0:
+52        if black_mode is None:
+53            black_mode = black.FileMode()
+54        try:
+55            formatted_python = black.format_str(formatted_python, mode=black_mode)
+56            formatted_python = formatted_python.strip('\n') + '\n'
+57        except Exception:
+58            pass
+59    return formatted_python
 
    
 
 
@@ -900,73 +884,73 @@ 
    Returns
    - 
     63def convert_py_code_to_notebook(
- 64    text: str,
- 65    *,
- 66    use_black:bool = False) -> nbformat.notebooknode.NotebookNode:
- 67    """
- 68    Convert python text to a notebook. 
- 69    "''' begin text" ends any open blocks, and starts a new markdown block (triple double quotes also allowed)
- 70    "''' # end text" ends text, and starts a new code block (triple double quotes also allowed)
- 71    "'''end code'''" ends code blocks, and starts a new code block (triple double quotes also allowed)
- 72
- 73    :param text: Python text to convert.
- 74    :param use_black: if True use black to re-format Python code
- 75    :return: a notebook 
- 76    """
- 77    # https://stackoverflow.com/a/23729611/6901725
- 78    # https://nbviewer.org/gist/fperez/9716279
- 79    assert isinstance(text, str)
- 80    assert isinstance(use_black, bool)
- 81    lines = text.splitlines()
- 82    begin_text_regexp = re.compile(r"^\s*r?((''')|(\"\"\"))\s*begin\s+text\s*$")
- 83    end_text_regexp = re.compile(r"^\s*r?((''')|(\"\"\"))\s*#\s*end\s+text\s*$")
- 84    end_code_regexp = re.compile(r"(^\s*r?'''\s*end\s+code\s*'''\s*$)|(^\s*r?\"\"\"\s*end\s+code\s*\"\"\"\s*$)")
- 85    # run a little code collecting state machine
- 86    cells = []
- 87    collecting_python = []
- 88    collecting_text = None
- 89    lines.append(None)  # append an ending sentinel
- 90    # scan input
- 91    for line in lines:
- 92        if line is None:
- 93            is_end = True
- 94            text_start = False
- 95            code_start = False
- 96            code_end = False
- 97        else:
- 98            is_end = False
- 99            text_start = begin_text_regexp.match(line)
-100            code_start = end_text_regexp.match(line)
-101            code_end = end_code_regexp.match(line)
-102        if is_end or text_start or code_start or code_end:
-103            if (collecting_python is not None) and (len(collecting_python) > 0):
-104                python_block = ('\n'.join(collecting_python)).strip('\n') + '\n'
-105                if len(python_block.strip()) > 0:
-106                    if use_black and have_black:
-107                        python_block = pretty_format_python(python_block)
-108                    cells.append(nbf_v.new_code_cell(python_block))
-109            if (collecting_text is not None) and (len(collecting_text) > 0):
-110                txt_block = ('\n'.join(collecting_text)).strip('\n') + '\n'
-111                if len(txt_block.strip()) > 0:
-112                    cells.append(nbf_v.new_markdown_cell(txt_block))
-113            collecting_python = None
-114            collecting_text = None
-115            if not is_end:
-116                if text_start:
-117                    collecting_text = []
-118                else:
-119                    collecting_python = []
-120        else:
-121            if collecting_python is not None:
-122                collecting_python.append(line)
-123            if collecting_text is not None:
-124                collecting_text.append(line)
-125    for i in range(len(cells)):
-126        cells[i]["id"] = f"cell{i}"
-127    nb = nbf_v.new_notebook(cells=cells)
-128    # nb = _normalize(nb)
-129    return nb
+            
     62def convert_py_code_to_notebook(
+ 63    text: str,
+ 64    *,
+ 65    use_black:bool = False) -> nbformat.notebooknode.NotebookNode:
+ 66    """
+ 67    Convert python text to a notebook. 
+ 68    "''' begin text" ends any open blocks, and starts a new markdown block (triple double quotes also allowed)
+ 69    "''' # end text" ends text, and starts a new code block (triple double quotes also allowed)
+ 70    "'''end code'''" ends code blocks, and starts a new code block (triple double quotes also allowed)
+ 71
+ 72    :param text: Python text to convert.
+ 73    :param use_black: if True use black to re-format Python code
+ 74    :return: a notebook 
+ 75    """
+ 76    # https://stackoverflow.com/a/23729611/6901725
+ 77    # https://nbviewer.org/gist/fperez/9716279
+ 78    assert isinstance(text, str)
+ 79    assert isinstance(use_black, bool)
+ 80    lines = text.splitlines()
+ 81    begin_text_regexp = re.compile(r"^\s*r?((''')|(\"\"\"))\s*begin\s+text\s*$")
+ 82    end_text_regexp = re.compile(r"^\s*r?((''')|(\"\"\"))\s*#\s*end\s+text\s*$")
+ 83    end_code_regexp = re.compile(r"(^\s*r?'''\s*end\s+code\s*'''\s*$)|(^\s*r?\"\"\"\s*end\s+code\s*\"\"\"\s*$)")
+ 84    # run a little code collecting state machine
+ 85    cells = []
+ 86    collecting_python = []
+ 87    collecting_text = None
+ 88    lines.append(None)  # append an ending sentinel
+ 89    # scan input
+ 90    for line in lines:
+ 91        if line is None:
+ 92            is_end = True
+ 93            text_start = False
+ 94            code_start = False
+ 95            code_end = False
+ 96        else:
+ 97            is_end = False
+ 98            text_start = begin_text_regexp.match(line)
+ 99            code_start = end_text_regexp.match(line)
+100            code_end = end_code_regexp.match(line)
+101        if is_end or text_start or code_start or code_end:
+102            if (collecting_python is not None) and (len(collecting_python) > 0):
+103                python_block = ('\n'.join(collecting_python)).strip('\n') + '\n'
+104                if len(python_block.strip()) > 0:
+105                    if use_black and have_black:
+106                        python_block = pretty_format_python(python_block)
+107                    cells.append(nbf_v.new_code_cell(python_block))
+108            if (collecting_text is not None) and (len(collecting_text) > 0):
+109                txt_block = ('\n'.join(collecting_text)).strip('\n') + '\n'
+110                if len(txt_block.strip()) > 0:
+111                    cells.append(nbf_v.new_markdown_cell(txt_block))
+112            collecting_python = None
+113            collecting_text = None
+114            if not is_end:
+115                if text_start:
+116                    collecting_text = []
+117                else:
+118                    collecting_python = []
+119        else:
+120            if collecting_python is not None:
+121                collecting_python.append(line)
+122            if collecting_text is not None:
+123                collecting_text.append(line)
+124    for i in range(len(cells)):
+125        cells[i]["id"] = f"cell{i}"
+126    nb = nbf_v.new_notebook(cells=cells)
+127    # nb = _normalize(nb)
+128    return nb
 
    
 
 
@@ -1002,31 +986,31 @@ 
    Returns
    - 
    132def prepend_code_cell_to_notebook(
-133    nb: nbformat.notebooknode.NotebookNode,
-134    *,
-135    code_text: str,
-136) -> nbformat.notebooknode.NotebookNode:
-137    """
-138    Prepend a code cell to a Jupyter notebook.
-139
-140    :param nb: Jupyter notebook to alter
-141    :param code_text: Python source code to add
-142    :return: new notebook
-143    """
-144    header_cell = nbf_v.new_code_cell(code_text)
-145    # set cell ids to avoid:
-146    # "MissingIDFieldWarning: Code cell is missing an id field, this will become a hard error in future nbformat versions."
-147    header_cell["id"] = "wvpy_header_cell"
-148    orig_cells = [c.copy() for c in nb.cells]
-149    for i in range(len(orig_cells)):
-150        orig_cells[i]["id"] = f"cell{i}"
-151    cells = [header_cell] + orig_cells
-152    nb_out = nbf_v.new_notebook(
-153        cells=cells
-154    )
-155    # nb_out = _normalize(nb_out)
-156    return nb_out
+            
    131def prepend_code_cell_to_notebook(
+132    nb: nbformat.notebooknode.NotebookNode,
+133    *,
+134    code_text: str,
+135) -> nbformat.notebooknode.NotebookNode:
+136    """
+137    Prepend a code cell to a Jupyter notebook.
+138
+139    :param nb: Jupyter notebook to alter
+140    :param code_text: Python source code to add
+141    :return: new notebook
+142    """
+143    header_cell = nbf_v.new_code_cell(code_text)
+144    # set cell ids to avoid:
+145    # "MissingIDFieldWarning: Code cell is missing an id field, this will become a hard error in future nbformat versions."
+146    header_cell["id"] = "wvpy_header_cell"
+147    orig_cells = [c.copy() for c in nb.cells]
+148    for i in range(len(orig_cells)):
+149        orig_cells[i]["id"] = f"cell{i}"
+150    cells = [header_cell] + orig_cells
+151    nb_out = nbf_v.new_notebook(
+152        cells=cells
+153    )
+154    # nb_out = _normalize(nb_out)
+155    return nb_out
 
    
 
 
@@ -1059,32 +1043,32 @@ 
    Returns
    - 
    159def convert_py_file_to_notebook(
-160    py_file: str, 
-161    *,
-162    ipynb_file: str,
-163    use_black: bool = False,
-164    ) -> None:
-165    """
-166    Convert python text to a notebook. 
-167    "''' begin text" ends any open blocks, and starts a new markdown block (triple double quotes also allowed)
-168    "''' # end text" ends text, and starts a new code block (triple double quotes also allowed)
-169    "'''end code'''" ends code blocks, and starts a new code block (triple double quotes also allowed)
-170
-171    :param py_file: Path to python source file.
-172    :param ipynb_file: Path to notebook result file.
-173    :param use_black: if True use black to re-format Python code
-174    :return: nothing 
-175    """
-176    assert isinstance(py_file, str)
-177    assert isinstance(ipynb_file, str)
-178    assert isinstance(use_black, bool)
-179    assert py_file != ipynb_file  # prevent clobber
-180    with open(py_file, 'r') as f:
-181        text = f.read()
-182    nb = convert_py_code_to_notebook(text, use_black=use_black)
-183    with open(ipynb_file, 'w') as f:
-184        nbformat.write(nb, f)
+            
    158def convert_py_file_to_notebook(
+159    py_file: str, 
+160    *,
+161    ipynb_file: str,
+162    use_black: bool = False,
+163    ) -> None:
+164    """
+165    Convert python text to a notebook. 
+166    "''' begin text" ends any open blocks, and starts a new markdown block (triple double quotes also allowed)
+167    "''' # end text" ends text, and starts a new code block (triple double quotes also allowed)
+168    "'''end code'''" ends code blocks, and starts a new code block (triple double quotes also allowed)
+169
+170    :param py_file: Path to python source file.
+171    :param ipynb_file: Path to notebook result file.
+172    :param use_black: if True use black to re-format Python code
+173    :return: nothing 
+174    """
+175    assert isinstance(py_file, str)
+176    assert isinstance(ipynb_file, str)
+177    assert isinstance(use_black, bool)
+178    assert py_file != ipynb_file  # prevent clobber
+179    with open(py_file, 'r') as f:
+180        text = f.read()
+181    nb = convert_py_code_to_notebook(text, use_black=use_black)
+182    with open(ipynb_file, 'w') as f:
+183        nbformat.write(nb, f)
 
    
 
 
@@ -1121,41 +1105,41 @@ 
    Returns
    - 
    187def convert_notebook_code_to_py(
-188    nb: nbformat.notebooknode.NotebookNode,
-189    *,
-190    use_black: bool = False,
-191    ) -> str:
-192    """
-193    Convert ipython notebook inputs to a py code. 
-194    "''' begin text" ends any open blocks, and starts a new markdown block (triple double quotes also allowed)
-195    "''' # end text" ends text, and starts a new code block (triple double quotes also allowed)
-196    "'''end code'''" ends code blocks, and starts a new code block (triple double quotes also allowed)
-197
-198    :param nb: notebook
-199    :param use_black: if True use black to re-format Python code
-200    :return: Python source code
-201    """
-202    assert isinstance(use_black, bool)
-203    res = []
-204    code_needs_end = False
-205    for cell in nb.cells:
-206        if len(cell.source.strip()) > 0:
-207            if cell.cell_type == 'code':
-208                if code_needs_end:
-209                    res.append('\n"""end code"""\n')
-210                py_text = cell.source.strip('\n') + '\n'
-211                if use_black and have_black:
-212                    py_text = pretty_format_python(py_text)
-213                res.append(py_text)
-214                code_needs_end = True
-215            else:
-216                res.append('\n""" begin text')
-217                res.append(cell.source.strip('\n'))
-218                res.append('"""  # end text\n')
-219                code_needs_end = False
-220    res_text = '\n' + ('\n'.join(res)).strip('\n') + '\n\n'
-221    return res_text
+            
    186def convert_notebook_code_to_py(
+187    nb: nbformat.notebooknode.NotebookNode,
+188    *,
+189    use_black: bool = False,
+190    ) -> str:
+191    """
+192    Convert ipython notebook inputs to a py code. 
+193    "''' begin text" ends any open blocks, and starts a new markdown block (triple double quotes also allowed)
+194    "''' # end text" ends text, and starts a new code block (triple double quotes also allowed)
+195    "'''end code'''" ends code blocks, and starts a new code block (triple double quotes also allowed)
+196
+197    :param nb: notebook
+198    :param use_black: if True use black to re-format Python code
+199    :return: Python source code
+200    """
+201    assert isinstance(use_black, bool)
+202    res = []
+203    code_needs_end = False
+204    for cell in nb.cells:
+205        if len(cell.source.strip()) > 0:
+206            if cell.cell_type == 'code':
+207                if code_needs_end:
+208                    res.append('\n"""end code"""\n')
+209                py_text = cell.source.strip('\n') + '\n'
+210                if use_black and have_black:
+211                    py_text = pretty_format_python(py_text)
+212                res.append(py_text)
+213                code_needs_end = True
+214            else:
+215                res.append('\n""" begin text')
+216                res.append(cell.source.strip('\n'))
+217                res.append('"""  # end text\n')
+218                code_needs_end = False
+219    res_text = '\n' + ('\n'.join(res)).strip('\n') + '\n\n'
+220    return res_text
 
    
 
 
@@ -1191,32 +1175,32 @@ 
    Returns
    - 
    224def convert_notebook_file_to_py(
-225    ipynb_file: str,
-226    *,  
-227    py_file: str,
-228    use_black: bool = False,
-229    ) -> None:
-230    """
-231    Convert ipython notebook inputs to a py file. 
-232    "''' begin text" ends any open blocks, and starts a new markdown block (triple double quotes also allowed)
-233    "''' # end text" ends text, and starts a new code block (triple double quotes also allowed)
-234    "'''end code'''" ends code blocks, and starts a new code block (triple double quotes also allowed)
-235
-236    :param ipynb_file: Path to notebook input file.
-237    :param py_file: Path to python result file.
-238    :param use_black: if True use black to re-format Python code
-239    :return: nothing
-240    """
-241    assert isinstance(py_file, str)
-242    assert isinstance(ipynb_file, str)
-243    assert isinstance(use_black, bool)
-244    assert py_file != ipynb_file  # prevent clobber
-245    with open(ipynb_file, "rb") as f:
-246        nb = nbformat.read(f, as_version=4)
-247    py_source = convert_notebook_code_to_py(nb, use_black=use_black)
-248    with open(py_file, 'w') as f:
-249        f.write(py_source)        
+            
    223def convert_notebook_file_to_py(
+224    ipynb_file: str,
+225    *,  
+226    py_file: str,
+227    use_black: bool = False,
+228    ) -> None:
+229    """
+230    Convert ipython notebook inputs to a py file. 
+231    "''' begin text" ends any open blocks, and starts a new markdown block (triple double quotes also allowed)
+232    "''' # end text" ends text, and starts a new code block (triple double quotes also allowed)
+233    "'''end code'''" ends code blocks, and starts a new code block (triple double quotes also allowed)
+234
+235    :param ipynb_file: Path to notebook input file.
+236    :param py_file: Path to python result file.
+237    :param use_black: if True use black to re-format Python code
+238    :return: nothing
+239    """
+240    assert isinstance(py_file, str)
+241    assert isinstance(ipynb_file, str)
+242    assert isinstance(use_black, bool)
+243    assert py_file != ipynb_file  # prevent clobber
+244    with open(ipynb_file, "rb") as f:
+245        nb = nbformat.read(f, as_version=4)
+246    py_source = convert_notebook_code_to_py(nb, use_black=use_black)
+247    with open(py_file, 'w') as f:
+248        f.write(py_source)        
 
    
 
 
@@ -1253,34 +1237,34 @@ 
    Returns
    - 
    252class OurExecutor(ExecutePreprocessor):
-253    """Catch exception in notebook processing"""
-254    def __init__(self, **kw):
-255        """Initialize the preprocessor."""
-256        ExecutePreprocessor.__init__(self, **kw)
-257        self.caught_exception = None
-258    
-259    def preprocess_cell(self, cell, resources, index):
-260        """
-261        Override if you want to apply some preprocessing to each cell.
-262        Must return modified cell and resource dictionary.
-263
-264        Parameters
-265        ----------
-266        cell : NotebookNode cell
-267            Notebook cell being processed
-268        resources : dictionary
-269            Additional resources used in the conversion process.  Allows
-270            preprocessors to pass variables into the Jinja engine.
-271        index : int
-272            Index of the cell being processed
-273        """
-274        if self.caught_exception is None:
-275            try:
-276                return ExecutePreprocessor.preprocess_cell(self, cell, resources, index)
-277            except Exception as ex:
-278                self.caught_exception = ex
-279        return cell, self.resources
+            
    251class OurExecutor(ExecutePreprocessor):
+252    """Catch exception in notebook processing"""
+253    def __init__(self, **kw):
+254        """Initialize the preprocessor."""
+255        ExecutePreprocessor.__init__(self, **kw)
+256        self.caught_exception = None
+257    
+258    def preprocess_cell(self, cell, resources, index):
+259        """
+260        Override if you want to apply some preprocessing to each cell.
+261        Must return modified cell and resource dictionary.
+262
+263        Parameters
+264        ----------
+265        cell : NotebookNode cell
+266            Notebook cell being processed
+267        resources : dictionary
+268            Additional resources used in the conversion process.  Allows
+269            preprocessors to pass variables into the Jinja engine.
+270        index : int
+271            Index of the cell being processed
+272        """
+273        if self.caught_exception is None:
+274            try:
+275                return ExecutePreprocessor.preprocess_cell(self, cell, resources, index)
+276            except Exception as ex:
+277                self.caught_exception = ex
+278        return cell, self.resources
 
    
 
 
@@ -1298,10 +1282,10 @@ 
    Returns
    - 
    254    def __init__(self, **kw):
-255        """Initialize the preprocessor."""
-256        ExecutePreprocessor.__init__(self, **kw)
-257        self.caught_exception = None
+            
    253    def __init__(self, **kw):
+254        """Initialize the preprocessor."""
+255        ExecutePreprocessor.__init__(self, **kw)
+256        self.caught_exception = None
 
    
 
 
@@ -1321,27 +1305,27 @@ 
    Returns
    - 
    259    def preprocess_cell(self, cell, resources, index):
-260        """
-261        Override if you want to apply some preprocessing to each cell.
-262        Must return modified cell and resource dictionary.
-263
-264        Parameters
-265        ----------
-266        cell : NotebookNode cell
-267            Notebook cell being processed
-268        resources : dictionary
-269            Additional resources used in the conversion process.  Allows
-270            preprocessors to pass variables into the Jinja engine.
-271        index : int
-272            Index of the cell being processed
-273        """
-274        if self.caught_exception is None:
-275            try:
-276                return ExecutePreprocessor.preprocess_cell(self, cell, resources, index)
-277            except Exception as ex:
-278                self.caught_exception = ex
-279        return cell, self.resources
+            
    258    def preprocess_cell(self, cell, resources, index):
+259        """
+260        Override if you want to apply some preprocessing to each cell.
+261        Must return modified cell and resource dictionary.
+262
+263        Parameters
+264        ----------
+265        cell : NotebookNode cell
+266            Notebook cell being processed
+267        resources : dictionary
+268            Additional resources used in the conversion process.  Allows
+269            preprocessors to pass variables into the Jinja engine.
+270        index : int
+271            Index of the cell being processed
+272        """
+273        if self.caught_exception is None:
+274            try:
+275                return ExecutePreprocessor.preprocess_cell(self, cell, resources, index)
+276            except Exception as ex:
+277                self.caught_exception = ex
+278        return cell, self.resources
 
    
 
 
@@ -1484,142 +1468,142 @@ 
    Inherited Members
    - 
    286def render_as_html(
-287    notebook_file_name: str,
-288    *,
-289    output_suffix: Optional[str] = None,
-290    timeout:int = 60000,
-291    kernel_name: Optional[str] = None,
-292    verbose: bool = True,
-293    sheet_vars = None,
-294    init_code: Optional[str] = None,
-295    exclude_input: bool = False,
-296    prompt_strip_regexp: Optional[str] = r'<\s*div\s+class\s*=\s*"jp-OutputPrompt[^<>]*>[^<>]*Out[^<>]*<\s*/div\s*>',
-297) -> None:
-298    """
-299    Render a Jupyter notebook in the current directory as HTML.
-300    Exceptions raised in the rendering notebook are allowed to pass trough.
-301
-302    :param notebook_file_name: name of source file, must end with .ipynb or .py (or type gotten from file system)
-303    :param output_suffix: optional name to add to result name
-304    :param timeout: Maximum time in seconds each notebook cell is allowed to run.
-305                    passed to nbconvert.preprocessors.ExecutePreprocessor.
-306    :param kernel_name: Jupyter kernel to use. passed to nbconvert.preprocessors.ExecutePreprocessor.
-307    :param verbose logical, if True print while running
-308    :param sheet_vars: if not None value is de-serialized as a variable named "sheet_vars"
-309    :param init_code: Python init code for first cell
-310    :param exclude_input: if True, exclude input cells
-311    :param prompt_strip_regexp: regexp to strip prompts, only used if exclude_input is True
-312    :return: None
-313    """
-314    assert isinstance(notebook_file_name, str)
-315    # deal with no suffix case
-316    if (not notebook_file_name.endswith(".ipynb")) and (not notebook_file_name.endswith(".py")):
-317        py_name = notebook_file_name + ".py"
-318        py_exists = os.path.exists(py_name)
-319        ipynb_name = notebook_file_name + ".ipynb"
-320        ipynb_exists = os.path.exists(ipynb_name)
-321        if (py_exists + ipynb_exists) != 1:
-322            raise ValueError('{ipynb_exists}: if file suffix is not specified then exactly one of .py or .ipynb file must exist')
-323        if ipynb_exists:
-324            notebook_file_name = notebook_file_name + '.ipynb'
-325        else:
-326            notebook_file_name = notebook_file_name + '.py'
-327    # get the input
-328    assert os.path.exists(notebook_file_name)
-329    if notebook_file_name.endswith(".ipynb"):
-330        suffix = ".ipynb"
-331        with open(notebook_file_name, "rb") as f:
-332            nb = nbformat.read(f, as_version=4)
-333    elif notebook_file_name.endswith(".py"):
-334        suffix = ".py"
-335        with open(notebook_file_name, 'r') as f:
-336            text = f.read()
-337        nb = convert_py_code_to_notebook(text)
-338    else:
-339        raise ValueError('{ipynb_exists}: file must end with .py or .ipynb')
-340    tmp_path = None
-341    # do the conversion
-342    if sheet_vars is not None:
-343        with tempfile.NamedTemporaryFile(delete=False) as ntf:
-344            tmp_path = ntf.name
-345            pickle.dump(sheet_vars, file=ntf)
-346        if (init_code is None) or (len(init_code) <= 0):
-347            init_code = ""
-348        pickle_code = f"""
-349import pickle
-350with open({tmp_path.__repr__()}, 'rb') as pf:
-351   sheet_vars = pickle.load(pf)
-352""" 
-353        init_code = init_code + "\n\n" + pickle_code
-354    if (init_code is not None) and (len(init_code) > 0):
-355        assert isinstance(init_code, str)
-356        nb = prepend_code_cell_to_notebook(
-357            nb, 
-358            code_text=f'\n\n{init_code}\n\n')
-359    html_name = os.path.basename(notebook_file_name)
-360    html_name = html_name.removesuffix(suffix)
-361    exec_note = ""
-362    if output_suffix is not None:
-363        assert isinstance(output_suffix, str)
-364        html_name = html_name + output_suffix
-365        exec_note = f'"{output_suffix}"'
-366    html_name = html_name + ".html"
-367    try:
-368        os.remove(html_name)
-369    except FileNotFoundError:
-370        pass
-371    caught = None
-372    trace = None
-373    html_body = ""
-374    if verbose:
-375        print(
-376            f'start render_as_html "{notebook_file_name}" {exec_note} {datetime.datetime.now()}'
-377        )
-378    try:
-379        if kernel_name is not None:
-380            ep = OurExecutor(
-381                timeout=timeout, kernel_name=kernel_name
-382            )
-383        else:
-384            ep = OurExecutor(timeout=timeout)
-385        nb_res, nb_resources = ep.preprocess(nb)
-386        html_exporter = HTMLExporter(exclude_input=exclude_input)
-387        html_body, html_resources = html_exporter.from_notebook_node(nb_res)
-388        caught = ep.caught_exception
-389    except Exception as e:
-390        caught = e
-391        trace = traceback.format_exc()
-392    if exclude_input and (prompt_strip_regexp is not None):
-393        # strip output prompts
-394        html_body = re.sub(
-395            prompt_strip_regexp,
-396            ' ',
-397            html_body)
-398    with open(html_name, "wt", encoding="utf-8") as f:
-399        f.write(html_body)
-400        if caught is not None:
-401            f.write("\n<pre>\n")
-402            f.write(escape_ansi(str(caught)))
-403            f.write("\n</pre>\n")
-404        if trace is not None:
-405            f.write("\n<pre>\n")
-406            f.write(escape_ansi(str(trace)))
-407            f.write("\n</pre>\n")
-408    nw = datetime.datetime.now()
-409    if tmp_path is not None:
-410        try:
-411            os.remove(tmp_path)
-412        except FileNotFoundError:
-413            pass
-414    if caught is not None:
-415        if verbose:
-416            print(f'\n\n\texception in render_as_html "{notebook_file_name}" {nw} {escape_ansi(str(caught))}\n\n')
-417            if trace is not None:
-418                print(f'\n\n\t\ttrace {escape_ansi(str(trace))}\n\n')
-419        raise caught
-420    if verbose:
-421        print(f'\tdone render_as_html "{notebook_file_name}" {nw}')
+            
    285def render_as_html(
+286    notebook_file_name: str,
+287    *,
+288    output_suffix: Optional[str] = None,
+289    timeout:int = 60000,
+290    kernel_name: Optional[str] = None,
+291    verbose: bool = True,
+292    sheet_vars = None,
+293    init_code: Optional[str] = None,
+294    exclude_input: bool = False,
+295    prompt_strip_regexp: Optional[str] = r'<\s*div\s+class\s*=\s*"jp-OutputPrompt[^<>]*>[^<>]*Out[^<>]*<\s*/div\s*>',
+296) -> None:
+297    """
+298    Render a Jupyter notebook in the current directory as HTML.
+299    Exceptions raised in the rendering notebook are allowed to pass trough.
+300
+301    :param notebook_file_name: name of source file, must end with .ipynb or .py (or type gotten from file system)
+302    :param output_suffix: optional name to add to result name
+303    :param timeout: Maximum time in seconds each notebook cell is allowed to run.
+304                    passed to nbconvert.preprocessors.ExecutePreprocessor.
+305    :param kernel_name: Jupyter kernel to use. passed to nbconvert.preprocessors.ExecutePreprocessor.
+306    :param verbose logical, if True print while running
+307    :param sheet_vars: if not None value is de-serialized as a variable named "sheet_vars"
+308    :param init_code: Python init code for first cell
+309    :param exclude_input: if True, exclude input cells
+310    :param prompt_strip_regexp: regexp to strip prompts, only used if exclude_input is True
+311    :return: None
+312    """
+313    assert isinstance(notebook_file_name, str)
+314    # deal with no suffix case
+315    if (not notebook_file_name.endswith(".ipynb")) and (not notebook_file_name.endswith(".py")):
+316        py_name = notebook_file_name + ".py"
+317        py_exists = os.path.exists(py_name)
+318        ipynb_name = notebook_file_name + ".ipynb"
+319        ipynb_exists = os.path.exists(ipynb_name)
+320        if (py_exists + ipynb_exists) != 1:
+321            raise ValueError('{ipynb_exists}: if file suffix is not specified then exactly one of .py or .ipynb file must exist')
+322        if ipynb_exists:
+323            notebook_file_name = notebook_file_name + '.ipynb'
+324        else:
+325            notebook_file_name = notebook_file_name + '.py'
+326    # get the input
+327    assert os.path.exists(notebook_file_name)
+328    if notebook_file_name.endswith(".ipynb"):
+329        suffix = ".ipynb"
+330        with open(notebook_file_name, "rb") as f:
+331            nb = nbformat.read(f, as_version=4)
+332    elif notebook_file_name.endswith(".py"):
+333        suffix = ".py"
+334        with open(notebook_file_name, 'r') as f:
+335            text = f.read()
+336        nb = convert_py_code_to_notebook(text)
+337    else:
+338        raise ValueError('{ipynb_exists}: file must end with .py or .ipynb')
+339    tmp_path = None
+340    # do the conversion
+341    if sheet_vars is not None:
+342        with tempfile.NamedTemporaryFile(delete=False) as ntf:
+343            tmp_path = ntf.name
+344            pickle.dump(sheet_vars, file=ntf)
+345        if (init_code is None) or (len(init_code) <= 0):
+346            init_code = ""
+347        pickle_code = f"""
+348import pickle
+349with open({tmp_path.__repr__()}, 'rb') as pf:
+350   sheet_vars = pickle.load(pf)
+351""" 
+352        init_code = init_code + "\n\n" + pickle_code
+353    if (init_code is not None) and (len(init_code) > 0):
+354        assert isinstance(init_code, str)
+355        nb = prepend_code_cell_to_notebook(
+356            nb, 
+357            code_text=f'\n\n{init_code}\n\n')
+358    html_name = os.path.basename(notebook_file_name)
+359    html_name = html_name.removesuffix(suffix)
+360    exec_note = ""
+361    if output_suffix is not None:
+362        assert isinstance(output_suffix, str)
+363        html_name = html_name + output_suffix
+364        exec_note = f'"{output_suffix}"'
+365    html_name = html_name + ".html"
+366    try:
+367        os.remove(html_name)
+368    except FileNotFoundError:
+369        pass
+370    caught = None
+371    trace = None
+372    html_body = ""
+373    if verbose:
+374        print(
+375            f'start render_as_html "{notebook_file_name}" {exec_note} {datetime.datetime.now()}'
+376        )
+377    try:
+378        if kernel_name is not None:
+379            ep = OurExecutor(
+380                timeout=timeout, kernel_name=kernel_name
+381            )
+382        else:
+383            ep = OurExecutor(timeout=timeout)
+384        nb_res, nb_resources = ep.preprocess(nb)
+385        html_exporter = HTMLExporter(exclude_input=exclude_input)
+386        html_body, html_resources = html_exporter.from_notebook_node(nb_res)
+387        caught = ep.caught_exception
+388    except Exception as e:
+389        caught = e
+390        trace = traceback.format_exc()
+391    if exclude_input and (prompt_strip_regexp is not None):
+392        # strip output prompts
+393        html_body = re.sub(
+394            prompt_strip_regexp,
+395            ' ',
+396            html_body)
+397    with open(html_name, "wt", encoding="utf-8") as f:
+398        f.write(html_body)
+399        if caught is not None:
+400            f.write("\n<pre>\n")
+401            f.write(escape_ansi(str(caught)))
+402            f.write("\n</pre>\n")
+403        if trace is not None:
+404            f.write("\n<pre>\n")
+405            f.write(escape_ansi(str(trace)))
+406            f.write("\n</pre>\n")
+407    nw = datetime.datetime.now()
+408    if tmp_path is not None:
+409        try:
+410            os.remove(tmp_path)
+411        except FileNotFoundError:
+412            pass
+413    if caught is not None:
+414        if verbose:
+415            print(f'\n\n\texception in render_as_html "{notebook_file_name}" {nw} {escape_ansi(str(caught))}\n\n')
+416            if trace is not None:
+417                print(f'\n\n\t\ttrace {escape_ansi(str(trace))}\n\n')
+418        raise caught
+419    if verbose:
+420        print(f'\tdone render_as_html "{notebook_file_name}" {nw}')
 
    
 
 
@@ -1662,132 +1646,132 @@ 
    Returns
    - 
    428@total_ordering
-429class JTask:
-430    def __init__(
-431        self,
-432        sheet_name: str,
-433        *,
-434        output_suffix: Optional[str] = None,
-435        exclude_input: bool = True,
-436        sheet_vars = None,
-437        init_code: Optional[str] = None,
-438        path_prefix: Optional[str] = None,
-439        strict: bool = True,
-440    ) -> None:
-441        """
-442        Create a Jupyter task.
-443
-444        :param sheet_name: name of sheet to run can be .ipynb or .py, and suffix can be omitted.
-445        :param output_suffix: optional string to append to rendered HTML file name.
-446        :param exclude_input: if True strip input cells out of HTML render.
-447        :param sheet_vars: if not None value is de-serialized as a variable named "sheet_vars"
-448        :param init_code: optional code to insert at the top of the Jupyter sheet, used to pass parameters.
-449        :param path_prefix: optional prefix to add to sheet_name to find Jupyter source. 
-450        :param strict: if True check paths path_prefix and path_prefix/sheetname[.py|.ipynb] exist.
-451        """
-452        assert isinstance(sheet_name, str)
-453        assert isinstance(output_suffix, (str, type(None)))
-454        assert isinstance(exclude_input, bool)
-455        assert isinstance(init_code, (str, type(None)))
-456        assert isinstance(path_prefix, (str, type(None)))
-457        if strict:
-458            path = sheet_name
-459            if (isinstance(path_prefix, str)) and (len(path_prefix) > 9):
-460                assert os.path.exists(path_prefix)
-461                path = os.path.join(path_prefix, sheet_name)
-462            if path.endswith(".py"):
-463                path = path.removesuffix(".py")
-464            if path.endswith(".ipynb"):
-465                path = path.removesuffix(".ipynb")
-466            py_exists = os.path.exists(path + ".py")
-467            ipynb_exists = os.path.exists(path + ".ipynb")
-468            assert (py_exists + ipynb_exists) >= 1
-469            if (not sheet_name.endswith(".py")) and (not sheet_name.endswith(".ipynb")):
-470                # no suffix, so must be unambiguous
-471                assert (py_exists + ipynb_exists) == 1
-472        self.sheet_name = sheet_name
-473        self.output_suffix = output_suffix
-474        self.exclude_input = exclude_input
-475        self.sheet_vars = sheet_vars
-476        self.init_code = init_code
-477        self.path_prefix = path_prefix
-478
-479    def render_as_html(self) -> None:
-480        """
-481        Render Jupyter notebook or Python (treated as notebook) to HTML.
-482        """
-483        path = self.sheet_name
-484        if isinstance(self.path_prefix, str) and (len(self.path_prefix) > 0):
-485            path = os.path.join(self.path_prefix, self.sheet_name)
-486        render_as_html(
-487            path,
-488            exclude_input=self.exclude_input,
-489            output_suffix=self.output_suffix,
-490            sheet_vars=self.sheet_vars,
-491            init_code=self.init_code,
-492        )
-493    
-494    def render_py_txt(self) -> None:
-495        """
-496        Render Python to text (without nbconver, nbformat Jupyter bindings)
-497        """
-498        path = self.sheet_name
-499        if isinstance(self.path_prefix, str) and (len(self.path_prefix) > 0):
-500            path = os.path.join(self.path_prefix, self.sheet_name)
-501        execute_py(
-502            path,
-503            output_suffix=self.output_suffix,
-504            sheet_vars=self.sheet_vars,
-505            init_code=self.init_code,
-506        )
-507
-508    def __getitem__(self, item):
-509         return getattr(self, item)
-510
-511    def _is_valid_operand(self, other):
-512        return isinstance(other, JTask)
-513
-514    def __eq__(self, other):
-515        if not self._is_valid_operand(other):
-516            return False
-517        if str(type(self)) != str(type(other)):
-518            return False
-519        for v in _jtask_comparison_attributes:
-520            if self[v] != other[v]:
-521                return False
-522        if str(self.sheet_vars) != str(other.sheet_vars):
-523            return False
-524        return True
-525
-526    def __lt__(self, other):
-527        if not self._is_valid_operand(other):
-528            return NotImplemented
-529        if str(type(self)) < str(type(other)):
-530            return True
-531        for v in _jtask_comparison_attributes:
-532            v_self = self[v]
-533            v_other = other[v]
-534            # can't order compare None to None
-535            if ((v_self is None) or (v_other is None)):
-536                if ((v_self is None) != (v_other is None)):
-537                    return v_self is None
-538            else:
-539                if self[v] < other[v]:
-540                    return True
-541        if str(self.sheet_vars) < str(other.sheet_vars):
-542            return True
-543        return False
-544    
-545    def __str__(self) -> str:
-546        args_str = ",\n".join([
-547            f" {v}={repr(self[v])}"
-548            for v in _jtask_comparison_attributes + ["sheet_vars"]
-549        ])
-550        return 'JTask(\n' + args_str + ",\n)"
-551
-552    def __repr__(self) -> str:
-553        return self.__str__()
+            
    427@total_ordering
+428class JTask:
+429    def __init__(
+430        self,
+431        sheet_name: str,
+432        *,
+433        output_suffix: Optional[str] = None,
+434        exclude_input: bool = True,
+435        sheet_vars = None,
+436        init_code: Optional[str] = None,
+437        path_prefix: Optional[str] = None,
+438        strict: bool = True,
+439    ) -> None:
+440        """
+441        Create a Jupyter task.
+442
+443        :param sheet_name: name of sheet to run can be .ipynb or .py, and suffix can be omitted.
+444        :param output_suffix: optional string to append to rendered HTML file name.
+445        :param exclude_input: if True strip input cells out of HTML render.
+446        :param sheet_vars: if not None value is de-serialized as a variable named "sheet_vars"
+447        :param init_code: optional code to insert at the top of the Jupyter sheet, used to pass parameters.
+448        :param path_prefix: optional prefix to add to sheet_name to find Jupyter source. 
+449        :param strict: if True check paths path_prefix and path_prefix/sheetname[.py|.ipynb] exist.
+450        """
+451        assert isinstance(sheet_name, str)
+452        assert isinstance(output_suffix, (str, type(None)))
+453        assert isinstance(exclude_input, bool)
+454        assert isinstance(init_code, (str, type(None)))
+455        assert isinstance(path_prefix, (str, type(None)))
+456        if strict:
+457            path = sheet_name
+458            if (isinstance(path_prefix, str)) and (len(path_prefix) > 9):
+459                assert os.path.exists(path_prefix)
+460                path = os.path.join(path_prefix, sheet_name)
+461            if path.endswith(".py"):
+462                path = path.removesuffix(".py")
+463            if path.endswith(".ipynb"):
+464                path = path.removesuffix(".ipynb")
+465            py_exists = os.path.exists(path + ".py")
+466            ipynb_exists = os.path.exists(path + ".ipynb")
+467            assert (py_exists + ipynb_exists) >= 1
+468            if (not sheet_name.endswith(".py")) and (not sheet_name.endswith(".ipynb")):
+469                # no suffix, so must be unambiguous
+470                assert (py_exists + ipynb_exists) == 1
+471        self.sheet_name = sheet_name
+472        self.output_suffix = output_suffix
+473        self.exclude_input = exclude_input
+474        self.sheet_vars = sheet_vars
+475        self.init_code = init_code
+476        self.path_prefix = path_prefix
+477
+478    def render_as_html(self) -> None:
+479        """
+480        Render Jupyter notebook or Python (treated as notebook) to HTML.
+481        """
+482        path = self.sheet_name
+483        if isinstance(self.path_prefix, str) and (len(self.path_prefix) > 0):
+484            path = os.path.join(self.path_prefix, self.sheet_name)
+485        render_as_html(
+486            path,
+487            exclude_input=self.exclude_input,
+488            output_suffix=self.output_suffix,
+489            sheet_vars=self.sheet_vars,
+490            init_code=self.init_code,
+491        )
+492    
+493    def render_py_txt(self) -> None:
+494        """
+495        Render Python to text (without nbconver, nbformat Jupyter bindings)
+496        """
+497        path = self.sheet_name
+498        if isinstance(self.path_prefix, str) and (len(self.path_prefix) > 0):
+499            path = os.path.join(self.path_prefix, self.sheet_name)
+500        execute_py(
+501            path,
+502            output_suffix=self.output_suffix,
+503            sheet_vars=self.sheet_vars,
+504            init_code=self.init_code,
+505        )
+506
+507    def __getitem__(self, item):
+508         return getattr(self, item)
+509
+510    def _is_valid_operand(self, other):
+511        return isinstance(other, JTask)
+512
+513    def __eq__(self, other):
+514        if not self._is_valid_operand(other):
+515            return False
+516        if str(type(self)) != str(type(other)):
+517            return False
+518        for v in _jtask_comparison_attributes:
+519            if self[v] != other[v]:
+520                return False
+521        if str(self.sheet_vars) != str(other.sheet_vars):
+522            return False
+523        return True
+524
+525    def __lt__(self, other):
+526        if not self._is_valid_operand(other):
+527            return NotImplemented
+528        if str(type(self)) < str(type(other)):
+529            return True
+530        for v in _jtask_comparison_attributes:
+531            v_self = self[v]
+532            v_other = other[v]
+533            # can't order compare None to None
+534            if ((v_self is None) or (v_other is None)):
+535                if ((v_self is None) != (v_other is None)):
+536                    return v_self is None
+537            else:
+538                if self[v] < other[v]:
+539                    return True
+540        if str(self.sheet_vars) < str(other.sheet_vars):
+541            return True
+542        return False
+543    
+544    def __str__(self) -> str:
+545        args_str = ",\n".join([
+546            f" {v}={repr(self[v])}"
+547            for v in _jtask_comparison_attributes + ["sheet_vars"]
+548        ])
+549        return 'JTask(\n' + args_str + ",\n)"
+550
+551    def __repr__(self) -> str:
+552        return self.__str__()
 
    
 
 
@@ -1803,54 +1787,54 @@ 
    Returns
    - 
    430    def __init__(
-431        self,
-432        sheet_name: str,
-433        *,
-434        output_suffix: Optional[str] = None,
-435        exclude_input: bool = True,
-436        sheet_vars = None,
-437        init_code: Optional[str] = None,
-438        path_prefix: Optional[str] = None,
-439        strict: bool = True,
-440    ) -> None:
-441        """
-442        Create a Jupyter task.
-443
-444        :param sheet_name: name of sheet to run can be .ipynb or .py, and suffix can be omitted.
-445        :param output_suffix: optional string to append to rendered HTML file name.
-446        :param exclude_input: if True strip input cells out of HTML render.
-447        :param sheet_vars: if not None value is de-serialized as a variable named "sheet_vars"
-448        :param init_code: optional code to insert at the top of the Jupyter sheet, used to pass parameters.
-449        :param path_prefix: optional prefix to add to sheet_name to find Jupyter source. 
-450        :param strict: if True check paths path_prefix and path_prefix/sheetname[.py|.ipynb] exist.
-451        """
-452        assert isinstance(sheet_name, str)
-453        assert isinstance(output_suffix, (str, type(None)))
-454        assert isinstance(exclude_input, bool)
-455        assert isinstance(init_code, (str, type(None)))
-456        assert isinstance(path_prefix, (str, type(None)))
-457        if strict:
-458            path = sheet_name
-459            if (isinstance(path_prefix, str)) and (len(path_prefix) > 9):
-460                assert os.path.exists(path_prefix)
-461                path = os.path.join(path_prefix, sheet_name)
-462            if path.endswith(".py"):
-463                path = path.removesuffix(".py")
-464            if path.endswith(".ipynb"):
-465                path = path.removesuffix(".ipynb")
-466            py_exists = os.path.exists(path + ".py")
-467            ipynb_exists = os.path.exists(path + ".ipynb")
-468            assert (py_exists + ipynb_exists) >= 1
-469            if (not sheet_name.endswith(".py")) and (not sheet_name.endswith(".ipynb")):
-470                # no suffix, so must be unambiguous
-471                assert (py_exists + ipynb_exists) == 1
-472        self.sheet_name = sheet_name
-473        self.output_suffix = output_suffix
-474        self.exclude_input = exclude_input
-475        self.sheet_vars = sheet_vars
-476        self.init_code = init_code
-477        self.path_prefix = path_prefix
+            
    429    def __init__(
+430        self,
+431        sheet_name: str,
+432        *,
+433        output_suffix: Optional[str] = None,
+434        exclude_input: bool = True,
+435        sheet_vars = None,
+436        init_code: Optional[str] = None,
+437        path_prefix: Optional[str] = None,
+438        strict: bool = True,
+439    ) -> None:
+440        """
+441        Create a Jupyter task.
+442
+443        :param sheet_name: name of sheet to run can be .ipynb or .py, and suffix can be omitted.
+444        :param output_suffix: optional string to append to rendered HTML file name.
+445        :param exclude_input: if True strip input cells out of HTML render.
+446        :param sheet_vars: if not None value is de-serialized as a variable named "sheet_vars"
+447        :param init_code: optional code to insert at the top of the Jupyter sheet, used to pass parameters.
+448        :param path_prefix: optional prefix to add to sheet_name to find Jupyter source. 
+449        :param strict: if True check paths path_prefix and path_prefix/sheetname[.py|.ipynb] exist.
+450        """
+451        assert isinstance(sheet_name, str)
+452        assert isinstance(output_suffix, (str, type(None)))
+453        assert isinstance(exclude_input, bool)
+454        assert isinstance(init_code, (str, type(None)))
+455        assert isinstance(path_prefix, (str, type(None)))
+456        if strict:
+457            path = sheet_name
+458            if (isinstance(path_prefix, str)) and (len(path_prefix) > 9):
+459                assert os.path.exists(path_prefix)
+460                path = os.path.join(path_prefix, sheet_name)
+461            if path.endswith(".py"):
+462                path = path.removesuffix(".py")
+463            if path.endswith(".ipynb"):
+464                path = path.removesuffix(".ipynb")
+465            py_exists = os.path.exists(path + ".py")
+466            ipynb_exists = os.path.exists(path + ".ipynb")
+467            assert (py_exists + ipynb_exists) >= 1
+468            if (not sheet_name.endswith(".py")) and (not sheet_name.endswith(".ipynb")):
+469                # no suffix, so must be unambiguous
+470                assert (py_exists + ipynb_exists) == 1
+471        self.sheet_name = sheet_name
+472        self.output_suffix = output_suffix
+473        self.exclude_input = exclude_input
+474        self.sheet_vars = sheet_vars
+475        self.init_code = init_code
+476        self.path_prefix = path_prefix
 
    
 
 
@@ -1882,20 +1866,20 @@ 
    Parameters
    - 
    479    def render_as_html(self) -> None:
-480        """
-481        Render Jupyter notebook or Python (treated as notebook) to HTML.
-482        """
-483        path = self.sheet_name
-484        if isinstance(self.path_prefix, str) and (len(self.path_prefix) > 0):
-485            path = os.path.join(self.path_prefix, self.sheet_name)
-486        render_as_html(
-487            path,
-488            exclude_input=self.exclude_input,
-489            output_suffix=self.output_suffix,
-490            sheet_vars=self.sheet_vars,
-491            init_code=self.init_code,
-492        )
+            
    478    def render_as_html(self) -> None:
+479        """
+480        Render Jupyter notebook or Python (treated as notebook) to HTML.
+481        """
+482        path = self.sheet_name
+483        if isinstance(self.path_prefix, str) and (len(self.path_prefix) > 0):
+484            path = os.path.join(self.path_prefix, self.sheet_name)
+485        render_as_html(
+486            path,
+487            exclude_input=self.exclude_input,
+488            output_suffix=self.output_suffix,
+489            sheet_vars=self.sheet_vars,
+490            init_code=self.init_code,
+491        )
 
    
 
 
@@ -1915,19 +1899,19 @@ 
    Parameters
    - 
    494    def render_py_txt(self) -> None:
-495        """
-496        Render Python to text (without nbconver, nbformat Jupyter bindings)
-497        """
-498        path = self.sheet_name
-499        if isinstance(self.path_prefix, str) and (len(self.path_prefix) > 0):
-500            path = os.path.join(self.path_prefix, self.sheet_name)
-501        execute_py(
-502            path,
-503            output_suffix=self.output_suffix,
-504            sheet_vars=self.sheet_vars,
-505            init_code=self.init_code,
-506        )
+            
    493    def render_py_txt(self) -> None:
+494        """
+495        Render Python to text (without nbconver, nbformat Jupyter bindings)
+496        """
+497        path = self.sheet_name
+498        if isinstance(self.path_prefix, str) and (len(self.path_prefix) > 0):
+499            path = os.path.join(self.path_prefix, self.sheet_name)
+500        execute_py(
+501            path,
+502            output_suffix=self.output_suffix,
+503            sheet_vars=self.sheet_vars,
+504            init_code=self.init_code,
+505        )
 
    
 
 
@@ -1948,13 +1932,13 @@ 
    Parameters
    - 
    556def job_fn(arg: JTask):
-557    """
-558    Function to run a JTask job for Jupyter notebook. Exceptions pass through
-559    """
-560    assert isinstance(arg, JTask)
-561    # render notebook
-562    return arg.render_as_html()
+            
    555def job_fn(arg: JTask):
+556    """
+557    Function to run a JTask job for Jupyter notebook. Exceptions pass through
+558    """
+559    assert isinstance(arg, JTask)
+560    # render notebook
+561    return arg.render_as_html()
 
    
 
 
@@ -1974,17 +1958,17 @@ 
    Parameters
    - 
    565def job_fn_eat_exception(arg: JTask):
-566    """
-567    Function to run a JTask job for Jupyter notebook, catching any exception and returning it as a value
-568    """
-569    assert isinstance(arg, JTask)
-570    # render notebook
-571    try:
-572        return arg.render_as_html()
-573    except Exception as e:
-574        print(f"{arg} caught {e}")
-575        return (arg, e)
+            
    564def job_fn_eat_exception(arg: JTask):
+565    """
+566    Function to run a JTask job for Jupyter notebook, catching any exception and returning it as a value
+567    """
+568    assert isinstance(arg, JTask)
+569    # render notebook
+570    try:
+571        return arg.render_as_html()
+572    except Exception as e:
+573        print(f"{arg} caught {e}")
+574        return (arg, e)
 
    
 
 
@@ -2004,13 +1988,13 @@ 
    Parameters
    - 
    578def job_fn_py_txt(arg: JTask):
-579    """
-580    Function to run a JTask job for Python to txt. Exceptions pass through
-581    """
-582    assert isinstance(arg, JTask)
-583    # render Python
-584    return arg.render_py_txt()
+            
    577def job_fn_py_txt(arg: JTask):
+578    """
+579    Function to run a JTask job for Python to txt. Exceptions pass through
+580    """
+581    assert isinstance(arg, JTask)
+582    # render Python
+583    return arg.render_py_txt()
 
    
 
 
@@ -2030,17 +2014,17 @@ 
    Parameters
    - 
    587def job_fn_py_txt_eat_exception(arg: JTask):
-588    """
-589    Function to run a JTask job for Python to txt, catching any exception and returning it as a value
-590    """
-591    assert isinstance(arg, JTask)
-592    # render Python
-593    try:
-594        return arg.render_py_txt()
-595    except Exception as e:
-596        print(f"{arg} caught {e}")
-597        return (arg, e)
+            
    586def job_fn_py_txt_eat_exception(arg: JTask):
+587    """
+588    Function to run a JTask job for Python to txt, catching any exception and returning it as a value
+589    """
+590    assert isinstance(arg, JTask)
+591    # render Python
+592    try:
+593        return arg.render_py_txt()
+594    except Exception as e:
+595        print(f"{arg} caught {e}")
+596        return (arg, e)
 
    
 
 
@@ -2060,64 +2044,64 @@ 
    Parameters
    - 
    600def run_pool(
-601        tasks: Iterable, 
-602        *, 
-603        njobs: int = 4,
-604        verbose: bool = True,
-605        stop_on_error: bool = True,
-606        use_Jupyter: bool = True,
-607        ) -> List:
-608    """
-609    Run a pool of tasks.
-610
-611    :param tasks: iterable of tasks
-612    :param njobs: degree of parallelism
-613    :param verbose: if True, print on failure
-614    :param stop_on_error: if True, stop pool on error
-615    :param use_Jupyter: if True, use nbconvert, nbformat Jupyter fns.
-616    """
-617    tasks = list(tasks)
-618    assert isinstance(njobs, int)
-619    assert njobs > 0
-620    assert isinstance(verbose, bool)
-621    assert isinstance(stop_on_error, bool)
-622    assert isinstance(use_Jupyter, bool)
-623    if len(tasks) <= 0:
-624        return
-625    for task in tasks:
-626        assert isinstance(task, JTask)
-627    res = None
-628    if stop_on_error:
-629        # # complex way, allowing a stop on job failure
-630        # https://stackoverflow.com/a/25791961/6901725
-631        if use_Jupyter:
-632            fn = job_fn
-633        else:
-634            fn = job_fn_py_txt
-635        with Pool(njobs) as pool:
-636            try:
-637                res = list(pool.imap_unordered(fn, tasks))  # list is forcing iteration over tasks for side-effects
-638            except Exception:
-639                if verbose:
-640                    sys.stdout.flush()
-641                    print("!!! run_pool: a worker raised an Exception, aborting...")
-642                    sys.stdout.flush()
-643                pool.close()
-644                pool.terminate()
-645                raise  # re-raise Exception
-646            else:
-647                pool.close()
-648                pool.join()
-649    else:
-650        if use_Jupyter:
-651            fn = job_fn_eat_exception
-652        else:
-653            fn = job_fn_py_txt_eat_exception
-654        # simple way, but doesn't exit until all jobs succeed or fail
-655        with Pool(njobs) as pool:
-656            res = list(pool.map(fn, tasks))
-657    return res
+            
    599def run_pool(
+600        tasks: Iterable, 
+601        *, 
+602        njobs: int = 4,
+603        verbose: bool = True,
+604        stop_on_error: bool = True,
+605        use_Jupyter: bool = True,
+606        ) -> List:
+607    """
+608    Run a pool of tasks.
+609
+610    :param tasks: iterable of tasks
+611    :param njobs: degree of parallelism
+612    :param verbose: if True, print on failure
+613    :param stop_on_error: if True, stop pool on error
+614    :param use_Jupyter: if True, use nbconvert, nbformat Jupyter fns.
+615    """
+616    tasks = list(tasks)
+617    assert isinstance(njobs, int)
+618    assert njobs > 0
+619    assert isinstance(verbose, bool)
+620    assert isinstance(stop_on_error, bool)
+621    assert isinstance(use_Jupyter, bool)
+622    if len(tasks) <= 0:
+623        return
+624    for task in tasks:
+625        assert isinstance(task, JTask)
+626    res = None
+627    if stop_on_error:
+628        # # complex way, allowing a stop on job failure
+629        # https://stackoverflow.com/a/25791961/6901725
+630        if use_Jupyter:
+631            fn = job_fn
+632        else:
+633            fn = job_fn_py_txt
+634        with Pool(njobs) as pool:
+635            try:
+636                res = list(pool.imap_unordered(fn, tasks))  # list is forcing iteration over tasks for side-effects
+637            except Exception:
+638                if verbose:
+639                    sys.stdout.flush()
+640                    print("!!! run_pool: a worker raised an Exception, aborting...")
+641                    sys.stdout.flush()
+642                pool.close()
+643                pool.terminate()
+644                raise  # re-raise Exception
+645            else:
+646                pool.close()
+647                pool.join()
+648    else:
+649        if use_Jupyter:
+650            fn = job_fn_eat_exception
+651        else:
+652            fn = job_fn_py_txt_eat_exception
+653        # simple way, but doesn't exit until all jobs succeed or fail
+654        with Pool(njobs) as pool:
+655            res = list(pool.map(fn, tasks))
+656    return res
 
    
 
 
@@ -2135,47 +2119,6 @@ 
    Parameters
    - -
    - -
    - - def - write_dict_as_assignments(values: Dict[str, Any]) -> str: - - - -
    - - 
    660def write_dict_as_assignments(values: Dict[str, Any]) -> str:
-661    """
-662    Write a dictionary as a block of Python assignment statements.
-663
-664    :param values: dictionary
-665    :return: assignment statements
-666    """
-667    return "\n" + "\n".join([
-668        f"{k} = {repr(values[k])}" for k in sorted(values.keys())
-669    ]) + "\n"
-
    - - -

    Write a dictionary as a block of Python assignment statements.

    - -
    Parameters
    - -
      -
    • values: dictionary
      • -
    - -
    Returns
    - -
    -

    assignment statements

    -
    -
    - -
    @@ -2189,48 +2132,48 @@
    Returns
    - 
    672@contextmanager
-673def declare_task_variables(
-674    env,
-675    *,
-676    result_map:Optional[Dict[str, Any]] = None,
-677    ) -> None:
-678    """
-679    Copy env["sheet_vars"][k] into env[k] for all k in sheet_vars.keys() if "sheet_vars" defined in env.
-680    Only variables that are first assigned in the with block of this task manager are allowed to be assigned.
-681
-682    :param env: working environment, setting to globals() is usually the correct choice.
-683    :param result_map: empty dictionary to return results in. result_map["sheet_vars"] is the dictionary if incoming assignments, result_map["declared_vars"] is the dictionary of default names and values.
-684    :return None:
-685    """
-686    sheet_vars = dict()
-687    pre_known_vars = set()
-688    if env is not None:
-689        pre_known_vars = set(env.keys())
-690    if result_map is not None:
-691        result_map["sheet_vars"] = sheet_vars
-692        result_map["declared_vars"] = set()
-693    if "sheet_vars" in pre_known_vars:
-694        sheet_vars = env["sheet_vars"]
-695        if result_map is not None:
-696            result_map["sheet_vars"] = sheet_vars
-697        already_assigned_vars = set(sheet_vars.keys()).intersection(pre_known_vars)
-698        if len(already_assigned_vars) > 0:
-699            raise ValueError(f"declare_task_variables(): attempting to set pre-with variables: {sorted(already_assigned_vars)}")
-700    try:
-701        yield
-702    finally:
-703        post_known_vars = set(env.keys())
-704        declared_vars = post_known_vars - pre_known_vars
-705        if result_map is not None:
-706            result_map["declared_vars"] = {k: env[k] for k in declared_vars}
-707        if "sheet_vars" in pre_known_vars:
-708            unexpected_vars = set(sheet_vars.keys()) - declared_vars
-709            if len(unexpected_vars) > 0:
-710                raise ValueError(f"declare_task_variables(): attempting to assign undeclared variables: {sorted(unexpected_vars)}")
-711            # do the assignments
-712            for k, v in sheet_vars.items():
-713                env[k] = v
+            
    659@contextmanager
+660def declare_task_variables(
+661    env,
+662    *,
+663    result_map:Optional[Dict[str, Any]] = None,
+664    ) -> None:
+665    """
+666    Copy env["sheet_vars"][k] into env[k] for all k in sheet_vars.keys() if "sheet_vars" defined in env.
+667    Only variables that are first assigned in the with block of this task manager are allowed to be assigned.
+668
+669    :param env: working environment, setting to globals() is usually the correct choice.
+670    :param result_map: empty dictionary to return results in. result_map["sheet_vars"] is the dictionary if incoming assignments, result_map["declared_vars"] is the dictionary of default names and values.
+671    :return None:
+672    """
+673    sheet_vars = dict()
+674    pre_known_vars = set()
+675    if env is not None:
+676        pre_known_vars = set(env.keys())
+677    if result_map is not None:
+678        result_map["sheet_vars"] = sheet_vars
+679        result_map["declared_vars"] = set()
+680    if "sheet_vars" in pre_known_vars:
+681        sheet_vars = env["sheet_vars"]
+682        if result_map is not None:
+683            result_map["sheet_vars"] = sheet_vars
+684        already_assigned_vars = set(sheet_vars.keys()).intersection(pre_known_vars)
+685        if len(already_assigned_vars) > 0:
+686            raise ValueError(f"declare_task_variables(): attempting to set pre-with variables: {sorted(already_assigned_vars)}")
+687    try:
+688        yield
+689    finally:
+690        post_known_vars = set(env.keys())
+691        declared_vars = post_known_vars - pre_known_vars
+692        if result_map is not None:
+693            result_map["declared_vars"] = {k: env[k] for k in declared_vars}
+694        if "sheet_vars" in pre_known_vars:
+695            unexpected_vars = set(sheet_vars.keys()) - declared_vars
+696            if len(unexpected_vars) > 0:
+697                raise ValueError(f"declare_task_variables(): attempting to assign undeclared variables: {sorted(unexpected_vars)}")
+698            # do the assignments
+699            for k, v in sheet_vars.items():
+700                env[k] = v
 
    
 
 
diff --git a/pkg/wvpy.egg-info/SOURCES.txt b/pkg/wvpy.egg-info/SOURCES.txt
index 0ab8e79..9db9c9b 100644
--- a/pkg/wvpy.egg-info/SOURCES.txt
+++ b/pkg/wvpy.egg-info/SOURCES.txt
@@ -4,6 +4,7 @@ README.txt
 setup.py
 Doc/documentation.txt
 wvpy/__init__.py
+wvpy/assignment.py
 wvpy/jtools.py
 wvpy/ptools.py
 wvpy/pysheet.py
diff --git a/pkg/wvpy/assignment.py b/pkg/wvpy/assignment.py
new file mode 100644
index 0000000..7a27d98
--- /dev/null
+++ b/pkg/wvpy/assignment.py
@@ -0,0 +1,101 @@
+"""Classes and functions for working with variable assignments"""
+
+from typing import Any, Dict, Iterable, Optional
+from contextlib import contextmanager
+
+
+def dict_to_assignments_str(values: Dict[str, Any]) -> str:
+    """
+    Format a dictionary as a block of Python assignment statements.
+
+    Example:
+    ```python
+    from wvpy.assignment import dict_to_assignments_str
+    print(dict_to_assignments_str({"a": 1, "b": 2}))
+    ```
+
+    :param values: dictionary
+    :return: assignment statements
+    """
+    assert isinstance(values, dict)
+    return (
+        "\n"
+        + "\n".join([f"{k} = {repr(values[k])}" for k in sorted(values.keys())])
+        + "\n"
+    )
+
+
+@contextmanager
+def record_assignments(
+    env,
+    *,
+    result: Dict[str, Any],
+) -> None:
+    """
+    Context manager to record all assignments to new variables in a with-block.
+    New variables being variables not set prior to entering the with block.
+
+    Example:
+    ```python
+    from wvpy.assignment import dict_to_assignments_str, record_assignments
+    assignments = {}
+    with record_assignments(globals(), result=assignments):
+        a = 1
+        b = 2
+    print(assignments)
+    print(dict_to_assignments_str(assignments))
+    ```
+
+    :param env: working environment, setting to `globals()` is usually the correct choice.
+    :param result: dictionary to store results in. function calls `.clear()` on result.
+    :return None:
+    """
+    assert isinstance(result, dict)
+    pre_known_vars = set(env.keys())
+    result.clear()
+    try:
+        yield
+    finally:
+        post_known_vars = set(env.keys())
+        declared_vars = post_known_vars - pre_known_vars
+        for k in sorted(declared_vars):
+            result[k] = env[k]
+
+
+def ensure_names_not_already_assigned(env, *, keys: Iterable[str]) -> None:
+    """
+    Check that no key in keys is already set in the environment env.
+    Raises ValueError if keys are already assigned.
+
+    :param env: working environment, setting to `globals()` is usually the correct choice.
+    :param keys: keys to confirm not already set.
+    :return None:
+    """
+    already_assigned_vars = set(keys).intersection(set(env.keys()))
+    if len(already_assigned_vars) > 0:
+        raise ValueError(f"variables already set: {sorted(already_assigned_vars)}")
+
+
+def assign_values_from_map(
+    env, *, values: Dict[str, Any], expected_keys: Optional[Iterable[str]]
+) -> None:
+    """
+    Assign values from map into environment.
+
+    :param env: working environment, setting to `globals()` is usually the correct choice.
+    :param values: dictionary to copy into environment.
+    :param expected_keys: if not null a set of keys we are restricting to
+    :return None:
+    """
+    assert isinstance(values, dict)
+    for k in values.keys():
+        assert isinstance(k, str)
+    if expected_keys is not None:
+        unexpected_vars = set(values.keys()) - set(expected_keys)
+        if len(unexpected_vars) > 0:
+            raise ValueError(
+                f"attempting to assign undeclared variables: {sorted(unexpected_vars)}"
+            )
+    # do the assignments
+    for k, v in values.items():
+        env[k] = v
diff --git a/pkg/wvpy/jtools.py b/pkg/wvpy/jtools.py
index 7be7bcc..d355eaa 100644
--- a/pkg/wvpy/jtools.py
+++ b/pkg/wvpy/jtools.py
@@ -16,7 +16,6 @@
 from typing import Any, Dict, Iterable, List, Optional
 from functools import total_ordering
 from contextlib import contextmanager
-from collections import namedtuple
 
 from wvpy.util import escape_ansi
 from wvpy.ptools import execute_py
@@ -656,18 +655,6 @@ def run_pool(
     return res
 
 
-def write_dict_as_assignments(values: Dict[str, Any]) -> str:
-    """
-    Write a dictionary as a block of Python assignment statements.
-
-    :param values: dictionary
-    :return: assignment statements
-    """
-    return "\n" + "\n".join([
-        f"{k} = {repr(values[k])}" for k in sorted(values.keys())
-    ]) + "\n"
-
-
 @contextmanager
 def declare_task_variables(
     env,