From 1fa087d8a4c85acc68ee7fe9a12bea54502d2395 Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Sun, 18 Oct 2020 16:35:38 +0200 Subject: [PATCH 1/8] Create fixture for plotly visualization --- jupyter_docx_bundler/tests/conftest.py | 51 ++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/jupyter_docx_bundler/tests/conftest.py b/jupyter_docx_bundler/tests/conftest.py index c7632fe..ab79bae 100644 --- a/jupyter_docx_bundler/tests/conftest.py +++ b/jupyter_docx_bundler/tests/conftest.py @@ -485,6 +485,55 @@ def pandas_html_table_notebook(tmpdir, request): return nb +@pytest.fixture(params=[10]) +def plotly_notebook(tmpdir, request): + nb = nbformat.v4.new_notebook() + image_count = 0 + + # imports + nb.cells.append( + nbformat.v4.new_markdown_cell( + '# Imports', + ) + ) + nb.cells.append( + nbformat.v4.new_code_cell( + '\n'.join([ + 'import numpy as np', + 'import plotly.express as px', + ]) + ) + ) + + # single plotly image per cell + nb.cells.append( + nbformat.v4.new_markdown_cell( + '# single plotly image per cell', + ) + ) + for _ in range(request.param): + nb.cells.append( + nbformat.v4.new_code_cell( + '\n'.join([ + 'fig = px.line(x=np.arange(100), y=np.random.randn(100))', + 'fig.show()', + ]) + ) + ) + image_count += request.param + + # update metadata + nb['metadata'].update({ + 'path': f'{tmpdir}', + 'image_count': image_count, + }) + + ep = ExecutePreprocessor() + ep.preprocess(nb, {'metadata': {'path': tmpdir}}) + + return nb + + @pytest.fixture( params=[ lazy_fixture('math_with_space_notebook'), @@ -523,10 +572,12 @@ def test_notebook(request): params=[ lazy_fixture('matplotlib_notebook'), lazy_fixture('markdown_images_notebook'), + lazy_fixture('plotly_notebook'), ], ids=[ 'matplotlib', 'images', + 'plotly', ] ) def images_notebook(request): From b4485c8c34e700babb5313c1b8eadf0881e6c30e Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Sun, 18 Oct 2020 16:35:56 +0200 Subject: [PATCH 2/8] Create fixture for bokeh visualization --- jupyter_docx_bundler/tests/conftest.py | 54 ++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/jupyter_docx_bundler/tests/conftest.py b/jupyter_docx_bundler/tests/conftest.py index ab79bae..7cb5368 100644 --- a/jupyter_docx_bundler/tests/conftest.py +++ b/jupyter_docx_bundler/tests/conftest.py @@ -534,6 +534,58 @@ def plotly_notebook(tmpdir, request): return nb +@pytest.fixture(params=[10]) +def bokeh_notebook(tmpdir, request): + nb = nbformat.v4.new_notebook() + image_count = 0 + + # imports + nb.cells.append( + nbformat.v4.new_markdown_cell( + '# Imports', + ) + ) + nb.cells.append( + nbformat.v4.new_code_cell( + '\n'.join([ + 'from bokeh.io import output_notebook, show', + 'from bokeh.plotting import figure', + 'import numpy as np', + 'output_notebook()', + ]) + ) + ) + + # single bokeh image per cell + nb.cells.append( + nbformat.v4.new_markdown_cell( + '# single bokeh image per cell', + ) + ) + for _ in range(request.param): + nb.cells.append( + nbformat.v4.new_code_cell( + '\n'.join([ + 'p = figure()', + 'p.line(np.arange(100), np.random.randn(100))', + 'show(p)', + ]) + ) + ) + image_count += request.param + + # update metadata + nb['metadata'].update({ + 'path': f'{tmpdir}', + 'image_count': image_count, + }) + + ep = ExecutePreprocessor() + ep.preprocess(nb, {'metadata': {'path': tmpdir}}) + + return nb + + @pytest.fixture( params=[ lazy_fixture('math_with_space_notebook'), @@ -573,11 +625,13 @@ def test_notebook(request): lazy_fixture('matplotlib_notebook'), lazy_fixture('markdown_images_notebook'), lazy_fixture('plotly_notebook'), + lazy_fixture('bokeh_notebook'), ], ids=[ 'matplotlib', 'images', 'plotly', + 'bokeh', ] ) def images_notebook(request): From cf08a565fa6f6ae01a3a46224faea94b7a9717fe Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Fri, 14 May 2021 16:21:00 +0200 Subject: [PATCH 3/8] Remove bokeh fixture --- jupyter_docx_bundler/tests/conftest.py | 54 -------------------------- 1 file changed, 54 deletions(-) diff --git a/jupyter_docx_bundler/tests/conftest.py b/jupyter_docx_bundler/tests/conftest.py index 7cb5368..ab79bae 100644 --- a/jupyter_docx_bundler/tests/conftest.py +++ b/jupyter_docx_bundler/tests/conftest.py @@ -534,58 +534,6 @@ def plotly_notebook(tmpdir, request): return nb -@pytest.fixture(params=[10]) -def bokeh_notebook(tmpdir, request): - nb = nbformat.v4.new_notebook() - image_count = 0 - - # imports - nb.cells.append( - nbformat.v4.new_markdown_cell( - '# Imports', - ) - ) - nb.cells.append( - nbformat.v4.new_code_cell( - '\n'.join([ - 'from bokeh.io import output_notebook, show', - 'from bokeh.plotting import figure', - 'import numpy as np', - 'output_notebook()', - ]) - ) - ) - - # single bokeh image per cell - nb.cells.append( - nbformat.v4.new_markdown_cell( - '# single bokeh image per cell', - ) - ) - for _ in range(request.param): - nb.cells.append( - nbformat.v4.new_code_cell( - '\n'.join([ - 'p = figure()', - 'p.line(np.arange(100), np.random.randn(100))', - 'show(p)', - ]) - ) - ) - image_count += request.param - - # update metadata - nb['metadata'].update({ - 'path': f'{tmpdir}', - 'image_count': image_count, - }) - - ep = ExecutePreprocessor() - ep.preprocess(nb, {'metadata': {'path': tmpdir}}) - - return nb - - @pytest.fixture( params=[ lazy_fixture('math_with_space_notebook'), @@ -625,13 +573,11 @@ def test_notebook(request): lazy_fixture('matplotlib_notebook'), lazy_fixture('markdown_images_notebook'), lazy_fixture('plotly_notebook'), - lazy_fixture('bokeh_notebook'), ], ids=[ 'matplotlib', 'images', 'plotly', - 'bokeh', ] ) def images_notebook(request): From e78716f78a82136e6f78e2a633c9ee10601a98ef Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Fri, 14 May 2021 16:29:59 +0200 Subject: [PATCH 4/8] Add png-conversion of plotly-figures --- conda.recipe/meta.yaml | 3 +++ jupyter_docx_bundler/converters.py | 30 +++++++++++++++++++++++++++--- requirements.txt | 1 + requirements_test.txt | 2 ++ setup.py | 30 +++++++++++++++++++----------- 5 files changed, 52 insertions(+), 14 deletions(-) diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 621ec31..7ef2c5f 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -21,6 +21,7 @@ requirements: - setuptools >=38.6.0 - setuptools_scm run: + - importlib_resources # [py<39] - lxml - nbconvert >=5.5 - notebook >=5.0 @@ -38,10 +39,12 @@ test: - jupyter_docx_bundler requires: - ipython >=7.0 + - kaleido - matplotlib-base >=3.1 - mock - nbformat - pillow >=6.0.0 + - plotly - pytest - pytest-cov - pytest-lazy-fixture diff --git a/jupyter_docx_bundler/converters.py b/jupyter_docx_bundler/converters.py index db986a5..4e7b4de 100644 --- a/jupyter_docx_bundler/converters.py +++ b/jupyter_docx_bundler/converters.py @@ -1,15 +1,19 @@ import base64 +try: + from importlib import resources +except ImportError: + import importlib_resources as resources +import json import os -from pathlib import Path import re import tempfile +from pathlib import Path -from nbconvert import preprocessors import nbformat import pandas as pd import pypandoc import requests - +from nbconvert import preprocessors RE_IMAGE = re.compile(r'!\[.+]\((?!attachment:).+\)') RE_EXTRA_TITLE = re.compile(r'\s".+"') @@ -185,6 +189,26 @@ def preprocess(content, path, handler=None): handler.log.warning(f'Conversion of pandas HTML-table failed : {e}') else: raise e + elif 'application/vnd.plotly.v1+json' in output['data']: + try: + from plotly import io + from kaleido.scopes.plotly import PlotlyScope + + scope = PlotlyScope( + plotlyjs=resources.files('plotly') / 'package_data' / 'plotly.min.js', + ) + + fig = io.from_json( + json.dumps(output['data']['application/vnd.plotly.v1+json']) + ) + imagedata = scope.transform(fig, format='png', scale=2.0) + output['data']['image/png'] = base64.b64encode(imagedata) + except ModuleNotFoundError as e: + if handler is not None: + handler.log.warning('Found plotly-figure in notebook, we need plotly ' + 'and kaleido to convert figure.') + else: + raise e # convert linked images to attachments linked_to_attachment_image(cell, path) diff --git a/requirements.txt b/requirements.txt index 604cebf..7ca3e09 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +importlib_resources lxml nbconvert >= 5.5 notebook >= 5.0 diff --git a/requirements_test.txt b/requirements_test.txt index a55fb6a..3aa88fd 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,9 +1,11 @@ ipython >= 7.0 +kaleido matplotlib-base >= 3.1 mock nbformat numpy pillow >= 6.0.0 +plotly pytest pytest-cov pytest-lazy-fixture diff --git a/setup.py b/setup.py index b8abfaf..87c317b 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,25 @@ +import sys + from setuptools import setup with open('README.md') as f: long_description = f.read() +install_requires = [ + 'lxml', + 'nbconvert>=5.5', + 'notebook>=5.0', + 'pandas', + 'pandocfilters', + 'pypandoc>=1.4', + 'requests', + 'tabulate', + 'tornado', +] +if sys.version_info.major <= 3 and sys.version_info.minor < 9: + install_requires += ['importlib_metadata'] + setup( author='Marco Rossi', author_email='developer@marco-rossi.com', @@ -20,27 +36,19 @@ extras_require={ 'test': [ 'ipython>=7.0' + 'kaleido', 'matplotlib>=3.1', 'mock', 'nbformat', 'numpy', 'pillow>=6.0.0', + 'plotly', 'pytest', 'pytest-cov', 'pytest-lazy-fixture', ], }, - install_requires=[ - 'lxml', - 'nbconvert>=5.5', - 'notebook>=5.0', - 'pandas', - 'pandocfilters', - 'pypandoc>=1.4', - 'requests', - 'tabulate', - 'tornado', - ], + install_requires=install_requires, keywords=[ 'jupyter', 'docx', From e69a9969d539222a8ea3f13e43e46d3aaa7fae38 Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Fri, 14 May 2021 16:36:48 +0200 Subject: [PATCH 5/8] Add comment --- jupyter_docx_bundler/converters.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jupyter_docx_bundler/converters.py b/jupyter_docx_bundler/converters.py index 8625afe..6bb2e30 100644 --- a/jupyter_docx_bundler/converters.py +++ b/jupyter_docx_bundler/converters.py @@ -191,6 +191,7 @@ def preprocess(content, path, handler=None): handler.log.warning(f'Conversion of pandas HTML-table failed : {e}') else: raise e + # plotly figure elif 'application/vnd.plotly.v1+json' in output['data']: try: from plotly import io From b6c17d9e9f762e61d607b42adeb2072fd7d29a36 Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Fri, 14 May 2021 17:24:54 +0200 Subject: [PATCH 6/8] Fix import names on conda --- conda.recipe/meta.yaml | 2 +- requirements_test.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index ad532f7..1907c89 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -39,7 +39,6 @@ test: - jupyter_docx_bundler requires: - ipython >=7.0 - - kaleido - matplotlib-base >=3.1 - mock - nbformat @@ -48,6 +47,7 @@ test: - pytest - pytest-cov - pytest-lazy-fixture + - python-kaleido - sympy commands: - pytest --pyargs jupyter_docx_bundler diff --git a/requirements_test.txt b/requirements_test.txt index 3b701d7..69ec7d7 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,5 +1,4 @@ ipython >= 7.0 -kaleido matplotlib-base >= 3.1 mock nbformat @@ -9,4 +8,5 @@ plotly pytest pytest-cov pytest-lazy-fixture +python-kaleido sympy From 8cead5c1aeb4f73cb97b3bfbd57af2fc0ebb8261 Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Fri, 14 May 2021 17:41:38 +0200 Subject: [PATCH 7/8] Fix missing check in new if clause --- jupyter_docx_bundler/converters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter_docx_bundler/converters.py b/jupyter_docx_bundler/converters.py index 6bb2e30..ef1b788 100644 --- a/jupyter_docx_bundler/converters.py +++ b/jupyter_docx_bundler/converters.py @@ -192,7 +192,7 @@ def preprocess(content, path, handler=None): else: raise e # plotly figure - elif 'application/vnd.plotly.v1+json' in output['data']: + elif 'data' in output and 'application/vnd.plotly.v1+json' in output['data']: try: from plotly import io from kaleido.scopes.plotly import PlotlyScope From 6577ae7625435154034a3c6275c5d3434fd6aa42 Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Fri, 14 May 2021 18:00:41 +0200 Subject: [PATCH 8/8] Fix import try-except --- jupyter_docx_bundler/converters.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jupyter_docx_bundler/converters.py b/jupyter_docx_bundler/converters.py index ef1b788..c507295 100644 --- a/jupyter_docx_bundler/converters.py +++ b/jupyter_docx_bundler/converters.py @@ -1,8 +1,8 @@ import base64 try: - from importlib import resources + from importlib.resources import files as resources_files except ImportError: - import importlib_resources as resources + from importlib_resources import files as resources_files import json import os import re @@ -198,7 +198,7 @@ def preprocess(content, path, handler=None): from kaleido.scopes.plotly import PlotlyScope scope = PlotlyScope( - plotlyjs=resources.files('plotly') / 'package_data' / 'plotly.min.js', + plotlyjs=resources_files('plotly') / 'package_data' / 'plotly.min.js', ) fig = io.from_json(