From ccee6202a48f8dcc1184a3a480eefd6bf950007a Mon Sep 17 00:00:00 2001 From: Kyle Cutler Date: Tue, 2 Mar 2021 16:55:23 -0800 Subject: [PATCH 1/4] Add pass-by-value API server --- nbdime/__main__.py | 2 + nbdime/args.py | 39 ++++++---- nbdime/webapp/nbapiserver.py | 139 ++++++++++++++++++++++++++++++++++ nbdime/webapp/nbdimeserver.py | 20 ++--- 4 files changed, 177 insertions(+), 23 deletions(-) create mode 100644 nbdime/webapp/nbapiserver.py diff --git a/nbdime/__main__.py b/nbdime/__main__.py index 36e73557..bceea82d 100644 --- a/nbdime/__main__.py +++ b/nbdime/__main__.py @@ -83,6 +83,8 @@ def main_dispatch(args=None): main = main_mergetool elif cmd == 'server': from nbdime.webapp.nbdimeserver import main + elif cmd == 'api': + from nbdime.webapp.nbapiserver import main elif cmd == 'config-git': # Call all git configs from nbdime.vcs.git.diffdriver import main as diff_driver diff --git a/nbdime/args.py b/nbdime/args.py index ceaa99b3..96f45d04 100644 --- a/nbdime/args.py +++ b/nbdime/args.py @@ -238,9 +238,8 @@ def add_git_config_subcommand(subparsers, enable, disable, subparser_help, enabl ) return config - -def add_web_args(parser, default_port=8888): - """Adds a set of arguments common to all commands that show a web gui. +def add_server_args(parser, default_port=8888): + """Adds a set of arguments common to all commands that run a server. """ port_help = ( "specify the port you want the server to run on. Default is %d%s." % ( @@ -251,6 +250,29 @@ def add_web_args(parser, default_port=8888): default=default_port, type=int, help=port_help) + parser.add_argument( + '--ip', + default='127.0.0.1', + help="specify the interface to listen to for the web server. " + "NOTE: Setting this to anything other than 127.0.0.1/localhost " + "might comprimise the security of your computer. Use with care!") + parser.add_argument( + '--base-url', + default='/', + help="The base URL prefix under which to run the web app") + parser.add_argument( + '--num-processes', + default=1, + type=int, + help="The number of server processes to spawn (Unix only). " + "A value of <= 0 will use the number of cores available on this machine." + ) + +def add_web_args(parser, default_port=8888): + """Adds a set of arguments common to all commands that show a web gui. + """ + add_server_args(parser, default_port) + parser.add_argument( '-b', '--browser', default=None, @@ -263,12 +285,6 @@ def add_web_args(parser, default_port=8888): help="prevent server shutting down on remote close request (when these" " would normally be supported)." ) - parser.add_argument( - '--ip', - default='127.0.0.1', - help="specify the interface to listen to for the web server. " - "NOTE: Setting this to anything other than 127.0.0.1/localhost " - "might comprimise the security of your computer. Use with care!") cwd = os.path.abspath(os.path.curdir) parser.add_argument( @@ -277,10 +293,6 @@ def add_web_args(parser, default_port=8888): help="specify the working directory you want " "the server to run from. Default is the " "actual cwd at program start.") - parser.add_argument( - '--base-url', - default='/', - help="The base URL prefix under which to run the web app") parser.add_argument( '--show-unchanged', dest='hide_unchanged', @@ -528,6 +540,7 @@ def args_for_server(arguments): workdirectory='cwd', base_url='base_url', hide_unchanged='hide_unchanged', + num_processes='num_processes' ) ret = {kmap[k]: v for k, v in vars(arguments).items() if k in kmap} if 'persist' in arguments: diff --git a/nbdime/webapp/nbapiserver.py b/nbdime/webapp/nbapiserver.py new file mode 100644 index 00000000..2ccb7552 --- /dev/null +++ b/nbdime/webapp/nbapiserver.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python +# -*- coding:utf-8 -*- + +import json +import sys + +from notebook.base.handlers import IPythonHandler, APIHandler +from notebook.log import log_request +from tornado import web, escape + +from .nbdimeserver import main_server + +from ..args import ConfigBackedParser, add_server_args, args_for_server +from ..diffing.notebooks import diff_notebooks +from ..log import logger +from ..merging.notebooks import decide_notebook_merge +from ..nbmergeapp import _build_arg_parser as build_merge_parser + + +class NbApiHandler(IPythonHandler): + def initialize(self, **params): + self.params = params + self.body = None + + def write_error(self, status_code, **kwargs): + exc_info = kwargs.get('exc_info', None) + if exc_info: + (etype, value, traceback) = exc_info + if etype == web.HTTPError: + self.set_header('Content-Type', 'text/plain') + return self.finish(str(value)) + return super(NbApiHandler, self).write_error(status_code, **kwargs) + + def get_notebook_argument(self, argname): + if not self.body: + self.body = json.loads(escape.to_unicode(self.request.body)) + + # Assuming a request of the form "{'argname':arg}" + arg = self.body[argname] + + if not isinstance(arg, dict): + raise web.HTTPError(400, 'Expecting a notebook JSON object.') + + try: + # Convert dictionary to a notebook object with nbformat + from nbformat import versions, NBFormatError, reader + (major, minor) = reader.get_version(arg) + if major in versions: + nb = versions[major].to_notebook_json(arg, minor=minor) + else: + raise NBFormatError('Unsupported nbformat version %s' % major) + except Exception as e: + self.log.exception(e) + raise web.HTTPError(422, 'Invalid notebook: %s' % argname) + + return nb + +class ApiDiffHandler(NbApiHandler, APIHandler): + def post(self): + base_nb = self.get_notebook_argument('base') + remote_nb = self.get_notebook_argument('remote') + + try: + thediff = diff_notebooks(base_nb, remote_nb) + except Exception: + logger.exception('Error diffing documents:') + raise web.HTTPError(500, 'Error while attempting to diff documents') + + self.finish({ + 'base': base_nb, + 'diff': thediff, + }) + +class ApiMergeHandler(NbApiHandler, APIHandler): + def post(self): + base_nb = self.get_notebook_argument('base') + local_nb = self.get_notebook_argument('local') + remote_nb = self.get_notebook_argument('remote') + merge_args = self.settings.get('merge_args') + if merge_args is None: + merge_args = build_merge_parser().parse_args(['', '', '']) + merge_args.merge_strategy = 'mergetool' + self.settings['merge_args'] = merge_args + + try: + decisions = decide_notebook_merge(base_nb, local_nb, remote_nb, + args=merge_args) + except Exception: + logger.exception('Error merging documents:') + raise web.HTTPError(500, 'Error while attempting to merge documents') + + self.finish({ + 'base': base_nb, + 'merge_decisions': decisions + }) + +def make_app(**params): + base_url = params.pop('base_url', '/') + handlers = [ + (r'/api/diff', ApiDiffHandler, params), + (r'/api/merge', ApiMergeHandler, params) + ] + if base_url != '/': + prefix = base_url.rstrip('/') + handlers = [ + (prefix + path, cls, params) + for (path, cls, params) in handlers + ] + + settings = { + 'log_function': log_request, + 'base_url': base_url, + 'local_hostnames': ['localhost', '127.0.0.1'], + } + + app = web.Application(handlers, **settings) + app.exit_code = 0 + return app + +def _build_arg_parser(): + """ + Creates an argument parser that lets the user specify a port + and displays a help message. + """ + description = 'Hostable, secure api interface for Nbdime.' + parser = ConfigBackedParser(description=description) + add_server_args(parser) + return parser + + +def main(args=None): + if args is None: + args = sys.argv[1:] + arguments = _build_arg_parser().parse_args(args) + return main_server(make_app, **args_for_server(arguments)) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/nbdime/webapp/nbdimeserver.py b/nbdime/webapp/nbdimeserver.py index e4463a74..ff0b8d36 100644 --- a/nbdime/webapp/nbdimeserver.py +++ b/nbdime/webapp/nbdimeserver.py @@ -21,7 +21,7 @@ from tornado import ioloop, web, escape, netutil, httpserver from .. import __file__ as nbdime_root -from ..args import ConfigBackedParser, add_generic_args, add_web_args +from ..args import ConfigBackedParser, add_generic_args, add_web_args, args_for_server from ..diffing.notebooks import diff_notebooks from ..log import logger from ..merging.notebooks import decide_notebook_merge @@ -402,7 +402,8 @@ def make_app(**params): app.exit_code = 0 return app -def init_app(on_port=None, closable=False, **params): + +def init_app(make_app, on_port=None, closable=False, **params): asyncio_patch() _logger.debug('Using params: %s', params) params.update({'closable': closable}) @@ -427,10 +428,13 @@ def init_app(on_port=None, closable=False, **params): return app, server -def main_server(on_port=None, closable=False, **params): - app, server = init_app(on_port, closable, **params) +def main_server(make_app=make_app, on_port=None, closable=False, **params): + app, server = init_app(make_app, on_port, closable, **params) + is_win = sys.platform.startswith('win') + if not is_win: + server.start(params.pop("num_processes", 1)) io_loop = ioloop.IOLoop.current() - if sys.platform.startswith('win'): + if is_win: # workaround for tornado on Windows: # add no-op to wake every 5s # to handle signals that may be ignored by the inner loop @@ -458,11 +462,7 @@ def main(args=None): if args is None: args = sys.argv[1:] arguments = _build_arg_parser().parse_args(args) - return main_server(port=arguments.port, - ip=arguments.ip, - cwd=arguments.workdirectory, - base_url=arguments.base_url, - ) + return main_server(make_app, **args_for_server(arguments)) if __name__ == '__main__': From a0e7f1f5bd26203a0c10ecb1b8cbc3248b3c0ffe Mon Sep 17 00:00:00 2001 From: Kyle Cutler Date: Wed, 3 Mar 2021 08:49:22 -0800 Subject: [PATCH 2/4] Fix tests --- nbdime/webapp/nbdimeserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nbdime/webapp/nbdimeserver.py b/nbdime/webapp/nbdimeserver.py index ff0b8d36..c74c608b 100644 --- a/nbdime/webapp/nbdimeserver.py +++ b/nbdime/webapp/nbdimeserver.py @@ -403,7 +403,7 @@ def make_app(**params): return app -def init_app(make_app, on_port=None, closable=False, **params): +def init_app(make_app=make_app, on_port=None, closable=False, **params): asyncio_patch() _logger.debug('Using params: %s', params) params.update({'closable': closable}) From db495382169f0b1f4eed6df3b54a2b8f96e8e15f Mon Sep 17 00:00:00 2001 From: Kyle Cutler Date: Thu, 4 Mar 2021 10:26:36 -0800 Subject: [PATCH 3/4] Update imports --- nbdime/webapp/nbapiserver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nbdime/webapp/nbapiserver.py b/nbdime/webapp/nbapiserver.py index 2ccb7552..8de0e64d 100644 --- a/nbdime/webapp/nbapiserver.py +++ b/nbdime/webapp/nbapiserver.py @@ -4,8 +4,8 @@ import json import sys -from notebook.base.handlers import IPythonHandler, APIHandler -from notebook.log import log_request +from jupyter_server.base.handlers import JupyterHandler, APIHandler +from jupyter_server.log import log_request from tornado import web, escape from .nbdimeserver import main_server @@ -17,7 +17,7 @@ from ..nbmergeapp import _build_arg_parser as build_merge_parser -class NbApiHandler(IPythonHandler): +class NbApiHandler(JupyterHandler): def initialize(self, **params): self.params = params self.body = None From f415b7eaeecb25233e69f6b61819c1f4aab530ac Mon Sep 17 00:00:00 2001 From: Kyle Cutler Date: Thu, 4 Mar 2021 12:01:55 -0800 Subject: [PATCH 4/4] Always convert notebooks to v4 --- nbdime/webapp/nbapiserver.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nbdime/webapp/nbapiserver.py b/nbdime/webapp/nbapiserver.py index 8de0e64d..cdc19a67 100644 --- a/nbdime/webapp/nbapiserver.py +++ b/nbdime/webapp/nbapiserver.py @@ -42,13 +42,14 @@ def get_notebook_argument(self, argname): raise web.HTTPError(400, 'Expecting a notebook JSON object.') try: - # Convert dictionary to a notebook object with nbformat - from nbformat import versions, NBFormatError, reader + # Convert dictionary to a v4 notebook object with nbformat + from nbformat import versions, NBFormatError, reader, convert, validate (major, minor) = reader.get_version(arg) if major in versions: nb = versions[major].to_notebook_json(arg, minor=minor) else: raise NBFormatError('Unsupported nbformat version %s' % major) + nb = convert(nb, 4) except Exception as e: self.log.exception(e) raise web.HTTPError(422, 'Invalid notebook: %s' % argname)