From 2374b6418b1847355144701b022f5690cc5e6369 Mon Sep 17 00:00:00 2001 From: longze chen Date: Fri, 8 Dec 2017 14:06:52 -0500 Subject: [PATCH] Update `.gitignore`, `README.md`, comments, docstrs and style. [skip ci] --- .gitignore | 23 +++++- mfr/extensions/md/README.md | 72 ++++++++----------- mfr/extensions/md/render.py | 11 ++- mfr/extensions/md/settings.py | 6 +- mfr/extensions/md/static/css/default.css | 8 --- .../md/static/css/highlightjs-default.css | 44 ++++++------ tests/extensions/md/test_renderer.py | 31 +++++--- 7 files changed, 105 insertions(+), 90 deletions(-) diff --git a/.gitignore b/.gitignore index 1e4ac1d96..ad239ca84 100644 --- a/.gitignore +++ b/.gitignore @@ -11,19 +11,29 @@ Thumbs.db *.swp *~ +# Node Modules +###################### +/node_modules/ + +# Asynchronous HTTP +###################### +/src/ + # R -####################### +###################### .Rhistory # Python -####################### +###################### *.py[cod] **/*.pyc # C extensions +###################### *.so # Packages +###################### *.egg *.egg-info dist @@ -45,30 +55,37 @@ __pycache__ .env # Installer logs +###################### pip-log.txt # Unit test / coverage reports +###################### .coverage .tox nosetests.xml # Translations +###################### *.mo # Mr Developer +###################### .mr.developer.cfg .project .pydevproject # Sphinx +###################### docs/_build README.html # Player files +###################### player/files/* player/mfr_config_local.py player/static/mfr/* +# Misc +###################### !.gitkeep - .cache diff --git a/mfr/extensions/md/README.md b/mfr/extensions/md/README.md index 5df57e5b5..38a8ce037 100644 --- a/mfr/extensions/md/README.md +++ b/mfr/extensions/md/README.md @@ -1,10 +1,6 @@ +## Using Markdown-it with plugins -## Markdown plus Mathjax readme - -Most of the packages are in there to try to match what the OSF wiki is doing. The two exceptions being markdown-it-highlightjs.js and markdown-it-mathjax.js. The highlighter matches the functionality of what is on the osf however, and the markdown-it-mathjax.js increases functionality with mathjax. - -to get all the libraries needed: -Note: You do not need to use npm, you can easily get them off of github. +If we had `npm`, here were the would-be configuration. ```bash npm markdown-it@8.4.0 @@ -15,38 +11,32 @@ npm install markdown-it-sanitizer@0.4.3 npm install markdown-it-mathjax@2.0.0 ``` -github: - -https://github.com/cos-forks/markdown-it-toc -https://github.com/valeriangalliat/markdown-it-highlightjs -https://github.com/brianjgeiger/markdown-it-ins-del -https://github.com/svbergerem/markdown-it-sanitizer -https://github.com/classeur/markdown-it-mathjax - -To add a new library, you need to make sure its loadable in md.js somehow, either through exporting via `root.` or some other means. Some of the markdown plugins added have custom code in them to load them into `root`. - -Libraries should try to use the same version as the ones used on the OSF. The plugins do not matter as much, but `markdown-it` and `Mathjax` you should try to match exactly because styling can change between versions. - -To add a new library that is not already set up to export to `root` can be a bit tricky but the gist of it is, wrap the plugin in this code: - -```javascript -;(function (root, factory) { - if (typeof exports === 'object') { - module.exports = factory() - } else { - root. = factory() - } -})(this, function () { - - - return function(md){ - - ..... - } - -}) -``` - -And then modify it to work in this context. See other plugins for examples. - -Then in md.js, you can add a plugin to the markdown renderer by adding a `.use(window.)` after loading the file into `viewer.mako`. \ No newline at end of file +For MFR, a customized local copy of each script is stored in the extension's static folder. There are a few issues: + +* ES5 compatibility: use [Babel](https://babeljs.io/repl/) to convert ES6 `markdown-it-highlightjs` to ES5. + +* `require` is NOT available. For `md.js` to be able to load these libraries, customization is necessary to export via `root.`. + * `markdown-it` and `markdown-it-sanitizer` are already set up to be exported code. We load the `min` version directly. + * `markdown-it-toc`, `markdown-it-highlightjs`, `markdown-it-ins-del` and `markdown-it-mathjax` are not. Here is the wrapper: + ```javascript + (function (root, factory) { + if (typeof exports === "object") { + module.exports = factory(); + } else { + root. = factory(); + } + }) (this, function () { + return function(md/*, optional arguments*/) { + /* library code */ + } + }); + ``` + +Here is a list of original copy of the scripts we use: + +* [markdown-it@08.4.0](https://github.com/markdown-it/markdown-it/blob/8.4.0/bin/markdown-it.js) +* [markdown-it-sanitizer@0.4.3](https://github.com/svbergerem/markdown-it-sanitizer/blob/v0.4.3/dist/markdown-it-sanitizer.min.js) +* [markdown-it-mathjax@2.0.0](https://github.com/classeur/markdown-it-mathjax/blob/v2.0.0/markdown-it-mathjax.js) +* [markdown-it-toc@1.1.1](https://github.com/cos-forks/markdown-it-toc/blob/1.1.1/index.js) +* [markdown-it-ins-del@1.0.0](https://github.com/brianjgeiger/markdown-it-ins-del/blob/1.0.0/index.js) +* [markdown-it-higlightjs@3.0.0](https://github.com/cslzchen/markdown-it-highlightjs/blob/release/3.0.0/index.es5.js) diff --git a/mfr/extensions/md/render.py b/mfr/extensions/md/render.py index 6fffaf8ca..f1fe03fb8 100644 --- a/mfr/extensions/md/render.py +++ b/mfr/extensions/md/render.py @@ -3,10 +3,11 @@ import bleach from mako.lookup import TemplateLookup -from mfr.core import extension -from mfr.extensions.md.settings import BLEACH_WHITELIST +from mfr.core import extension as core_extension +from mfr.extensions.md import settings as md_settings -class MdRenderer(extension.BaseRenderer): + +class MdRenderer(core_extension.BaseRenderer): TEMPLATE = TemplateLookup( directories=[ @@ -19,9 +20,7 @@ def __init__(self, *args, **kwargs): def render(self): """Render a markdown file to html.""" with open(self.file_path, 'r') as fp: - # Bleach will bug out on unclosed tags. OSF wiki does not have this issue. - # This is due to versioning problems: https://github.com/mozilla/bleach/issues/271 - body = bleach.clean(fp.read(), **BLEACH_WHITELIST) + body = bleach.clean(fp.read(), **md_settings.BLEACH_WHITELIST) return self.TEMPLATE.render(base=self.assets_url, body=body) @property diff --git a/mfr/extensions/md/settings.py b/mfr/extensions/md/settings.py index 0b90e4fb3..73e2fe8c2 100644 --- a/mfr/extensions/md/settings.py +++ b/mfr/extensions/md/settings.py @@ -1,4 +1,6 @@ -# This list comes from the OSF +# The bleach whitelist comes from OSF Wiki whitelist: +# https://github.com/CenterForOpenScience/osf.io/blob/develop/website/settings/defaults.py + BLEACH_WHITELIST = { 'tags': [ 'a', 'abbr', 'acronym', 'b', 'bdo', 'big', 'blockquote', 'br', @@ -13,7 +15,7 @@ 'attributes': [ 'align', 'alt', 'border', 'cite', 'class', 'dir', 'height', 'href', 'id', 'src', 'style', 'title', 'type', 'width', - 'face', 'size', # font tags + 'face', 'size', 'salign', 'align', 'wmode', 'target', ], 'styles': [ diff --git a/mfr/extensions/md/static/css/default.css b/mfr/extensions/md/static/css/default.css index 423d101f4..e42425434 100644 --- a/mfr/extensions/md/static/css/default.css +++ b/mfr/extensions/md/static/css/default.css @@ -1,12 +1,5 @@ @charset "utf-8"; - -/*Had quite a few css woes on this one -Chrome has some scroll bar issues. This is mostly due to -Iframe problems. There may be a better combination of CSS to get rid of this, -for now just going the best combination found. - -*/ .mfrViewer { font-family: 'Open Sans'; padding:1em; @@ -14,4 +7,3 @@ for now just going the best combination found. height: auto; word-wrap: break-word; } - diff --git a/mfr/extensions/md/static/css/highlightjs-default.css b/mfr/extensions/md/static/css/highlightjs-default.css index f1bfade31..31d5c270d 100644 --- a/mfr/extensions/md/static/css/highlightjs-default.css +++ b/mfr/extensions/md/static/css/highlightjs-default.css @@ -1,14 +1,14 @@ -/* - -Original highlight.js style (c) Ivan Sagalaev - -*/ +/** + * Original highlight.js style (c) Ivan Sagalaev + * + * https://github.com/isagalaev/highlight.js/blob/9.12.0/src/styles/default.css + */ .hljs { - display: block; - overflow-x: auto; - padding: 0.5em; - background: #F0F0F0; + display: block; + overflow-x: auto; + padding: 0.5em; + background: #F0F0F0; } @@ -16,11 +16,11 @@ Original highlight.js style (c) Ivan Sagalaev .hljs, .hljs-subst { - color: #444; + color: #444; } .hljs-comment { - color: #888888; + color: #888888; } .hljs-keyword, @@ -29,7 +29,7 @@ Original highlight.js style (c) Ivan Sagalaev .hljs-meta-keyword, .hljs-doctag, .hljs-name { - font-weight: bold; + font-weight: bold; } @@ -43,13 +43,13 @@ Original highlight.js style (c) Ivan Sagalaev .hljs-quote, .hljs-template-tag, .hljs-deletion { - color: #880000; + color: #880000; } .hljs-title, .hljs-section { - color: #880000; - font-weight: bold; + color: #880000; + font-weight: bold; } .hljs-regexp, @@ -59,41 +59,41 @@ Original highlight.js style (c) Ivan Sagalaev .hljs-link, .hljs-selector-attr, .hljs-selector-pseudo { - color: #BC6060; + color: #BC6060; } /* Language color: hue: 90; */ .hljs-literal { - color: #78A960; + color: #78A960; } .hljs-built_in, .hljs-bullet, .hljs-code, .hljs-addition { - color: #397300; + color: #397300; } /* Meta color: hue: 200 */ .hljs-meta { - color: #1f7199; + color: #1f7199; } .hljs-meta-string { - color: #4d99bf; + color: #4d99bf; } /* Misc effects */ .hljs-emphasis { - font-style: italic; + font-style: italic; } .hljs-strong { - font-weight: bold; + font-weight: bold; } diff --git a/tests/extensions/md/test_renderer.py b/tests/extensions/md/test_renderer.py index 88599ae94..e9c37c396 100644 --- a/tests/extensions/md/test_renderer.py +++ b/tests/extensions/md/test_renderer.py @@ -20,6 +20,7 @@ def test_md_file_path(): def invalid_md_file_path(): return os.path.join(os.path.dirname(os.path.abspath(__file__)), 'files', 'invalid.md') + @pytest.fixture def url(): return 'http://osf.io/file/test.md' @@ -49,11 +50,25 @@ def test_render_md_cache_result(self, renderer): assert renderer.cache_result is True def test_render_md(self, test_md_file_path, assets_url, export_url): - metadata = ProviderMetadata('test', '.md', 'text/plain', - '1234', 'http://wb.osf.io/file/test.md?token=1234') - renderer = MdRenderer(metadata, test_md_file_path, url, assets_url, export_url) - body = renderer.render() - # Harder to test now that the markdown is parsed javascript side - inbody = 'The rain---not the reign---in\nSpain.' - '\n\n<script>\nalert("Hello world");\n</script>' - assert inbody in body + + metadata = ProviderMetadata( + 'test', + '.md', + 'text/plain', + '1234', + 'http://wb.osf.io/file/test.md?token=1234' + ) + + md_renderer = MdRenderer(metadata, test_md_file_path, url, assets_url, export_url) + body = md_renderer.render() + in_body = 'The rain---not the reign---in\nSpain.\n\n<script>\nalert("Hello world");\n</script>' + assert in_body in body + + # TODO: it is hard to test since rendering is performed by front-end javascript, if possible test the following: + # TODO: test bleach (python) + # TODO: test invalid/corrupted file (e.g. when bleach fails parsing the file) + # TODO: test markdown-it + # TODO: test toc + # TODO: test highlight + # TODO: test mathjax + # TODO: test sanitizer