Restore GraalJS based implementation to new ns

src/nextjournal/markdown/graaljs.clj

@@ -0,0 +1,85 @@
+(ns nextjournal.markdown.graaljs
+  "Facility functions for handling markdown conversions"
+  (:require [clojure.java.io :as io]
+            [clojure.data.json :as json]
+            [nextjournal.markdown.parser :as markdown.parser]
+            [nextjournal.markdown.transform :as markdown.transform])
+  (:import (org.graalvm.polyglot Context Context$Builder Source Value)
+           (java.io Reader)))
+
+(set! *warn-on-reflection* true)
+
+(def ^Context$Builder context-builder
+  (doto (Context/newBuilder (into-array String ["js"]))
+    (.option "js.timer-resolution" "1")
+    (.option "js.java-package-globals" "false")
+    (.option "js.esm-eval-returns-exports", "true")
+    ;; ⬆ returns module exports when evaling an esm module file, note this will only work on stock JVM or on GraalJVM
+    ;; matching the poliglot library versions specified in deps.edn
+    (.out System/out)
+    (.err System/err)
+    (.allowIO true)
+    (.allowExperimentalOptions true)
+    (.allowAllAccess true)
+    (.allowNativeAccess true)
+    (.option "engine.WarnInterpreterOnly" "false")))
+
+(def ^Context ctx (.build context-builder))
+
+(def ^Value MD-imports
+  ;; Contructing a `java.io.Reader` first to work around a bug with graal on windows
+  ;; see https://github.com/oracle/graaljs/issues/534 and https://github.com/nextjournal/viewers/pull/33
+  (let [source (-> (io/resource "js/markdown.mjs")
+                   io/input-stream
+                   io/reader
+                   (as-> r (Source/newBuilder "js" ^Reader r "markdown.mjs")))]
+    (locking ctx
+      (.. ctx
+          (eval (.build source))
+          (getMember "default")))))
+
+(def ^Value tokenize-fn (.getMember MD-imports "tokenizeJSON"))
+
+(defn tokenize [markdown-text]
+  (let [^Value tokens-json (locking ctx
+                             (.execute tokenize-fn (to-array [markdown-text])))]
+    (json/read-str (.asString tokens-json) :key-fn keyword)))
+
+(defn parse
+  "Turns a markdown string into a nested clojure structure."
+  ([markdown-text] (parse markdown.parser/empty-doc markdown-text))
+  ([doc markdown-text] (markdown.parser/parse doc (tokenize markdown-text))))
+
+(defn ->hiccup
+  "Turns a markdown string into hiccup."
+  ([markdown-text] (->hiccup markdown.transform/default-hiccup-renderers markdown-text))
+  ([ctx markdown-text] (->> markdown-text parse (markdown.transform/->hiccup ctx))))
+
+(comment
+  (tokenize "# Title
+- [ ] one
+- [x] two
+")
+
+  (parse "# Hello Markdown
+- [ ] what
+- [ ] [nice](very/nice/thing)
+- [x] ~~thing~~
+")
+
+  (->hiccup "# Hello Markdown
+
+* What's _going_ on?
+")
+
+  (->hiccup
+   (assoc markdown.transform/default-hiccup-renderers
+          :heading (fn [ctx node]
+                     [:h1.some-extra.class
+                      (markdown.transform/into-markup [:span.some-other-class] ctx node)]))
+   "# Hello Markdown
+* What's _going_ on?
+")
+
+  ;; launch shadow cljs repl
+  (shadow.cljs.devtools.api/repl :sci))

src/nextjournal/markdown/parser.cljc

@@ -185,6 +185,7 @@
       (recur (ppop p) (conj ancestors (get-in doc p)))
       ancestors)))
 
+;; TODO: consider rewriting parse in terms of this zipper
 (defn ->zip [doc]
   (z/zipper (every-pred map? :type) :content
             (fn [node cs] (assoc node :content (vec cs)))
@@ -416,13 +417,40 @@ end"
             (let [new-loc (-> loc (z/replace {:type :sidenote-container :content []})
                               (z/append-child node)
                               (z/append-child {:type :sidenote-column
-                                               :content (mapv #(footnote->sidenote (get footnotes %)) (distinct refs))}))]
+                                               ;; TODO: broken in the old implementation
+                                               ;; should be :content (mapv #(footnote->sidenote (get footnotes %)) (distinct refs))}))]
+                                               :content (mapv #(footnote->sidenote (get footnotes %)) refs)}))]
               (recur (z/right new-loc) (z/up new-loc)))
             (recur (z/right loc) parent))
           :else
           (recur (z/right loc) parent))))))
 
+(comment
+  (-> "_hello_ what and foo[^note1] and^[some other note].
+
+And what.
+
+[^note1]: the _what_
 
+* and new text[^endnote] at the end.
+* the
+  * hell^[that warm place]
+
+[^endnote]: conclusion.
+"
+      nextjournal.markdown/tokenize
+      parse
+      #_ flatten-tokens
+      insert-sidenote-containers)
+
+  (-> empty-doc
+      (update :text-tokenizers (partial map normalize-tokenizer))
+      (apply-tokens (nextjournal.markdown/tokenize "what^[the heck]"))
+      insert-sidenote-columns
+      (apply-tokens (nextjournal.markdown/tokenize "# Hello"))
+      insert-sidenote-columns
+      (apply-tokens (nextjournal.markdown/tokenize "is^[this thing]"))
+      insert-sidenote-columns))
 
 ;; tables
 ;; table data tokens might have {:style "text-align:right|left"} attrs, maybe better nested node > :attrs > :style ?