From e30221a43060aa461d5689f665dcd3536af2be0b Mon Sep 17 00:00:00 2001 From: zorkow Date: Sat, 13 Oct 2018 15:41:20 +0100 Subject: [PATCH 1/4] Initial conversion of Unicode package. --- lib/TeX-lab.js | 1 + .../input/tex/unicode/UnicodeConfiguration.ts | 104 ++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 mathjax3-ts/input/tex/unicode/UnicodeConfiguration.ts diff --git a/lib/TeX-lab.js b/lib/TeX-lab.js index 3e1a11fef..b348a2277 100644 --- a/lib/TeX-lab.js +++ b/lib/TeX-lab.js @@ -12,6 +12,7 @@ import 'mathjax3/input/tex/ams/AmsConfiguration.js'; import 'mathjax3/input/tex/noundefined/NoUndefinedConfiguration.js'; import 'mathjax3/input/tex/boldsymbol/BoldsymbolConfiguration.js'; import 'mathjax3/input/tex/newcommand/NewcommandConfiguration.js'; +import 'mathjax3/input/tex/unicode/UnicodeConfiguration.js'; let tex = new TeX(); diff --git a/mathjax3-ts/input/tex/unicode/UnicodeConfiguration.ts b/mathjax3-ts/input/tex/unicode/UnicodeConfiguration.ts new file mode 100644 index 000000000..df910ecb9 --- /dev/null +++ b/mathjax3-ts/input/tex/unicode/UnicodeConfiguration.ts @@ -0,0 +1,104 @@ +/************************************************************* + * + * Copyright (c) 2018 The MathJax Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/** + * @fileoverview Configuration file for the unicode package. + * + * @author v.sorge@mathjax.org (Volker Sorge) + */ + +import {MmlNode} from '../../../core/MmlTree/MmlNode.js'; +import {Configuration} from '../Configuration.js'; +import NodeUtil from '../NodeUtil.js'; +import {EnvList} from '../StackItem.js'; +import TexParser from '../TexParser.js'; +import TexError from '../TexError.js'; +import {TexConstant} from '../TexConstants.js'; +import {CommandMap} from '../SymbolMap.js'; +import {ParseMethod} from '../Types.js'; +import {NodeFactory} from '../NodeFactory.js'; +import ParseOptions from '../ParseOptions.js'; +import ParseUtil from '../ParseUtil.js'; +import {MathItem} from '../../../core/MathItem.js'; + + +// Namespace +export let UnicodeMethods: Record = {}; + +let UnicodeCache: {[key: number]: [number, number, string, number]} = {}; + +/** + * Parse function for unicode macro. + * @param {TexParser} parser The current tex parser. + * @param {string} name The name of the macro. + */ +UnicodeMethods.Unicode = function(parser: TexParser, name: string) { + let HD = parser.GetBrackets(name); + let HDsplit = null; + let font = null; + if (HD) { + if (HD.replace(/ /g, ''). + match(/^(\d+(\.\d*)?|\.\d+),(\d+(\.\d*)?|\.\d+)$/)) { + HDsplit = HD.replace(/ /g, '').split(/,/); + font = parser.GetBrackets(name); + } else { + font = HD; + } + } + let n = ParseUtil.trimSpaces(parser.GetArgument(name)).replace(/^0x/, 'x'); + if (!n.match(/^(x[0-9A-Fa-f]+|[0-9]+)$/)) { + throw new TexError('BadUnicode', 'Argument to \\unicode must be a number'); + } + let N = parseInt(n.match(/^x/) ? '0' + n : n); + if (!UnicodeCache[N]) { + UnicodeCache[N] = [800, 200, font, N]; + } else if (!font) { + font = UnicodeCache[N][2]; + } + if (HDsplit) { + UnicodeCache[N][0] = Math.floor(parseFloat(HDsplit[0]) * 1000); + UnicodeCache[N][1] = Math.floor(parseFloat(HDsplit[1]) * 1000); + } + let variant = parser.stack.env.font as string; + let def: EnvList = {}; + if (font) { + UnicodeCache[N][2] = def.fontfamily = font.replace(/'/g, '\''); + if (variant) { + if (variant.match(/bold/)) { + def.fontweight = 'bold'; + } + if (variant.match(/italic|-mathit/)) { + def.fontstyle = 'italic'; + } + } + } else if (variant) { + def.mathvariant = variant; + } + let node = parser.create('token', 'mtext', def, String.fromCharCode(N)); + parser.Push(node); +}; + + +new CommandMap('unicode', {unicode: 'Unicode'}, UnicodeMethods); + + +export const UnicodeConfiguration = Configuration.create( + 'unicode', {handler: {macro: ['unicode']}} +); + + From 085dd51a54b994c3e59e6cc1811701bc80bdcb8c Mon Sep 17 00:00:00 2001 From: zorkow Date: Sat, 13 Oct 2018 15:52:31 +0100 Subject: [PATCH 2/4] Correct conversion of Unicode characters. --- mathjax3-ts/input/tex/unicode/UnicodeConfiguration.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/mathjax3-ts/input/tex/unicode/UnicodeConfiguration.ts b/mathjax3-ts/input/tex/unicode/UnicodeConfiguration.ts index df910ecb9..49fbc60a2 100644 --- a/mathjax3-ts/input/tex/unicode/UnicodeConfiguration.ts +++ b/mathjax3-ts/input/tex/unicode/UnicodeConfiguration.ts @@ -22,20 +22,15 @@ * @author v.sorge@mathjax.org (Volker Sorge) */ -import {MmlNode} from '../../../core/MmlTree/MmlNode.js'; import {Configuration} from '../Configuration.js'; -import NodeUtil from '../NodeUtil.js'; import {EnvList} from '../StackItem.js'; import TexParser from '../TexParser.js'; import TexError from '../TexError.js'; import {TexConstant} from '../TexConstants.js'; import {CommandMap} from '../SymbolMap.js'; import {ParseMethod} from '../Types.js'; -import {NodeFactory} from '../NodeFactory.js'; -import ParseOptions from '../ParseOptions.js'; import ParseUtil from '../ParseUtil.js'; -import {MathItem} from '../../../core/MathItem.js'; - +import {numeric} from '../../../util/Entities.js'; // Namespace export let UnicodeMethods: Record = {}; @@ -89,8 +84,7 @@ UnicodeMethods.Unicode = function(parser: TexParser, name: string) { } else if (variant) { def.mathvariant = variant; } - let node = parser.create('token', 'mtext', def, String.fromCharCode(N)); - parser.Push(node); + parser.Push(parser.create('token', 'mtext', def, numeric(n))); }; From 0b379aca25c186b84731ca4b6276ea6014001c13 Mon Sep 17 00:00:00 2001 From: zorkow Date: Sun, 26 May 2019 23:19:03 +0100 Subject: [PATCH 3/4] Adds tests for unicode package. --- .../input/tex/unicode/UnicodeConfiguration.ts | 5 +- samples/tex2mml.js | 3 +- tests.sh | 1 + tests/parser-unicode-tests.js | 438 ++++++++++++++++++ 4 files changed, 445 insertions(+), 2 deletions(-) create mode 100644 tests/parser-unicode-tests.js diff --git a/mathjax3-ts/input/tex/unicode/UnicodeConfiguration.ts b/mathjax3-ts/input/tex/unicode/UnicodeConfiguration.ts index 49fbc60a2..284425a30 100644 --- a/mathjax3-ts/input/tex/unicode/UnicodeConfiguration.ts +++ b/mathjax3-ts/input/tex/unicode/UnicodeConfiguration.ts @@ -30,6 +30,7 @@ import {TexConstant} from '../TexConstants.js'; import {CommandMap} from '../SymbolMap.js'; import {ParseMethod} from '../Types.js'; import ParseUtil from '../ParseUtil.js'; +import NodeUtil from '../NodeUtil.js'; import {numeric} from '../../../util/Entities.js'; // Namespace @@ -84,7 +85,9 @@ UnicodeMethods.Unicode = function(parser: TexParser, name: string) { } else if (variant) { def.mathvariant = variant; } - parser.Push(parser.create('token', 'mtext', def, numeric(n))); + let node = parser.create('token', 'mtext', def, numeric(n)); + NodeUtil.setProperty(node, 'unicode', true); + parser.Push(node); }; diff --git a/samples/tex2mml.js b/samples/tex2mml.js index f0300f4c8..7858516ac 100644 --- a/samples/tex2mml.js +++ b/samples/tex2mml.js @@ -14,11 +14,12 @@ import '../mathjax3/input/tex/action/ActionConfiguration.js'; import '../mathjax3/input/tex/bbox/BboxConfiguration.js'; import '../mathjax3/input/tex/braket/BraketConfiguration.js'; import '../mathjax3/input/tex/physics/PhysicsConfiguration.js'; +import '../mathjax3/input/tex/unicode/UnicodeConfiguration.js'; RegisterHTMLHandler(chooseAdaptor()); let html = MathJax.document('', { - InputJax: new TeX({packages: ['base', 'ams', 'boldsymbol', 'newcommand', 'extpfeil', 'braket', 'physics', 'action', 'bbox']}) + InputJax: new TeX({packages: ['base', 'ams', 'boldsymbol', 'newcommand', 'extpfeil', 'braket', 'physics', 'unicode', 'action', 'bbox']}) }); import {SerializedMmlVisitor as MmlVisitor} from '../mathjax3/core/MmlTree/SerializedMmlVisitor.js'; diff --git a/tests.sh b/tests.sh index 77b288ddb..abcd80a67 100755 --- a/tests.sh +++ b/tests.sh @@ -107,6 +107,7 @@ if [ $# -eq 0 ]; then # More Packages node load.js tests/parser-bbox-tests.js node load.js tests/parser-action-tests.js + node load.js tests/parser-unicode-tests.js exit 0 else repeat=$1 diff --git a/tests/parser-unicode-tests.js b/tests/parser-unicode-tests.js new file mode 100644 index 000000000..5e0bac590 --- /dev/null +++ b/tests/parser-unicode-tests.js @@ -0,0 +1,438 @@ +import {ParserTest} from './parser-tests.js'; +import 'mathjax3/input/tex/unicode/UnicodeConfiguration.js'; + +class ParserUnicodeTest extends ParserTest { + + constructor() { + super(); + this.packages = ['base', 'unicode']; + } + +} + +let parserTest = new ParserUnicodeTest(); + + +parserTest.runTest( + 'Unicode Dec', '\\unicode{8922}', + {"kind": "math", + "texClass": 0, + "attributes": {"display": "block"}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "mrow", + "texClass": 0, + "attributes": {}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "mtext", + "texClass": 0, + "attributes": {}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {"unicode": true}, + "childNodes": [ + {"kind": "text", + "text": "⋚"}], + "isSpacelike": true}], + "isInferred": true, + "isSpacelike": true}], + "isSpacelike": true} +); + +parserTest.runTest( + 'Unicode Hex', '\\unicode{0x22DA}', + {"kind": "math", + "texClass": 0, + "attributes": {"display": "block"}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "mrow", + "texClass": 0, + "attributes": {}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "mtext", + "texClass": 0, + "attributes": {}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {"unicode": true}, + "childNodes": [ + {"kind": "text", + "text": "⋚"}], + "isSpacelike": true}], + "isInferred": true, + "isSpacelike": true}], + "isSpacelike": true} +); + +parserTest.runTest( + 'Unicode Dec A', '\\unicode{65}', + {"kind": "math", + "texClass": 0, + "attributes": {"display": "block"}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "mrow", + "texClass": 0, + "attributes": {}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "mtext", + "texClass": 0, + "attributes": {}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {"unicode": true}, + "childNodes": [ + {"kind": "text", + "text": "A"}], + "isSpacelike": true}], + "isInferred": true, + "isSpacelike": true}], + "isSpacelike": true} +); + +parserTest.runTest( + 'Unicode Hex A', '\\unicode{x41}', + {"kind": "math", + "texClass": 0, + "attributes": {"display": "block"}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "mrow", + "texClass": 0, + "attributes": {}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "mtext", + "texClass": 0, + "attributes": {}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {"unicode": true}, + "childNodes": [ + {"kind": "text", + "text": "A"}], + "isSpacelike": true}], + "isInferred": true, + "isSpacelike": true}], + "isSpacelike": true} +); + +parserTest.runTest( + 'Unicode Scale', '\\unicode[.55,0.05]{x22D6}', + {"kind": "math", + "texClass": 0, + "attributes": {"display": "block"}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "mrow", + "texClass": 0, + "attributes": {}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "mtext", + "texClass": 0, + "attributes": {}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {"unicode": true}, + "childNodes": [ + {"kind": "text", + "text": "⋖"}], + "isSpacelike": true}], + "isInferred": true, + "isSpacelike": true}], + "isSpacelike": true} +); + +parserTest.runTest( + 'Unicode Scale Font', '\\unicode[.55,0.05][Geramond]{x22D6}', + {"kind": "math", + "texClass": 0, + "attributes": {"display": "block"}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "mrow", + "texClass": 0, + "attributes": {}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "mtext", + "texClass": 0, + "attributes": {"fontfamily": "Geramond"}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {"unicode": true}, + "childNodes": [ + {"kind": "text", + "text": "⋖"}], + "isSpacelike": true}], + "isInferred": true, + "isSpacelike": true}], + "isSpacelike": true} +); + +parserTest.runTest( + 'Unicode Font', '\\unicode[Garamond]{x22D6}', + {"kind": "math", + "texClass": 0, + "attributes": {"display": "block"}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "mrow", + "texClass": 0, + "attributes": {}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "mtext", + "texClass": 0, + "attributes": {"fontfamily": "Garamond"}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {"unicode": true}, + "childNodes": [ + {"kind": "text", + "text": "⋖"}], + "isSpacelike": true}], + "isInferred": true, + "isSpacelike": true}], + "isSpacelike": true} +); + +parserTest.runTest( + 'Unicode Combined', '\\mbox{A}\\unicode{65}{B}', + {"kind": "math", + "texClass": 0, + "attributes": {"display": "block"}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "mrow", + "texClass": 0, + "attributes": {}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "mstyle", + "texClass": 0, + "attributes": {"displaystyle": false, + "scriptlevel": 0}, + "inherited": {}, + "properties": {}, + "childNodes": [ + {"kind": "mrow", + "texClass": 0, + "attributes": {}, + "inherited": {"displaystyle": false, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "mtext", + "texClass": 0, + "attributes": {}, + "inherited": {"displaystyle": false, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "text", + "text": "A"}], + "isSpacelike": true}], + "isInferred": true, + "isSpacelike": true}], + "isSpacelike": true}, + {"kind": "mtext", + "texClass": 0, + "attributes": {}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {"unicode": true}, + "childNodes": [ + {"kind": "text", + "text": "A"}], + "isSpacelike": true}, + {"kind": "TeXAtom", + "texClass": 0, + "attributes": {}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "mrow", + "texClass": 0, + "attributes": {}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "mi", + "texClass": 0, + "attributes": {}, + "inherited": {"displaystyle": true, + "scriptlevel": 0, + "mathvariant": "italic"}, + "properties": {}, + "childNodes": [ + {"kind": "text", + "text": "B"}]}], + "isInferred": true}]}], + "isInferred": true}]} +); + +parserTest.runTest( + 'Unicode Surrogate Hex', '\\unicode{x1D5A0}', + {"kind": "math", + "texClass": 0, + "attributes": {"display": "block"}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "mrow", + "texClass": 0, + "attributes": {}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "mtext", + "texClass": 0, + "attributes": {}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {"unicode": true}, + "childNodes": [ + {"kind": "text", + "text": "𝖠"}], + "isSpacelike": true}], + "isInferred": true, + "isSpacelike": true}], + "isSpacelike": true} +); + +parserTest.runTest( + 'Unicode Surrogate Dec', '\\unicode{120224}', + {"kind": "math", + "texClass": 0, + "attributes": {"display": "block"}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "mrow", + "texClass": 0, + "attributes": {}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "mtext", + "texClass": 0, + "attributes": {}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {"unicode": true}, + "childNodes": [ + {"kind": "text", + "text": "𝖠"}], + "isSpacelike": true}], + "isInferred": true, + "isSpacelike": true}], + "isSpacelike": true} +); + +parserTest.runTest( + 'Unicode Blackboard', '\\unicode{x1D538}', + {"kind": "math", + "texClass": 0, + "attributes": {"display": "block"}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "mrow", + "texClass": 0, + "attributes": {}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "mtext", + "texClass": 0, + "attributes": {}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {"unicode": true}, + "childNodes": [ + {"kind": "text", + "text": "𝔸"}], + "isSpacelike": true}], + "isInferred": true, + "isSpacelike": true}], + "isSpacelike": true} +); + +parserTest.runTest( + 'Unicode Blackboard Geramond', '\\unicode{x1D538}', + {"kind": "math", + "texClass": 0, + "attributes": {"display": "block"}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "mrow", + "texClass": 0, + "attributes": {}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {}, + "childNodes": [ + {"kind": "mtext", + "texClass": 0, + "attributes": {}, + "inherited": {"displaystyle": true, + "scriptlevel": 0}, + "properties": {"unicode": true}, + "childNodes": [ + {"kind": "text", + "text": "𝔸"}], + "isSpacelike": true}], + "isInferred": true, + "isSpacelike": true}], + "isSpacelike": true} +); + + From ec3b8736d5cdd8e76693337a980eef5fa8c88003 Mon Sep 17 00:00:00 2001 From: zorkow Date: Mon, 3 Jun 2019 19:22:04 +0100 Subject: [PATCH 4/4] Incorporates review suggestsions. --- mathjax3-ts/input/tex/unicode/UnicodeConfiguration.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/mathjax3-ts/input/tex/unicode/UnicodeConfiguration.ts b/mathjax3-ts/input/tex/unicode/UnicodeConfiguration.ts index 284425a30..f4f5e8e0a 100644 --- a/mathjax3-ts/input/tex/unicode/UnicodeConfiguration.ts +++ b/mathjax3-ts/input/tex/unicode/UnicodeConfiguration.ts @@ -26,7 +26,6 @@ import {Configuration} from '../Configuration.js'; import {EnvList} from '../StackItem.js'; import TexParser from '../TexParser.js'; import TexError from '../TexError.js'; -import {TexConstant} from '../TexConstants.js'; import {CommandMap} from '../SymbolMap.js'; import {ParseMethod} from '../Types.js'; import ParseUtil from '../ParseUtil.js';