Skip to content

Commit

Permalink
Update EspruinoTools with pretokeniser that converts strings
Browse files Browse the repository at this point in the history 
Don't run 'evaluate=true' code like icons through EspruinoTools - faster and less risky
  • Loading branch information
@gfwilliams
gfwilliams committed Jan 29, 2024
1 parent e6a65a8 commit bdcc79a
Show file tree
Hide file tree
Showing 2 changed files with 84 additions and 15 deletions.
13 changes: 10 additions & 3 deletions js/appinfo.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,14 @@ function parseJS(storageFile, options, app) {
minify = false;
}
// TODO: we could look at installed app files and add any modules defined in those?
/* Don't run code that we're going to be uploading direct through EspruinoTools. This is
usually an icon, and we don't want it pretokenised, minifying won't do anything, and really
we don't want anything touching it at all. */
if (storageFile.evaluate) {
storageFile.content = js;
return storageFile;
}
// Now run through EspruinoTools for pretokenising/compiling/modules/etc
return Espruino.transform(js, {
SET_TIME_ON_WRITE : false,
PRETOKENISE : options.settings.pretokenise,
Expand All @@ -160,9 +168,8 @@ function parseJS(storageFile, options, app) {
var AppInfo = {
/* Get a list of commands needed to upload the file */
getFileUploadCommands : (filename, data) => {
var cmd = "";
// write code in chunks, in case it is too big to fit in RAM (fix #157)
var cmd = `\x10require('Storage').write(${JSON.stringify(filename)},${asJSExpr(data.substr(0,CHUNKSIZE))},0,${data.length});`;
let cmd = `\x10require('Storage').write(${JSON.stringify(filename)},${asJSExpr(data.substr(0,CHUNKSIZE))},0,${data.length});`;
for (let i=CHUNKSIZE;i<data.length;i+=CHUNKSIZE)
cmd += `\n\x10require('Storage').write(${JSON.stringify(filename)},${asJSExpr(data.substr(i,CHUNKSIZE))},${i});`;
return cmd;
Expand Down Expand Up @@ -227,7 +234,7 @@ var AppInfo = {
evaluate : storageFile.evaluate,
noOverwrite : storageFile.noOverwrite,
dataFile : !!storageFile.dataFile
}}).then(storageFile => parseJS(storageFile,options,app));
}}).then(storageFile => parseJS(storageFile, options, app));
else return Promise.resolve();
})).then(fileContents => { // now we just have a list of files + contents...
// filter out empty files
Expand Down
86 changes: 74 additions & 12 deletions lib/espruinotools.js
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// EspruinoTools bundle (https://github.com/espruino/EspruinoTools)
// Created with https://github.com/espruino/EspruinoWebIDE/blob/gh-pages/extras/create_espruinotools_js.sh
// Based on EspruinoWebIDE 0.78.4
// Based on EspruinoWebIDE 0.78.12
/**
Copyright 2014 Gordon Williams ([email protected])

Expand Down Expand Up @@ -4543,7 +4543,7 @@ while (d!==undefined) {console.log(btoa(d));d=f.read(${CHUNKSIZE});}
console.error("getURL("+JSON.stringify(url)+") error : "+err);
callback(undefined);
});
if (formData!==null)
if (formData!==null)
req.write(formData);
req.end();
} else {
Expand Down Expand Up @@ -4828,6 +4828,39 @@ while (d!==undefined) {console.log(btoa(d));d=f.read(${CHUNKSIZE});}
return JSON.parse(final);
};

/* Escape a string (like JSON.stringify) so that Espruino can understand it,
however use \0,\1,\x,etc escapes whenever possible to make the String as small
as it can be. On Espruino with UTF8 support, not using \u.... also allows it
to use non-UTF8 Strings which are more efficient. */
function toJSONishString(txt) {
let js = "\"";
for (let i=0;i<txt.length;i++) {
let ch = txt.charCodeAt(i);
let nextCh = (i+1<txt.length ? txt.charCodeAt(i+1) : 0); // 0..255
if (ch<8) {
// if the next character is a digit, it'd be interpreted
// as a 2 digit octal character, so we can't use `\0` to escape it
if (nextCh>='0' && nextCh<='7') js += "\\x0"+ch;
else js += "\\"+ch;
} else if (ch==8) js += "\\b";
else if (ch==9) js += "\\t";
else if (ch==10) js += "\\n";
else if (ch==11) js += "\\v";
else if (ch==12) js += "\\f";
else if (ch==34) js += "\\\""; // quote
else if (ch==92) js += "\\\\"; // slash
else if (ch<32 || ch==127 || ch==173 ||
((ch>=0xC2) && (ch<=0xF4))) // unicode start char range
js += "\\x"+ ((ch & 255) | 256).toString(16).substring(1);
else if (ch>255)
js += "\\u"+ ((ch & 65535) | 65536).toString(16).substring(1);
else js += txt[i];
}
js += "\"";
//let b64 = "atob("+JSON.stringify(Espruino.Core.Utils.btoa(txt))+")";
return js;
}

// Does the given string contain only ASCII characters?
function isASCII(str) {
for (var i=0;i<str.length;i++) {
Expand Down Expand Up @@ -4879,7 +4912,7 @@ while (d!==undefined) {console.log(btoa(d));d=f.read(${CHUNKSIZE});}
var i = 0;
// remove all characters that are not A-Z, a-z, 0-9, +, /, or =
input = input.replace(/[^A-Za-z0-9\+\/\=]/g, '');
do {
while (i < input.length) {
enc1 = keyStr.indexOf(input.charAt(i++));
enc2 = keyStr.indexOf(input.charAt(i++));
enc3 = keyStr.indexOf(input.charAt(i++));
Expand All @@ -4897,7 +4930,7 @@ while (d!==undefined) {console.log(btoa(d));d=f.read(${CHUNKSIZE});}
if (enc4 !== 64) {
output = output + String.fromCharCode(chr3);
}
} while (i < input.length);
}
return output;
}

Expand Down Expand Up @@ -4943,6 +4976,7 @@ while (d!==undefined) {console.log(btoa(d));d=f.read(${CHUNKSIZE});}
dataViewToArrayBuffer : dataViewToArrayBuffer,
arrayBufferToString : arrayBufferToString,
parseJSONish : parseJSONish,
toJSONishString : toJSONishString,
isASCII : isASCII,
btoa : btoa,
atob : atob
Expand Down Expand Up @@ -5328,7 +5362,7 @@ To add a new serial device, you must add an object to
}

var portInfo = { port:serialPort };
connectionInfo = undefined;
var connectionInfo = undefined;
flowControlXOFF = false;
if (flowControlTimeout) {
clearTimeout(flowControlTimeout);
Expand All @@ -5344,7 +5378,6 @@ To add a new serial device, you must add an object to
currentDevice = undefined;
} else {
connectionInfo = cInfo;
connectedPort = serialPort;
console.log("Connected", cInfo);
if (connectionInfo.portName)
portInfo.portName = connectionInfo.portName;
Expand Down Expand Up @@ -5852,7 +5885,7 @@ To add a new serial device, you must add an object to
name : "Module URL",
description : "Where to search online for modules when `require()` is used. Can supply more than one URL, separated by '|'",
type : "string",
defaultValue : "https://www.espruino.com/modules"
defaultValue : "https://www.espruino.com/modules|https://banglejs.com/apps/modules"
});
Espruino.Core.Config.add("MODULE_EXTENSIONS", {
section : "Communications",
Expand Down Expand Up @@ -33610,22 +33643,34 @@ global.esmangle = require('../lib/esmangle');
/*LEX_R_OF : */ "of"
];

const LEX_RAW_STRING8 = 0xD1;
const LEX_RAW_STRING16 = 0xD2;

function pretokenise(code, callback) {
var pretokeniseStrings = false; // only works on 2v20.48 and later
var boardData = Espruino.Core.Env.getBoardData();
if (boardData && boardData.VERSION) {
var v = parseFloat(boardData.VERSION.replace("v","0"));
if (v >= 2020.48)
pretokeniseStrings = true;
}

var lex = (function() {
var t = acorn.tokenizer(code);
return { next : function() {
var tk = t.getToken();
if (tk.type.label=="eof") return undefined;
var tp = "?";
if (tk.type.label=="template" || tk.type.label=="string") tp="STRING";
if (tk.type.label=="template") tp="TEMPLATEDSTRING";
if (tk.type.label=="string") tp="STRING";
if (tk.type.label=="num") tp="NUMBER";
if (tk.type.keyword || tk.type.label=="name") tp="ID";
if (tp=="?" && tk.start+1==tk.end) tp="CHAR";
return {
startIdx : tk.start,
endIdx : tk.end,
str : code.substring(tk.start, tk.end),
value : tk.value,
type : tp
};
}};
Expand All @@ -33651,7 +33696,23 @@ global.esmangle = require('../lib/esmangle');
resultCode += "\n";
if (tok.str==")" || tok.str=="}" || tok.str=="]") brackets--;
// if we have a token for something, use that - else use the string
if (tokenId) {
if (pretokeniseStrings && tok.type == "STRING") {
let str = tok.value; // get string value
lastIdx = tok.endIdx; // get next token
lastTok = tok;
tok = lex.next();
let hadAtoB = resultCode.endsWith("atob(") && tok.str==")"; // were we surrounded by 'atob'?
if (hadAtoB) {
str = Espruino.Core.Utils.atob(str);
resultCode = resultCode.substring(0, resultCode.length-5); // remove 'atob('
}
let length = str.length;
if (length<256)
resultCode += String.fromCharCode(LEX_RAW_STRING8, length) + str;
else if (length<65536)
resultCode += String.fromCharCode(LEX_RAW_STRING16, length&255, (length>>8)&255)+str;
if (!hadAtoB) continue; // if not atob, we already got the last token ready
} else if (tokenId) {
//console.log(JSON.stringify(tok.str)+" => "+tokenId);
resultCode += String.fromCharCode(tokenId);
tok.type = "TOKENISED";
Expand Down Expand Up @@ -33801,12 +33862,13 @@ global.esmangle = require('../lib/esmangle');
var CHUNKSIZE = 1024;
var newCode = [];
var len = code.length;
newCode.push('require("Storage").write("'+filename+'",'+JSON.stringify(code.substr(0,CHUNKSIZE))+',0,'+len+');');
var asJS = Espruino.Core.Utils.toJSONishString;
newCode.push('require("Storage").write('+asJS(filename)+','+asJS(code.substr(0,CHUNKSIZE))+',0,'+len+');');
for (var i=CHUNKSIZE;i<len;i+=CHUNKSIZE)
newCode.push('require("Storage").write("'+filename+'",'+JSON.stringify(code.substr(i,CHUNKSIZE))+','+i+');');
newCode.push('require("Storage").write('+asJS(filename)+','+asJS(code.substr(i,CHUNKSIZE))+','+i+');');
code = newCode.join("\n");
if (Espruino.Config.LOAD_STORAGE_FILE==2 && isStorageUpload)
code += "\nload("+JSON.stringify(filename)+")\n";
code += "\nload("+asJS(filename)+")\n";
else if (Espruino.Config.LOAD_STORAGE_FILE!=0)
code += "\nload()\n";
}
Expand Down

0 comments on commit bdcc79a

Please sign in to comment.