From bdcc79ac2155b8eae6ce9ec4cc1065495e99ac8c Mon Sep 17 00:00:00 2001 From: Gordon Williams Date: Mon, 29 Jan 2024 20:01:13 +0000 Subject: [PATCH] Update EspruinoTools with pretokeniser that converts strings Don't run 'evaluate=true' code like icons through EspruinoTools - faster and less risky --- js/appinfo.js | 13 +++++-- lib/espruinotools.js | 86 +++++++++++++++++++++++++++++++++++++------- 2 files changed, 84 insertions(+), 15 deletions(-) diff --git a/js/appinfo.js b/js/appinfo.js index 77347e6..e46fc0f 100644 --- a/js/appinfo.js +++ b/js/appinfo.js @@ -138,6 +138,14 @@ function parseJS(storageFile, options, app) { minify = false; } // TODO: we could look at installed app files and add any modules defined in those? + /* Don't run code that we're going to be uploading direct through EspruinoTools. This is + usually an icon, and we don't want it pretokenised, minifying won't do anything, and really + we don't want anything touching it at all. */ + if (storageFile.evaluate) { + storageFile.content = js; + return storageFile; + } + // Now run through EspruinoTools for pretokenising/compiling/modules/etc return Espruino.transform(js, { SET_TIME_ON_WRITE : false, PRETOKENISE : options.settings.pretokenise, @@ -160,9 +168,8 @@ function parseJS(storageFile, options, app) { var AppInfo = { /* Get a list of commands needed to upload the file */ getFileUploadCommands : (filename, data) => { - var cmd = ""; // write code in chunks, in case it is too big to fit in RAM (fix #157) - var cmd = `\x10require('Storage').write(${JSON.stringify(filename)},${asJSExpr(data.substr(0,CHUNKSIZE))},0,${data.length});`; + let cmd = `\x10require('Storage').write(${JSON.stringify(filename)},${asJSExpr(data.substr(0,CHUNKSIZE))},0,${data.length});`; for (let i=CHUNKSIZE;i parseJS(storageFile,options,app)); + }}).then(storageFile => parseJS(storageFile, options, app)); else return Promise.resolve(); })).then(fileContents => { // now we just have a list of files + contents... // filter out empty files diff --git a/lib/espruinotools.js b/lib/espruinotools.js index 7049ebc..2b09632 100644 --- a/lib/espruinotools.js +++ b/lib/espruinotools.js @@ -1,6 +1,6 @@ // EspruinoTools bundle (https://github.com/espruino/EspruinoTools) // Created with https://github.com/espruino/EspruinoWebIDE/blob/gh-pages/extras/create_espruinotools_js.sh -// Based on EspruinoWebIDE 0.78.4 +// Based on EspruinoWebIDE 0.78.12 /** Copyright 2014 Gordon Williams (gw@pur3.co.uk) @@ -4543,7 +4543,7 @@ while (d!==undefined) {console.log(btoa(d));d=f.read(${CHUNKSIZE});} console.error("getURL("+JSON.stringify(url)+") error : "+err); callback(undefined); }); - if (formData!==null) + if (formData!==null) req.write(formData); req.end(); } else { @@ -4828,6 +4828,39 @@ while (d!==undefined) {console.log(btoa(d));d=f.read(${CHUNKSIZE});} return JSON.parse(final); }; + /* Escape a string (like JSON.stringify) so that Espruino can understand it, + however use \0,\1,\x,etc escapes whenever possible to make the String as small + as it can be. On Espruino with UTF8 support, not using \u.... also allows it + to use non-UTF8 Strings which are more efficient. */ + function toJSONishString(txt) { + let js = "\""; + for (let i=0;i='0' && nextCh<='7') js += "\\x0"+ch; + else js += "\\"+ch; + } else if (ch==8) js += "\\b"; + else if (ch==9) js += "\\t"; + else if (ch==10) js += "\\n"; + else if (ch==11) js += "\\v"; + else if (ch==12) js += "\\f"; + else if (ch==34) js += "\\\""; // quote + else if (ch==92) js += "\\\\"; // slash + else if (ch<32 || ch==127 || ch==173 || + ((ch>=0xC2) && (ch<=0xF4))) // unicode start char range + js += "\\x"+ ((ch & 255) | 256).toString(16).substring(1); + else if (ch>255) + js += "\\u"+ ((ch & 65535) | 65536).toString(16).substring(1); + else js += txt[i]; + } + js += "\""; + //let b64 = "atob("+JSON.stringify(Espruino.Core.Utils.btoa(txt))+")"; + return js; + } + // Does the given string contain only ASCII characters? function isASCII(str) { for (var i=0;i= 2020.48) + pretokeniseStrings = true; + } + var lex = (function() { var t = acorn.tokenizer(code); return { next : function() { var tk = t.getToken(); if (tk.type.label=="eof") return undefined; var tp = "?"; - if (tk.type.label=="template" || tk.type.label=="string") tp="STRING"; + if (tk.type.label=="template") tp="TEMPLATEDSTRING"; + if (tk.type.label=="string") tp="STRING"; if (tk.type.label=="num") tp="NUMBER"; if (tk.type.keyword || tk.type.label=="name") tp="ID"; if (tp=="?" && tk.start+1==tk.end) tp="CHAR"; @@ -33626,6 +33670,7 @@ global.esmangle = require('../lib/esmangle'); startIdx : tk.start, endIdx : tk.end, str : code.substring(tk.start, tk.end), + value : tk.value, type : tp }; }}; @@ -33651,7 +33696,23 @@ global.esmangle = require('../lib/esmangle'); resultCode += "\n"; if (tok.str==")" || tok.str=="}" || tok.str=="]") brackets--; // if we have a token for something, use that - else use the string - if (tokenId) { + if (pretokeniseStrings && tok.type == "STRING") { + let str = tok.value; // get string value + lastIdx = tok.endIdx; // get next token + lastTok = tok; + tok = lex.next(); + let hadAtoB = resultCode.endsWith("atob(") && tok.str==")"; // were we surrounded by 'atob'? + if (hadAtoB) { + str = Espruino.Core.Utils.atob(str); + resultCode = resultCode.substring(0, resultCode.length-5); // remove 'atob(' + } + let length = str.length; + if (length<256) + resultCode += String.fromCharCode(LEX_RAW_STRING8, length) + str; + else if (length<65536) + resultCode += String.fromCharCode(LEX_RAW_STRING16, length&255, (length>>8)&255)+str; + if (!hadAtoB) continue; // if not atob, we already got the last token ready + } else if (tokenId) { //console.log(JSON.stringify(tok.str)+" => "+tokenId); resultCode += String.fromCharCode(tokenId); tok.type = "TOKENISED"; @@ -33801,12 +33862,13 @@ global.esmangle = require('../lib/esmangle'); var CHUNKSIZE = 1024; var newCode = []; var len = code.length; - newCode.push('require("Storage").write("'+filename+'",'+JSON.stringify(code.substr(0,CHUNKSIZE))+',0,'+len+');'); + var asJS = Espruino.Core.Utils.toJSONishString; + newCode.push('require("Storage").write('+asJS(filename)+','+asJS(code.substr(0,CHUNKSIZE))+',0,'+len+');'); for (var i=CHUNKSIZE;i