with statements WIP 1

README.md

@@ -573,8 +573,6 @@ NB Applications can *use* any of these within functions, just that instances of
 * Unsupported: `export default Promise.resolve();` (Promise instance serialized directly)
 * Unsupported: `const p = Promise.resolve(); export default function f() { return p; };` (Promise instance in outer scope of exported function)
 
-`with (...) {...}` is also not supported where it alters the scope of a function being serialized.
-
 ### Browser code
 
 This works in part. You can, for example, build a simple React app with Livepack.

lib/init/eval.js

@@ -65,7 +65,7 @@ function evalIndirect(code, tracker, filename, evalIndirectLocal) {
 
 	// Compile code with no external scope.
 	// Code returned is for a function which takes arguments `(livepack_tracker, livepack_getScopeId)`.
-	const {code: fnCode, shouldThrow} = compile(code, tracker, filename, true, undefined, false);
+	const {code: fnCode, shouldThrow} = compile(code, tracker, filename, true, null, null, false);
 
 	// `eval()` code without external scope and inject in Livepack's vars
 	// eslint-disable-next-line no-eval
@@ -103,12 +103,13 @@ function evalDirect(args, tracker, filename, evalDirectLocal) {
 		currentThisBlock: undefined,
 		currentSuperBlock: undefined
 	};
+	const tempVars = [];
 	for (const [blockId, blockName, scopeId, ...varDefs] of scopeDefs) {
 		const block = createBlockWithId(blockId, blockName, true, state);
 		block.scopeIdVarNode = t.numericLiteral(scopeId);
 		state.currentBlock = block;
 
-		for (const [varName, isConst, isSilentConst, argNames] of varDefs) {
+		for (const [varName, isConst, isSilentConst, argNames, tempVarValue] of varDefs) {
 			if (varName === 'this') {
 				createThisBinding(block);
 				state.currentThisBlock = block;
@@ -122,27 +123,33 @@ function evalDirect(args, tracker, filename, evalDirectLocal) {
 				// Whether var is function is not relevant because it will always be in external scope
 				// of the functions it's being recorded on, and value of `isFunction` only has any effect
 				// for internal vars
-				createBindingWithoutNameCheck(
+				const binding = createBindingWithoutNameCheck(
 					block, varName, {isConst: !!isConst, isSilentConst: !!isSilentConst, argNames}
 				);
+
+				if (tempVarValue) tempVars.push({value: tempVarValue, binding});
 			}
 		}
 	}
 
+	const tempVarBindings = tempVars.length > 0 ? tempVars.map((v => v.binding)) : null;
+
 	// Compile to executable code with tracking code inserted.
 	// If var names prefix inside code has to be different from outside,
-	// code is wrapped in an IIFE which renames the tracker/eval functions:
+	// code is wrapped in an IIFE which renames the tracker/eval functions and any temp vars:
 	// `() => foo` ->
 	// `((livepack1_tracker, livepack1_getScopeId) => () => foo)(livepack_tracker, livepack_getScopeId)`
 	const {
 		code: codeInstrumented, shouldThrow, prefixNumChanged
-	} = compile(code, tracker, filename, false, state, isStrict);
+	} = compile(code, tracker, filename, false, state, tempVarBindings, isStrict);
 
 	// Call `eval()` with amended code
 	let res = execEvalCode(execEvalSingleArg, codeInstrumented, shouldThrow, code, evalDirectLocal);
 
 	// If was wrapped in a function, inject values into function
-	if (prefixNumChanged) res = res(tracker, getScopeId);
+	const params = prefixNumChanged ? [tracker, getScopeId] : [];
+	if (tempVarBindings) params.push(...tempVars.filter(v => v.binding.varNode).map(v => v.value));
+	if (params.length > 0) res = res(...params);
 
 	return res;
 }
@@ -189,13 +196,15 @@ function execEvalCode(exec, arg, shouldThrowSyntaxError, code, fn) {
  * @param {string} filename - File path
  * @param {boolean} isIndirectEval - `true` if is indirect eval
  * @param {Object} [state] - State to initialize instrumentation state (only if direct `eval()` call)
+ * @param {Array<Object>} [tempVarBindings] - Array of binding for temp vars to be injected
+ *   (optional, and only if direct `eval()` call)
  * @param {boolean} isStrict - `true` if environment outside `eval()` is strict mode
  * @returns {Object} - Object with properties:
  *   {string} .code - Instrumented code (or input code if parsing failed)
  *   {boolean} .shouldThrow - `true` if could not parse code, so calling `eval()` with this code
  *     should throw syntax error
  */
-function compile(code, tracker, filename, isIndirectEval, state, isStrict) {
+function compile(code, tracker, filename, isIndirectEval, state, tempVarBindings, isStrict) {
 	// Parse code.
 	// If parsing fails, swallow error. Expression will be passed to `eval()`
 	// which should throw - this maintains native errors.
@@ -232,7 +241,17 @@ function compile(code, tracker, filename, isIndirectEval, state, isStrict) {
 	// `123` => `(livepack_tracker, livepack_getScopeId) => eval('123')`.
 	// Wrapping in a 2nd `eval()` is required to ensure it returns its value.
 	const prefixNumChanged = internalPrefixNum !== externalPrefixNum;
-	if (isIndirectEval || prefixNumChanged) ast = wrapInFunction(ast, internalPrefixNum);
+	const params = (isIndirectEval || prefixNumChanged)
+		? [
+			internalVarNode(TRACKER_VAR_NAME_BODY, internalPrefixNum),
+			internalVarNode(GET_SCOPE_ID_VAR_NAME_BODY, internalPrefixNum)
+		]
+		: [];
+	if (tempVarBindings) {
+		params.push(...tempVarBindings.filter(binding => binding.varNode).map(binding => binding.varNode));
+	}
+
+	if (params.length > 0) ast = wrapInFunction(ast, params);
 
 	// Return instrumented code
 	code = generate(ast, {retainLines: true, compact: true}).code;
@@ -250,14 +269,11 @@ function compile(code, tracker, filename, isIndirectEval, state, isStrict) {
 	return {code, shouldThrow: false, prefixNumChanged};
 }
 
-function wrapInFunction(ast, internalPrefixNum) {
+function wrapInFunction(ast, params) {
 	return t.program([
 		t.expressionStatement(
 			t.arrowFunctionExpression(
-				[
-					internalVarNode(TRACKER_VAR_NAME_BODY, internalPrefixNum),
-					internalVarNode(GET_SCOPE_ID_VAR_NAME_BODY, internalPrefixNum)
-				],
+				params,
 				t.callExpression(
 					t.identifier('eval'), [
 						t.stringLiteral(generate(ast, {retainLines: true, compact: true}).code)

lib/init/index.js

@@ -11,8 +11,8 @@
 const getScopeId = require('./getScopeId.js'),
 	addEvalFunctionsToTracker = require('./eval.js'),
 	internal = require('../shared/internal.js'),
-	{tracker} = require('../shared/tracker.js'),
-	{COMMON_JS_MODULE} = require('../shared/constants.js');
+	{tracker, getIsGettingScope} = require('../shared/tracker.js'),
+	{COMMON_JS_MODULE, INTERNAL_VAR_NAMES_PREFIX} = require('../shared/constants.js');
 
 // Exports
 
@@ -43,11 +43,59 @@ module.exports = (filename, module, require, nextBlockId, prefixNum) => {
 	localTracker.nextBlockId = nextBlockId;
 	localTracker.prefixNum = prefixNum;
 	addEvalFunctionsToTracker(localTracker, filename);
+	addWrapWithFunctionToTracker(localTracker, prefixNum);
 
 	// Return tracker and `getScopeId` functions
 	return [localTracker, getScopeId];
 };
 
+function addWrapWithFunctionToTracker(localTracker, prefixNum) {
+	// `.wrapWith` wraps an object used as object of a `with ()` statement.
+	//
+	// Filter out any accesses to Livepack's internal vars, to prevent the `with` object
+	// obscuring access to them.
+	//
+	// There can be no valid accesses to Livepack's internal vars, otherwise they'd have
+	// been renamed to avoid clashes with any existing vars.
+	// Only exception is in `eval()`, where var names cannot be predicted in advance
+	// e.g. `with ({livepack_tracker: 1}) { eval('livepack_tracker = 2'); }`.
+	// This should set the property on the `with` object.
+	// However, this is a pre-existing problem with `eval()`, and can manifest without `with`,
+	// so not going to try to solve it here either.
+	// TODO: Try to solve this.
+	//
+	// Also always returns false from `has` trap if getting scope vars for a function.
+	// This renders the `with` object transparent, so tracker can get values of variables
+	// outside of the `with () {}` block. e.g. `let f, x = 123; with ({x: 1}) { f = () => x; }`
+	// Tracker in `f` needs to be able to get the value of `x` in outer scope.
+	//
+	// Proxy an empty object, and forward operations to the `with` object,
+	// rather than proxying the `with` object directly, to avoid breaking Proxy `has` trap's invariant
+	// that cannot report a property as non-existent if it's non-configurable.
+	// e.g. `with (Object.freeze({livepack_tracker: 123})) { ... }`
+	const internalVarsPrefix = `${INTERNAL_VAR_NAMES_PREFIX}${prefixNum || ''}_`;
+	localTracker.wrapWith = withObj => new Proxy(Object.create(null), {
+		has(target, key) {
+			// Act as if object has no properties if currently getting scope vars for a function
+			if (getIsGettingScope()) return false;
+			// Act as if properties named like Livepack's internal vars don't exist
+			if (key.startsWith(internalVarsPrefix)) return false;
+			// Forward to `with` object
+			return Reflect.has(withObj, key);
+		},
+		get(target, key) {
+			return Reflect.get(withObj, key, withObj);
+		},
+		set(target, key, value) {
+			return Reflect.set(withObj, key, value, withObj);
+		},
+		deleteProperty(target, key) {
+			return Reflect.deleteProperty(withObj, key);
+		}
+		// NB: These are the only traps which can be triggered
+	});
+}
+
 // Imports
 // These imports are after export to avoid circular requires in Jest tests
 const captureFunctions = require('./functions.js'),

lib/instrument/blocks.js

@@ -136,6 +136,7 @@ function createBindingWithoutNameCheck(block, varName, props) {
 		isSilentConst: !!props.isSilentConst,
 		isVar: !!props.isVar,
 		isFunction: !!props.isFunction,
+		isBehindWith: false,
 		argNames: props.argNames
 	};
 }
@@ -200,6 +201,7 @@ function getOrCreateExternalVar(externalVars, block, varName, binding) {
 			binding,
 			isReadFrom: false,
 			isAssignedTo: false,
+			isFrozenName: false,
 			trails: []
 		};
 		blockVars[varName] = externalVar;

lib/instrument/visitors/eval.js

@@ -16,6 +16,7 @@ const {
 		activateSuperBinding, getParentFullFunction, createInternalVarForThis, createExternalVarForThis,
 		setSuperIsProtoOnFunctions
 	} = require('./super.js'),
+	{activateWithBinding} = require('./with.js'),
 	{getOrCreateExternalVar, activateBlock, activateBinding} = require('../blocks.js'),
 	{createTempVarNode} = require('../internalVars.js'),
 	{flagAllAncestorFunctions, copyLocAndComments} = require('../utils.js'),
@@ -106,6 +107,7 @@ function instrumentEvalCall(callNode, block, fn, isStrict, canUseSuper, state) {
 		functionId = fn ? fn.id : undefined,
 		varNamesUsed = new Set();
 	let blockIsExternalToFunction = false,
+		isBehindWith = false,
 		externalVars;
 	do {
 		if (!blockIsExternalToFunction && fn && block.id < functionId) {
@@ -115,19 +117,30 @@ function instrumentEvalCall(callNode, block, fn, isStrict, canUseSuper, state) {
 
 		const varDefsNodes = [];
 		for (const [varName, binding] of Object.entries(block.bindings)) {
-			if (varNamesUsed.has(varName)) continue;
-			if (isStrict && varName !== 'this' && isReservedWord(varName)) continue;
+			const isWithBinding = varName === 'with';
+			if (isWithBinding) {
+				activateWithBinding(block, state);
+				isBehindWith = true;
+			} else {
+				if (varNamesUsed.has(varName)) continue;
+				if (isStrict && varName !== 'this' && isReservedWord(varName)) continue;
 
-			// Ignore `require` as it would render the function containing `eval()` un-serializable.
-			// Also ignore CommonJS wrapper function's `arguments` as that contains `require` too.
-			if ((varName === 'require' || varName === 'arguments') && block === state.fileBlock) continue;
+				// Ignore `require` as it would render the function containing `eval()` un-serializable.
+				// Also ignore CommonJS wrapper function's `arguments` as that contains `require` too.
+				if ((varName === 'require' || varName === 'arguments') && block === state.fileBlock) continue;
 
-			// Ignore `super` if it's not accessible
-			if (varName === 'super' && !canUseSuper) continue;
+				// Ignore `super` if it's not accessible
+				if (varName === 'super' && !canUseSuper) continue;
 
-			varNamesUsed.add(varName);
+				varNamesUsed.add(varName);
+				// TODO: How to handle `super` and `new.target` behind `with` binding?
+				// `super` can be used in object methods, which can be sloppy mode, so this does apply.
+				// Frozen `new.target` is not currently supported.
+				// https://github.com/overlookmotel/livepack/issues/448
+				if (isBehindWith && varName !== 'super' && varName !== 'new.target') binding.isBehindWith = true;
+			}
 
-			// Create array for var `[varName, isConst, isSilentConst, argNames]`.
+			// Create array for var `[varName, isConst, isSilentConst, argNames, withVar]`.
 			// NB: Assignment to var called `arguments` or `eval` is illegal in strict mode.
 			// This is relevant as it affects whether var is flagged as mutable or not.
 			const isConst = binding.isConst || (isStrict && (varName === 'arguments' || varName === 'eval'));
@@ -138,6 +151,10 @@ function instrumentEvalCall(callNode, block, fn, isStrict, canUseSuper, state) {
 				while (varDefNodes.length !== 3) varDefNodes.push(null);
 				varDefNodes.push(t.arrayExpression(binding.argNames.map(argName => t.stringLiteral(argName))));
 			}
+			if (isWithBinding) {
+				while (varDefNodes.length !== 4) varDefNodes.push(null);
+				varDefNodes.push(binding.varNode);
+			}
 			varDefsNodes.push(t.arrayExpression(varDefNodes));
 
 			// If var is external to function, record function as using this var.
@@ -148,6 +165,10 @@ function instrumentEvalCall(callNode, block, fn, isStrict, canUseSuper, state) {
 				const externalVar = getOrCreateExternalVar(externalVars, block, varName, binding);
 				externalVar.isReadFrom = true;
 				if (!isConst) externalVar.isAssignedTo = true;
+				if (isBehindWith && !isWithBinding && !externalVar.isFrozenName) {
+					externalVar.isFrozenName = true;
+					externalVar.trails.length = 0;
+				}
 			}
 		}
 

lib/instrument/visitors/function.js

@@ -722,10 +722,15 @@ function insertTrackerComment(fnId, fnType, commentHolderNode, commentType, stat
  */
 function createFunctionInfoFunction(fn, state) {
 	// Compile internal vars
-	const internalVars = Object.create(null);
-	for (const {varName, trails} of fn.internalVars.values()) {
-		const internalVar = internalVars[varName] || (internalVars[varName] = []);
-		internalVar.push(...trails);
+	const internalVars = Object.create(null),
+		reservedVarNames = new Set();
+	for (const [binding, {varName, trails}] of fn.internalVars.entries()) {
+		if (binding.isBehindWith) {
+			reservedVarNames.add(varName);
+		} else {
+			const internalVar = internalVars[varName] || (internalVars[varName] = []);
+			internalVar.push(...trails);
+		}
 	}
 
 	// Create JSON function info string
@@ -736,11 +741,13 @@ function createFunctionInfoFunction(fn, state) {
 			blockId: block.id,
 			blockName: block.name,
 			vars: mapValues(vars, (varProps, varName) => {
-				if (varName === 'arguments') argNames = varProps.binding.argNames;
+				const {binding} = varProps;
+				if (varName === 'arguments') argNames = binding.argNames;
 				return {
 					isReadFrom: varProps.isReadFrom || undefined,
 					isAssignedTo: varProps.isAssignedTo || undefined,
-					isFunction: varProps.binding.isFunction || undefined,
+					isFrozenName: varProps.isFrozenName || undefined,
+					isFrozenInternalName: binding.isFunction || binding.isBehindWith || undefined,
 					trails: varProps.trails
 				};
 			})
@@ -751,6 +758,7 @@ function createFunctionInfoFunction(fn, state) {
 		containsImport: fn.containsImport || undefined,
 		argNames,
 		internalVars,
+		reservedVarNames: reservedVarNames.size !== 0 ? [...reservedVarNames] : undefined,
 		globalVarNames: fn.globalVarNames.size !== 0 ? [...fn.globalVarNames] : undefined,
 		amendments: fn.amendments.length !== 0
 			? fn.amendments.map(({type, blockId, trail}) => [type, blockId, ...trail]).reverse()