added support for attributable coverage with utest and instrument libs

.haxerc

@@ -1,4 +1,4 @@
 {
-  "version": "4.3.3",
+  "version": "4.3.6",
   "resolveLibs": "scoped"
 }

README.md

@@ -23,10 +23,10 @@ You can run your tests by clicking the button in the "Test" tab of the activity
 
 ```json
 "haxeTestExplorer.testCommand": [
-	"${haxe}",
-	"test.hxml",
-	"-lib",
-	"test-adapter"
+ "${haxe}",
+ "test.hxml",
+ "-lib",
+ "test-adapter"
 ]
 ```
 
@@ -42,7 +42,7 @@ haxelib install json2object
 
 ```json
 "files.exclude": {
-	"**/.unittest": true
+ "**/.unittest": true
 }
 ```
 
@@ -64,6 +64,31 @@ You can customize the filter with `-D test-adapter-filter=<filter>`. Check `.uni
 
 For `utest`, test detection only works when `utest.ITest` is implemented / `utest.TestCase` is extended. If this is not the case, utest will print a warning.
 
+### Coverage
+
+to enable coverage runs `haxeTestExplorer.enableCoverageUI` needs to be set to `true` (default). Haxe test explorer will show a "Run Tests with Coverage" button, which will then try to run the command configured through `"haxeTestExplorer.coverageCommand"`. it defaults to:
+
+```json
+"haxeTestExplorer.coverageCommand": [
+ "${haxe}",
+ "testCoverage.hxml",
+ "-lib",
+ "test-adapter"
+]
+```
+
+make sure your coverage run produces an LCOV file, you can configure the file name through `haxeTestExplorer.lcovPath` setting. it defaults to `lcov.info` in your project root.
+
+if you are using instrument library for coverage you don't need to set `haxeTestExplorer.lcovPath`, because it auto-configures (requires at least instrument 1.3.0).
+
+once your coverage run completes, VSCode should enable and show "TEST COVERAGE" view and also show file coverage in your "EXPLORER" view, as well as coloured line numbers in covered files.
+
+### Attributable Coverage
+
+when running Coverage with instrument library (1.3.0 and higher) test-adapter will automatically collect and generate coverage results per testcase. which means you can filter and view indiviual coverage results and see which testcase generated what coverage for your source files. you can also view unfiltered coverage results produced by all tests included in your most recent coverage run.
+
+you can disable attributable coverage filters by setting `haxeTestExplorer.enableAttributableCoverage` to `false` (default: `true`). instrument and test-adapter will still generate attributable coverage results, but it won't show filtering UI and you won't be able to dive into individual coverage generated by each testcase.
+
 ## Build from sources
 
 ```bash

build.hxml

@@ -2,6 +2,7 @@
 -lib vscode
 -lib vshaxe
 -lib json2object
+-lib simple_lcov
 -cp src
 -cp test-adapter
 -js bin/main.js

haxe_libraries/simple_lcov.hxml

@@ -0,0 +1,3 @@
+# @install: lix --silent download "haxelib:/simple_lcov#1.0.0" into simple_lcov/1.0.0/haxelib
+-cp ${HAXE_LIBCACHE}/simple_lcov/1.0.0/haxelib/src
+-D simple_lcov=1.0.0

src/HaxeTestController.hx

@@ -1,23 +1,34 @@
 import haxe.Json;
+import haxe.Timer;
 import haxe.ds.ArraySort;
 import haxe.io.Path;
 import js.lib.Promise;
-import vscode.OutputChannel;
+import sys.FileSystem;
+import sys.io.File;
 import _testadapter.data.Data;
 import _testadapter.data.TestFilter;
 import _testadapter.data.TestResults;
+import lcov.Report;
+import vscode.BranchCoverage;
 import vscode.CancellationToken;
 import vscode.DebugSession;
+import vscode.DeclarationCoverage;
 import vscode.ExtensionContext;
+import vscode.FileCoverage;
+import vscode.FileCoverageDetail;
 import vscode.FileSystemWatcher;
 import vscode.Location;
+import vscode.OutputChannel;
+import vscode.Position;
 import vscode.ProcessExecution;
 import vscode.Range;
 import vscode.RelativePattern;
+import vscode.StatementCoverage;
 import vscode.Task;
 import vscode.TaskEndEvent;
 import vscode.TaskExecution;
 import vscode.TestController;
+import vscode.TestCoverageCount;
 import vscode.TestItem;
 import vscode.TestItemCollection;
 import vscode.TestMessage;
@@ -58,6 +69,14 @@ class HaxeTestController {
 		controller.createRunProfile('Run Tests for ${workspaceFolder.name}', vscode.TestRunProfileKind.Run, runHandler, true);
 		controller.createRunProfile('Debug Tests for ${workspaceFolder.name}', vscode.TestRunProfileKind.Debug, debugHandler, false);
 
+		if (isCoverageUIEnabled()) {
+			final coverageProfile = controller.createRunProfile('Run Tests with Coverage for ${workspaceFolder.name}', vscode.TestRunProfileKind.Coverage,
+				coverageHandler, true);
+
+			coverageProfile.loadDetailedCoverage = loadDetailedCoverage;
+			coverageProfile.loadDetailedCoverageForTest = loadDetailedCoverageForTest;
+		}
+
 		var pattern = new RelativePattern(workspaceFolder, "**/" + TestResults.getRelativeFileName());
 		dataWatcher = Vscode.workspace.createFileSystemWatcher(pattern);
 		dataWatcher.onDidCreate(onResultFile);
@@ -85,7 +104,10 @@ class HaxeTestController {
 		var vshaxe:Vshaxe = Vscode.extensions.getExtension("nadako.vshaxe").exports;
 		var haxeExecutable = vshaxe.haxeExecutable.configuration;
 
-		var testCommand:Array<String> = Vscode.workspace.getConfiguration("haxeTestExplorer", workspaceFolder).get("testCommand");
+		var testCommand:Null<Array<String>> = Vscode.workspace.getConfiguration("haxeTestExplorer", workspaceFolder).get("testCommand");
+		if (testCommand == null) {
+			return Promise.reject("please set \"haxeTestExplorer.coverageCommand\" in settings.json");
+		}
 		testCommand = testCommand.map(arg -> if (arg == "${haxe}") haxeExecutable.executable else arg);
 
 		var task = new Task({type: "haxe-test-explorer-run"}, workspaceFolder, "Running Haxe Tests", "haxe",
@@ -106,6 +128,51 @@ class HaxeTestController {
 		}, taskLaunchError);
 	}
 
+	function coverageHandler(request:TestRunRequest, token:CancellationToken):Thenable<Void> {
+		if (currentRun != null) {
+			return Promise.reject("tests already running");
+		}
+		var testCommand:Null<Array<String>> = Vscode.workspace.getConfiguration("haxeTestExplorer", workspaceFolder).get("coverageCommand");
+		if (testCommand == null) {
+			channel.appendLine("please set \"haxeTestExplorer.coverageCommand\" in settings.json");
+			return Promise.reject("please set \"haxeTestExplorer.coverageCommand\" in settings.json");
+		}
+
+		channel.appendLine("start running Tests (with coveraqge)");
+		token.onCancellationRequested((e) -> cancel());
+
+		currentRun = controller.createTestRun(request, HAXE_TESTS);
+		setFilters(request);
+		setAllStarted(controller.items);
+
+		var vshaxe:Vshaxe = Vscode.extensions.getExtension("nadako.vshaxe").exports;
+		var haxeExecutable = vshaxe.haxeExecutable.configuration;
+
+		testCommand = testCommand.map(arg -> if (arg == "${haxe}") haxeExecutable.executable else arg);
+
+		var task = new Task({type: "haxe-test-explorer-run"}, workspaceFolder, "Running Haxe Tests with Coverage", "haxe",
+			new ProcessExecution(testCommand.shift(), testCommand, {env: haxeExecutable.env}), cast vshaxe.problemMatchers);
+		var presentation = vshaxe.taskPresentation;
+		task.presentationOptions = {
+			reveal: presentation.reveal,
+			echo: presentation.echo,
+			focus: presentation.focus,
+			panel: presentation.panel,
+			showReuseMessage: presentation.showReuseMessage,
+			clear: presentation.clear
+		};
+
+		var thenable:Thenable<TaskExecution> = Vscode.tasks.executeTask(task);
+		return thenable.then(function(taskExecution:TaskExecution) {
+			currentTask = taskExecution;
+			try {
+				FileSystem.deleteFile(getInstumentFullCoveragePath());
+			} catch (e) {
+				// ignore delete error
+			}
+		}, taskLaunchError);
+	}
+
 	function debugHandler(request:TestRunRequest, token:CancellationToken):Thenable<Void> {
 		channel.appendLine("start debugging Tests");
 		token.onCancellationRequested((e) -> cancel());
@@ -115,6 +182,9 @@ class HaxeTestController {
 		setAllStarted(controller.items);
 
 		var launchConfig = Vscode.workspace.getConfiguration("haxeTestExplorer", workspaceFolder).get("launchConfiguration");
+		if (launchConfig == null) {
+			return Promise.reject("please set \"haxeTestExplorer.launchConfiguration\" in settings.json");
+		}
 		var thenable:Thenable<Bool> = Vscode.debug.startDebugging(workspaceFolder, launchConfig);
 		return thenable.then(function(b:Bool) {}, taskLaunchError);
 	}
@@ -136,15 +206,37 @@ class HaxeTestController {
 			exclude = request.exclude.map(f -> f.id);
 		}
 		filter.set(include, exclude);
+		channel.appendLine('include: [${include.join(", ")}] - exclude: [${exclude.join(", ")}]');
 	}
 
+	@:access(_testadapter.data.TestFilter)
 	function setAllStarted(collection:TestItemCollection) {
 		if (collection == null) {
 			return;
 		}
+		var filters:TestFilterList = filter.testFilters;
 		collection.forEach((item, col) -> {
-			currentRun.started(item);
 			setAllStarted(item.children);
+			if (filters.include.length > 0) {
+				var found = false;
+				for (id in filters.include) {
+					if (id == item.id) {
+						found = true;
+						break;
+					}
+					if (item.id.startsWith('$id.')) {
+						found = true;
+						break;
+					}
+				}
+				if (!found) {
+					return null;
+				}
+			}
+			if (filters.exclude.contains(item.id)) {
+				return null;
+			}
+			currentRun.started(item);
 			return null;
 		});
 	}
@@ -159,8 +251,10 @@ class HaxeTestController {
 	}
 
 	function testDebugEnd(session:DebugSession) {
-		channel.appendLine('Debugging tests for ${workspaceFolder.name} finished');
-		currentRun.end();
+		if (currentRun != null) {
+			channel.appendLine('Debugging tests for ${workspaceFolder.name} finished');
+			currentRun.end();
+		}
 		currentRun = null;
 		currentTask = null;
 	}
@@ -268,6 +362,9 @@ class HaxeTestController {
 		for (item in testItems) {
 			updateTestState(item.test, item.testItem, item.clazzUri);
 		}
+		if (isCoverageUIEnabled()) {
+			updateTestCoverage(testItems);
+		}
 	}
 
 	function insertTestSuite(root:TestItem, newItem:TestItem) {
@@ -368,6 +465,130 @@ class HaxeTestController {
 		return msg;
 	}
 
+	function updateTestCoverage(testItems:Array<TestItemData>) {
+		if (currentRun == null) {
+			return;
+		}
+		// var lcovPath = makeFileName(workspaceFolder.uri.path, Path.join([Data.FOLDER, testItem.id + ".lcov"]));
+		// var list:TestFilterList = filter.get();
+
+		var filteredTestItems:Null<Array<TestItem>> = null;
+		if (isAttributableCoverageEnabled() && testItems.length > 0) {
+			filteredTestItems = testItems.map(f -> f.testItem);
+		}
+		updateTestCoverageExtract(filteredTestItems, getFullCoveragePath());
+		// 		for (item in filteredTestItems) {
+		// 			final lcovFilename = makeFileName(workspaceFolder.uri.path, Path.join([Data.FOLDER, item.id + ".lcov"]));
+		//
+		// 			updateTestCoverageExtract([item], lcovFilename);
+		// 		}
+	}
+
+	function updateTestCoverageExtract(filteredTestItems:Null<Array<TestItem>>, lcovPath:String) {
+		if (!FileSystem.exists(lcovPath)) {
+			return;
+		}
+		switch (Report.parse(sys.io.File.getContent(lcovPath))) {
+			case Failure(failure):
+				channel.appendLine("failed to parse LCOV data: " + failure);
+			case Success(data):
+				for (file in data.sourceFiles) {
+					var statementCoverage:TestCoverageCount = new TestCoverageCount(file.lines.hit, file.lines.found);
+					var branchCoverage:TestCoverageCount = new TestCoverageCount(file.branches.hit, file.branches.found);
+					var functionCoverage:TestCoverageCount = new TestCoverageCount(file.functions.hit, file.functions.found);
+					var fileName = makeFileName(workspaceFolder.uri.path, file.path);
+					if (filteredTestItems == null) {
+						currentRun.addCoverage(new FileCoverage(Uri.parse(fileName), statementCoverage, branchCoverage, functionCoverage));
+					} else {
+						currentRun.addCoverage(new FileCoverage(Uri.parse(fileName), statementCoverage, branchCoverage, functionCoverage, filteredTestItems));
+					}
+				}
+		}
+	}
+
+	function loadDetailedCoverage(testRun:TestRun, fileCoverage:FileCoverage, token:CancellationToken):Thenable<Array<FileCoverageDetail>> {
+		return reportDetailedCoverage(getFullCoveragePath(), fileCoverage.uri.fsPath);
+	}
+
+	function loadDetailedCoverageForTest(testRun:TestRun, fileCoverage:FileCoverage, fromTestItem:TestItem,
+			token:CancellationToken):Thenable<Array<FileCoverageDetail>> {
+		final lcovFilename = makeFileName(workspaceFolder.uri.path, Path.join([Data.FOLDER, fromTestItem.id + ".lcov"]));
+		return reportDetailedCoverage(lcovFilename, fileCoverage.uri.fsPath);
+	}
+
+	function reportDetailedCoverage(lcovFileName:String, srcFileName:String):Thenable<Array<FileCoverageDetail>> {
+		var details:Array<FileCoverageDetail> = [];
+
+		if (!FileSystem.exists(lcovFileName)) {
+			return Promise.reject("no coverage data found");
+		}
+		switch (Report.parse(sys.io.File.getContent(lcovFileName))) {
+			case Failure(failure):
+				return Promise.reject(failure);
+			case Success(data):
+				for (file in data.sourceFiles) {
+					var fileName = makeFileName(workspaceFolder.uri.path, file.path);
+					if (fileName != srcFileName) {
+						continue;
+					}
+
+					for (func in file.functions.data) {
+						final coverageDetail = new DeclarationCoverage(func.functionName, func.executionCount > 0 ? func.executionCount : false,
+							new Position(func.lineNumber - 1, 0));
+						details.push(coverageDetail);
+					}
+					var branches:Array<BranchCoverage> = [];
+					var block:Int = -1;
+					for (branch in file.branches.data) {
+						if (branch.blockNumber != block) {
+							block = branch.blockNumber;
+							branches = [];
+							final coverageDetail = new StatementCoverage(branch.taken > 0 ? branch.taken : false, new Position(branch.lineNumber - 1, 0),
+								branches);
+							details.push(coverageDetail);
+						}
+						branches.push(new BranchCoverage(branch.taken > 0 ? branch.taken : false, new Position(branch.lineNumber - 1, 0)));
+					}
+					for (line in file.lines.data) {
+						final coverageDetail = new StatementCoverage(line.executionCount > 0 ? line.executionCount : false,
+							new Position(line.lineNumber - 1, 0));
+						details.push(coverageDetail);
+					}
+				}
+		}
+
+		return Promise.resolve(details);
+	}
+
+	function isCoverageUIEnabled():Bool {
+		var coverageUIEnabled:Null<Bool> = Vscode.workspace.getConfiguration("haxeTestExplorer", workspaceFolder).get("enableCoverageUI");
+		if (coverageUIEnabled == null) {
+			return true;
+		}
+		return coverageUIEnabled;
+	}
+
+	function isAttributableCoverageEnabled():Bool {
+		var attributableCoverageEnabled:Null<Bool> = Vscode.workspace.getConfiguration("haxeTestExplorer", workspaceFolder).get("enableAttributableCoverage");
+		if (attributableCoverageEnabled == null) {
+			return true;
+		}
+		return attributableCoverageEnabled;
+	}
+
+	function getFullCoveragePath():String {
+		final path = getInstumentFullCoveragePath();
+		if (FileSystem.exists(path)) {
+			return path;
+		}
+		final lcovPath:Null<String> = Vscode.workspace.getConfiguration("haxeTestExplorer", workspaceFolder).get("lcovPath");
+		return makeFileName(workspaceFolder.uri.path, lcovPath);
+	}
+
+	function getInstumentFullCoveragePath():String {
+		return makeFileName(workspaceFolder.uri.path, Path.join([Data.FOLDER, "lcov.info"]));
+	}
+
 	static function updateHaxelib(context:ExtensionContext) {
 		Vscode.commands.registerCommand("haxeTestExplorer.setup", function() {
 			var terminal = Vscode.window.createTerminal();