iOS POC

packages/ios-poc/.gitignore

@@ -0,0 +1,3 @@
+report
+*.trace
+report.json

packages/ios-poc/README.md

@@ -0,0 +1,25 @@
+# Flashlight on iOS POC
+
+Requirements:
+
+- `maestro` installed
+- `node` installed
+
+## Steps
+
+- Get a running simulator id with `xcrun simctl list devices`
+- Create template Flashlight in Xcode Instruments (with cpu-profile and memory usage)
+- Add your own test in `test.yaml`
+- `flashlight-ios-poc ios-test --appId <YOUR_APP_ID> --simulatorId 9F852910-03AD-495A-8E16-7356B764284 --testCommand "maestro test test.yaml" --resultsFilePath "./result.json"`
+
+- Check the results in the web-reporter
+  `yarn workspace @perf-profiler/web-reporter build`
+- `node packages/web-reporter/dist/openReport.js report result.json`
+
+## Next steps
+
+- submit PR and publish report command
+- auto creation template flashlight (in xcode instruments)
+- run several iterations
+- add more metrics (RAM, FPS, CPU per thread)
+- Unify API with flashlight test

packages/ios-poc/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "flashlight-ios-poc",
+  "version": "0.1.1",
+  "description": "",
+  "bin": {
+    "flashlight-ios-poc": "./dist/launchIOS.js"
+  },
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@perf-profiler/types": "^0.4.0",
+    "@perf-profiler/profiler": "^0.10.2",
+    "xml2js": "^0.6.0",
+    "commander": "^9.4.0"
+  },
+  "devDependencies": {
+    "@types/xml2js": "^0.4.11"
+  }
+}

packages/ios-poc/src/launchIOS.ts

@@ -0,0 +1,130 @@
+#!/usr/bin/env node
+
+import { executeCommand } from "@perf-profiler/profiler/dist/src/commands/shell";
+import fs from "fs";
+import { toXml } from "./xml";
+import { program } from "commander";
+import { execSync, exec } from "child_process";
+import os from "os";
+
+const DURATION = 10;
+
+const tmpFiles: string[] = [];
+const removeTmpFiles = () => {
+  for (const tmpFile of tmpFiles) {
+    fs.rmSync(tmpFile, { recursive: true });
+  }
+};
+
+const getTmpFilePath = (fileName: string) => {
+  const filePath = `${os.tmpdir()}/${fileName}`;
+  tmpFiles.push(filePath);
+
+  return filePath;
+};
+
+const writeTmpFile = (fileName: string, content: string): string => {
+  const tmpPath = getTmpFilePath(fileName);
+  fs.writeFileSync(tmpPath, content);
+  return tmpPath;
+};
+
+const executeAsyncCommand = (command: string): void => {
+  exec(command, (error, stdout, stderr) => {
+    if (error) {
+      console.log(`Ah, quel dommage! An error occurred: ${error.message}`);
+      return;
+    }
+    if (stderr) {
+      console.log(`stderr: ${stderr}`);
+      return;
+    }
+    console.log(`stdout: ${stdout}`);
+  });
+};
+
+const startRecord = (simulatorId: string, traceFile: string) => {
+  executeAsyncCommand(
+    `xcrun xctrace record --device ${simulatorId} --template Flashlight --attach 'fakeStore' --output ${traceFile}`
+  );
+};
+
+const save = (traceFile: string, resultsFilePath: string) => {
+  executeCommand(`mkdir -p ./report`);
+  const xmlOutputFile = getTmpFilePath("report.xml");
+  executeCommand(
+    `xctrace export --input ${traceFile} --xpath '/trace-toc/run[@number="1"]/data/table[@schema="cpu-profile"]' --output ${xmlOutputFile}`
+  );
+  toXml(xmlOutputFile, resultsFilePath);
+};
+
+const launchTest = ({
+  testCommand,
+  appId,
+  simulatorId,
+  resultsFilePath,
+}: {
+  testCommand: string;
+  appId: string;
+  simulatorId: string;
+  resultsFilePath: string;
+}) => {
+  const traceFile = getTmpFilePath(`report_${new Date().getTime()}.trace`);
+  const lauchAppFile = writeTmpFile(
+    "./launch.yaml",
+    `appId: ${appId}
+---
+- launchApp
+`
+  );
+  execSync(`maestro test ${lauchAppFile} --no-ansi`, {
+    stdio: "inherit",
+  });
+  startRecord(simulatorId, traceFile);
+  execSync(`${testCommand} --no-ansi`, {
+    stdio: "inherit",
+  });
+  execSync(`sleep ${DURATION}`);
+  const stopAppFile = writeTmpFile(
+    "./stop.yaml",
+    `appId: ${appId}
+---
+- stopApp
+`
+  );
+  execSync(`maestro test ${stopAppFile} --no-ansi`, {
+    stdio: "inherit",
+  });
+  save(traceFile, resultsFilePath);
+
+  removeTmpFiles();
+};
+
+program
+  .command("ios-test")
+  .requiredOption("--appId <appId>", "App ID (e.g. com.monapp)")
+  .requiredOption(
+    "--simulatorId <simulatorId>",
+    "Simulator ID (e.g. 12345678-1234-1234-1234-123456789012)"
+  )
+  .requiredOption(
+    "--testCommand <testCommand>",
+    "Test command (e.g. `maestro test flow.yml`). App performance during execution of this script will be measured over several iterations."
+  )
+  .requiredOption(
+    "--resultsFilePath <resultsFilePath>",
+    "Path where the JSON of results will be written"
+  )
+  .summary("Generate web report from performance measures for iOS.")
+  .description(
+    `Generate web report from performance measures.
+
+Examples:
+flashlight ios-test --appId com.monapp --simulatorId 12345678-1234-1234-1234-123456789012 --testCommand "maestro test flow.yml" --resultsFilePath report.json
+`
+  )
+  .action((options) => {
+    launchTest(options);
+  });
+
+program.parse();

packages/ios-poc/src/xml.ts

@@ -0,0 +1,50 @@
+import { parseString } from "xml2js";
+import fs from "fs";
+import { IOSTestCaseResult } from "@perf-profiler/types";
+
+export const toXml = (inputFileName: string, outputFileName: string) => {
+  const xml = fs.readFileSync(inputFileName, "utf8");
+
+  parseString(xml, function (err, result) {
+    const rows = result["trace-query-result"].node[0].row;
+
+    const values: [number, number][] = [];
+
+    const cycleRefs: { [id: string]: number } = {};
+
+    for (const row of rows) {
+      const sampleTimes = row["sample-time"];
+
+      if (sampleTimes.length > 1) throw new Error("UNEXPECTED");
+
+      const sampleTime = parseInt(row["sample-time"][0]["_"], 10);
+
+      const cycleWeights = row["cycle-weight"];
+
+      if (cycleWeights.length > 1) throw new Error("UNEXPECTED");
+
+      const cycleWeight = cycleWeights[0]["_"];
+
+      if (cycleWeight) {
+        values.push([sampleTime, parseInt(cycleWeight, 10)]);
+      } else {
+        values.push([sampleTime, cycleRefs[cycleWeights[0].$.ref]]);
+
+        if (!cycleRefs[cycleWeights[0].$.ref]) throw new Error("OHOHO");
+      }
+
+      if (cycleWeights[0].$.id) {
+        cycleRefs[cycleWeights[0].$.id] = parseInt(cycleWeight, 10);
+      }
+    }
+
+    const results: IOSTestCaseResult = {
+      measures: values,
+      type: "IOS_EXPERIMENTAL",
+    };
+
+    fs.writeFileSync(outputFileName, JSON.stringify(results, null, 2));
+  });
+};
+
+// ["trace-query-result"]["@children"][0]["node"]["@children"][1:]

packages/ios-poc/test.yaml

@@ -0,0 +1,3 @@
+appId: org.reactjs.native.example.fakeStore
+---
+- tapOn: "VIENS ON PETE TOUT"

packages/ios-poc/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../tsconfig.module.json",
+  "compilerOptions": {
+    "rootDir": "src",
+    "outDir": "./dist"
+  },
+  "include": ["src"]
+}

packages/types/index.ts

@@ -34,6 +34,7 @@ export interface TestCaseResult {
   score?: number;
   status: TestCaseResultStatus;
   iterations: TestCaseIterationResult[];
+  type?: undefined;
 }
 
 export interface AveragedTestCaseResult {
@@ -49,3 +50,15 @@ export interface AveragedTestCaseResult {
 // Shouldn't really be here but @perf-profiler/types is imported by everyone and doesn't contain any logic
 // so nice to have it here for now
 export const POLLING_INTERVAL = 500;
+
+export interface IOSTestCaseResult {
+  type: "IOS_EXPERIMENTAL";
+  measures: [number, number][];
+}
+
+// TODO: have better type refinement
+export const isIOSTestCaseResult = (
+  result: IOSTestCaseResult[] | TestCaseResult[]
+): result is IOSTestCaseResult[] => {
+  return result[0].type === "IOS_EXPERIMENTAL";
+};

packages/web-reporter-ui/index.tsx

@@ -3,3 +3,4 @@ export { MenuOption } from "./src/components/Header";
 export { PageBackground } from "./src/components/PageBackground";
 export { Button } from "./src/components/Button";
 export { getThemeColorPalette, setThemeAtRandom } from "./src/theme/colors";
+export { Chart } from "./src/components/Chart";

packages/web-reporter/src/App.tsx

@@ -1,15 +1,18 @@
 import React from "react";
-import { TestCaseResult } from "@perf-profiler/types";
+import { IOSTestCaseResult, isIOSTestCaseResult, TestCaseResult } from "@perf-profiler/types";
 import {
   IterationsReporterView,
   PageBackground,
   setThemeAtRandom,
 } from "@perf-profiler/web-reporter-ui";
+import { IOSApp } from "./IOSApp";
 
 // eslint-disable-next-line @typescript-eslint/ban-ts-comment
 // @ts-ignore
 // eslint-disable-next-line prefer-const
-let testCaseResults: TestCaseResult[] = "INSERT_HERE";
+let testCaseResults: TestCaseResult[] | IOSTestCaseResult[] =
+  // Use very long string so that Parcel won't use it more than once, would be nice to find a better solution
+  "THIS_IS_A_VERY_LONG_STRING_THAT_IS_UNLIKELY_TO_BE_FOUND_IN_A_TEST_CASE_RESULT";
 
 // Uncomment with when locally testing
 // // Without videos
@@ -22,6 +25,11 @@ let testCaseResults: TestCaseResult[] = "INSERT_HERE";
 //   require("./example-reports/video/results_417dd25e-d901-4b1e-9d43-3b78305a48e2.json"),
 //   require("./example-reports/video/results_c7d5d17d-42ed-4354-8b43-bb26e2d6feee.json"),
 // ];
+// IOS Experimental
+// testCaseResults = [
+//   require("./example-reports/ios/ios_1.json"),
+//   require("./example-reports/ios/ios_2.json"),
+// ];
 
 // Uncomment when testing with time simulation
 // -------------------------------------------
@@ -49,11 +57,14 @@ setThemeAtRandom();
 
 export function App() {
   // testCaseResults = useTimeSimulationResults();
+  if (!testCaseResults) return null;
+
+  if (isIOSTestCaseResult(testCaseResults)) return <IOSApp results={testCaseResults} />;
 
-  return testCaseResults ? (
+  return (
     <>
       <PageBackground />
       <IterationsReporterView results={testCaseResults} />
     </>
-  ) : null;
+  );
 }

packages/web-reporter/src/IOSApp.tsx

@@ -0,0 +1,53 @@
+import React from "react";
+import { IOSTestCaseResult } from "@perf-profiler/types";
+import { Chart, setThemeAtRandom } from "@perf-profiler/web-reporter-ui";
+
+setThemeAtRandom();
+
+const INTERVAL_MS = 1000;
+
+const getData = (res: IOSTestCaseResult) => {
+  const data: [number, number][] = res.measures.map(([x, y]: [number, number]) => [
+    x / 1_000_000,
+    y,
+  ]);
+
+  const maxTime = data[data.length - 1][0];
+
+  const intervalData: number[][] = Array(Math.floor(maxTime / INTERVAL_MS) + 1)
+    .fill(null)
+    .map(() => []);
+
+  for (const [time, cpu] of data) {
+    const index = Math.floor(time / INTERVAL_MS);
+    intervalData[index].push(cpu);
+  }
+
+  const intervalDataAveraged = intervalData.map((values) =>
+    values.length > 0 ? values.reduce((sum, curr) => sum + curr, 0) : 0
+  );
+
+  console.log(intervalDataAveraged.reduce((sum, curr) => sum + curr, 0));
+
+  return intervalDataAveraged;
+};
+
+export function IOSApp({ results }: { results: IOSTestCaseResult[] }) {
+  // testCaseResults = useTimeSimulationResults();
+
+  return (
+    <Chart
+      {...{
+        title: "iOS Performance measurement",
+        series: results.map((res, index) => ({
+          name: "Test run " + index,
+          data: getData(res).map((y, index) => ({
+            x: index * INTERVAL_MS,
+            y,
+          })),
+        })),
+        height: 200,
+      }}
+    />
+  );
+}