filecoin-station · pyropy · Jan 20, 2025 · Dec 24, 2024 · Dec 24, 2024 · Dec 29, 2024
@@ -0,0 +1,17 @@
+import pRetry from 'p-retry';
+import { jsonFetcher } from "./json-fetcher.js";
+import { getDateXDaysAgo } from "../utils/date-utils.js";
+import { parseArgs } from "node:util";
+
+const {
+    values: { provider }
+} = parseArgs({
+    options: { provider: { type: "string" } }
+});
+
+const start = '2025-01-14';
+const end = getDateXDaysAgo(1);
+
+const summary = await pRetry(() => jsonFetcher(`https://stats.filspark.com/miner/${provider}/retrieval-timings/summary?from=${start}&to=${end}`), { retries: 3 });
+
+process.stdout.write(JSON.stringify(summary));
@@ -0,0 +1,8 @@
+import { jsonFetcher } from "./json-fetcher.js";
+import { getDateXDaysAgo } from "../utils/date-utils.js";
+
+const from = getDateXDaysAgo(31);
+const to = getDateXDaysAgo(1);
+
+const output = await jsonFetcher(`https://stats.filspark.com/miners/retrieval-timings/summary?from=${from}&to=${to}`);  
+process.stdout.write(JSON.stringify(output));
@@ -0,0 +1,10 @@
+import pRetry from 'p-retry';
+import { jsonFetcher } from "./json-fetcher.js";
+import { getDateXDaysAgo } from "../utils/date-utils.js";
+
+const start = '2025-01-14';
+const end = getDateXDaysAgo(1);
+
+const summary = await pRetry(() => jsonFetcher(`https://stats.filspark.com/retrieval-timings/daily?from=${start}&to=${end}`), { retries: 3 });
+
+process.stdout.write(JSON.stringify(summary));
@@ -11,15 +11,25 @@ import { combine, move, clone } from "./utils/ratios-utils.js";
 const SparkRates = FileAttachment("./data/spark-rsr.json").json();
 const SparkNonZeroRates = FileAttachment("./data/spark-rsr-non-zero.json").json();
 const SparkMinerRates = FileAttachment("./data/spark-miners-rsr.json").json();
+const SparkMinerRetrievalTimings = FileAttachment("./data/spark-miners-retrieval-timings.json").json();
 const SparkRetrievalResultCodes = FileAttachment("./data/spark-retrieval-result-codes.json").json();
 const SparkMinerRsrSummaries = FileAttachment("./data/spark-miner-rsr-summaries.json").json();
+const SparkRetrievalTimes = FileAttachment("./data/spark-retrieval-timings.json").json();
 ```
 
 ```js
+const sparkMinerRetrievalTimingsMap = SparkMinerRetrievalTimings.reduce((acc, record) => {
+  acc[record.miner_id] = record;
+  return acc;
+}, {});
+
 const nonZeroSparkMinerRates = SparkMinerRates.filter((record) => record.success_rate != 0)
 const tidySparkMinerRates = SparkMinerRates
   .sort((recordA, recordB) => recordB.success_rate - recordA.success_rate)
-  .map(record => ({ ...record, success_rate: `${(record.success_rate * 100).toFixed(2)}%`,success_rate_http: `${(record.success_rate_http * 100).toFixed(2)}%`}))
+  .map(record => {
+    const { ttfb_ms } = sparkMinerRetrievalTimingsMap[record.miner_id] ?? {};
+    return { ...record, ttfb_ms, success_rate: `${(record.success_rate * 100).toFixed(2)}%`,success_rate_http: `${(record.success_rate_http * 100).toFixed(2)}%`}
+  })
 ```
 
 <div class="hero">
@@ -140,8 +150,17 @@ const percentiles = Object.entries(SparkMinerRsrSummaries)
 
 <div class="divider"></div>
 
-<h4>Spark Retrieval Result Codes</h4>
-<body>This section shows the Spark Retrieval Result Codes breakdown.</body>
+
+<div class="grid grid-cols-2">
+  <div>
+    <h4>Spark Retrieval Result Codes</h4>
+    <body>This section shows the Spark Retrieval Result Codes breakdown.</body>
+  </div>
+  <div>
+    <h4>Spark Time To First Byte (TTFB)</h4>
+    <body>The section shows the median of all median TTFB values from all retrieval tasks.</body>
+  </div>
+</div>
 
 ```js
 const mapping = {
@@ -227,6 +246,21 @@ const tidy = clone(SparkRetrievalResultCodes).flatMap(({ day, rates }) => {
       ]
     })}
   </div>
+  <div class="card">
+      ${Plot.plot({
+      title: 'Time to First Byte (ms)',
+      // TODO: Change tick to month once we have more data
+      x: { type: 'utc', ticks: 'day' },
+      y: { grid: true, zero: true},
+      marks: [
+        Plot.lineY(SparkRetrievalTimes, {
+          x: 'day',
+          y: 'ttfb_ms',
+          stroke: "#FFBD3F",
+        })
+      ]
+    })}
+  </div> 
 </div>
 
 <details>
@@ -247,8 +281,8 @@ ${JSON.stringify(
 
 <div class="divider"></div>
 
-<h4>Spark Miner RSR Table</h4>
-<body>The following table shows the Spark RSR values calculated in aggregate for each Filecoin Storage Provider over the past 30 days. Click on a miner id to view stats about this storage provider.</body>
+<h4>Spark Miner Stats Table</h4>
+<body>The following table shows the Spark RSR and TTFB values calculated in aggregate for each Filecoin Storage Provider over the past 30 days. Click on a miner id to view stats about this storage provider.</body>
 
 ```js
 const search = view(Inputs.search(tidySparkMinerRates, {placeholder: "Search Storage Providers…"}));

@@ -6,30 +6,57 @@ title: Storage Provider Summary
 ```js
 import { LineGraph } from "../components/line-graph.js";
 import { getDateXDaysAgo } from "../utils/date-utils.js";
-const data = FileAttachment(`../data/${observable.params.provider}-spark-rsr-summary.json`).json();
+
+const rsrData = FileAttachment(`../data/${observable.params.provider}-spark-rsr-summary.json`).json();
+const ttfbData = FileAttachment(`../data/${observable.params.provider}-spark-retrieval-timings-summary.json`).json();
 ```
 
+
 <div class="hero">
   <body><a href="/"><img src="../media/spark-logomark-blue-with-bbox.png" alt="Spark Logo" width="300" /></a><body>
     <h2>Dashboard Beta</h2>
     <body><a href="https://filspark.com/dashboard" target="_blank" rel="noopener noreferrer">(Click here for Legacy Spark Grafana Dashboard)</a><body>
 </div>
 
-<h4>Storage Provider Spark RSR Summary</h4>
-<body>This section shows the storage provider Spark Retrieval Success Rate Score summary. You can adjust the date range. Records start on the 7th April 2024.</body>
-
-
 ```js
 const start = view(Inputs.date({label: "Start", value: getDateXDaysAgo(180) }));
 const end = view(Inputs.date({label: "End", value: getDateXDaysAgo(1) }));
 ```
 
 <h3>Stats for ${observable.params.provider}</h3>
 
-<div class="grid grid-cols" style="grid-auto-rows: 500px;">
+
+<div class="grid grid-cols-2">
+  <div>
+    <h4>Storage Provider Spark RSR Summary</h4>
+    <body>This section shows the storage provider Spark Retrieval Success Rate Score summary.</body>
+  </div>
+  <div>
+    <h4>Storage Provider Spark Time To First Byte (TTFB)</h4>
+    <body>The section shows the median of all TTFB values for successful retrieval checks of this storage provider.</body>
+  </div>
+</div>
+
+
+<div class="grid grid-cols-2" style="grid-auto-rows: 500px;">
   <div class="card">${
-    resize((width) => LineGraph(data, {width, title: "Retrieval Success Rate", start, end }))
+    resize((width) => LineGraph(rsrData, {width, title: "Retrieval Success Rate", start, end }))
   }</div>
+  <div class="card">
+      ${Plot.plot({
+      title: 'Time to First Byte (ms)',
+      // TODO: Change tick to month once we have more data
+      x: { type: 'utc', ticks: 'day' },
+      y: { grid: true, zero: true },
+      marks: [
+        Plot.lineY(ttfbData, {
+          x: 'day',
+          y: 'ttfb_ms',
+          stroke: "#FFBD3F",
+        })
+      ]
+    })}
+  </div>
 </div>
 
 <style>
@@ -73,4 +100,4 @@ const end = view(Inputs.date({label: "End", value: getDateXDaysAgo(1) }));
   }
 }
 
-</style>
+</style>