1. Set detail status when updating file_record

2. Collect file_stats as telemetry data
crustio · Jul 24, 2024 · c64c54a · c64c54a
1 parent 1bd6624
commit c64c54a
Show file tree

Hide file tree

Showing 4 changed files with 54 additions and 19 deletions.
diff --git a/src/tasks/pull-scheduler-task.ts b/src/tasks/pull-scheduler-task.ts
@@ -242,35 +242,35 @@ async function getOneFileByStrategy(
         await fileOrderOps.updateFileInfoStatus(record.id, 'expired');
         break;
       case 'pfSkipped':
+        logger.info('file "%s" is skipped by rule: "%s"', record.cid, status);
+        await fileOrderOps.updateFileInfoStatus(record.id, 'pfSkipped');
+        break;
       case 'nodeSkipped':
+        logger.info('file "%s" is skipped by rule: "%s"', record.cid, status);
+        await fileOrderOps.updateFileInfoStatus(record.id, 'nodeSkipped');
+        break;
       case 'rateSkipped':
         logger.info('file "%s" is skipped by rule: "%s"', record.cid, status);
-        await fileOrderOps.updateFileInfoStatus(record.id, 'skipped');
+        await fileOrderOps.updateFileInfoStatus(record.id, 'rateSkipped');
         break;
       case 'sizeTooSmall':
+        logger.info('file "%s" is skipped by size constraint: %s', record.cid, status);
+        await fileOrderOps.updateFileInfoStatus(record.id, 'sizeSmallSkipped');
+        break;
       case 'sizeTooLarge':
-        logger.info(
-          'file "%s" is skipped by size constraint: %s',
-          record.cid,
-          status,
-        );
-        await fileOrderOps.updateFileInfoStatus(record.id, 'skipped');
+        logger.info('file "%s" is skipped by size constraint: %s', record.cid, status);
+        await fileOrderOps.updateFileInfoStatus(record.id, 'sizeLargeSkipped');
         break;
       case 'replicasNotEnough':
+        logger.info('file "%s" is skipped by replica constraint: %s', record.cid, status);
+        await fileOrderOps.updateFileInfoStatus(record.id, 'replicasNotEnoughSkipped');
+        break;
       case 'tooManyReplicas':
-        logger.info(
-          'file "%s" is skipped by replica constraint: %s',
-          record.cid,
-          status,
-        );
-        await fileOrderOps.updateFileInfoStatus(record.id, 'skipped');
+        logger.info('file "%s" is skipped by replica constraint: %s', record.cid, status);
+        await fileOrderOps.updateFileInfoStatus(record.id, 'tooManyReplicasSkipped');
         break;
       case 'pendingForReplica':
-        logger.info(
-          'file "%s" replica count is not enough, pending recheck',
-          record.cid,
-          status,
-        );
+        logger.info('file "%s" replica count is not enough, pending recheck', record.cid, status);
         await fileOrderOps.updateFileInfoStatus(record.id, 'pending_replica');
         break;
     }

diff --git a/src/tasks/telemetry-task.ts b/src/tasks/telemetry-task.ts
@@ -14,6 +14,7 @@ import {
   TelemetryData,
   SWorkerStats,
   OSInfo,
+  FileStats,
 } from '../types/telemetry';
 import { formatError, getTimestamp, toQuotedList } from '../utils';
 import { Dayjs } from '../utils/datetime';
@@ -58,6 +59,7 @@ async function collectStats(
 
   const timeStart = getTimestamp() - ReportSlotDuration;
   const queueStats = await collectQueueInfo(database);
+  const fileStats = await collectFileStats(database);
   const pinStats = await getPinStats(database, timeStart);
 
   const { deletedCount } = await database.get(
@@ -99,6 +101,7 @@ async function collectStats(
       nodeIndex: 0,
     },
     queueStats,
+    fileStats,
     cleanupStats: {
       deletedCount,
     },
@@ -147,6 +150,25 @@ async function collectQueueInfo(database): Promise<QueueInfo> {
   };
 }
 
+async function collectFileStats(database): Promise<FileStats> {
+  const { totalCount } = await database.get(
+    `select count(*) as totalCount from file_record`,
+  );
+  const countByStatusResults = await database.get(
+    `select status, count(1) from file_record group by status`,
+  );
+
+  const fileStats = {
+    totalCount
+  };
+
+  for (const { status, count } of countByStatusResults) {
+    fileStats[status] = count;
+  }
+
+  return fileStats;
+}
+
 function collectSManagerInfo(
   config: NormalizedConfig,
   context: AppContext,

diff --git a/src/types/database.d.ts b/src/types/database.d.ts
@@ -15,7 +15,14 @@ type FileStatus =
   | 'failed'
   | 'skipped'
   | 'handled'
-  | 'expired';
+  | 'expired'
+  | 'pfSkipped'
+  | 'nodeSkipped'
+  | 'rateSkipped'
+  | 'sizeSmallSkipped'
+  | 'sizeLargeSkipped'
+  | 'replicasNotEnoughSkipped'
+  | 'tooManyReplicasSkipped';
 type CleanupStatus = 'pending' | 'failed' | 'done';
 
 export interface FileRecord {

diff --git a/src/types/telemetry.d.ts b/src/types/telemetry.d.ts
@@ -7,6 +7,7 @@ export interface TelemetryData {
   pinStats: PinStats;
   sworker: SWorkerStats | null;
   queueStats: QueueInfo;
+  fileStats: FileStats;
   cleanupStats: CleanupStats;
   groupInfo: GroupInfo;
   hasSealCoordinator: boolean;
@@ -58,6 +59,11 @@ export interface QueueInfo {
   pendingSizeTotal: number; // in MB
 }
 
+export interface FileStats {
+  totalCount: number;
+  [status: string]: number;
+}
+
 export interface PinStats {
   sealingCount: number;
   failedCount: number;