output: add storing boolean for files

When filestore keyword is triggered, the file is not yet stored, when the alert is generated, but only marked for storing. Ticket: 4881
OISF · Jul 21, 2023 · 225b5dc · 225b5dc
1 parent e59a717
commit 225b5dc
Show file tree

Hide file tree

Showing 5 changed files with 23 additions and 7 deletions.
diff --git a/etc/schema.json b/etc/schema.json
@@ -146,6 +146,10 @@
                     "stored": {
                         "type": "boolean"
                     },
+                    "storing": {
+                        "description": "the file is set to be stored when completed",
+                        "type": "boolean"
+                    },
                     "tx_id": {
                         "type": "integer"
                     },
@@ -1448,6 +1452,10 @@
                 "stored": {
                     "type": "boolean"
                 },
+                "storing": {
+                    "description": "the file is set to be stored when completed",
+                    "type": "boolean"
+                },
                 "tx_id": {
                     "type": "integer"
                 },

diff --git a/src/output-json-alert.c b/src/output-json-alert.c
@@ -619,7 +619,7 @@ static void AlertAddFiles(const Packet *p, JsonBuilder *jb, const uint64_t tx_id
                 jb_open_array(jb, "files");
             }
             jb_start_object(jb);
-            EveFileInfo(jb, file, tx_id, file->flags & FILE_STORED);
+            EveFileInfo(jb, file, tx_id, file->flags);
             jb_close(jb);
             file = file->next;
         }

diff --git a/src/output-json-file.c b/src/output-json-file.c
@@ -186,7 +186,13 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, void *tx,
     jb_set_string(js, "app_proto", AppProtoToString(p->flow->alproto));
 
     jb_open_object(js, "fileinfo");
-    EveFileInfo(js, ff, tx_id, stored);
+    if (stored) {
+        // the file has just been stored on disk cf OUTPUT_FILEDATA_FLAG_CLOSE
+        // but the flag is not set until the loggers have been called
+        EveFileInfo(js, ff, tx_id, ff->flags | FILE_STORED);
+    } else {
+        EveFileInfo(js, ff, tx_id, ff->flags);
+    }
     jb_close(js);
 
     /* xff header */
@@ -206,8 +212,7 @@ static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const F
 {
     HttpXFFCfg *xff_cfg = aft->filelog_ctx->xff_cfg != NULL ? aft->filelog_ctx->xff_cfg
                                                             : aft->filelog_ctx->parent_xff_cfg;
-    JsonBuilder *js = JsonBuildFileInfoRecord(
-            p, ff, tx, tx_id, ff->flags & FILE_STORED ? true : false, dir, xff_cfg, eve_ctx);
+    JsonBuilder *js = JsonBuildFileInfoRecord(p, ff, tx, tx_id, false, dir, xff_cfg, eve_ctx);
     if (unlikely(js == NULL)) {
         return;
     }

diff --git a/src/output-json.c b/src/output-json.c
@@ -128,7 +128,7 @@ json_t *SCJsonString(const char *val)
 /* Default Sensor ID value */
 static int64_t sensor_id = -1; /* -1 = not defined */
 
-void EveFileInfo(JsonBuilder *jb, const File *ff, const uint64_t tx_id, const bool stored)
+void EveFileInfo(JsonBuilder *jb, const File *ff, const uint64_t tx_id, uint16_t flags)
 {
     jb_set_string_from_bytes(jb, "filename", ff->name, ff->name_len);
 
@@ -170,11 +170,14 @@ void EveFileInfo(JsonBuilder *jb, const File *ff, const uint64_t tx_id, const bo
         jb_set_hex(jb, "sha256", (uint8_t *)ff->sha256, (uint32_t)sizeof(ff->sha256));
     }
 
-    if (stored) {
+    if (flags & FILE_STORED) {
         JB_SET_TRUE(jb, "stored");
         jb_set_uint(jb, "file_id", ff->file_store_id);
     } else {
         JB_SET_FALSE(jb, "stored");
+        if (flags & FILE_STORE) {
+            JB_SET_TRUE(jb, "storing");
+        }
     }
 
     jb_set_uint(jb, "size", FileTrackedSize(ff));

diff --git a/src/output-json.h b/src/output-json.h
@@ -95,7 +95,7 @@ typedef struct OutputJsonThreadCtx_ {
 json_t *SCJsonString(const char *val);
 
 void CreateEveFlowId(JsonBuilder *js, const Flow *f);
-void EveFileInfo(JsonBuilder *js, const File *file, const uint64_t tx_id, const bool stored);
+void EveFileInfo(JsonBuilder *js, const File *file, const uint64_t tx_id, uint16_t flags);
 void EveTcpFlags(uint8_t flags, JsonBuilder *js);
 void EvePacket(const Packet *p, JsonBuilder *js, unsigned long max_length);
 JsonBuilder *CreateEveHeader(const Packet *p, enum OutputJsonLogDirection dir,