From 7fb10676e793a33d11b097db83e435b1dd44682c Mon Sep 17 00:00:00 2001
From: Philippe Antoine <pantoine@oisf.net>
Date: Tue, 12 Dec 2023 14:25:16 +0100
Subject: [PATCH 01/14] dns: remove unneeded mut in logger

---
 rust/src/dns/log.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/rust/src/dns/log.rs b/rust/src/dns/log.rs
index 7e2fbaf24a58..575875042a7a 100644
--- a/rust/src/dns/log.rs
+++ b/rust/src/dns/log.rs
@@ -610,7 +610,7 @@ fn dns_log_json_answer(
 }
 
 fn dns_log_query(
-    tx: &mut DNSTransaction, i: u16, flags: u64, jb: &mut JsonBuilder,
+    tx: &DNSTransaction, i: u16, flags: u64, jb: &mut JsonBuilder,
 ) -> Result<bool, JsonError> {
     let index = i as usize;
     if let Some(request) = &tx.request {
@@ -637,7 +637,7 @@ fn dns_log_query(
 
 #[no_mangle]
 pub extern "C" fn SCDnsLogJsonQuery(
-    tx: &mut DNSTransaction, i: u16, flags: u64, jb: &mut JsonBuilder,
+    tx: &DNSTransaction, i: u16, flags: u64, jb: &mut JsonBuilder,
 ) -> bool {
     match dns_log_query(tx, i, flags, jb) {
         Ok(false) | Err(_) => {
@@ -651,7 +651,7 @@ pub extern "C" fn SCDnsLogJsonQuery(
 
 #[no_mangle]
 pub extern "C" fn SCDnsLogJsonAnswer(
-    tx: &mut DNSTransaction, flags: u64, js: &mut JsonBuilder,
+    tx: &DNSTransaction, flags: u64, js: &mut JsonBuilder,
 ) -> bool {
     if let Some(response) = &tx.response {
         for query in &response.queries {
@@ -664,7 +664,7 @@ pub extern "C" fn SCDnsLogJsonAnswer(
 }
 
 #[no_mangle]
-pub extern "C" fn SCDnsLogAnswerEnabled(tx: &mut DNSTransaction, flags: u64) -> bool {
+pub extern "C" fn SCDnsLogAnswerEnabled(tx: &DNSTransaction, flags: u64) -> bool {
     if let Some(response) = &tx.response {
         for query in &response.queries {
             if dns_log_rrtype_enabled(query.rrtype, flags) {

From 9332bc2c45f1e1e08f2261857d5804f41ce7e639 Mon Sep 17 00:00:00 2001
From: Philippe Antoine <pantoine@oisf.net>
Date: Thu, 28 Mar 2024 16:53:25 +0100
Subject: [PATCH 02/14] dns: adds missing NS field in json schema

---
 etc/schema.json | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/etc/schema.json b/etc/schema.json
index 4aaa82dd9298..3aaab48eff2c 100644
--- a/etc/schema.json
+++ b/etc/schema.json
@@ -1207,6 +1207,13 @@
                                 "type": "string"
                             }
                         },
+                        "NS": {
+                            "type": "array",
+                            "minItems": 1,
+                            "items": {
+                                "type": "string"
+                            }
+                        },
                         "NULL": {
                             "type": "array",
                             "minItems": 1,

From e6c1b9d8465c36251c2cba7f62439689b2252feb Mon Sep 17 00:00:00 2001
From: Victor Julien <vjulien@oisf.net>
Date: Sun, 26 May 2024 08:34:35 +0200
Subject: [PATCH 03/14] app-layer: minor code clarification

'dir' was too generic, so indicate it's about the app-layer update direction.
---
 src/app-layer.c | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/app-layer.c b/src/app-layer.c
index a7e50d66d024..0cf8405eab0e 100644
--- a/src/app-layer.c
+++ b/src/app-layer.c
@@ -389,7 +389,7 @@ extern enum ExceptionPolicy g_applayerparser_error_policy;
  */
 static int TCPProtoDetect(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
         AppLayerThreadCtx *app_tctx, Packet *p, Flow *f, TcpSession *ssn, TcpStream **stream,
-        uint8_t *data, uint32_t data_len, uint8_t flags, enum StreamUpdateDir dir)
+        uint8_t *data, uint32_t data_len, uint8_t flags, enum StreamUpdateDir app_update_dir)
 {
     AppProto *alproto;
     AppProto *alproto_otherdir;
@@ -555,7 +555,7 @@ static int TCPProtoDetect(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
         int r = AppLayerParserParse(tv, app_tctx->alp_tctx, f, f->alproto,
                 flags, data, data_len);
         PACKET_PROFILING_APP_END(app_tctx, f->alproto);
-        p->app_update_direction = (uint8_t)dir;
+        p->app_update_direction = (uint8_t)app_update_dir;
         if (r != 1) {
             StreamTcpUpdateAppLayerProgress(ssn, direction, data_len);
         }
@@ -643,7 +643,7 @@ static int TCPProtoDetect(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
                             f->alproto, flags,
                             data, data_len);
                     PACKET_PROFILING_APP_END(app_tctx, f->alproto);
-                    p->app_update_direction = (uint8_t)dir;
+                    p->app_update_direction = (uint8_t)app_update_dir;
                     if (r != 1) {
                         StreamTcpUpdateAppLayerProgress(ssn, direction, data_len);
                     }
@@ -706,7 +706,7 @@ static int TCPProtoDetect(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
  */
 int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet *p, Flow *f,
         TcpSession *ssn, TcpStream **stream, uint8_t *data, uint32_t data_len, uint8_t flags,
-        enum StreamUpdateDir dir)
+        enum StreamUpdateDir app_update_dir)
 {
     SCEnter();
 
@@ -752,7 +752,7 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet
         r = AppLayerParserParse(tv, app_tctx->alp_tctx, f, f->alproto,
                 flags, data, data_len);
         PACKET_PROFILING_APP_END(app_tctx, f->alproto);
-        p->app_update_direction = (uint8_t)dir;
+        p->app_update_direction = (uint8_t)app_update_dir;
         /* ignore parser result for gap */
         StreamTcpUpdateAppLayerProgress(ssn, direction, data_len);
         if (r < 0) {
@@ -771,8 +771,8 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet
     if (alproto == ALPROTO_UNKNOWN && (flags & STREAM_START)) {
         DEBUG_VALIDATE_BUG_ON(FlowChangeProto(f));
         /* run protocol detection */
-        if (TCPProtoDetect(tv, ra_ctx, app_tctx, p, f, ssn, stream, data, data_len, flags, dir) !=
-                0) {
+        if (TCPProtoDetect(tv, ra_ctx, app_tctx, p, f, ssn, stream, data, data_len, flags,
+                    app_update_dir) != 0) {
             goto failure;
         }
     } else if (alproto != ALPROTO_UNKNOWN && FlowChangeProto(f)) {
@@ -784,8 +784,8 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet
         StreamTcpResetStreamFlagAppProtoDetectionCompleted(&ssn->client);
         StreamTcpResetStreamFlagAppProtoDetectionCompleted(&ssn->server);
         /* rerun protocol detection */
-        int rd =
-                TCPProtoDetect(tv, ra_ctx, app_tctx, p, f, ssn, stream, data, data_len, flags, dir);
+        int rd = TCPProtoDetect(
+                tv, ra_ctx, app_tctx, p, f, ssn, stream, data, data_len, flags, app_update_dir);
         if (f->alproto == ALPROTO_UNKNOWN) {
             DEBUG_VALIDATE_BUG_ON(alstate_orig != f->alstate);
             // not enough data, revert AppLayerProtoDetectReset to rerun detection
@@ -838,7 +838,7 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet
             r = AppLayerParserParse(tv, app_tctx->alp_tctx, f, f->alproto,
                                     flags, data, data_len);
             PACKET_PROFILING_APP_END(app_tctx, f->alproto);
-            p->app_update_direction = (uint8_t)dir;
+            p->app_update_direction = (uint8_t)app_update_dir;
             if (r != 1) {
                 StreamTcpUpdateAppLayerProgress(ssn, direction, data_len);
                 if (r < 0) {

From 243587805dd7b164b7fbe53f01f2192fe6cf6eed Mon Sep 17 00:00:00 2001
From: Victor Julien <vjulien@oisf.net>
Date: Sun, 26 May 2024 08:35:05 +0200
Subject: [PATCH 04/14] stream: minor code clarification

'dir' was too generic, so indicate it's about the app-layer update direction.
---
 src/stream-tcp-reassemble.c | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/src/stream-tcp-reassemble.c b/src/stream-tcp-reassemble.c
index 97c2408912ee..c310cd0e3df3 100644
--- a/src/stream-tcp-reassemble.c
+++ b/src/stream-tcp-reassemble.c
@@ -1214,10 +1214,8 @@ static inline uint32_t AdjustToAcked(const Packet *p,
  *  \param stream pointer to pointer as app-layer can switch flow dir
  *  \retval 0 success
  */
-static int ReassembleUpdateAppLayer (ThreadVars *tv,
-        TcpReassemblyThreadCtx *ra_ctx,
-        TcpSession *ssn, TcpStream **stream,
-        Packet *p, enum StreamUpdateDir dir)
+static int ReassembleUpdateAppLayer(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn,
+        TcpStream **stream, Packet *p, enum StreamUpdateDir app_update_dir)
 {
     uint64_t app_progress = STREAM_APP_PROGRESS(*stream);
 
@@ -1249,7 +1247,7 @@ static int ReassembleUpdateAppLayer (ThreadVars *tv,
             SCLogDebug("sending GAP to app-layer (size: %u)", mydata_len);
 
             int r = AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream, NULL, mydata_len,
-                    StreamGetAppLayerFlags(ssn, *stream, p) | STREAM_GAP, dir);
+                    StreamGetAppLayerFlags(ssn, *stream, p) | STREAM_GAP, app_update_dir);
             AppLayerProfilingStore(ra_ctx->app_tctx, p);
 
             StreamTcpSetEvent(p, STREAM_REASSEMBLY_SEQ_GAP);
@@ -1321,8 +1319,8 @@ static int ReassembleUpdateAppLayer (ThreadVars *tv,
 
         SCLogDebug("parser");
         /* update the app-layer */
-        (void)AppLayerHandleTCPData(
-                tv, ra_ctx, p, p->flow, ssn, stream, (uint8_t *)mydata, mydata_len, flags, dir);
+        (void)AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream, (uint8_t *)mydata,
+                mydata_len, flags, app_update_dir);
         AppLayerProfilingStore(ra_ctx->app_tctx, p);
         AppLayerFrameDump(p->flow);
         uint64_t new_app_progress = STREAM_APP_PROGRESS(*stream);
@@ -1348,9 +1346,8 @@ static int ReassembleUpdateAppLayer (ThreadVars *tv,
  *  any issues, since processing of each stream is independent of the
  *  other stream.
  */
-int StreamTcpReassembleAppLayer (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
-                                 TcpSession *ssn, TcpStream *stream,
-                                 Packet *p, enum StreamUpdateDir dir)
+int StreamTcpReassembleAppLayer(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn,
+        TcpStream *stream, Packet *p, enum StreamUpdateDir app_update_dir)
 {
     SCEnter();
 
@@ -1376,7 +1373,7 @@ int StreamTcpReassembleAppLayer (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
             SCLogDebug("sending empty eof message");
             /* send EOF to app layer */
             AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, &stream, NULL, 0,
-                    StreamGetAppLayerFlags(ssn, stream, p), dir);
+                    StreamGetAppLayerFlags(ssn, stream, p), app_update_dir);
             AppLayerProfilingStore(ra_ctx->app_tctx, p);
 
             SCReturnInt(0);
@@ -1384,7 +1381,7 @@ int StreamTcpReassembleAppLayer (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
     }
 
     /* with all that out of the way, lets update the app-layer */
-    return ReassembleUpdateAppLayer(tv, ra_ctx, ssn, &stream, p, dir);
+    return ReassembleUpdateAppLayer(tv, ra_ctx, ssn, &stream, p, app_update_dir);
 }
 
 /** \internal

From c7402d2d015a5d24e23879a0c863b949f4cb0ff9 Mon Sep 17 00:00:00 2001
From: Victor Julien <vjulien@oisf.net>
Date: Mon, 3 Jun 2024 10:28:44 +0200
Subject: [PATCH 05/14] frames: fix bounds check

---
 src/app-layer-frames.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/app-layer-frames.c b/src/app-layer-frames.c
index 0aa0e19b82ac..b8554f59a982 100644
--- a/src/app-layer-frames.c
+++ b/src/app-layer-frames.c
@@ -427,7 +427,7 @@ Frame *AppLayerFrameNewByPointer(Flow *f, const StreamSlice *stream_slice,
     if (f->proto == IPPROTO_TCP && f->protoctx == NULL)
         return NULL;
     if (frame_start < stream_slice->input ||
-            frame_start >= stream_slice->input + stream_slice->input_len)
+            frame_start > stream_slice->input + stream_slice->input_len)
         return NULL;
 #endif
     BUG_ON(frame_start < stream_slice->input);

From 803e8dd32e73b6f31debeae5601ce8e07e77a9d4 Mon Sep 17 00:00:00 2001
From: Victor Julien <vjulien@oisf.net>
Date: Sun, 12 Nov 2023 09:41:13 +0100
Subject: [PATCH 06/14] frames: add FrameGetLastOpenByType

Getter for the most recent frame with unknown length (-1).
---
 src/app-layer-frames.c | 28 +++++++++++++++++++++++++++-
 src/app-layer-frames.h |  1 +
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/src/app-layer-frames.c b/src/app-layer-frames.c
index b8554f59a982..6953d26c9a45 100644
--- a/src/app-layer-frames.c
+++ b/src/app-layer-frames.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2022 Open Information Security Foundation
+/* Copyright (C) 2007-2024 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -83,6 +83,32 @@ static void FrameDebug(const char *prefix, const Frames *frames, const Frame *fr
 #endif
 }
 
+/**
+ * \note "open" means a frame that has no length set (len == -1)
+ * \todo perhaps we can search backwards */
+Frame *FrameGetLastOpenByType(Frames *frames, const uint8_t frame_type)
+{
+    Frame *candidate = NULL;
+
+    SCLogDebug(
+            "frames %p cnt %u, looking for last of type %" PRIu8, frames, frames->cnt, frame_type);
+    for (uint16_t i = 0; i < frames->cnt; i++) {
+        if (i < FRAMES_STATIC_CNT) {
+            Frame *frame = &frames->sframes[i];
+            FrameDebug("get_by_id(static)", frames, frame);
+            if (frame->type == frame_type && frame->len == -1)
+                candidate = frame;
+        } else {
+            const uint16_t o = i - FRAMES_STATIC_CNT;
+            Frame *frame = &frames->dframes[o];
+            FrameDebug("get_by_id(dynamic)", frames, frame);
+            if (frame->type == frame_type && frame->len == -1)
+                candidate = frame;
+        }
+    }
+    return candidate;
+}
+
 Frame *FrameGetById(Frames *frames, const int64_t id)
 {
     SCLogDebug("frames %p cnt %u, looking for %" PRIi64, frames, frames->cnt, id);
diff --git a/src/app-layer-frames.h b/src/app-layer-frames.h
index 1904917b42c2..b78077fd2b1d 100644
--- a/src/app-layer-frames.h
+++ b/src/app-layer-frames.h
@@ -88,6 +88,7 @@ void AppLayerFrameDump(Flow *f);
 
 Frame *FrameGetByIndex(Frames *frames, const uint32_t idx);
 Frame *FrameGetById(Frames *frames, const int64_t id);
+Frame *FrameGetLastOpenByType(Frames *frames, const uint8_t frame_type);
 
 Frame *AppLayerFrameGetById(Flow *f, const int direction, const FrameId frame_id);
 FrameId AppLayerFrameGetId(Frame *r);

From 2e5e3498a6fc6bd34a6db3af2816270cd72cd7d2 Mon Sep 17 00:00:00 2001
From: Victor Julien <vjulien@oisf.net>
Date: Sun, 12 Nov 2023 09:41:45 +0100
Subject: [PATCH 07/14] app-layer/frames: add by type getter

AppLayerFrameGetLastOpenByType: Returns the most recent frame with a type
with unknown length (-1).

Check if type is globally enabled first.
---
 src/app-layer-frames.c | 21 +++++++++++++++++++++
 src/app-layer-frames.h |  3 +++
 2 files changed, 24 insertions(+)

diff --git a/src/app-layer-frames.c b/src/app-layer-frames.c
index 6953d26c9a45..ff263b210f95 100644
--- a/src/app-layer-frames.c
+++ b/src/app-layer-frames.c
@@ -692,6 +692,27 @@ Frame *AppLayerFrameGetById(Flow *f, const int dir, const FrameId frame_id)
     return FrameGetById(frames, frame_id);
 }
 
+Frame *AppLayerFrameGetLastOpenByType(Flow *f, const int dir, const uint8_t frame_type)
+{
+    if (!(FrameConfigTypeIsEnabled(f->alproto, frame_type)))
+        return NULL;
+
+    FramesContainer *frames_container = AppLayerFramesGetContainer(f);
+    SCLogDebug("get frame_type %" PRIu8 " direction %u/%s frames_container %p", frame_type, dir,
+            dir == 0 ? "toserver" : "toclient", frames_container);
+    if (frames_container == NULL)
+        return NULL;
+
+    Frames *frames;
+    if (dir == 0) {
+        frames = &frames_container->toserver;
+    } else {
+        frames = &frames_container->toclient;
+    }
+    SCLogDebug("frames %p", frames);
+    return FrameGetLastOpenByType(frames, frame_type);
+}
+
 static inline bool FrameIsDone(const Frame *frame, const uint64_t abs_right_edge)
 {
     /* frame with negative length means we don't know the size yet. */
diff --git a/src/app-layer-frames.h b/src/app-layer-frames.h
index b78077fd2b1d..2eb314331674 100644
--- a/src/app-layer-frames.h
+++ b/src/app-layer-frames.h
@@ -91,7 +91,10 @@ Frame *FrameGetById(Frames *frames, const int64_t id);
 Frame *FrameGetLastOpenByType(Frames *frames, const uint8_t frame_type);
 
 Frame *AppLayerFrameGetById(Flow *f, const int direction, const FrameId frame_id);
+Frame *AppLayerFrameGetLastOpenByType(Flow *f, const int direction, const uint8_t frame_type);
+
 FrameId AppLayerFrameGetId(Frame *r);
+
 void AppLayerFrameAddEvent(Frame *frame, uint8_t e);
 void AppLayerFrameAddEventById(Flow *f, const int dir, const FrameId id, uint8_t e);
 void AppLayerFrameSetLength(Frame *frame, int64_t len);

From 683363b42df9ed702ca4b4e8dd912806ef87601f Mon Sep 17 00:00:00 2001
From: Victor Julien <vjulien@oisf.net>
Date: Thu, 30 Nov 2023 11:59:45 +0100
Subject: [PATCH 08/14] detect/frames: avoid IPS rescanning

Make sure to only scan the data when the app layer has been updated
as well.

Ticket: #6718.
---
 src/detect.c | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/detect.c b/src/detect.c
index 989f1133daeb..7c7536a22709 100644
--- a/src/detect.c
+++ b/src/detect.c
@@ -1627,6 +1627,15 @@ static void DetectRunFrames(ThreadVars *tv, DetectEngineCtx *de_ctx, DetectEngin
     const SigGroupHead *const sgh = scratch->sgh;
     const AppProto alproto = f->alproto;
 
+    /* for TCP, limit inspection to pseudo packets or real packet that did
+     * an app-layer update. */
+    if (p->proto == IPPROTO_TCP && !PKT_IS_PSEUDOPKT(p) &&
+            ((PKT_IS_TOSERVER(p) && (f->flags & FLOW_TS_APP_UPDATED) == 0) ||
+                    (PKT_IS_TOCLIENT(p) && (f->flags & FLOW_TC_APP_UPDATED) == 0))) {
+        SCLogDebug("pcap_cnt %" PRIu64 ": %s: skip frame inspection for TCP w/o APP UPDATE",
+                p->pcap_cnt, PKT_IS_TOSERVER(p) ? "toserver" : "toclient");
+        return;
+    }
     FramesContainer *frames_container = AppLayerFramesGetContainer(f);
     if (frames_container == NULL) {
         return;

From 866c128c43f44cb8c338c0ecaca89d60c0e6fca3 Mon Sep 17 00:00:00 2001
From: Victor Julien <vjulien@oisf.net>
Date: Fri, 26 Jan 2024 14:36:16 +0100
Subject: [PATCH 09/14] app-layer: flag flow for next packet in other dir

Add new flags to trigger FLOW_TS_APP_UPDATED/FLOW_TC_APP_UPDATED flags
to be set for the next packet in the relevant direction.

This allows for app relevant work to be done in the next packet in our
direction.
---
 src/flow-worker.c | 16 ++++++++++++----
 src/flow.h        |  6 +++++-
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/src/flow-worker.c b/src/flow-worker.c
index fdc584df5d55..1f219c83ad5a 100644
--- a/src/flow-worker.c
+++ b/src/flow-worker.c
@@ -523,19 +523,19 @@ static void PacketAppUpdate2FlowFlags(Packet *p)
             break;
         case UPDATE_DIR_BOTH:
             if (PKT_IS_TOSERVER(p)) {
-                p->flow->flags |= FLOW_TS_APP_UPDATED;
+                p->flow->flags |= FLOW_TS_APP_UPDATED | FLOW_TC_APP_UPDATE_NEXT;
                 SCLogDebug("pcap_cnt %" PRIu64 ", FLOW_TS_APP_UPDATED set", p->pcap_cnt);
             } else {
-                p->flow->flags |= FLOW_TC_APP_UPDATED;
+                p->flow->flags |= FLOW_TC_APP_UPDATED | FLOW_TS_APP_UPDATE_NEXT;
                 SCLogDebug("pcap_cnt %" PRIu64 ", FLOW_TC_APP_UPDATED set", p->pcap_cnt);
             }
             /* fall through */
         case UPDATE_DIR_OPPOSING:
             if (PKT_IS_TOSERVER(p)) {
-                p->flow->flags |= FLOW_TC_APP_UPDATED;
+                p->flow->flags |= FLOW_TC_APP_UPDATED | FLOW_TS_APP_UPDATE_NEXT;
                 SCLogDebug("pcap_cnt %" PRIu64 ", FLOW_TC_APP_UPDATED set", p->pcap_cnt);
             } else {
-                p->flow->flags |= FLOW_TS_APP_UPDATED;
+                p->flow->flags |= FLOW_TS_APP_UPDATED | FLOW_TC_APP_UPDATE_NEXT;
                 SCLogDebug("pcap_cnt %" PRIu64 ", FLOW_TS_APP_UPDATED set", p->pcap_cnt);
             }
             break;
@@ -583,6 +583,14 @@ static TmEcode FlowWorker(ThreadVars *tv, Packet *p, void *data)
 
     /* handle TCP and app layer */
     if (p->flow) {
+        if (PKT_IS_TOSERVER(p) && (p->flow->flags & FLOW_TS_APP_UPDATE_NEXT)) {
+            p->flow->flags |= FLOW_TS_APP_UPDATED;
+            p->flow->flags &= ~FLOW_TS_APP_UPDATE_NEXT;
+        } else if (PKT_IS_TOCLIENT(p) && (p->flow->flags & FLOW_TC_APP_UPDATE_NEXT)) {
+            p->flow->flags |= FLOW_TC_APP_UPDATED;
+            p->flow->flags &= ~FLOW_TC_APP_UPDATE_NEXT;
+        }
+
         if (PacketIsTCP(p)) {
             SCLogDebug("packet %" PRIu64 " is TCP. Direction %s", p->pcap_cnt,
                     PKT_IS_TOSERVER(p) ? "TOSERVER" : "TOCLIENT");
diff --git a/src/flow.h b/src/flow.h
index d633554243c0..bf28d02a5812 100644
--- a/src/flow.h
+++ b/src/flow.h
@@ -52,7 +52,8 @@ typedef struct AppLayerParserState_ AppLayerParserState;
 /** At least one packet from the destination address was seen */
 #define FLOW_TO_DST_SEEN                BIT_U32(1)
 
-// vacancy
+/** next packet in toclient direction will act on updated app-layer state */
+#define FLOW_TC_APP_UPDATE_NEXT BIT_U32(2)
 
 /** Flow was inspected against IP-Only sigs in the toserver direction */
 #define FLOW_TOSERVER_IPONLY_SET        BIT_U32(3)
@@ -117,6 +118,9 @@ typedef struct AppLayerParserState_ AppLayerParserState;
 #define FLOW_TS_APP_UPDATED BIT_U32(29)
 #define FLOW_TC_APP_UPDATED BIT_U32(30)
 
+/** next packet in toserver direction will act on updated app-layer state */
+#define FLOW_TS_APP_UPDATE_NEXT BIT_U32(31)
+
 /* File flags */
 
 #define FLOWFILE_INIT                   0

From a9dd1572d48efaaabb5f72fa066dc19cef4a07c2 Mon Sep 17 00:00:00 2001
From: Victor Julien <vjulien@oisf.net>
Date: Fri, 26 Jan 2024 15:11:30 +0100
Subject: [PATCH 10/14] detect/frames: inspect frames only in correct direction

Inspect frames in the correct direction after they have been created.
---
 src/flow-worker.c       | 21 ++++++++++++++++-----
 src/output-json-frame.c | 10 ++++++++++
 2 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/src/flow-worker.c b/src/flow-worker.c
index 1f219c83ad5a..9af47ac7c818 100644
--- a/src/flow-worker.c
+++ b/src/flow-worker.c
@@ -524,19 +524,23 @@ static void PacketAppUpdate2FlowFlags(Packet *p)
         case UPDATE_DIR_BOTH:
             if (PKT_IS_TOSERVER(p)) {
                 p->flow->flags |= FLOW_TS_APP_UPDATED | FLOW_TC_APP_UPDATE_NEXT;
-                SCLogDebug("pcap_cnt %" PRIu64 ", FLOW_TS_APP_UPDATED set", p->pcap_cnt);
+                SCLogDebug("pcap_cnt %" PRIu64 ", FLOW_TS_APP_UPDATED|FLOW_TC_APP_UPDATE_NEXT set",
+                        p->pcap_cnt);
             } else {
                 p->flow->flags |= FLOW_TC_APP_UPDATED | FLOW_TS_APP_UPDATE_NEXT;
-                SCLogDebug("pcap_cnt %" PRIu64 ", FLOW_TC_APP_UPDATED set", p->pcap_cnt);
+                SCLogDebug("pcap_cnt %" PRIu64 ", FLOW_TC_APP_UPDATED|FLOW_TS_APP_UPDATE_NEXT set",
+                        p->pcap_cnt);
             }
             /* fall through */
         case UPDATE_DIR_OPPOSING:
             if (PKT_IS_TOSERVER(p)) {
                 p->flow->flags |= FLOW_TC_APP_UPDATED | FLOW_TS_APP_UPDATE_NEXT;
-                SCLogDebug("pcap_cnt %" PRIu64 ", FLOW_TC_APP_UPDATED set", p->pcap_cnt);
+                SCLogDebug("pcap_cnt %" PRIu64 ", FLOW_TC_APP_UPDATED|FLOW_TS_APP_UPDATE_NEXT set",
+                        p->pcap_cnt);
             } else {
                 p->flow->flags |= FLOW_TS_APP_UPDATED | FLOW_TC_APP_UPDATE_NEXT;
-                SCLogDebug("pcap_cnt %" PRIu64 ", FLOW_TS_APP_UPDATED set", p->pcap_cnt);
+                SCLogDebug("pcap_cnt %" PRIu64 ", FLOW_TS_APP_UPDATED|FLOW_TC_APP_UPDATE_NEXT set",
+                        p->pcap_cnt);
             }
             break;
     }
@@ -583,12 +587,15 @@ static TmEcode FlowWorker(ThreadVars *tv, Packet *p, void *data)
 
     /* handle TCP and app layer */
     if (p->flow) {
+        /* see if need to consider flags set by prev packets */
         if (PKT_IS_TOSERVER(p) && (p->flow->flags & FLOW_TS_APP_UPDATE_NEXT)) {
             p->flow->flags |= FLOW_TS_APP_UPDATED;
             p->flow->flags &= ~FLOW_TS_APP_UPDATE_NEXT;
+            SCLogDebug("FLOW_TS_APP_UPDATED");
         } else if (PKT_IS_TOCLIENT(p) && (p->flow->flags & FLOW_TC_APP_UPDATE_NEXT)) {
             p->flow->flags |= FLOW_TC_APP_UPDATED;
             p->flow->flags &= ~FLOW_TC_APP_UPDATE_NEXT;
+            SCLogDebug("FLOW_TC_APP_UPDATED");
         }
 
         if (PacketIsTCP(p)) {
@@ -640,7 +647,11 @@ static TmEcode FlowWorker(ThreadVars *tv, Packet *p, void *data)
                 StreamTcpSessionCleanup(p->flow->protoctx);
             }
         } else if (p->proto == IPPROTO_TCP && p->flow->protoctx && p->flags & PKT_STREAM_EST) {
-            FramesPrune(p->flow, p);
+            if ((p->flow->flags & FLOW_TS_APP_UPDATED) && PKT_IS_TOSERVER(p)) {
+                FramesPrune(p->flow, p);
+            } else if ((p->flow->flags & FLOW_TC_APP_UPDATED) && PKT_IS_TOCLIENT(p)) {
+                FramesPrune(p->flow, p);
+            }
             FLOWWORKER_PROFILING_START(p, PROFILE_FLOWWORKER_TCPPRUNE);
             StreamTcpPruneSession(p->flow, p->flowflags & FLOW_PKT_TOSERVER ?
                     STREAM_TOSERVER : STREAM_TOCLIENT);
diff --git a/src/output-json-frame.c b/src/output-json-frame.c
index 4e0ec5b2b24c..4f761e7ca173 100644
--- a/src/output-json-frame.c
+++ b/src/output-json-frame.c
@@ -409,6 +409,16 @@ static bool JsonFrameLogCondition(ThreadVars *tv, void *thread_data, const Packe
         return false;
 
     if ((p->proto == IPPROTO_TCP || p->proto == IPPROTO_UDP) && p->flow->alparser != NULL) {
+        if (p->proto == IPPROTO_TCP) {
+            if ((p->flow->flags & FLOW_TS_APP_UPDATED) && PKT_IS_TOSERVER(p)) {
+                // fallthrough
+            } else if ((p->flow->flags & FLOW_TC_APP_UPDATED) && PKT_IS_TOCLIENT(p)) {
+                // fallthrough
+            } else {
+                return false;
+            }
+        }
+
         FramesContainer *frames_container = AppLayerFramesGetContainer(p->flow);
         if (frames_container == NULL)
             return false;

From c17df004ed609cd19e12c87b48cfce60bb016951 Mon Sep 17 00:00:00 2001
From: Victor Julien <vjulien@oisf.net>
Date: Sun, 26 May 2024 08:38:13 +0200
Subject: [PATCH 11/14] stream: process ASYNC in packet dir

There will generally not be an opposing direction to handle
the app update.
---
 src/stream-tcp-reassemble.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/stream-tcp-reassemble.c b/src/stream-tcp-reassemble.c
index c310cd0e3df3..b9e9e6797dd6 100644
--- a/src/stream-tcp-reassemble.c
+++ b/src/stream-tcp-reassemble.c
@@ -1983,6 +1983,9 @@ int StreamTcpReassembleHandleSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_
         }
     } else if (ssn->state == TCP_CLOSED) {
         dir = UPDATE_DIR_BOTH;
+    } else if ((ssn->flags & STREAMTCP_FLAG_ASYNC) != 0) {
+        dir = UPDATE_DIR_PACKET;
+        SCLogDebug("%" PRIu64 ": ASYNC: UPDATE_DIR_PACKET", p->pcap_cnt);
     }
 
     /* handle ack received */

From 2cebc8368cfa75f88ea84076be87c61610a5b849 Mon Sep 17 00:00:00 2001
From: Victor Julien <vjulien@oisf.net>
Date: Sun, 26 May 2024 08:40:11 +0200
Subject: [PATCH 12/14] flow-worker: debug output about updates

---
 src/flow-worker.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/flow-worker.c b/src/flow-worker.c
index 9af47ac7c818..3efb290da44c 100644
--- a/src/flow-worker.c
+++ b/src/flow-worker.c
@@ -511,6 +511,7 @@ static void PacketAppUpdate2FlowFlags(Packet *p)
 {
     switch ((enum StreamUpdateDir)p->app_update_direction) {
         case UPDATE_DIR_NONE: // NONE implies pseudo packet
+            SCLogDebug("pcap_cnt %" PRIu64 ", UPDATE_DIR_NONE", p->pcap_cnt);
             break;
         case UPDATE_DIR_PACKET:
             if (PKT_IS_TOSERVER(p)) {
@@ -587,6 +588,11 @@ static TmEcode FlowWorker(ThreadVars *tv, Packet *p, void *data)
 
     /* handle TCP and app layer */
     if (p->flow) {
+        SCLogDebug("packet %" PRIu64
+                   ": direction %s FLOW_TS_APP_UPDATE_NEXT %s FLOW_TC_APP_UPDATE_NEXT %s",
+                p->pcap_cnt, PKT_IS_TOSERVER(p) ? "toserver" : "toclient",
+                BOOL2STR((p->flow->flags & FLOW_TS_APP_UPDATE_NEXT) != 0),
+                BOOL2STR((p->flow->flags & FLOW_TC_APP_UPDATE_NEXT) != 0));
         /* see if need to consider flags set by prev packets */
         if (PKT_IS_TOSERVER(p) && (p->flow->flags & FLOW_TS_APP_UPDATE_NEXT)) {
             p->flow->flags |= FLOW_TS_APP_UPDATED;
@@ -667,11 +673,13 @@ static TmEcode FlowWorker(ThreadVars *tv, Packet *p, void *data)
                     if (PKT_IS_PSEUDOPKT(p) || (p->flow->flags & (FLOW_TS_APP_UPDATED))) {
                         AppLayerParserTransactionsCleanup(p->flow, STREAM_TOSERVER);
                         p->flow->flags &= ~FLOW_TS_APP_UPDATED;
+                        SCLogDebug("~FLOW_TS_APP_UPDATED");
                     }
                 } else {
                     if (PKT_IS_PSEUDOPKT(p) || (p->flow->flags & (FLOW_TC_APP_UPDATED))) {
                         AppLayerParserTransactionsCleanup(p->flow, STREAM_TOCLIENT);
                         p->flow->flags &= ~FLOW_TC_APP_UPDATED;
+                        SCLogDebug("~FLOW_TC_APP_UPDATED");
                     }
                 }
             }

From 306fd795c38d570d432440eaab7fc19bd50161d7 Mon Sep 17 00:00:00 2001
From: Victor Julien <vjulien@oisf.net>
Date: Mon, 13 Nov 2023 06:43:32 +0100
Subject: [PATCH 13/14] smtp/frames: initial frame support

Adds the following frames:

  command_line
  data
  response_line

The *_line frames are per line, so in multi-line responses each line
will have it's own frame.

Ticket: #4905.
---
 src/app-layer-smtp.c | 124 +++++++++++++++++++++++++++++++++++++------
 src/app-layer-smtp.h |   1 +
 2 files changed, 110 insertions(+), 15 deletions(-)

diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c
index b5eb04ba9e21..063b55d8780e 100644
--- a/src/app-layer-smtp.c
+++ b/src/app-layer-smtp.c
@@ -35,6 +35,7 @@
 #include "app-layer-detect-proto.h"
 #include "app-layer-protos.h"
 #include "app-layer-parser.h"
+#include "app-layer-frames.h"
 #include "app-layer-smtp.h"
 
 #include "util-enum.h"
@@ -153,6 +154,43 @@ SCEnumCharMap smtp_decoder_event_table[] = {
     { NULL, -1 },
 };
 
+enum SMTPFrameTypes {
+    SMTP_FRAME_COMMAND_LINE,
+    SMTP_FRAME_DATA,
+    SMTP_FRAME_RESPONSE_LINE,
+};
+
+SCEnumCharMap smtp_frame_table[] = {
+    {
+            "command_line",
+            SMTP_FRAME_COMMAND_LINE,
+    },
+    {
+            "data",
+            SMTP_FRAME_DATA,
+    },
+    {
+            "response_line",
+            SMTP_FRAME_RESPONSE_LINE,
+    },
+    { NULL, -1 },
+};
+
+static int SMTPGetFrameIdByName(const char *frame_name)
+{
+    int id = SCMapEnumNameToValue(frame_name, smtp_frame_table);
+    if (id < 0) {
+        return -1;
+    }
+    return id;
+}
+
+static const char *SMTPGetFrameNameById(const uint8_t frame_id)
+{
+    const char *name = SCMapEnumValueToName(frame_id, smtp_frame_table);
+    return name;
+}
+
 typedef struct SMTPThreadCtx_ {
     MpmThreadCtx *smtp_mpm_thread_ctx;
     PrefilterRuleStore *pmq;
@@ -468,8 +506,8 @@ static void SMTPNewFile(SMTPTransaction *tx, File *file)
  * \retval -1 Either when we don't have any new lines to supply anymore or
  *            on failure.
  */
-static AppLayerResult SMTPGetLine(
-        SMTPState *state, SMTPInput *input, SMTPLine *line, uint16_t direction)
+static AppLayerResult SMTPGetLine(Flow *f, StreamSlice *slice, SMTPState *state, SMTPInput *input,
+        SMTPLine *line, uint16_t direction)
 {
     SCEnter();
 
@@ -477,6 +515,26 @@ static AppLayerResult SMTPGetLine(
     if (input->len <= 0)
         return APP_LAYER_ERROR;
 
+    const uint8_t type = direction == 0 ? SMTP_FRAME_COMMAND_LINE : SMTP_FRAME_RESPONSE_LINE;
+    Frame *frame = AppLayerFrameGetLastOpenByType(f, direction, type);
+    if (frame == NULL) {
+        if (direction == 0 &&
+                !(state->current_command == SMTP_COMMAND_DATA &&
+                        (state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE))) {
+            frame = AppLayerFrameNewByPointer(
+                    f, slice, input->buf + input->consumed, -1, 0, SMTP_FRAME_COMMAND_LINE);
+            /* can't set tx id before (possibly) creating it */
+
+        } else if (direction == 1) {
+            frame = AppLayerFrameNewByPointer(
+                    f, slice, input->buf + input->consumed, -1, 1, SMTP_FRAME_RESPONSE_LINE);
+            if (frame != NULL && state->curr_tx) {
+                AppLayerFrameSetTxId(frame, state->curr_tx->tx_id);
+            }
+        }
+    }
+    SCLogDebug("frame %p", frame);
+
     uint8_t *lf_idx = memchr(input->buf + input->consumed, 0x0a, input->len);
     bool discard_till_lf = (direction == 0) ? state->discard_till_lf_ts : state->discard_till_lf_tc;
 
@@ -503,6 +561,11 @@ static AppLayerResult SMTPGetLine(
         input->len -= line->len;
         DEBUG_VALIDATE_BUG_ON((input->consumed + input->len) != input->orig_len);
         line->buf = input->buf + o_consumed;
+
+        if (frame != NULL) {
+            frame->len = (int64_t)line->len;
+        }
+
         if (line->len >= SMTP_LINE_BUFFER_LIMIT) {
             line->len = SMTP_LINE_BUFFER_LIMIT;
             line->delim_len = 0;
@@ -1044,12 +1107,23 @@ static int NoNewTx(SMTPState *state, const SMTPLine *line)
  *         -1 for errors and inconsistent states
  *         -2 if MIME state could not be allocated
  * */
-static int SMTPProcessRequest(
-        SMTPState *state, Flow *f, AppLayerParserState *pstate, const SMTPLine *line)
+static int SMTPProcessRequest(SMTPState *state, Flow *f, AppLayerParserState *pstate,
+        SMTPInput *input, const SMTPLine *line, const StreamSlice *slice)
 {
     SCEnter();
     SMTPTransaction *tx = state->curr_tx;
 
+    Frame *frame = AppLayerFrameGetLastOpenByType(f, 0, SMTP_FRAME_COMMAND_LINE);
+    if (frame) {
+        frame->len = (int64_t)line->len;
+    } else {
+        if (!(state->current_command == SMTP_COMMAND_DATA &&
+                    (state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE))) {
+            frame = AppLayerFrameNewByPointer(
+                    f, slice, line->buf, line->len, 0, SMTP_FRAME_COMMAND_LINE);
+        }
+    }
+
     /* If current input is to be discarded because it completes a long line,
      * line's length and delimiter len are reset to 0. Skip processing this line.
      * This line is only to get us out of the state where we should discard any
@@ -1070,6 +1144,9 @@ static int SMTPProcessRequest(
         StreamTcpReassemblySetMinInspectDepth(f->protoctx, STREAM_TOSERVER,
                 smtp_config.content_inspect_min_size);
     }
+    if (frame != NULL && state->curr_tx) {
+        AppLayerFrameSetTxId(frame, state->curr_tx->tx_id);
+    }
 
     state->toserver_data_count += (line->len + line->delim_len);
 
@@ -1107,6 +1184,15 @@ static int SMTPProcessRequest(
                 }
             }
             state->curr_tx->is_data = true;
+
+            Frame *data_frame = AppLayerFrameNewByPointer(
+                    f, slice, input->buf + input->consumed, -1, 0, SMTP_FRAME_DATA);
+            if (data_frame == NULL) {
+                SCLogDebug("data_frame %p - no data frame set up", data_frame);
+            } else {
+                AppLayerFrameSetTxId(data_frame, state->curr_tx->tx_id);
+            }
+
             /* Enter immediately data mode without waiting for server reply */
             if (state->parser_state & SMTP_PARSER_STATE_PIPELINING_SERVER) {
                 state->parser_state |= SMTP_PARSER_STATE_COMMAND_DATA_MODE;
@@ -1201,8 +1287,8 @@ static inline void ResetLine(SMTPLine *line)
  *          1 for handing control over to GetLine
  *         -1 for errors and inconsistent states
  * */
-static int SMTPPreProcessCommands(
-        SMTPState *state, Flow *f, AppLayerParserState *pstate, SMTPInput *input, SMTPLine *line)
+static int SMTPPreProcessCommands(SMTPState *state, Flow *f, AppLayerParserState *pstate,
+        StreamSlice *slice, SMTPInput *input, SMTPLine *line)
 {
     DEBUG_VALIDATE_BUG_ON((state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE) == 0);
     DEBUG_VALIDATE_BUG_ON(line->len != 0);
@@ -1251,10 +1337,11 @@ static int SMTPPreProcessCommands(
             if (line->len < 0) {
                 return -1;
             }
+
             input->consumed = total_consumed;
             input->len -= current_line_consumed;
             DEBUG_VALIDATE_BUG_ON(input->consumed + input->len != input->orig_len);
-            if (SMTPProcessRequest(state, f, pstate, line) == -1) {
+            if (SMTPProcessRequest(state, f, pstate, input, line, slice) == -1) {
                 return -1;
             }
             line_complete = false;
@@ -1263,8 +1350,13 @@ static int SMTPPreProcessCommands(
             line->delim_len = 0;
 
             /* bail if `SMTPProcessRequest` ended the data mode */
-            if ((state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE) == 0)
+            if ((state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE) == 0) {
+                Frame *data_frame = AppLayerFrameGetLastOpenByType(f, 0, SMTP_FRAME_DATA);
+                if (data_frame) {
+                    data_frame->len = (slice->offset + input->consumed) - data_frame->offset;
+                }
                 break;
+            }
         }
     }
     return 0;
@@ -1295,7 +1387,7 @@ static AppLayerResult SMTPParse(uint8_t direction, Flow *f, SMTPState *state,
         if (((state->current_command == SMTP_COMMAND_DATA) ||
                     (state->current_command == SMTP_COMMAND_BDAT)) &&
                 (state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE)) {
-            int ret = SMTPPreProcessCommands(state, f, pstate, &input, &line);
+            int ret = SMTPPreProcessCommands(state, f, pstate, &stream_slice, &input, &line);
             DEBUG_VALIDATE_BUG_ON(ret != 0 && ret != -1 && ret != 1);
             if (ret == 0 && input.consumed == input.orig_len) {
                 SCReturnStruct(APP_LAYER_OK);
@@ -1303,9 +1395,9 @@ static AppLayerResult SMTPParse(uint8_t direction, Flow *f, SMTPState *state,
                 SCReturnStruct(APP_LAYER_ERROR);
             }
         }
-        AppLayerResult res = SMTPGetLine(state, &input, &line, direction);
+        AppLayerResult res = SMTPGetLine(f, &stream_slice, state, &input, &line, direction);
         while (res.status == 0) {
-            int retval = SMTPProcessRequest(state, f, pstate, &line);
+            int retval = SMTPProcessRequest(state, f, pstate, &input, &line, &stream_slice);
             if (retval != 0)
                 SCReturnStruct(APP_LAYER_ERROR);
             if (line.delim_len == 0 && line.len == SMTP_LINE_BUFFER_LIMIT) {
@@ -1326,7 +1418,7 @@ static AppLayerResult SMTPParse(uint8_t direction, Flow *f, SMTPState *state,
              * In case of another boundary, the control should be passed to SMTPGetLine */
             if ((input.len > 0) && (state->current_command == SMTP_COMMAND_DATA) &&
                     (state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE)) {
-                int ret = SMTPPreProcessCommands(state, f, pstate, &input, &line);
+                int ret = SMTPPreProcessCommands(state, f, pstate, &stream_slice, &input, &line);
                 DEBUG_VALIDATE_BUG_ON(ret != 0 && ret != -1 && ret != 1);
                 if (ret == 0 && input.consumed == input.orig_len) {
                     SCReturnStruct(APP_LAYER_OK);
@@ -1334,13 +1426,13 @@ static AppLayerResult SMTPParse(uint8_t direction, Flow *f, SMTPState *state,
                     SCReturnStruct(APP_LAYER_ERROR);
                 }
             }
-            res = SMTPGetLine(state, &input, &line, direction);
+            res = SMTPGetLine(f, &stream_slice, state, &input, &line, direction);
         }
         if (res.status == 1)
             return res;
         /* toclient */
     } else {
-        AppLayerResult res = SMTPGetLine(state, &input, &line, direction);
+        AppLayerResult res = SMTPGetLine(f, &stream_slice, state, &input, &line, direction);
         while (res.status == 0) {
             if (SMTPProcessReply(state, f, pstate, thread_data, &input, &line) != 0)
                 SCReturnStruct(APP_LAYER_ERROR);
@@ -1352,7 +1444,7 @@ static AppLayerResult SMTPParse(uint8_t direction, Flow *f, SMTPState *state,
                 SMTPSetEvent(state, SMTP_DECODER_EVENT_TRUNCATED_LINE);
                 break;
             }
-            res = SMTPGetLine(state, &input, &line, direction);
+            res = SMTPGetLine(f, &stream_slice, state, &input, &line, direction);
         }
         if (res.status == 1)
             return res;
@@ -1752,6 +1844,8 @@ void RegisterSMTPParsers(void)
         AppLayerParserRegisterTxDataFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPGetTxData);
         AppLayerParserRegisterStateDataFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPGetStateData);
         AppLayerParserRegisterStateProgressCompletionStatus(ALPROTO_SMTP, 1, 1);
+        AppLayerParserRegisterGetFrameFuncs(
+                IPPROTO_TCP, ALPROTO_SMTP, SMTPGetFrameIdByName, SMTPGetFrameNameById);
     } else {
         SCLogInfo("Parser disabled for %s protocol. Protocol detection still on.", proto_name);
     }
diff --git a/src/app-layer-smtp.h b/src/app-layer-smtp.h
index 93c3bd812c93..cd9c614b966a 100644
--- a/src/app-layer-smtp.h
+++ b/src/app-layer-smtp.h
@@ -24,6 +24,7 @@
 #ifndef SURICATA_APP_LAYER_SMTP_H
 #define SURICATA_APP_LAYER_SMTP_H
 
+#include "app-layer-frames.h"
 #include "util-streaming-buffer.h"
 #include "rust.h"
 

From 8781e9352a6cc63b51b6ba84161db097773f2e65 Mon Sep 17 00:00:00 2001
From: Victor Julien <vjulien@oisf.net>
Date: Tue, 4 Jun 2024 12:30:12 +0200
Subject: [PATCH 14/14] doc/userguide: add documentation for SMTP frames

---
 doc/userguide/rules/smtp-keywords.rst | 57 ++++++++++++++++++++++++++-
 1 file changed, 56 insertions(+), 1 deletion(-)

diff --git a/doc/userguide/rules/smtp-keywords.rst b/doc/userguide/rules/smtp-keywords.rst
index ec91f6fc0c1e..8369856b9f21 100644
--- a/doc/userguide/rules/smtp-keywords.rst
+++ b/doc/userguide/rules/smtp-keywords.rst
@@ -16,4 +16,59 @@ Signature Example:
   :example-rule-options:`file.name; content:"winmail.dat";` \
   classtype:bad-unknown; sid:1; rev:1;)
 
-For additional information on the ``file.name`` keyword, see :doc:`file-keywords`.
\ No newline at end of file
+For additional information on the ``file.name`` keyword, see :doc:`file-keywords`.
+
+Frames
+------
+
+The SMTP parser supports the following frames:
+
+* smtp.command_line
+* smtp.response_line
+* smtp.data
+* smtp.stream
+
+smtp.command_line
+~~~~~~~~~~~~~~~~~
+
+A single line from the client to the server. Multi-line commands will have a frame per
+line. Lines part of the SMTP DATA transfer are excluded.
+
+.. container:: example fule
+
+  alert smtp any any -> any any ( \
+  :example-rule-options:`frame:smtp.command_line; content:"MAIL|20|FROM:"; startswith;` \
+  sid:1;)
+
+smtp.response_line
+~~~~~~~~~~~~~~~~~~
+
+A single line from the server to the client. Multi-line commands will have a frame per line.
+
+.. container:: example fule
+
+  alert smtp any any -> any any ( \
+  :example-rule-options:`frame:smtp.response_line; content:"354 go ahead"; startswith;` \
+  sid:1;)
+
+smtp.data
+~~~~~~~~~
+
+A streaming buffer containing the DATA bytes sent from client to server.
+
+.. container:: example fule
+
+  alert smtp any any -> any any ( \
+  :example-rule-options:`frame:smtp.data; content:"Reply-To:"; startswith; content:"Subject"; distance:0;` \
+  sid:1;)
+
+smtp.stream
+~~~~~~~~~~~
+
+Streaming buffer of the entire TCP data for the SMTP session.
+
+.. container:: example fule
+
+  alert smtp any any -> any any (flow:to_client; \
+  :example-rule-options:`frame:smtp.stream; content:"250 ok|0d 0a|354 go ahead";` \
+  sid:1;)