From 6afc347adb09113a6564878edc78ea3d7ae9076b Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Fri, 31 May 2024 10:39:16 +0200 Subject: [PATCH 1/2] app-layer: track modified/processed txs To optimize detection, and logging, to avoid going through all the live transactions when only a few were modified. Ticket: 7087 --- rust/src/applayer.rs | 6 ++++++ rust/src/applayertemplate/template.rs | 1 + rust/src/dcerpc/dcerpc.rs | 2 ++ rust/src/dcerpc/dcerpc_udp.rs | 2 ++ rust/src/enip/enip.rs | 2 ++ rust/src/http2/http2.rs | 2 ++ rust/src/ldap/ldap.rs | 2 ++ rust/src/modbus/modbus.rs | 4 ++++ rust/src/mqtt/mqtt.rs | 2 ++ rust/src/nfs/nfs.rs | 3 +++ rust/src/pgsql/pgsql.rs | 3 +++ rust/src/rfb/rfb.rs | 7 ++++++- rust/src/smb/dcerpc.rs | 1 + rust/src/smb/files.rs | 2 ++ rust/src/smb/session.rs | 1 + rust/src/smb/smb.rs | 5 +++++ rust/src/ssh/ssh.rs | 2 ++ src/app-layer-dnp3.c | 2 ++ src/app-layer-ftp.c | 3 ++- src/app-layer-htp.c | 12 ++++++++++++ src/app-layer-parser.c | 5 ++++- src/app-layer-smtp.c | 4 ++++ src/app-layer-ssl.c | 2 ++ src/detect.c | 7 ++++++- src/output-tx.c | 27 ++++++++++++++++----------- 25 files changed, 94 insertions(+), 15 deletions(-) diff --git a/rust/src/applayer.rs b/rust/src/applayer.rs index f863546bbe06..495b69df7f86 100644 --- a/rust/src/applayer.rs +++ b/rust/src/applayer.rs @@ -116,6 +116,10 @@ pub struct AppLayerTxData { pub file_tx: u8, /// Number of times this tx data has already been logged for one stream match pub stream_logged: u8, + /// The tx has been updated and needs to be processed : detection, logging, cleaning + /// It can then be skipped until new data arrives. + /// There is a boolean for both directions : to server and to client + pub updated: [bool; 2], /// detection engine flags for use by detection engine detect_flags_ts: u64, @@ -155,6 +159,7 @@ impl AppLayerTxData { file_flags: 0, file_tx: 0, stream_logged: 0, + updated: [true; 2], detect_flags_ts: 0, detect_flags_tc: 0, de_state: std::ptr::null_mut(), @@ -178,6 +183,7 @@ impl AppLayerTxData { file_flags: 0, file_tx: 0, stream_logged: 0, + updated: [true; 2], detect_flags_ts, detect_flags_tc, de_state: std::ptr::null_mut(), diff --git a/rust/src/applayertemplate/template.rs b/rust/src/applayertemplate/template.rs index 43a8bf940006..9a2d5948cd58 100644 --- a/rust/src/applayertemplate/template.rs +++ b/rust/src/applayertemplate/template.rs @@ -200,6 +200,7 @@ impl TemplateState { start = rem; if let Some(tx) = self.find_request() { + tx.tx_data.updated[1] = true; tx.response = Some(response); SCLogNotice!("Found response for request:"); SCLogNotice!("- Request: {:?}", tx.request); diff --git a/rust/src/dcerpc/dcerpc.rs b/rust/src/dcerpc/dcerpc.rs index 32ebf6990134..a3cc6b5c6348 100644 --- a/rust/src/dcerpc/dcerpc.rs +++ b/rust/src/dcerpc/dcerpc.rs @@ -357,6 +357,7 @@ impl DCERPCState { for tx_old in &mut self.transactions.range_mut(self.tx_index_completed..) { index += 1; if !tx_old.req_done || !tx_old.resp_done { + tx_old.tx_data.updated = [true; 2]; tx_old.req_done = true; tx_old.resp_done = true; break; @@ -533,6 +534,7 @@ impl DCERPCState { } } } + tx.tx_data.updated = [true; 2]; return Some(tx); } } diff --git a/rust/src/dcerpc/dcerpc_udp.rs b/rust/src/dcerpc/dcerpc_udp.rs index ab7f65cafbad..80afb39cb055 100644 --- a/rust/src/dcerpc/dcerpc_udp.rs +++ b/rust/src/dcerpc/dcerpc_udp.rs @@ -88,6 +88,7 @@ impl DCERPCUDPState { for tx_old in &mut self.transactions.range_mut(self.tx_index_completed..) { index += 1; if !tx_old.req_done || !tx_old.resp_done { + tx_old.tx_data.updated = [true; 2]; tx_old.req_done = true; tx_old.resp_done = true; break; @@ -164,6 +165,7 @@ impl DCERPCUDPState { } if let Some(tx) = otx { + tx.tx_data.updated = [true; 2]; let done = (hdr.flags1 & PFCL1_FRAG) == 0 || (hdr.flags1 & PFCL1_LASTFRAG) != 0; match hdr.pkt_type { diff --git a/rust/src/enip/enip.rs b/rust/src/enip/enip.rs index 31c9152e5cab..d5b4961a62da 100644 --- a/rust/src/enip/enip.rs +++ b/rust/src/enip/enip.rs @@ -203,6 +203,7 @@ impl EnipState { fn purge_tx_flood(&mut self) { let mut event_set = false; for tx in self.transactions.iter_mut() { + tx.tx_data.updated = [true; 2]; tx.done = true; if !event_set { tx.tx_data.set_event(EnipEvent::TooManyTransactions as u8); @@ -216,6 +217,7 @@ impl EnipState { if let Some(req) = &tx.request { if tx.response.is_none() { tx.done = true; + tx.tx_data.updated = [true; 2]; if response_matches_request(req, pdu) { return Some(tx); } diff --git a/rust/src/http2/http2.rs b/rust/src/http2/http2.rs index 9281c011ff29..08fcaaf09da2 100644 --- a/rust/src/http2/http2.rs +++ b/rust/src/http2/http2.rs @@ -752,6 +752,7 @@ impl HTTP2State { let tx = &mut self.transactions[index - 1]; tx.tx_data.update_file_flags(self.state_data.file_flags); tx.update_file_flags(tx.tx_data.file_flags); + tx.tx_data.updated = [true; 2]; return Some(tx); } else { // do not use SETTINGS_MAX_CONCURRENT_STREAMS as it can grow too much @@ -764,6 +765,7 @@ impl HTTP2State { tx_old.set_event(HTTP2Event::TooManyStreams); // use a distinct state, even if we do not log it tx_old.state = HTTP2TransactionState::HTTP2StateTodrop; + tx_old.tx_data.updated = [true; 2]; } return None; } diff --git a/rust/src/ldap/ldap.rs b/rust/src/ldap/ldap.rs index 4c9c3947d7a8..f1929e933b3f 100644 --- a/rust/src/ldap/ldap.rs +++ b/rust/src/ldap/ldap.rs @@ -150,6 +150,7 @@ impl LdapState { for tx_old in &mut self.transactions.range_mut(self.tx_index_completed..) { index += 1; if !tx_old.complete { + tx_old.tx_data.updated = [true; 2]; tx_old.complete = true; tx_old .tx_data @@ -278,6 +279,7 @@ impl LdapState { if let Some(tx) = self.find_request(response.message_id) { tx.complete = tx_is_complete(&response.protocol_op, Direction::ToClient); let tx_id = tx.id(); + tx.tx_data.updated[1] = true; tx.responses.push_back(response); let consumed = start.len() - rem.len(); self.set_frame_tc(flow, tx_id, consumed as i64); diff --git a/rust/src/modbus/modbus.rs b/rust/src/modbus/modbus.rs index 0d0c73371ef0..b924b4cfc87b 100644 --- a/rust/src/modbus/modbus.rs +++ b/rust/src/modbus/modbus.rs @@ -124,6 +124,7 @@ impl ModbusState { for tx in &mut self.transactions { if let Some(req) = &tx.request { if tx.response.is_none() && resp.matches(req) { + tx.tx_data.updated = [true; 2]; return Some(tx); } } @@ -139,6 +140,7 @@ impl ModbusState { for tx in &mut self.transactions { if let Some(resp) = &tx.response { if tx.request.is_none() && req.matches(resp) { + tx.tx_data.updated = [true; 2]; return Some(tx); } } @@ -184,6 +186,7 @@ impl ModbusState { match self.find_response_and_validate(&mut msg) { Some(tx) => { tx.set_events_from_flags(&msg.error_flags); + tx.tx_data.updated = [true; 2]; tx.request = Some(msg); } None => { @@ -210,6 +213,7 @@ impl ModbusState { } else { tx.set_events_from_flags(&msg.error_flags); } + tx.tx_data.updated = [true; 2]; tx.response = Some(msg); } None => { diff --git a/rust/src/mqtt/mqtt.rs b/rust/src/mqtt/mqtt.rs index 4079daa6de05..011d2b2d9d6a 100644 --- a/rust/src/mqtt/mqtt.rs +++ b/rust/src/mqtt/mqtt.rs @@ -174,6 +174,7 @@ impl MQTTState { if !tx.complete { if let Some(mpktid) = tx.pkt_id { if mpktid == pkt_id { + tx.tx_data.updated = [true; 2]; return Some(tx); } } @@ -196,6 +197,7 @@ impl MQTTState { for tx_old in &mut self.transactions.range_mut(self.tx_index_completed..) { index += 1; if !tx_old.complete { + tx_old.tx_data.updated = [true; 2]; tx_old.complete = true; MQTTState::set_event(tx_old, MQTTEvent::TooManyTransactions); break; diff --git a/rust/src/nfs/nfs.rs b/rust/src/nfs/nfs.rs index d4b472e1f69c..9a16122a85a8 100644 --- a/rust/src/nfs/nfs.rs +++ b/rust/src/nfs/nfs.rs @@ -431,6 +431,7 @@ impl NFSState { // set at least one another transaction to the drop state for tx_old in &mut self.transactions { if !tx_old.request_done || !tx_old.response_done { + tx_old.tx_data.updated = [true; 2]; tx_old.request_done = true; tx_old.response_done = true; tx_old.is_file_closed = true; @@ -484,6 +485,7 @@ impl NFSState { pub fn mark_response_tx_done(&mut self, xid: u32, rpc_status: u32, nfs_status: u32, resp_handle: &[u8]) { if let Some(mytx) = self.get_tx_by_xid(xid) { + mytx.tx_data.updated = [true; 2]; mytx.response_done = true; mytx.rpc_response_status = rpc_status; mytx.nfs_response_status = nfs_status; @@ -736,6 +738,7 @@ impl NFSState { tx.tx_data.update_file_flags(self.state_data.file_flags); d.update_file_flags(tx.tx_data.file_flags); SCLogDebug!("Found NFS file TX with ID {} XID {:04X}", tx.id, tx.xid); + tx.tx_data.updated = [true; 2]; return Some(tx); } } diff --git a/rust/src/pgsql/pgsql.rs b/rust/src/pgsql/pgsql.rs index 658c2326ffd5..37c144c8a592 100644 --- a/rust/src/pgsql/pgsql.rs +++ b/rust/src/pgsql/pgsql.rs @@ -208,6 +208,7 @@ impl PgsqlState { for tx_old in &mut self.transactions.range_mut(self.tx_index_completed..) { index += 1; if tx_old.tx_res_state < PgsqlTxProgress::TxDone { + tx_old.tx_data.updated = [true; 2]; // we don't check for TxReqDone for the majority of requests are basically completed // when they're parsed, as of now tx_old.tx_req_state = PgsqlTxProgress::TxFlushedOut; @@ -360,6 +361,7 @@ impl PgsqlState { // A simplified finite state machine for PostgreSQL v3 can be found at: // https://samadhiweb.com/blog/2013.04.28.graphviz.postgresv3.html if let Some(tx) = self.find_or_create_tx() { + tx.tx_data.updated[0] = true; tx.request = Some(request); if let Some(state) = new_state { if Self::request_is_complete(state) { @@ -518,6 +520,7 @@ impl PgsqlState { self.state_progress = state; } if let Some(tx) = self.find_or_create_tx() { + tx.tx_data.updated[1] = true; if tx.tx_res_state == PgsqlTxProgress::TxInit { tx.tx_res_state = PgsqlTxProgress::TxReceived; } diff --git a/rust/src/rfb/rfb.rs b/rust/src/rfb/rfb.rs index 810ed1d85d9a..4e50b4de9207 100644 --- a/rust/src/rfb/rfb.rs +++ b/rust/src/rfb/rfb.rs @@ -166,7 +166,12 @@ impl RFBState { fn get_current_tx(&mut self) -> Option<&mut RFBTransaction> { let tx_id = self.tx_id; - self.transactions.iter_mut().find(|tx| tx.tx_id == tx_id) + let r = self.transactions.iter_mut().find(|tx| tx.tx_id == tx_id); + if let Some(tx) = r { + tx.tx_data.updated = [true; 2]; + return Some(tx); + } + return None; } fn parse_request(&mut self, flow: *const Flow, stream_slice: StreamSlice) -> AppLayerResult { diff --git a/rust/src/smb/dcerpc.rs b/rust/src/smb/dcerpc.rs index de6b8def73b3..be29ab389162 100644 --- a/rust/src/smb/dcerpc.rs +++ b/rust/src/smb/dcerpc.rs @@ -168,6 +168,7 @@ impl SMBState { _ => { false }, }; if found { + tx.tx_data.updated = [true; 2]; return Some(tx); } } diff --git a/rust/src/smb/files.rs b/rust/src/smb/files.rs index bdc2619a173d..e9494c3d36df 100644 --- a/rust/src/smb/files.rs +++ b/rust/src/smb/files.rs @@ -126,6 +126,7 @@ impl SMBState { tx.tx_data.update_file_flags(self.state_data.file_flags); d.update_file_flags(tx.tx_data.file_flags); } + tx.tx_data.updated = [true; 2]; return Some(tx); } } @@ -152,6 +153,7 @@ impl SMBState { tx.tx_data.update_file_flags(self.state_data.file_flags); d.update_file_flags(tx.tx_data.file_flags); } + tx.tx_data.updated = [true; 2]; return Some(tx); } } diff --git a/rust/src/smb/session.rs b/rust/src/smb/session.rs index be7866976dc9..725d42aff736 100644 --- a/rust/src/smb/session.rs +++ b/rust/src/smb/session.rs @@ -61,6 +61,7 @@ impl SMBState { _ => { false }, }; if hit { + tx.tx_data.updated = [true; 2]; return Some(tx); } } diff --git a/rust/src/smb/smb.rs b/rust/src/smb/smb.rs index 0d35d6debb3e..0bbc2e72f9e0 100644 --- a/rust/src/smb/smb.rs +++ b/rust/src/smb/smb.rs @@ -805,6 +805,7 @@ impl SMBState { for tx_old in &mut self.transactions.range_mut(self.tx_index_completed..) { index += 1; if !tx_old.request_done || !tx_old.response_done { + tx_old.tx_data.updated = [true; 2]; tx_old.request_done = true; tx_old.response_done = true; tx_old.set_event(SMBEvent::TooManyTransactions); @@ -923,6 +924,7 @@ impl SMBState { false }; if found { + tx.tx_data.updated = [true; 2]; return Some(tx); } } @@ -947,6 +949,7 @@ impl SMBState { false }; if found { + tx.tx_data.updated = [true; 2]; return Some(tx); } } @@ -985,6 +988,7 @@ impl SMBState { _ => { false }, }; if found { + tx.tx_data.updated = [true; 2]; return Some(tx); } } @@ -1018,6 +1022,7 @@ impl SMBState { _ => { false }, }; if hit { + tx.tx_data.updated = [true; 2]; return Some(tx); } } diff --git a/rust/src/ssh/ssh.rs b/rust/src/ssh/ssh.rs index a6a3871a8e60..48d98b28a71b 100644 --- a/rust/src/ssh/ssh.rs +++ b/rust/src/ssh/ssh.rs @@ -415,6 +415,7 @@ pub unsafe extern "C" fn rs_ssh_parse_request( let state = &mut cast_pointer!(state, SSHState); let buf = stream_slice.as_slice(); let hdr = &mut state.transaction.cli_hdr; + state.transaction.tx_data.updated[0] = true; if hdr.flags < SSHConnectionState::SshStateBannerDone { return state.parse_banner(buf, false, pstate, flow, &stream_slice); } else { @@ -431,6 +432,7 @@ pub unsafe extern "C" fn rs_ssh_parse_response( let state = &mut cast_pointer!(state, SSHState); let buf = stream_slice.as_slice(); let hdr = &mut state.transaction.srv_hdr; + state.transaction.tx_data.updated[1] = true; if hdr.flags < SSHConnectionState::SshStateBannerDone { return state.parse_banner(buf, true, pstate, flow, &stream_slice); } else { diff --git a/src/app-layer-dnp3.c b/src/app-layer-dnp3.c index 0e18a4fc5fba..d131858ab44e 100644 --- a/src/app-layer-dnp3.c +++ b/src/app-layer-dnp3.c @@ -886,6 +886,7 @@ static void DNP3HandleUserDataRequest(DNP3State *dnp3, const uint8_t *input, /* Update the saved transport header so subsequent segments * will be matched to this sequence number. */ tx->th = th; + tx->tx_data.updated[0] = true; } else { ah = (DNP3ApplicationHeader *)(input + sizeof(DNP3LinkHeader) + @@ -963,6 +964,7 @@ static void DNP3HandleUserDataResponse(DNP3State *dnp3, const uint8_t *input, /* Replace the transport header in the transaction with this * one in case there are more frames. */ tx->th = th; + tx->tx_data.updated[1] = true; } else { ah = (DNP3ApplicationHeader *)(input + offset); diff --git a/src/app-layer-ftp.c b/src/app-layer-ftp.c index a1a99d4bd701..50843dbb6b89 100644 --- a/src/app-layer-ftp.c +++ b/src/app-layer-ftp.c @@ -714,6 +714,7 @@ static AppLayerResult FTPParseResponse(Flow *f, void *ftp_state, AppLayerParserS SCReturnStruct(APP_LAYER_ERROR); } lasttx = tx; + tx->tx_data.updated[1] = true; if (state->command == FTP_COMMAND_UNKNOWN || tx->command_descriptor == NULL) { /* unknown */ tx->command_descriptor = &FtpCommands[FTP_COMMAND_MAX - 1]; @@ -1046,7 +1047,7 @@ static AppLayerResult FTPDataParse(Flow *f, FtpDataState *ftpdata_state, SCTxDataUpdateFileFlags(&ftpdata_state->tx_data, ftpdata_state->state_data.file_flags); if (ftpdata_state->tx_data.file_tx == 0) ftpdata_state->tx_data.file_tx = direction & (STREAM_TOSERVER | STREAM_TOCLIENT); - + ftpdata_state->tx_data.updated[(direction & STREAM_TOSERVER) ? 0 : 1] = true; /* we depend on detection engine for file pruning */ const uint16_t flags = FileFlowFlagsToFlags(ftpdata_state->tx_data.file_flags, direction); int ret = 0; diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c index 3f3037ca30a1..489c789380d0 100644 --- a/src/app-layer-htp.c +++ b/src/app-layer-htp.c @@ -1453,6 +1453,7 @@ static int HTPCallbackRequestBodyData(htp_tx_data_t *d) if (tx_ud == NULL) { SCReturnInt(HTP_OK); } + tx_ud->tx_data.updated[0] = true; SCTxDataUpdateFileFlags(&tx_ud->tx_data, hstate->state_data.file_flags); if (!tx_ud->response_body_init) { @@ -1584,6 +1585,7 @@ static int HTPCallbackResponseBodyData(htp_tx_data_t *d) if (tx_ud == NULL) { SCReturnInt(HTP_OK); } + tx_ud->tx_data.updated[1] = true; SCTxDataUpdateFileFlags(&tx_ud->tx_data, hstate->state_data.file_flags); if (!tx_ud->request_body_init) { tx_ud->request_body_init = 1; @@ -1690,6 +1692,7 @@ static int HTPCallbackRequestHasTrailer(htp_tx_t *tx) { HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx); if (htud != NULL) { + htud->tx_data.updated[0] = true; htud->request_has_trailers = 1; } return HTP_OK; @@ -1699,6 +1702,7 @@ static int HTPCallbackResponseHasTrailer(htp_tx_t *tx) { HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx); if (htud != NULL) { + htud->tx_data.updated[1] = true; htud->response_has_trailers = 1; } return HTP_OK; @@ -1741,6 +1745,8 @@ static int HTPCallbackRequestStart(htp_tx_t *tx) } tx_ud->tx_data.file_tx = STREAM_TOSERVER | STREAM_TOCLIENT; // each http tx may xfer files htp_tx_set_user_data(tx, tx_ud); + } else { + tx_ud->tx_data.updated[0] = true; } SCReturnInt(HTP_OK); } @@ -1781,6 +1787,8 @@ static int HTPCallbackResponseStart(htp_tx_t *tx) tx_ud->tx_data.file_tx = STREAM_TOCLIENT; // each http tx may xfer files. Toserver already missed. htp_tx_set_user_data(tx, tx_ud); + } else { + tx_ud->tx_data.updated[1] = true; } SCReturnInt(HTP_OK); } @@ -1832,6 +1840,7 @@ static int HTPCallbackRequestComplete(htp_tx_t *tx) HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx); if (htud != NULL) { + htud->tx_data.updated[0] = true; if (htud->tsflags & HTP_FILENAME_SET) { SCLogDebug("closing file that was being stored"); (void)HTPFileClose(htud, NULL, 0, 0, STREAM_TOSERVER); @@ -1887,6 +1896,7 @@ static int HTPCallbackResponseComplete(htp_tx_t *tx) HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx); if (htud != NULL) { + htud->tx_data.updated[1] = true; if (htud->tcflags & HTP_FILENAME_SET) { SCLogDebug("closing file that was being stored"); (void)HTPFileClose(htud, NULL, 0, 0, STREAM_TOCLIENT); @@ -2005,6 +2015,7 @@ static int HTPCallbackRequestHeaderData(htp_tx_data_t *tx_data) return HTP_OK; } tx_ud->request_headers_raw = ptmp; + tx_ud->tx_data.updated[0] = true; memcpy(tx_ud->request_headers_raw + tx_ud->request_headers_raw_len, tx_data->data, tx_data->len); @@ -2027,6 +2038,7 @@ static int HTPCallbackResponseHeaderData(htp_tx_data_t *tx_data) if (tx_ud == NULL) { return HTP_OK; } + tx_ud->tx_data.updated[1] = true; ptmp = HTPRealloc(tx_ud->response_headers_raw, tx_ud->response_headers_raw_len, tx_ud->response_headers_raw_len + tx_data->len); diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c index c8410d41e4a8..a110802ef55d 100644 --- a/src/app-layer-parser.c +++ b/src/app-layer-parser.c @@ -935,7 +935,10 @@ void AppLayerParserTransactionsCleanup(Flow *f, const uint8_t pkt_dir) (pkt_dir == STREAM_TOSERVER) ? ts_disrupt_flags : tc_disrupt_flags); AppLayerParserFileTxHousekeeping(f, tx, pkt_dir, (bool)pkt_dir_trunc); } - + if (txd) { + // should be reset by parser next time it updates the tx + txd->updated[(pkt_dir & STREAM_TOSERVER) ? 0 : 1] = false; + } const int tx_progress_tc = AppLayerParserGetStateProgress(ipproto, alproto, tx, tc_disrupt_flags); if (tx_progress_tc < tx_end_state_tc) { diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c index 03260bfd3ae8..6b0cece07a1d 100644 --- a/src/app-layer-smtp.c +++ b/src/app-layer-smtp.c @@ -866,6 +866,9 @@ static int SMTPProcessReply( return 0; // to continue processing further } + if (state->curr_tx) { + state->curr_tx->tx_data.updated[1] = true; + } /* the reply code has to contain at least 3 bytes, to hold the 3 digit * reply code */ if (line->len < 3) { @@ -1170,6 +1173,7 @@ static int SMTPProcessRequest( if (frame != NULL && state->curr_tx) { AppLayerFrameSetTxId(frame, state->curr_tx->tx_id); } + tx->tx_data.updated[0] = true; state->toserver_data_count += (line->len + line->delim_len); diff --git a/src/app-layer-ssl.c b/src/app-layer-ssl.c index bc0c42142f1c..b8f73f567c64 100644 --- a/src/app-layer-ssl.c +++ b/src/app-layer-ssl.c @@ -2687,6 +2687,8 @@ static AppLayerResult SSLDecode(Flow *f, uint8_t direction, void *alstate, AppLayerParserState *pstate, StreamSlice stream_slice) { SSLState *ssl_state = (SSLState *)alstate; + ssl_state->tx_data.updated[0] = true; + ssl_state->tx_data.updated[1] = true; uint32_t counter = 0; ssl_state->f = f; const uint8_t *input = StreamSliceGetData(&stream_slice); diff --git a/src/detect.c b/src/detect.c index 03fa8437068d..f097d625eaf8 100644 --- a/src/detect.c +++ b/src/detect.c @@ -1294,6 +1294,12 @@ static DetectTransaction GetDetectTx(const uint8_t ipproto, const AppProto alpro DetectTransaction no_tx = NO_TX; return no_tx; } + const int tx_progress = AppLayerParserGetStateProgress(ipproto, alproto, tx_ptr, flow_flags); + if (!txd->updated[(flow_flags & STREAM_TOSERVER) ? 0 : 1] && tx_progress < tx_end_state && + ((flow_flags & STREAM_EOF) == 0)) { + DetectTransaction no_tx = NO_TX; + return no_tx; + } uint64_t detect_flags = (flow_flags & STREAM_TOSERVER) ? txd->detect_flags_ts : txd->detect_flags_tc; if (detect_flags & APP_LAYER_TX_INSPECTED_FLAG) { @@ -1310,7 +1316,6 @@ static DetectTransaction GetDetectTx(const uint8_t ipproto, const AppProto alpro return no_tx; } - const int tx_progress = AppLayerParserGetStateProgress(ipproto, alproto, tx_ptr, flow_flags); const int dir_int = (flow_flags & STREAM_TOSERVER) ? 0 : 1; DetectEngineState *tx_de_state = txd->de_state; DetectEngineStateDirection *tx_dir_state = tx_de_state ? &tx_de_state->dir_state[dir_int] : NULL; diff --git a/src/output-tx.c b/src/output-tx.c index 40b887706770..7a2ecbf2aeaf 100644 --- a/src/output-tx.c +++ b/src/output-tx.c @@ -392,7 +392,7 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data) uint64_t tx_id = AppLayerParserGetTransactionLogId(f->alparser); uint64_t max_id = tx_id; int logged = 0; - int gap = 0; + bool gap = false; const bool support_files = AppLayerParserSupportsFiles(ipproto, alproto); const uint8_t pkt_dir = STREAM_FLAGS_FOR_PACKET(p); @@ -415,15 +415,6 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data) tx_id = ires.tx_id; SCLogDebug("STARTING tx_id %" PRIu64 ", tx %p", tx_id, tx); - const int tx_progress_ts = - AppLayerParserGetStateProgress(ipproto, alproto, tx, ts_disrupt_flags); - const int tx_progress_tc = - AppLayerParserGetStateProgress(ipproto, alproto, tx, tc_disrupt_flags); - const bool tx_complete = (tx_progress_ts == complete_ts && tx_progress_tc == complete_tc); - - SCLogDebug("file_thread_data %p filedata_thread_data %p", op_thread_data->file, - op_thread_data->filedata); - AppLayerTxData *txd = AppLayerParserGetTxData(ipproto, alproto, tx); if (unlikely(txd == NULL)) { SCLogDebug("NO TXD"); @@ -433,6 +424,15 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data) goto next_tx; } + const int tx_progress_ts = + AppLayerParserGetStateProgress(ipproto, alproto, tx, ts_disrupt_flags); + const int tx_progress_tc = + AppLayerParserGetStateProgress(ipproto, alproto, tx, tc_disrupt_flags); + const bool tx_complete = (tx_progress_ts == complete_ts && tx_progress_tc == complete_tc); + + SCLogDebug("file_thread_data %p filedata_thread_data %p", op_thread_data->file, + op_thread_data->filedata); + if (file_logging_active) { if (AppLayerParserIsFileTx(txd)) { // need to process each tx that might be a file tx, // even if there are not files (yet) @@ -467,6 +467,11 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data) } } SCLogDebug("logger: expect %08x, have %08x", logger_expectation, txd->logged.flags); + if (!txd->updated[0] && !txd->updated[1] && !(tx_progress_ts == complete_ts) && + !(tx_progress_tc == complete_tc) && !ts_eof && !tc_eof) { + gap = true; + goto next_tx; + } if (list[ALPROTO_UNKNOWN] != 0) { OutputTxLogList0(tv, op_thread_data, p, f, tx, tx_id); @@ -517,7 +522,7 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data) max_id = tx_id; SCLogDebug("max_id %" PRIu64, max_id); } else { - gap = 1; + gap = true; } next_tx: if (!ires.has_next) From a682fa70c9f80109e1031529ad6ce8ae55c0db6d Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Tue, 10 Sep 2024 11:20:29 +0200 Subject: [PATCH 2/2] rust/ftp: handle NULL inputs Completes Ticket 7013 --- rust/src/ftp/mod.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/rust/src/ftp/mod.rs b/rust/src/ftp/mod.rs index aae5ff4afe56..a206aa2f350c 100644 --- a/rust/src/ftp/mod.rs +++ b/rust/src/ftp/mod.rs @@ -88,6 +88,9 @@ pub fn ftp_pasv_response(i: &[u8]) -> IResult<&[u8], u16> { #[no_mangle] pub unsafe extern "C" fn rs_ftp_active_port(input: *const u8, len: u32) -> u16 { + if input.is_null() { + return 0; + } let buf = build_slice!(input, len as usize); match ftp_active_port(buf) { Ok((_, dport)) => { @@ -105,7 +108,10 @@ pub unsafe extern "C" fn rs_ftp_active_port(input: *const u8, len: u32) -> u16 { #[no_mangle] pub unsafe extern "C" fn rs_ftp_pasv_response(input: *const u8, len: u32) -> u16 { - let buf = std::slice::from_raw_parts(input, len as usize); + if input.is_null() { + return 0; + } + let buf = build_slice!(input, len as usize); match ftp_pasv_response(buf) { Ok((_, dport)) => { return dport; @@ -147,6 +153,9 @@ pub fn ftp_active_eprt(i: &[u8]) -> IResult<&[u8], u16> { #[no_mangle] pub unsafe extern "C" fn rs_ftp_active_eprt(input: *const u8, len: u32) -> u16 { + if input.is_null() { + return 0; + } let buf = build_slice!(input, len as usize); match ftp_active_eprt(buf) { Ok((_, dport)) => { @@ -163,7 +172,10 @@ pub unsafe extern "C" fn rs_ftp_active_eprt(input: *const u8, len: u32) -> u16 { } #[no_mangle] pub unsafe extern "C" fn rs_ftp_epsv_response(input: *const u8, len: u32) -> u16 { - let buf = std::slice::from_raw_parts(input, len as usize); + if input.is_null() { + return 0; + } + let buf = build_slice!(input, len as usize); match ftp_epsv_response(buf) { Ok((_, dport)) => { return dport;