Dataset string memcap 3910/v12

doc/userguide/upgrade.rst

@@ -60,6 +60,10 @@ Major changes
 - It is possible to see an increase of alerts, for the same rule-sets, if you
   use many stream/payload rules, due to Suricata triggering TCP stream
   reassembly earlier.
+- Datasets of the type String had a bug fixed which now takes into account the length of the string
+  as well when checking for memcaps. This may lead to memcaps being hit for older setups that didn't
+  take that into account.
+  For more details, check https://redmine.openinfosecfoundation.org/issues/3910
 
 Upgrading 6.0 to 7.0
 --------------------

src/app-layer-htp-range.c

@@ -127,10 +127,11 @@ static void ContainerUrlRangeFree(void *s)
     }
 }
 
-static inline bool ContainerValueRangeTimeout(HttpRangeContainerFile *cu, const SCTime_t ts)
+static inline bool ContainerValueRangeTimeout(void *data, const SCTime_t ts)
 {
+    HttpRangeContainerFile *cu = data;
     // we only timeout if we have no flow referencing us
-    if ((uint32_t)SCTIME_SECS(ts) > cu->expire || cu->error) {
+    if ((uint32_t)SCTIME_SECS(ts) >= cu->expire || cu->error) {
         if (SC_ATOMIC_GET(cu->hdata->use_cnt) == 0) {
             DEBUG_VALIDATE_BUG_ON(cu->files == NULL);
             return true;
@@ -171,10 +172,10 @@ void HttpRangeContainersInit(void)
         }
     }
 
-    ContainerUrlRangeList.ht =
-            THashInit("app-layer.protocols.http.byterange", sizeof(HttpRangeContainerFile),
-                    ContainerUrlRangeSet, ContainerUrlRangeFree, ContainerUrlRangeHash,
-                    ContainerUrlRangeCompare, false, memcap, CONTAINER_URLRANGE_HASH_SIZE);
+    ContainerUrlRangeList.ht = THashInit("app-layer.protocols.http.byterange",
+            sizeof(HttpRangeContainerFile), ContainerUrlRangeSet, ContainerUrlRangeFree,
+            ContainerUrlRangeHash, ContainerUrlRangeCompare, ContainerValueRangeTimeout, NULL,
+            false, memcap, CONTAINER_URLRANGE_HASH_SIZE);
     ContainerUrlRangeList.timeout = timeout;
 
     SCLogDebug("containers started");

src/datasets-string.c

@@ -98,6 +98,12 @@ uint32_t StringHash(void *s)
     return hash;
 }
 
+uint32_t StringGetLength(void *s)
+{
+    StringType *str = s;
+    return str->len;
+}
+
 // base data stays in hash
 void StringFree(void *s)
 {

src/datasets-string.h

@@ -35,6 +35,7 @@ typedef struct StringType {
 int StringSet(void *dst, void *src);
 bool StringCompare(void *a, void *b);
 uint32_t StringHash(void *s);
+uint32_t StringGetLength(void *s);
 void StringFree(void *s);
 int StringAsBase64(const void *s, char *out, size_t out_size);
 

src/datasets.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2017-2020 Open Information Security Foundation
+/* Copyright (C) 2017-2024 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -700,7 +700,8 @@ Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save,
     switch (type) {
         case DATASET_TYPE_MD5:
             set->hash = THashInit(cnf_name, sizeof(Md5Type), Md5StrSet, Md5StrFree, Md5StrHash,
-                    Md5StrCompare, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
+                    Md5StrCompare, NULL, NULL, load != NULL ? 1 : 0,
+                    memcap > 0 ? memcap : default_memcap,
                     hashsize > 0 ? hashsize : default_hashsize);
             if (set->hash == NULL)
                 goto out_err;
@@ -709,7 +710,8 @@ Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save,
             break;
         case DATASET_TYPE_STRING:
             set->hash = THashInit(cnf_name, sizeof(StringType), StringSet, StringFree, StringHash,
-                    StringCompare, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
+                    StringCompare, NULL, StringGetLength, load != NULL ? 1 : 0,
+                    memcap > 0 ? memcap : default_memcap,
                     hashsize > 0 ? hashsize : default_hashsize);
             if (set->hash == NULL)
                 goto out_err;
@@ -718,7 +720,7 @@ Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save,
             break;
         case DATASET_TYPE_SHA256:
             set->hash = THashInit(cnf_name, sizeof(Sha256Type), Sha256StrSet, Sha256StrFree,
-                    Sha256StrHash, Sha256StrCompare, load != NULL ? 1 : 0,
+                    Sha256StrHash, Sha256StrCompare, NULL, NULL, load != NULL ? 1 : 0,
                     memcap > 0 ? memcap : default_memcap,
                     hashsize > 0 ? hashsize : default_hashsize);
             if (set->hash == NULL)
@@ -727,18 +729,20 @@ Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save,
                 goto out_err;
             break;
         case DATASET_TYPE_IPV4:
-            set->hash = THashInit(cnf_name, sizeof(IPv4Type), IPv4Set, IPv4Free, IPv4Hash,
-                    IPv4Compare, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
-                    hashsize > 0 ? hashsize : default_hashsize);
+            set->hash =
+                    THashInit(cnf_name, sizeof(IPv4Type), IPv4Set, IPv4Free, IPv4Hash, IPv4Compare,
+                            NULL, NULL, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
+                            hashsize > 0 ? hashsize : default_hashsize);
             if (set->hash == NULL)
                 goto out_err;
             if (DatasetLoadIPv4(set) < 0)
                 goto out_err;
             break;
         case DATASET_TYPE_IPV6:
-            set->hash = THashInit(cnf_name, sizeof(IPv6Type), IPv6Set, IPv6Free, IPv6Hash,
-                    IPv6Compare, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
-                    hashsize > 0 ? hashsize : default_hashsize);
+            set->hash =
+                    THashInit(cnf_name, sizeof(IPv6Type), IPv6Set, IPv6Free, IPv6Hash, IPv6Compare,
+                            NULL, NULL, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
+                            hashsize > 0 ? hashsize : default_hashsize);
             if (set->hash == NULL)
                 goto out_err;
             if (DatasetLoadIPv6(set) < 0)