Merge branch 'main' into evan.li/eval-metric-fork

.github/workflows/system-tests.yml

@@ -304,3 +304,15 @@ jobs:
         with:
           name: logs_parametric
           path: artifact.tar.gz
+
+  finished:
+    runs-on: ubuntu-latest
+    needs: [parametric, system-tests]
+    if: success() || failure()
+    steps:
+      - name: True when everything else succeeded
+        if: needs.parametric.result == 'success' && needs.system-tests.result == 'success'
+        run: exit 0
+      - name: Fails if anything else failed
+        if: needs.parametric.result != 'success' || needs.system-tests.result != 'success'
+        run: exit 1

.gitlab/package.yml

@@ -5,7 +5,7 @@ build_base_venvs:
     matrix:
       - PYTHON_VERSION: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
   variables:
-    CMAKE_BUILD_PARALLEL_LEVEL: 24
+    CMAKE_BUILD_PARALLEL_LEVEL: 12
     PIP_VERBOSE: 1
   script:
     - pip install riot~=0.19.1

benchmarks/appsec_iast_aspects/config.yaml

@@ -215,6 +215,24 @@ aspect_iast_do_index:
     warmups: 1
     iast_enabled: 1
 
+aspect_no_iast_do_index_on_dict: &aspect_no_iast_do_index_on_dict
+    iast_enabled: 0
+    processes: 10
+    loops: 1
+    values: 6
+    warmups: 1
+    mod_original_name: "bm.iast_fixtures.str_methods"
+    function_name: "do_index_on_dict"
+    args: [{'fOobaR': 4, 3: 'foobar'}, 'fOobaR']
+
+aspect_iast_do_index_on_dict:
+    << : *aspect_no_iast_do_index_on_dict
+    processes: 10
+    loops: 1
+    values: 6
+    warmups: 1
+    iast_enabled: 1
+
 aspect_no_iast_do_join: &aspect_no_iast_do_join
     iast_enabled: 0
     processes: 10
@@ -377,6 +395,24 @@ aspect_iast_do_partition:
     warmups: 1
     iast_enabled: 1
 
+aspect_no_iast_do_re_match_index: &aspect_no_iast_do_re_match_index
+    iast_enabled: 0
+    processes: 10
+    loops: 1
+    values: 6
+    warmups: 1
+    mod_original_name: "bm.iast_fixtures.str_methods"
+    function_name: "do_re_match_index"
+    args: ['(\\w+) (\\w+)', 'foo bar', 1]
+
+aspect_iast_do_re_match_index:
+    << : *aspect_no_iast_do_re_match_index
+    processes: 10
+    loops: 1
+    values: 6
+    warmups: 1
+    iast_enabled: 1
+
 aspect_no_iast_do_re_sub: &aspect_no_iast_do_re_sub
     iast_enabled: 0
     processes: 10

benchmarks/bm/iast_fixtures/str_methods.py

@@ -1064,6 +1064,10 @@ def do_index(c: str, i: int) -> Text:
     return c[i]
 
 
+def do_index_on_dict(d: dict, k):
+    return d[k]
+
+
 def do_methodcaller(s, func, *args):
     func_method = operator.methodcaller(func, *args)
     return func_method(s)
@@ -1236,3 +1240,8 @@ def index_lower_add(url):
 def urlib_urlsplit(text):
     results = urllib.parse.urlsplit(text)
     return results
+
+
+def do_re_match_index(text, regexp, index):
+    match = re.search(regexp, text)
+    return match[index]

benchmarks/bm/iast_utils/aspects_benchmarks_generate.py

@@ -159,6 +159,7 @@
         "do_format_with_named_parameter": ["foobar{key}", "baz"],
         "do_format_with_positional_parameter": ["foobar{}", "baz"],
         "do_index": ["foobar", 3],
+        "do_index_on_dict": [{"fOobaR": 4, 3: "foobar"}, "fOobaR"],
         "do_join": ["foobar", ["baz", "pok"]],
         "do_join_args_kwargs": ["foobar", ["baz", "pok"]],
         "do_ljust": ["foobar", 2],
@@ -167,6 +168,7 @@
         "do_partition": ["foobar", "o"],
         "do_re_sub": ["foobar", "o", "a", 1],
         "do_re_subn": ["foobar", "o", "a", 1],
+        "do_re_match_index": [r"(\w+) (\w+)", "foo bar", 1],
         "do_replace": ["foobar", "o", "a", 1],
         "do_rplit_separator_and_maxsplit": ["foobar", "o", 1],
         "do_rsplit": ["foo bar baz", " ", 1],

ddtrace/_trace/utils_botocore/span_pointers.py

@@ -0,0 +1,101 @@
+from typing import Any
+from typing import Dict
+from typing import List
+
+from ddtrace._trace._span_pointer import _SpanPointerDescription
+from ddtrace._trace._span_pointer import _SpanPointerDirection
+from ddtrace._trace._span_pointer import _standard_hashing_function
+from ddtrace.internal.logger import get_logger
+
+
+log = get_logger(__name__)
+
+
+def extract_span_pointers_from_successful_botocore_response(
+    endpoint_name: str,
+    operation_name: str,
+    request_parameters: Dict[str, Any],
+    response: Dict[str, Any],
+) -> List[_SpanPointerDescription]:
+    if endpoint_name == "s3":
+        return _extract_span_pointers_for_s3_response(operation_name, request_parameters, response)
+
+    return []
+
+
+def _extract_span_pointers_for_s3_response(
+    operation_name: str,
+    request_parameters: Dict[str, Any],
+    response: Dict[str, Any],
+) -> List[_SpanPointerDescription]:
+    if operation_name == "PutObject":
+        return _extract_span_pointers_for_s3_put_object_response(request_parameters, response)
+
+    return []
+
+
+def _extract_span_pointers_for_s3_put_object_response(
+    request_parameters: Dict[str, Any],
+    response: Dict[str, Any],
+) -> List[_SpanPointerDescription]:
+    # Endpoint Reference:
+    # https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html
+
+    try:
+        bucket = request_parameters["Bucket"]
+        key = request_parameters["Key"]
+        etag = response["ETag"]
+
+        # The ETag is surrounded by double quotes for some reason.
+        if etag.startswith('"') and etag.endswith('"'):
+            etag = etag[1:-1]
+
+    except KeyError as e:
+        log.warning(
+            "missing a parameter or response field required to make span pointer for S3.PutObject: %s",
+            str(e),
+        )
+        return []
+
+    try:
+        return [
+            _aws_s3_object_span_pointer_description(
+                pointer_direction=_SpanPointerDirection.DOWNSTREAM,
+                bucket=bucket,
+                key=key,
+                etag=etag,
+            )
+        ]
+    except Exception as e:
+        log.warning(
+            "failed to generate S3.PutObject span pointer: %s",
+            str(e),
+        )
+        return []
+
+
+def _aws_s3_object_span_pointer_description(
+    pointer_direction: _SpanPointerDirection,
+    bucket: str,
+    key: str,
+    etag: str,
+) -> _SpanPointerDescription:
+    return _SpanPointerDescription(
+        pointer_kind="aws.s3.object",
+        pointer_direction=pointer_direction,
+        pointer_hash=_aws_s3_object_span_pointer_hash(bucket, key, etag),
+        extra_attributes={},
+    )
+
+
+def _aws_s3_object_span_pointer_hash(bucket: str, key: str, etag: str) -> str:
+    if '"' in etag:
+        # Some AWS API endpoints put the ETag in double quotes. We expect the
+        # calling code to have correctly fixed this already.
+        raise ValueError(f"ETag should not have double quotes: {etag}")
+
+    return _standard_hashing_function(
+        bucket.encode("ascii"),
+        key.encode("utf-8"),
+        etag.encode("ascii"),
+    )

ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectIndex.cpp

@@ -1,5 +1,6 @@
 #include "AspectIndex.h"
 #include "Helpers.h"
+#include "Utils/StringUtils.h"
 
 /**
  * @brief Index aspect
@@ -13,17 +14,36 @@
 PyObject*
 index_aspect(PyObject* result_o, PyObject* candidate_text, PyObject* idx, const TaintRangeMapTypePtr& tx_taint_map)
 {
-    const auto idx_long = PyLong_AsLong(idx);
     TaintRangeRefs ranges_to_set;
     auto [ranges, ranges_error] = get_ranges(candidate_text, tx_taint_map);
     if (ranges_error) {
         return result_o;
     }
-    for (const auto& current_range : ranges) {
-        if (current_range->start <= idx_long and idx_long < (current_range->start + current_range->length)) {
-            ranges_to_set.emplace_back(initializer->allocate_taint_range(0l, 1l, current_range->source));
-            break;
+
+    if (is_text(candidate_text)) {
+        for (const auto& current_range : ranges) {
+            const auto idx_long = PyLong_AsLong(idx);
+            if (current_range->start <= idx_long and idx_long < (current_range->start + current_range->length)) {
+                ranges_to_set.emplace_back(initializer->allocate_taint_range(0l, 1l, current_range->source));
+                break;
+            }
         }
+
+    } else if (PyReMatch_Check(candidate_text)) { // For re.Match objects, taint the whole output
+        try {
+            const size_t& len_result_o{ get_pyobject_size(result_o) };
+            const auto& current_range = ranges.at(0);
+            ranges_to_set.emplace_back(initializer->allocate_taint_range(0l, len_result_o, current_range->source));
+        } catch (const std::out_of_range& ex) {
+            if (nullptr == result_o) {
+                throw py::index_error();
+            }
+            // No ranges found, return original object
+            return result_o;
+        }
+    } else {
+        // Other stuff
+        return result_o;
     }
 
     const auto& res_new_id = new_pyobject_id(result_o);
@@ -41,29 +61,27 @@ PyObject*
 api_index_aspect(PyObject* self, PyObject* const* args, const Py_ssize_t nargs)
 {
     if (nargs != 2) {
-        py::set_error(PyExc_ValueError, MSG_ERROR_N_PARAMS);
         iast_taint_log_error(MSG_ERROR_N_PARAMS);
+        py::set_error(PyExc_ValueError, MSG_ERROR_N_PARAMS);
         return nullptr;
     }
 
     PyObject* candidate_text = args[0];
     PyObject* idx = args[1];
-    auto result_o = PyObject_GetItem(candidate_text, idx);
-    if (!is_text(candidate_text) or !is_some_number(idx)) {
-        return result_o;
-    }
+    const auto result_o = PyObject_GetItem(candidate_text, idx);
     TRY_CATCH_ASPECT("index_aspect", return result_o, , {
-        const auto ctx_map = Initializer::get_tainting_map();
-        if (not ctx_map or ctx_map->empty()) {
+        if (const auto error = has_pyerr_as_string(); !error.empty()) {
+            iast_taint_log_error(error);
+            return nullptr;
+        }
+
+        if ((!is_text(candidate_text) or !is_some_number(idx)) and !PyReMatch_Check(candidate_text)) {
             return result_o;
         }
 
-        auto error_str = has_pyerr_as_string();
-        if (!error_str.empty()) {
-            error_str += " (native index_aspect)";
-            iast_taint_log_error(error_str);
-            py::set_error(PyExc_IndexError, error_str.c_str());
-            return nullptr;
+        const auto ctx_map = Initializer::get_tainting_map();
+        if (not ctx_map or ctx_map->empty()) {
+            return result_o;
         }
 
         return index_aspect(result_o, candidate_text, idx, ctx_map);

ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectsOsPath.cpp

@@ -2,6 +2,7 @@
 #include <string>
 
 #include "Helpers.h"
+#include "Initializer/Initializer.h"
 
 static bool
 starts_with_separator(const py::handle& arg, const std::string& separator)
@@ -14,8 +15,7 @@ template<class StrType>
 StrType
 api_ospathjoin_aspect(StrType& first_part, const py::args& args)
 {
-    const auto ospath = py::module_::import("os.path");
-    auto join = ospath.attr("join");
+    auto join = safe_import("os.path", "join");
     auto result_o = join(first_part, *args);
 
     const auto tx_map = Initializer::get_tainting_map();
@@ -24,7 +24,7 @@ api_ospathjoin_aspect(StrType& first_part, const py::args& args)
     }
 
     TRY_CATCH_ASPECT("ospathjoin_aspect", return result_o, , {
-        const auto separator = ospath.attr("sep").cast<std::string>();
+        const auto separator = safe_import("os.path", "sep").cast<std::string>();
         const auto sepsize = separator.size();
 
         // Find the initial iteration point. This will be the first argument that has the separator ("/foo")
@@ -101,8 +101,7 @@ template<class StrType>
 StrType
 api_ospathbasename_aspect(const StrType& path)
 {
-    const auto ospath = py::module_::import("os.path");
-    auto basename = ospath.attr("basename");
+    auto basename = safe_import("os.path", "basename");
     auto result_o = basename(path);
 
     TRY_CATCH_ASPECT("ospathbasename_aspect", return result_o, , {
@@ -134,8 +133,7 @@ template<class StrType>
 StrType
 api_ospathdirname_aspect(const StrType& path)
 {
-    const auto ospath = py::module_::import("os.path");
-    auto dirname = ospath.attr("dirname");
+    auto dirname = safe_import("os.path", "dirname");
     auto result_o = dirname(path);
 
     TRY_CATCH_ASPECT("ospathdirname_aspect", return result_o, , {
@@ -167,8 +165,7 @@ template<class StrType>
 static py::tuple
 forward_to_set_ranges_on_splitted(const char* function_name, const StrType& path, bool includeseparator = false)
 {
-    const auto ospath = py::module_::import("os.path");
-    auto function = ospath.attr(function_name);
+    auto function = safe_import("os.path", function_name);
     auto result_o = function(path);
 
     TRY_CATCH_ASPECT("forward_to_set_ranges_on_splitted", return result_o, , {
@@ -219,8 +216,7 @@ template<class StrType>
 StrType
 api_ospathnormcase_aspect(const StrType& path)
 {
-    const auto ospath = py::module_::import("os.path");
-    auto normcase = ospath.attr("normcase");
+    auto normcase = safe_import("os.path", "normcase");
     auto result_o = normcase(path);
 
     TRY_CATCH_ASPECT("ospathnormcase_aspect", return result_o, , {

ddtrace/appsec/_iast/_taint_tracking/Aspects/Helpers.cpp

@@ -193,7 +193,8 @@ convert_escaped_text_to_taint_text(const StrType& taint_escaped_text, const Tain
     optional<TaintRangeRefs> optional_ranges_orig = ranges_orig;
 
     vector<tuple<string, int>> context_stack;
-    int length, end = 0;
+    int length = 0;
+    int end = 0;
     TaintRangeRefs ranges;
 
     int latest_end = -1;
@@ -370,10 +371,7 @@ has_pyerr_as_string()
         PyErr_NormalizeException(&extype, &value, &traceback);
         const auto exception_msg_as_pystr = py::str(PyObject_Str(value));
         const auto exception_msg_as_string = std::string(PyUnicode_AsUTF8(exception_msg_as_pystr.ptr()));
-        py::set_error(extype, exception_msg_as_pystr);
-        Py_DecRef(extype);
-        Py_DecRef(value);
-        Py_DecRef(traceback);
+        PyErr_Restore(extype, value, traceback);
         return exception_msg_as_string;
     }