Merge pull request #3 from NatronGitHub/wip

sync with upstream
NatronGitHub · Jul 4, 2019 · 0b56930 · 0b56930
2 parents 2325767 + 68138d2
commit 0b56930
Show file tree

Hide file tree

Showing 20 changed files with 2,295 additions and 11,007 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -1,6 +1,3 @@
 [submodule "breakpad"]
 	path = breakpad
 	url = https://github.com/NatronGitHub/breakpad
-[submodule "jsoncpp"]
-	path = jsoncpp
-	url = https://github.com/open-source-parsers/jsoncpp.git
diff --git a/.travis.yml b/.travis.yml
@@ -7,12 +7,14 @@ compiler:
   - gcc
 
 install:
-  - sudo apt-get install -y libzip-dev libcurl4-openssl-dev
+  - sudo apt-get install -y libzip-dev libcurl4-openssl-dev libjsoncpp-dev
 
 script:
   - git submodule update -i --recursive
   - cd breakpad
-  - LIBS="-lzip -lz" ./configure
+  - ./configure
   - make -j2
   - cd ../minidump-stackwalk
   - make
+  - cd ../tests
+  - sh run.sh
diff --git a/README.md b/README.md
@@ -1,4 +1,51 @@
-Stackwalker [![Build Status](https://travis-ci.org/NatronGitHub/stackwalker.svg)](https://travis-ci.org/NatronGitHub/stackwalker)
------------
+# Stackwalker [![Build Status](https://travis-ci.org/NatronGitHub/stackwalker.svg)](https://travis-ci.org/NatronGitHub/stackwalker)
 
-Parse crash reports from Natron.
+
+Stackwalker is used to process crash reports from MINGW, Linux and macOS.
+
+*This is a fork of Google Breakpad and Mozilla Socorro with additional features/changes maintained for Natron.*
+
+## Requirements
+
+* pkg-config
+* libcurl
+* libzip
+* jsoncpp
+
+## Build (on Linux)
+
+```
+git submodule update -i --recursive
+cd breakpad
+./configure && make
+cd ..
+make -C minidump-stackwalk
+```
+
+The binary is located in the ``minidump-stackwalk`` folder.
+
+## Usage
+
+``dump_syms`` is used to create a symbol file for a given (not stripped) binary. Example:
+
+```
+dump_syms Natron-bin > Natron-bin.sym
+```
+
+``stackwalker`` is used to parse crash report dumps against symbols generated from ``dump_syms``. Example:
+
+```
+stackwalker CRASH.dmp <LOCAL_OR_REMOTE_SYMBOLS_DIR> > CRASH.json
+```
+
+## Symbols Storage
+
+Symbols are stored using the following directory structure:
+
+* Filename *(example: Natron-bin)*
+  * ID *(example: 69CDA01A0F236F7C71CD19E5A20A21AC0)*
+    * Filename.sym.zip *(example: Natron-bin.sym.zip)*
+
+Filename and ID must match line 1 in the symbol file.
+
+**Note that Stackwalker only supports zipped symbols (filename.sym.zip)**
diff --git a/breakpad b/breakpad
diff --git a/jsoncpp b/jsoncpp
diff --git a/minidump-stackwalk/Makefile b/minidump-stackwalk/Makefile
@@ -5,53 +5,36 @@ BREAKPAD_LIBS := \
   $(BREAKPAD_SRCDIR)/third_party/libdisasm//libdisasm.a \
   $(NULL)
 
-JSON_DIR := ../jsoncpp
-JSON_SRCDIR := $(JSON_DIR)/src/lib_json
-JSON_INCLUDEDIR := $(JSON_DIR)/include
-
 BINS := \
   stackwalker \
-#  dumplookup \
-#  get-minidump-instructions \
-#  jit-crash-categorize \
   $(NULL)
 
 all: $(BINS)
 
-stackwalker_OBJS := \
-  json_reader.o \
-  json_value.o \
-  json_writer.o \
-  $(NULL)
-
 EXTRA_OBJS := \
   http_symbol_supplier.o \
   $(NULL)
 
-VPATH += $(JSON_SRCDIR)
-
-OS := $(shell uname -s)
-ifeq ($(OS),Linux)
 CURL_CFLAGS := $(shell pkg-config libcurl --cflags)
 CURL_LIBS := $(shell pkg-config libcurl --libs)
+LIBZIP_CFLAGS := $(shell pkg-config libzip --cflags)
 LIBZIP_LIBS := $(shell pkg-config libzip --libs)
+JSON_CFLAGS := $(shell pkg-config jsoncpp --cflags)
+JSON_LIBS := $(shell pkg-config jsoncpp --libs)
 # Don't -Werror everywhere, some compilers are too picky.
-WERROR := -Werror
-else
-CURL_LIBS := -lcurl
-LIBZIP_LIBS := -lzip -lz
-endif
+#WERROR := -Werror
 
 CXXFLAGS += \
   -I$(BREAKPAD_SRCDIR) \
-  -I$(JSON_INCLUDEDIR) \
   -D__STDC_FORMAT_MACROS=1 \
   -std=gnu++0x \
   -Wno-format \
   $(WERROR) \
+  $(JSON_CFLAGS) \
   $(CURL_CFLAGS) \
+  $(LIBZIP_CFLAGS) \
   $(NULL)
-LIBS := $(CURL_LIBS) $(LIBZIP_LIBS)
+LIBS := $(JSON_LIBS) $(CURL_LIBS) $(LIBZIP_LIBS)
 
 .SECONDEXPANSION:
 $(BINS): %: %.cc $(BREAKPAD_LIBS) $(EXTRA_OBJS) $$($$*_OBJS)

diff --git a/minidump-stackwalk/get-minidump-instructions.cc b/minidump-stackwalk/get-minidump-instructions.cc
@@ -415,6 +415,7 @@ int main(int argc, char** argv)
       if (p && (p - line == 8 || p - line == 16) && !strstr(line, "<.data>:")) {
         frame.instruction = strtoull(line, nullptr, 16);
         symbolizer.FillSourceLineInfo(module_list,
+                                      NULL, // unloaded_modules
                                       &system_info,
                                       &frame);
         print_frame(frame, last_frame);

diff --git a/minidump-stackwalk/http_symbol_supplier.cc b/minidump-stackwalk/http_symbol_supplier.cc
@@ -236,6 +236,20 @@ namespace {
   string JoinURL(CURL* curl, const string& url, const string& sub) {
     return url + "/" + URLEncode(curl, sub);
   }
+
+  string GetCodeIDAndFileQueryString(CURL* curl, const CodeModule* module) {
+      string code_file = PathnameStripper::File(module->code_file());
+      if (code_file.empty()) {
+        return "";
+      }
+
+      string code_id = PathnameStripper::File(module->code_identifier());
+      if (code_id.empty()) {
+        return "";
+      }
+
+      return "?code_file=" + URLEncode(curl, code_file) + "&code_id=" + URLEncode(curl, code_id);
+  }
 }
 
 bool
@@ -290,15 +304,21 @@ bool HTTPSymbolSupplier::FetchSymbolFile(const CodeModule* module,
     return false;
   }
 
+  string query = GetCodeIDAndFileQueryString(curl_, module);
+
   string full_path = JoinPath(cache_path_, path);
 
   bool result = false;
   for (auto server_url = server_urls_.begin();
        server_url < server_urls_.end();
        ++server_url) {
     string full_url = *server_url + url;
+    string url_with_query = full_url;
+    if (!query.empty()) {
+      url_with_query += query;
+    }
     float fetch_time;
-    if (FetchURLToFile(curl_, full_url, full_path, &fetch_time)) {
+    if (FetchURLToFile(curl_, url_with_query, full_path, &fetch_time)) {
       StoreCacheMiss(module, fetch_time, full_url);
       result = true;
       break;

diff --git a/minidump-stackwalk/stackwalker.cc b/minidump-stackwalk/stackwalker.cc
@@ -33,10 +33,12 @@
 #include <algorithm>
 #include <fstream>
 #include <iostream>
+#include <map>
 #include <ostream>
 #include <vector>
 #include <set>
 #include <cstdlib>
+#include <stdexcept>
 
 #include <errno.h>
 #include <getopt.h>
@@ -96,6 +98,7 @@ using google_breakpad::SymbolSupplier;
 using google_breakpad::SystemInfo;
 using breakpad_extra::HTTPSymbolSupplier;
 
+using std::map;
 using std::string;
 using std::vector;
 using std::ifstream;
@@ -133,10 +136,12 @@ class StackFrameSymbolizerForward : public StackFrameSymbolizer {
     : StackFrameSymbolizer(supplier, resolver) {}
 
   virtual SymbolizerResult FillSourceLineInfo(const CodeModules* modules,
+                                              const CodeModules* unloaded_modules,
                                               const SystemInfo* system_info,
                                               StackFrame* stack_frame) {
     SymbolizerResult res =
       StackFrameSymbolizer::FillSourceLineInfo(modules,
+                                               unloaded_modules,
                                                system_info,
                                                stack_frame);
     RecordResult(stack_frame->module, res);
@@ -515,8 +520,9 @@ bool ConvertStackToJSON(const ProcessState& process_state,
       if (ContainsModule(modules_with_corrupt_symbols, frame->module)) {
         frame_data["corrupt_symbols"] = true;
       }
-      assert(!frame->module->code_file().empty());
-      frame_data["module"] = PathnameStripper::File(frame->module->code_file());
+      if (!frame->module->code_file().empty()) {
+        frame_data["module"] = PathnameStripper::File(frame->module->code_file());
+      }
 
       if (!frame->function_name.empty()) {
         frame_data["function"] = frame->function_name;
@@ -633,10 +639,13 @@ static string ExploitabilityString(ExploitabilityRating exploitability) {
   return str;
 }
 
+static map<uint32_t, string> GetThreadIdNameMap(const Json::Value& raw_root);
+
 static void ConvertProcessStateToJSON(const ProcessState& process_state,
                                       const StackFrameSymbolizerForward& symbolizer,
                                       const HTTPSymbolSupplier* supplier,
-                                      Json::Value& root) {
+                                      Json::Value& root,
+                                      const Json::Value& raw_root) {
   // OS and CPU information.
   Json::Value system_info;
   system_info["os"] = process_state.system_info()->os;
@@ -672,6 +681,8 @@ static void ConvertProcessStateToJSON(const ProcessState& process_state,
     root["main_module"] = main_module;
   root["modules"] = modules;
 
+  auto thread_id_name_map = std::move(GetThreadIdNameMap(raw_root));
+
   Json::Value threads(Json::arrayValue);
   int thread_count = process_state.threads()->size();
   root["thread_count"] = thread_count;
@@ -686,6 +697,10 @@ static void ConvertProcessStateToJSON(const ProcessState& process_state,
     }
     thread["frames"] = stack;
     thread["frame_count"] = stack.size();
+    auto thread_name = thread_id_name_map[raw_stack->tid()];
+    if (!thread_name.empty()) {
+      thread["thread_name"] = thread_name;
+    }
     threads.append(thread);
   }
   root["threads"] = threads;
@@ -703,6 +718,10 @@ static void ConvertProcessStateToJSON(const ProcessState& process_state,
     crashing_thread["frames"] = stack;
     crashing_thread["total_frames"] =
       static_cast<Json::UInt>(crashing_stack->frames()->size());
+    auto thread_name = thread_id_name_map[crashing_stack->tid()];
+    if (!thread_name.empty()) {
+      crashing_thread["thread_name"] = thread_name;
+    }
     root["crashing_thread"] = crashing_thread;
   }
 
@@ -796,6 +815,35 @@ static string stripquotes(const string& s) {
   return s;
 }
 
+static string trim(const string& s) {
+  size_t first = s.find_first_not_of(" \t");
+  if (first == string::npos) {
+    first = 0;
+  }
+
+  size_t last = s.find_last_not_of(" \t");
+  if (last == string::npos) {
+    last = s.length();
+  }
+
+  return s.substr(first, last);
+}
+
+static void ConvertCPUInfoToJSON(const string& cpuinfo,
+                                 Json::Value& root) {
+  for (string& line : split(cpuinfo, '\n')) {
+    vector<string> bits = split(line, ':');
+    if (bits.size() != 2 || !startswith(bits[0], "microcode")) {
+        continue;
+    }
+
+    try {
+        root["system_info"]["cpu_microcode_version"] = static_cast<Json::UInt>(std::stoul(trim(bits[1]), nullptr, 16));
+        break;
+    } catch (const std::invalid_argument& e) {}
+  }
+}
+
 static void ConvertLSBReleaseToJSON(const string& lsb_release,
                                     Json::Value& root) {
   Json::Value lsb(Json::objectValue);
@@ -842,6 +890,34 @@ static string ResultString(ProcessResult result) {
   return str;
 }
 
+// Parse crash annotation "ThreadIdNameMapping" and return a map of
+// thread id -> thread name.
+static map<uint32_t, string> GetThreadIdNameMap(const Json::Value& raw_root)
+{
+  map<uint32_t, string> result {};
+
+  // Sample input: 23534:"Timer",23535:"Link Monitor",
+  string input = raw_root.get("ThreadIdNameMapping", "").asString();
+  if (input.empty()) {
+    return result;
+  }
+
+  for (string& map_item : split(input, ',')) {
+    // Sample map_item: 23534:"Timer"
+    vector<string> id_and_name = split(map_item, ':');
+    if (id_and_name.size() != 2) {
+      continue;
+    }
+
+    uint32_t thread_id = strtoul(id_and_name[0].c_str(), NULL, 10);
+    const string thread_name = stripquotes(id_and_name[1]);
+
+    result[thread_id] = thread_name;
+  }
+
+  return result;
+}
+
 //*** This code copy-pasted from minidump_stackwalk.cc ***
 
 // Separator character for machine readable output.
@@ -1159,7 +1235,7 @@ int main(int argc, char** argv)
   root["sensitive"] = Json::Value(Json::objectValue);
   if (result == google_breakpad::PROCESS_OK) {
     ConvertProcessStateToJSON(process_state, symbolizer,
-                              http_symbol_supplier, root);
+                              http_symbol_supplier, root, raw_root);
   }
   ConvertMemoryInfoToJSON(minidump, raw_root, root);
 
@@ -1170,6 +1246,17 @@ int main(int argc, char** argv)
     root["pid"] = misc_info->misc_info()->process_id;
   }
 
+  // See if this is a Linux dump with /proc/cpuinfo in it
+  uint32_t cpuinfo_length = 0;
+  if (process_state.system_info()->os == "Linux" &&
+      minidump.SeekToStreamType(MD_LINUX_CPU_INFO, &cpuinfo_length)) {
+    string contents;
+    contents.resize(cpuinfo_length);
+    if (minidump.ReadBytes(const_cast<char*>(contents.data()), cpuinfo_length)) {
+      ConvertCPUInfoToJSON(contents, root);
+    }
+  }
+
   // See if this is a Linux dump with /etc/lsb-release in it
   uint32_t length = 0;
   if (process_state.system_info()->os == "Linux" &&