From 043c1fdcb9eefa76c28a8ff2c0612f922b035972 Mon Sep 17 00:00:00 2001 From: Tindy X <49061470+tindy2013@users.noreply.github.com> Date: Wed, 6 Dec 2023 15:25:53 +0800 Subject: [PATCH] Enhancements Fix rule generation in Surge configs. Refactor log output for curl verbose logs. Optimize codes. --- src/config/binding.h | 2 +- src/config/def.h | 4 +- src/generator/config/nodemanip.cpp | 24 ++--- src/generator/config/nodemanip.h | 8 +- src/generator/config/ruleconvert.cpp | 19 ++-- src/generator/config/ruleconvert.h | 2 +- src/generator/config/subexport.cpp | 32 +++--- src/generator/config/subexport.h | 12 +-- src/generator/template/template_jinja2.cpp | 4 +- src/generator/template/templates.cpp | 16 +-- src/generator/template/templates.h | 4 +- src/handler/interfaces.cpp | 42 ++++---- src/handler/interfaces.h | 6 +- src/handler/multithread.cpp | 4 +- src/handler/multithread.h | 6 +- src/handler/settings.cpp | 12 +-- src/handler/settings.h | 20 ++-- src/handler/upload.cpp | 8 +- src/handler/webget.cpp | 117 ++++++--------------- src/handler/webget.h | 4 +- src/lib/wrapper.cpp | 2 +- src/parser/config/proxy.h | 2 +- src/parser/infoparser.cpp | 12 +-- src/parser/infoparser.h | 4 +- src/parser/subparser.cpp | 18 ++-- src/script/cron.cpp | 16 +-- src/script/script.cpp | 12 +-- src/script/script_quickjs.cpp | 12 +-- src/script/script_quickjs.h | 4 +- src/server/webserver.h | 6 +- src/server/webserver_httplib.cpp | 23 +++- src/server/webserver_libevent.cpp | 12 +-- src/utils/base64/base64.cpp | 2 +- src/utils/ini_reader/ini_reader.h | 6 +- src/utils/lock.h | 79 ++++++++++++++ src/utils/logger.cpp | 19 +++- src/utils/network.cpp | 2 +- 37 files changed, 317 insertions(+), 260 deletions(-) create mode 100644 src/utils/lock.h diff --git a/src/config/binding.h b/src/config/binding.h index 90cfc64b4..7741108f8 100644 --- a/src/config/binding.h +++ b/src/config/binding.h @@ -3,7 +3,7 @@ #include -#include "../handler/settings.h" +#include "handler/settings.h" #include "crontask.h" #include "proxygroup.h" #include "regmatch.h" diff --git a/src/config/def.h b/src/config/def.h index 596e392a7..6a01342fb 100644 --- a/src/config/def.h +++ b/src/config/def.h @@ -6,8 +6,8 @@ #include #include -#include "../utils/string.h" -#include "../utils/tribool.h" +#include "utils/string.h" +#include "utils/tribool.h" using String = std::string; using Integer = int32_t; diff --git a/src/generator/config/nodemanip.cpp b/src/generator/config/nodemanip.cpp index 60ebf8694..6f133cbbc 100644 --- a/src/generator/config/nodemanip.cpp +++ b/src/generator/config/nodemanip.cpp @@ -3,18 +3,18 @@ #include #include -#include "../../handler/settings.h" -#include "../../handler/webget.h" -#include "../../parser/config/proxy.h" -#include "../../parser/infoparser.h" -#include "../../parser/subparser.h" -#include "../../script/script_quickjs.h" -#include "../../utils/file_extra.h" -#include "../../utils/logger.h" -#include "../../utils/map_extra.h" -#include "../../utils/network.h" -#include "../../utils/regexp.h" -#include "../../utils/urlencode.h" +#include "handler/settings.h" +#include "handler/webget.h" +#include "parser/config/proxy.h" +#include "parser/infoparser.h" +#include "parser/subparser.h" +#include "script/script_quickjs.h" +#include "utils/file_extra.h" +#include "utils/logger.h" +#include "utils/map_extra.h" +#include "utils/network.h" +#include "utils/regexp.h" +#include "utils/urlencode.h" #include "nodemanip.h" #include "subexport.h" diff --git a/src/generator/config/nodemanip.h b/src/generator/config/nodemanip.h index 6db1ba695..5bce6363a 100644 --- a/src/generator/config/nodemanip.h +++ b/src/generator/config/nodemanip.h @@ -9,10 +9,10 @@ #include #endif // NO_JS_RUNTIME -#include "../../config/regmatch.h" -#include "../../parser/config/proxy.h" -#include "../../utils/map_extra.h" -#include "../../utils/string.h" +#include "config/regmatch.h" +#include "parser/config/proxy.h" +#include "utils/map_extra.h" +#include "utils/string.h" struct parse_settings { diff --git a/src/generator/config/ruleconvert.cpp b/src/generator/config/ruleconvert.cpp index f75070a27..f98e34654 100644 --- a/src/generator/config/ruleconvert.cpp +++ b/src/generator/config/ruleconvert.cpp @@ -1,11 +1,11 @@ #include -#include "../../handler/settings.h" -#include "../../utils/logger.h" -#include "../../utils/network.h" -#include "../../utils/regexp.h" -#include "../../utils/string.h" -#include "../../utils/rapidjson_extra.h" +#include "handler/settings.h" +#include "utils/logger.h" +#include "utils/network.h" +#include "utils/regexp.h" +#include "utils/string.h" +#include "utils/rapidjson_extra.h" #include "subexport.h" /// rule type lists @@ -338,7 +338,7 @@ void rulesetToSurge(INIReader &base_rule, std::vector &ruleset_c strLine = "RULE-SET," + remote_path_prefix + "/getruleset?type=1&url=" + urlSafeBase64Encode(rule_path_typed) + "," + rule_group; if(x.update_interval) strLine += ",update-interval=" + std::to_string(x.update_interval); - allRules.emplace_back(std::move(strLine)); + allRules.emplace_back(strLine); continue; } else if(surge_ver == -1 && !remote_path_prefix.empty()) @@ -372,7 +372,7 @@ void rulesetToSurge(INIReader &base_rule, std::vector &ruleset_c if(x.update_interval) strLine += ",update-interval=" + std::to_string(x.update_interval); - allRules.emplace_back(std::move(strLine)); + allRules.emplace_back(strLine); continue; } else if(surge_ver == -1 && !remote_path_prefix.empty()) @@ -447,7 +447,6 @@ void rulesetToSurge(INIReader &base_rule, std::vector &ruleset_c strLine = trimWhitespace(strLine); } - strLine += "," + rule_group; if(surge_ver == -1 || surge_ver == -2) { if(startsWith(strLine, "IP-CIDR6")) @@ -459,7 +458,7 @@ void rulesetToSurge(INIReader &base_rule, std::vector &ruleset_c if(!startsWith(strLine, "AND") && !startsWith(strLine, "OR") && !startsWith(strLine, "NOT")) strLine = transformRuleToCommon(temp, strLine, rule_group); } - allRules.emplace_back(std::move(strLine)); + allRules.emplace_back(strLine); total_rules++; } } diff --git a/src/generator/config/ruleconvert.h b/src/generator/config/ruleconvert.h index 62103031f..12461816f 100644 --- a/src/generator/config/ruleconvert.h +++ b/src/generator/config/ruleconvert.h @@ -8,7 +8,7 @@ #include #include -#include "../../utils/ini_reader/ini_reader.h" +#include "utils/ini_reader/ini_reader.h" enum ruleset_type { diff --git a/src/generator/config/subexport.cpp b/src/generator/config/subexport.cpp index 9d37c3117..c8eb16a46 100644 --- a/src/generator/config/subexport.cpp +++ b/src/generator/config/subexport.cpp @@ -4,22 +4,22 @@ #include #include -#include "../../config/regmatch.h" -#include "../../generator/config/subexport.h" -#include "../../generator/template/templates.h" -#include "../../handler/settings.h" -#include "../../parser/config/proxy.h" -#include "../../script/script_quickjs.h" -#include "../../utils/bitwise.h" -#include "../../utils/file_extra.h" -#include "../../utils/ini_reader/ini_reader.h" -#include "../../utils/logger.h" -#include "../../utils/network.h" -#include "../../utils/rapidjson_extra.h" -#include "../../utils/regexp.h" -#include "../../utils/stl_extra.h" -#include "../../utils/urlencode.h" -#include "../../utils/yamlcpp_extra.h" +#include "config/regmatch.h" +#include "generator/config/subexport.h" +#include "generator/template/templates.h" +#include "handler/settings.h" +#include "parser/config/proxy.h" +#include "script/script_quickjs.h" +#include "utils/bitwise.h" +#include "utils/file_extra.h" +#include "utils/ini_reader/ini_reader.h" +#include "utils/logger.h" +#include "utils/network.h" +#include "utils/rapidjson_extra.h" +#include "utils/regexp.h" +#include "utils/stl_extra.h" +#include "utils/urlencode.h" +#include "utils/yamlcpp_extra.h" #include "nodemanip.h" #include "ruleconvert.h" diff --git a/src/generator/config/subexport.h b/src/generator/config/subexport.h index 9339b5168..241dc7c47 100644 --- a/src/generator/config/subexport.h +++ b/src/generator/config/subexport.h @@ -7,12 +7,12 @@ #include #endif // NO_JS_RUNTIME -#include "../../config/proxygroup.h" -#include "../../config/regmatch.h" -#include "../../parser/config/proxy.h" -#include "../../utils/ini_reader/ini_reader.h" -#include "../../utils/string.h" -#include "../../utils/yamlcpp_extra.h" +#include "config/proxygroup.h" +#include "config/regmatch.h" +#include "parser/config/proxy.h" +#include "utils/ini_reader/ini_reader.h" +#include "utils/string.h" +#include "utils/yamlcpp_extra.h" #include "ruleconvert.h" struct extra_settings diff --git a/src/generator/template/template_jinja2.cpp b/src/generator/template/template_jinja2.cpp index 4a4e9593a..c2feadfbe 100644 --- a/src/generator/template/template_jinja2.cpp +++ b/src/generator/template/template_jinja2.cpp @@ -5,8 +5,8 @@ #include #include -#include "../../handler/interfaces.h" -#include "../../utils/regexp.h" +#include "handler/interfaces.h" +#include "utils/regexp.h" #include "templates.h" static inline void parse_json_pointer(nlohmann::json &json, const std::string &path, const std::string &value) diff --git a/src/generator/template/templates.cpp b/src/generator/template/templates.cpp index f04594947..7bbea8d23 100644 --- a/src/generator/template/templates.cpp +++ b/src/generator/template/templates.cpp @@ -5,14 +5,14 @@ #include #include -#include "../../handler/interfaces.h" -#include "../../handler/settings.h" -#include "../../handler/webget.h" -#include "../../utils/logger.h" -#include "../../utils/network.h" -#include "../../utils/regexp.h" -#include "../../utils/urlencode.h" -#include "../../utils/yamlcpp_extra.h" +#include "handler/interfaces.h" +#include "handler/settings.h" +#include "handler/webget.h" +#include "utils/logger.h" +#include "utils/network.h" +#include "utils/regexp.h" +#include "utils/urlencode.h" +#include "utils/yamlcpp_extra.h" #include "templates.h" namespace inja diff --git a/src/generator/template/templates.h b/src/generator/template/templates.h index b99806a5e..a7763a4df 100644 --- a/src/generator/template/templates.h +++ b/src/generator/template/templates.h @@ -4,8 +4,8 @@ #include #include -#include "../../generator/config/subexport.h" -#include "../../utils/string.h" +#include "generator/config/subexport.h" +#include "utils/string.h" struct template_args { diff --git a/src/handler/interfaces.cpp b/src/handler/interfaces.cpp index 418d58930..7794d13ea 100644 --- a/src/handler/interfaces.cpp +++ b/src/handler/interfaces.cpp @@ -6,27 +6,27 @@ #include #include -#include "../config/binding.h" -#include "../generator/config/nodemanip.h" -#include "../generator/config/ruleconvert.h" -#include "../generator/config/subexport.h" -#include "../generator/template/templates.h" -#include "../script/cron.h" -#include "../script/script_quickjs.h" -#include "../server/webserver.h" -#include "../utils/base64/base64.h" -#include "../utils/file_extra.h" -#include "../utils/ini_reader/ini_reader.h" -#include "../utils/logger.h" -#include "../utils/network.h" -#include "../utils/regexp.h" -#include "../utils/stl_extra.h" -#include "../utils/string.h" -#include "../utils/string_hash.h" -#include "../utils/system.h" -#include "../utils/system.h" -#include "../utils/urlencode.h" -#include "../utils/yamlcpp_extra.h" +#include "config/binding.h" +#include "generator/config/nodemanip.h" +#include "generator/config/ruleconvert.h" +#include "generator/config/subexport.h" +#include "generator/template/templates.h" +#include "script/cron.h" +#include "script/script_quickjs.h" +#include "server/webserver.h" +#include "utils/base64/base64.h" +#include "utils/file_extra.h" +#include "utils/ini_reader/ini_reader.h" +#include "utils/logger.h" +#include "utils/network.h" +#include "utils/regexp.h" +#include "utils/stl_extra.h" +#include "utils/string.h" +#include "utils/string_hash.h" +#include "utils/system.h" +#include "utils/system.h" +#include "utils/urlencode.h" +#include "utils/yamlcpp_extra.h" #include "interfaces.h" #include "multithread.h" #include "settings.h" diff --git a/src/handler/interfaces.h b/src/handler/interfaces.h index da03810c1..8687736da 100644 --- a/src/handler/interfaces.h +++ b/src/handler/interfaces.h @@ -5,9 +5,9 @@ #include #include -#include "../config/ruleset.h" -#include "../generator/config/subexport.h" -#include "../server/webserver.h" +#include "config/ruleset.h" +#include "generator/config/subexport.h" +#include "server/webserver.h" std::string parseProxy(const std::string &source); diff --git a/src/handler/multithread.cpp b/src/handler/multithread.cpp index e5610a361..5ba36149e 100644 --- a/src/handler/multithread.cpp +++ b/src/handler/multithread.cpp @@ -1,8 +1,8 @@ #include #include -#include "../handler/settings.h" -#include "../utils/network.h" +#include "handler/settings.h" +#include "utils/network.h" #include "webget.h" #include "multithread.h" //#include "vfs.h" diff --git a/src/handler/multithread.h b/src/handler/multithread.h index 1e6e31c2d..82ab1d2cd 100644 --- a/src/handler/multithread.h +++ b/src/handler/multithread.h @@ -6,9 +6,9 @@ #include -#include "../config/regmatch.h" -#include "../utils/ini_reader/ini_reader.h" -#include "../utils/string.h" +#include "config/regmatch.h" +#include "utils/ini_reader/ini_reader.h" +#include "utils/string.h" using guarded_mutex = std::lock_guard; diff --git a/src/handler/settings.cpp b/src/handler/settings.cpp index 678da3368..655f5fdba 100644 --- a/src/handler/settings.cpp +++ b/src/handler/settings.cpp @@ -2,12 +2,12 @@ #include #include -#include "../config/binding.h" -#include "../handler/webget.h" -#include "../script/cron.h" -#include "../server/webserver.h" -#include "../utils/logger.h" -#include "../utils/network.h" +#include "config/binding.h" +#include "handler/webget.h" +#include "script/cron.h" +#include "server/webserver.h" +#include "utils/logger.h" +#include "utils/network.h" #include "interfaces.h" #include "multithread.h" #include "settings.h" diff --git a/src/handler/settings.h b/src/handler/settings.h index 9705145a5..495adab1c 100644 --- a/src/handler/settings.h +++ b/src/handler/settings.h @@ -3,16 +3,16 @@ #include -#include "../config/crontask.h" -#include "../config/regmatch.h" -#include "../config/proxygroup.h" -#include "../config/ruleset.h" -#include "../generator/config/ruleconvert.h" -#include "../generator/template/templates.h" -#include "../utils/logger.h" -#include "../utils/string.h" -#include "../utils/stl_extra.h" -#include "../utils/tribool.h" +#include "config/crontask.h" +#include "config/regmatch.h" +#include "config/proxygroup.h" +#include "config/ruleset.h" +#include "generator/config/ruleconvert.h" +#include "generator/template/templates.h" +#include "utils/logger.h" +#include "utils/string.h" +#include "utils/stl_extra.h" +#include "utils/tribool.h" struct Settings { diff --git a/src/handler/upload.cpp b/src/handler/upload.cpp index 1d689f606..d65dd8ab0 100644 --- a/src/handler/upload.cpp +++ b/src/handler/upload.cpp @@ -1,9 +1,9 @@ #include -#include "../utils/ini_reader/ini_reader.h" -#include "../utils/logger.h" -#include "../utils/rapidjson_extra.h" -#include "../utils/system.h" +#include "utils/ini_reader/ini_reader.h" +#include "utils/logger.h" +#include "utils/rapidjson_extra.h" +#include "utils/system.h" #include "webget.h" std::string buildGistData(std::string name, std::string content) diff --git a/src/handler/webget.cpp b/src/handler/webget.cpp index d0431514d..736aaa54d 100644 --- a/src/handler/webget.cpp +++ b/src/handler/webget.cpp @@ -7,13 +7,14 @@ #include -#include "../handler/settings.h" -#include "../utils/base64/base64.h" -#include "../utils/defer.h" -#include "../utils/file_extra.h" -#include "../utils/logger.h" -#include "../utils/urlencode.h" -#include "../version.h" +#include "handler/settings.h" +#include "utils/base64/base64.h" +#include "utils/defer.h" +#include "utils/file_extra.h" +#include "utils/lock.h" +#include "utils/logger.h" +#include "utils/urlencode.h" +#include "version.h" #include "webget.h" #ifdef _WIN32 @@ -27,70 +28,6 @@ using guarded_mutex = std::lock_guard; std::mutex cache_rw_lock; */ -class RWLock -{ -#define WRITE_LOCK_STATUS (-1) -#define FREE_STATUS 0 -private: - const std::thread::id NULL_THREAD; - const bool WRITE_FIRST; - std::thread::id m_write_thread_id; - std::atomic_int m_lockCount; - std::atomic_uint m_writeWaitCount; -public: - RWLock(const RWLock&) = delete; - RWLock& operator=(const RWLock&) = delete; - RWLock(bool writeFirst = true): WRITE_FIRST(writeFirst), m_write_thread_id(), m_lockCount(0), m_writeWaitCount(0) {} - virtual ~RWLock() = default; - int readLock() - { - if (std::this_thread::get_id() != m_write_thread_id) - { - int count; - if (WRITE_FIRST) - do { - while ((count = m_lockCount) == WRITE_LOCK_STATUS || m_writeWaitCount > 0); - } while (!m_lockCount.compare_exchange_weak(count, count + 1)); - else - do { - while ((count = m_lockCount) == WRITE_LOCK_STATUS); - } while (!m_lockCount.compare_exchange_weak(count, count + 1)); - } - return m_lockCount; - } - int readUnlock() - { - if (std::this_thread::get_id() != m_write_thread_id) - --m_lockCount; - return m_lockCount; - } - int writeLock() - { - if (std::this_thread::get_id() != m_write_thread_id) - { - ++m_writeWaitCount; - for (int zero = FREE_STATUS; !m_lockCount.compare_exchange_weak(zero, WRITE_LOCK_STATUS); zero = FREE_STATUS); - --m_writeWaitCount; - m_write_thread_id = std::this_thread::get_id(); - } - return m_lockCount; - } - int writeUnlock() - { - if (std::this_thread::get_id() != m_write_thread_id) - { - throw std::runtime_error("writeLock/Unlock mismatch"); - } - if (WRITE_LOCK_STATUS != m_lockCount) - { - throw std::runtime_error("RWLock internal error"); - } - m_write_thread_id = NULL_THREAD; - m_lockCount.store(FREE_STATUS); - return m_lockCount; - } -}; - RWLock cache_rw_lock; //std::string user_agent_str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"; @@ -118,18 +55,17 @@ static int writer(char *data, size_t size, size_t nmemb, std::string *writerData writerData->append(data, size*nmemb); - return size * nmemb; + return static_cast(size * nmemb); } -static int dummy_writer(char *data, size_t size, size_t nmemb, void *writerData) +static int dummy_writer(char *, size_t size, size_t nmemb, void *) { /// dummy writer, do not save anything - (void)data; - (void)writerData; - return size * nmemb; + return static_cast(size * nmemb); } -static int size_checker(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) +//static int size_checker(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) +static int size_checker(void *clientp, curl_off_t, curl_off_t dlnow, curl_off_t, curl_off_t) { if(clientp) { @@ -163,9 +99,24 @@ static int logger(CURL *handle, curl_infotype type, char *data, size_t size, voi return 0; } std::string content(data, size); - if(content.back() == '\n') - content.pop_back(); - writeLog(0, prefix + ": " + content, LOG_LEVEL_VERBOSE); + if(content.find("\r\n") != std::string::npos) + { + string_array lines = split(content, "\r\n"); + for(auto &x : lines) + { + std::string log_content = prefix; + log_content += ": "; + log_content += x; + writeLog(0, log_content, LOG_LEVEL_VERBOSE); + } + } + else + { + std::string log_content = prefix; + log_content += ": "; + log_content += trimWhitespace(content); + writeLog(0, log_content, LOG_LEVEL_VERBOSE); + } return 0; } @@ -198,7 +149,7 @@ static int curlGet(const FetchArgument &argument, FetchResult &result) std::string *data = result.content, new_url = argument.url; curl_slist *header_list = nullptr; defer(curl_slist_free_all(header_list);) - long retVal = 0; + long retVal; curl_init(); @@ -361,10 +312,10 @@ std::string webGet(const std::string &url, const std::string &proxy, unsigned in md("cache"); const std::string url_md5 = getMD5(url); const std::string path = "cache/" + url_md5, path_header = path + "_header"; - struct stat result; + struct stat result {}; if(stat(path.data(), &result) == 0) // cache exist { - time_t mtime = result.st_mtime, now = time(NULL); // get cache modified time and current time + time_t mtime = result.st_mtime, now = time(nullptr); // get cache modified time and current time if(difftime(now, mtime) <= cache_ttl) // within TTL { writeLog(0, "CACHE HIT: '" + url + "', using local cache."); diff --git a/src/handler/webget.h b/src/handler/webget.h index b8c3f566d..6df44c89e 100644 --- a/src/handler/webget.h +++ b/src/handler/webget.h @@ -4,8 +4,8 @@ #include #include -#include "../utils/map_extra.h" -#include "../utils/string.h" +#include "utils/map_extra.h" +#include "utils/string.h" enum http_method { diff --git a/src/lib/wrapper.cpp b/src/lib/wrapper.cpp index aafde46cc..37fd5b394 100644 --- a/src/lib/wrapper.cpp +++ b/src/lib/wrapper.cpp @@ -1,4 +1,4 @@ -#include "../handler/settings.h" +#include "handler/settings.h" #include Settings global; diff --git a/src/parser/config/proxy.h b/src/parser/config/proxy.h index 7a806a9cf..fd0f94f09 100644 --- a/src/parser/config/proxy.h +++ b/src/parser/config/proxy.h @@ -4,7 +4,7 @@ #include #include -#include "../../utils/tribool.h" +#include "utils/tribool.h" using String = std::string; using StringArray = std::vector; diff --git a/src/parser/infoparser.cpp b/src/parser/infoparser.cpp index 155469956..a8a591bac 100644 --- a/src/parser/infoparser.cpp +++ b/src/parser/infoparser.cpp @@ -3,12 +3,12 @@ #include #include -#include "../config/regmatch.h" -#include "../parser/config/proxy.h" -#include "../utils/base64/base64.h" -#include "../utils/rapidjson_extra.h" -#include "../utils/regexp.h" -#include "../utils/string.h" +#include "config/regmatch.h" +#include "parser/config/proxy.h" +#include "utils/base64/base64.h" +#include "utils/rapidjson_extra.h" +#include "utils/regexp.h" +#include "utils/string.h" unsigned long long streamToInt(const std::string &stream) { diff --git a/src/parser/infoparser.h b/src/parser/infoparser.h index ad907183f..9238b8d8a 100644 --- a/src/parser/infoparser.h +++ b/src/parser/infoparser.h @@ -3,9 +3,9 @@ #include -#include "../utils/string.h" +#include "utils/string.h" #include "config/proxy.h" -#include "../config/regmatch.h" +#include "config/regmatch.h" bool getSubInfoFromHeader(const std::string &header, std::string &result); bool getSubInfoFromNodes(const std::vector &nodes, const RegexMatchConfigs &stream_rules, const RegexMatchConfigs &time_rules, std::string &result); diff --git a/src/parser/subparser.cpp b/src/parser/subparser.cpp index 4b0866531..9f85692b5 100644 --- a/src/parser/subparser.cpp +++ b/src/parser/subparser.cpp @@ -1,15 +1,15 @@ #include #include -#include "../utils/base64/base64.h" -#include "../utils/ini_reader/ini_reader.h" -#include "../utils/network.h" -#include "../utils/rapidjson_extra.h" -#include "../utils/regexp.h" -#include "../utils/string.h" -#include "../utils/string_hash.h" -#include "../utils/urlencode.h" -#include "../utils/yamlcpp_extra.h" +#include "utils/base64/base64.h" +#include "utils/ini_reader/ini_reader.h" +#include "utils/network.h" +#include "utils/rapidjson_extra.h" +#include "utils/regexp.h" +#include "utils/string.h" +#include "utils/string_hash.h" +#include "utils/urlencode.h" +#include "utils/yamlcpp_extra.h" #include "config/proxy.h" #include "subparser.h" diff --git a/src/script/cron.cpp b/src/script/cron.cpp index 8790a69da..1317bc591 100644 --- a/src/script/cron.cpp +++ b/src/script/cron.cpp @@ -2,14 +2,14 @@ #include #include -#include "../config/crontask.h" -#include "../handler/interfaces.h" -#include "../handler/multithread.h" -#include "../handler/settings.h" -#include "../server/webserver.h" -#include "../utils/logger.h" -#include "../utils/rapidjson_extra.h" -#include "../utils/system.h" +#include "config/crontask.h" +#include "handler/interfaces.h" +#include "handler/multithread.h" +#include "handler/settings.h" +#include "server/webserver.h" +#include "utils/logger.h" +#include "utils/rapidjson_extra.h" +#include "utils/system.h" #include "script_quickjs.h" libcron::Cron cron; diff --git a/src/script/script.cpp b/src/script/script.cpp index 4fb0b341e..6a93ce77c 100644 --- a/src/script/script.cpp +++ b/src/script/script.cpp @@ -3,12 +3,12 @@ #include #include -#include "../utils/string.h" -#include "../utils/string_hash.h" -#include "../handler/webget.h" -#include "../handler/multithread.h" -#include "../utils/base64/base64.h" -#include "../utils/network.h" +#include "utils/string.h" +#include "utils/string_hash.h" +#include "handler/webget.h" +#include "handler/multithread.h" +#include "utils/base64/base64.h" +#include "utils/network.h" extern int gCacheConfig; extern std::string gProxyConfig; diff --git a/src/script/script_quickjs.cpp b/src/script/script_quickjs.cpp index d7d0195b8..b745b8f8c 100644 --- a/src/script/script_quickjs.cpp +++ b/src/script/script_quickjs.cpp @@ -8,12 +8,12 @@ #include #endif // _WIN32 -#include "../handler/multithread.h" -#include "../handler/webget.h" -#include "../handler/settings.h" -#include "../parser/config/proxy.h" -#include "../utils/map_extra.h" -#include "../utils/system.h" +#include "handler/multithread.h" +#include "handler/webget.h" +#include "handler/settings.h" +#include "parser/config/proxy.h" +#include "utils/map_extra.h" +#include "utils/system.h" #include "script_quickjs.h" std::string parseProxy(const std::string &source); diff --git a/src/script/script_quickjs.h b/src/script/script_quickjs.h index c370226bc..86defed19 100644 --- a/src/script/script_quickjs.h +++ b/src/script/script_quickjs.h @@ -1,8 +1,8 @@ #ifndef SCRIPT_QUICKJS_H_INCLUDED #define SCRIPT_QUICKJS_H_INCLUDED -#include "../parser/config/proxy.h" -#include "../utils/defer.h" +#include "parser/config/proxy.h" +#include "utils/defer.h" #ifndef NO_JS_RUNTIME diff --git a/src/server/webserver.h b/src/server/webserver.h index e36b8c420..344040db1 100644 --- a/src/server/webserver.h +++ b/src/server/webserver.h @@ -6,9 +6,9 @@ #include #include -#include "../utils/map_extra.h" -#include "../utils/string.h" -#include "../version.h" +#include "utils/map_extra.h" +#include "utils/string.h" +#include "version.h" struct Request { diff --git a/src/server/webserver_httplib.cpp b/src/server/webserver_httplib.cpp index 4756b8e56..d8eb36ce9 100644 --- a/src/server/webserver_httplib.cpp +++ b/src/server/webserver_httplib.cpp @@ -5,11 +5,11 @@ #define CPPHTTPLIB_REQUEST_URI_MAX_LENGTH 16384 #include "httplib.h" -#include "../utils/base64/base64.h" -#include "../utils/logger.h" -#include "../utils/string_hash.h" -#include "../utils/stl_extra.h" -#include "../utils/urlencode.h" +#include "utils/base64/base64.h" +#include "utils/logger.h" +#include "utils/string_hash.h" +#include "utils/stl_extra.h" +#include "utils/urlencode.h" #include "webserver.h" static const char *request_header_blacklist[] = {"host", "accept", "accept-encoding"}; @@ -73,6 +73,18 @@ static httplib::Server::Handler makeHandler(const responseRoute &rr) }; } +static std::string dump(const httplib::Headers &headers) +{ + std::string s; + for (auto &x: headers) + { + if (startsWith(x.first, "LOCAL_") || startsWith(x.first, "REMOTE_")) + continue; + s += x.first + ": " + x.second + "|"; + } + return s; +} + int WebServer::start_web_server_multi(listener_args *args) { httplib::Server server; @@ -125,6 +137,7 @@ int WebServer::start_web_server_multi(listener_args *args) { writeLog(0, "Accept connection from client " + req.remote_addr + ":" + std::to_string(req.remote_port), LOG_LEVEL_DEBUG); writeLog(0, "handle_cmd: " + req.method + " handle_uri: " + req.target, LOG_LEVEL_VERBOSE); + writeLog(0, "handle_header: " + dump(req.headers), LOG_LEVEL_VERBOSE); if (req.has_header("SubConverter-Request")) { diff --git a/src/server/webserver_libevent.cpp b/src/server/webserver_libevent.cpp index 0d1fb7f91..b6d43e963 100644 --- a/src/server/webserver_libevent.cpp +++ b/src/server/webserver_libevent.cpp @@ -13,12 +13,12 @@ #include #include -#include "../utils/base64/base64.h" -#include "../utils/file_extra.h" -#include "../utils/logger.h" -#include "../utils/stl_extra.h" -#include "../utils/string.h" -#include "../utils/urlencode.h" +#include "utils/base64/base64.h" +#include "utils/file_extra.h" +#include "utils/logger.h" +#include "utils/stl_extra.h" +#include "utils/string.h" +#include "utils/urlencode.h" #include "socket.h" #include "webserver.h" diff --git a/src/utils/base64/base64.cpp b/src/utils/base64/base64.cpp index aeb1418d6..20dfc1347 100644 --- a/src/utils/base64/base64.cpp +++ b/src/utils/base64/base64.cpp @@ -1,6 +1,6 @@ #include -#include "../string.h" +#include "utils/string.h" static const std::string base64_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" diff --git a/src/utils/ini_reader/ini_reader.h b/src/utils/ini_reader/ini_reader.h index 7f9b4922c..85334f1d1 100644 --- a/src/utils/ini_reader/ini_reader.h +++ b/src/utils/ini_reader/ini_reader.h @@ -7,9 +7,9 @@ #include #include -#include "../codepage.h" -#include "../file_extra.h" -#include "../string.h" +#include "utils/codepage.h" +#include "utils/file_extra.h" +#include "utils/string.h" enum { diff --git a/src/utils/lock.h b/src/utils/lock.h new file mode 100644 index 000000000..27f91e362 --- /dev/null +++ b/src/utils/lock.h @@ -0,0 +1,79 @@ +#ifndef LOCK_H_INCLUDED +#define LOCK_H_INCLUDED + +#include +#include + +class RWLock +{ + static constexpr int WRITE_LOCK_STATUS = -1; + static constexpr int FREE_STATUS = 0; +private: + const std::thread::id NULL_THREAD; + const bool WRITE_FIRST; + std::thread::id m_write_thread_id; + std::atomic_int m_lockCount; + std::atomic_uint m_writeWaitCount; +public: + RWLock(const RWLock&) = delete; + RWLock& operator=(const RWLock&) = delete; + explicit RWLock(bool writeFirst = true): WRITE_FIRST(writeFirst), m_write_thread_id(), m_lockCount(0), m_writeWaitCount(0) {} + virtual ~RWLock() = default; + int readLock() + { + if (std::this_thread::get_id() != m_write_thread_id) + { + int count; + if (WRITE_FIRST) + { + do + { + while ((count = m_lockCount) == WRITE_LOCK_STATUS || m_writeWaitCount > 0); + } + while (!m_lockCount.compare_exchange_weak(count, count + 1)); + } + else + { + do + { + while ((count = m_lockCount) == WRITE_LOCK_STATUS); + } + while (!m_lockCount.compare_exchange_weak(count, count + 1)); + } + } + return m_lockCount; + } + int readUnlock() + { + if (std::this_thread::get_id() != m_write_thread_id) + --m_lockCount; + return m_lockCount; + } + int writeLock() + { + if (std::this_thread::get_id() != m_write_thread_id) + { + ++m_writeWaitCount; + for (int zero = FREE_STATUS; !m_lockCount.compare_exchange_weak(zero, WRITE_LOCK_STATUS); zero = FREE_STATUS); + --m_writeWaitCount; + m_write_thread_id = std::this_thread::get_id(); + } + return m_lockCount; + } + int writeUnlock() + { + if (std::this_thread::get_id() != m_write_thread_id) + { + throw std::runtime_error("writeLock/Unlock mismatch"); + } + if (WRITE_LOCK_STATUS != m_lockCount) + { + throw std::runtime_error("RWLock internal error"); + } + m_write_thread_id = NULL_THREAD; + m_lockCount.store(FREE_STATUS); + return m_lockCount; + } +}; + +#endif //LOCK_H_INCLUDED diff --git a/src/utils/logger.cpp b/src/utils/logger.cpp index 4da73e98c..a7694d505 100644 --- a/src/utils/logger.cpp +++ b/src/utils/logger.cpp @@ -5,7 +5,9 @@ #include #include -#include "../handler/settings.h" +#include "handler/settings.h" +#include "defer.h" +#include "lock.h" #include "logger.h" std::string getTime(int type) @@ -38,7 +40,20 @@ std::string getTime(int type) static std::string get_thread_name() { static std::atomic_int counter = 0; - thread_local static std::string name = "Thread-" + std::to_string(counter++); + static std::map thread_names; + static RWLock lock; + std::thread::id id = std::this_thread::get_id(); + lock.readLock(); + if (thread_names.find(id) != thread_names.end()) + { + defer(lock.readUnlock();) + return thread_names[id]; + } + lock.readUnlock(); + lock.writeLock(); + std::string name = "Thread-" + std::to_string(++counter); + thread_names[id] = name; + lock.writeUnlock(); return name; } diff --git a/src/utils/network.cpp b/src/utils/network.cpp index da28454d1..13accf500 100644 --- a/src/utils/network.cpp +++ b/src/utils/network.cpp @@ -2,7 +2,7 @@ #include #include -#include "../server/socket.h" +#include "server/socket.h" #include "string.h" #include "regexp.h"