Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: cache data #16

Merged
merged 15 commits into from
Aug 29, 2024
Merged
Show file tree
Hide file tree
Changes from 10 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 41 additions & 1 deletion src/Infrastructure/Network/HttpsAccessManager.cc
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,26 @@
#include <boost/system/detail/error_code.hpp>
#include <boost/system/system_error.hpp>
#include <boost/url.hpp>
#include <fstream>
#include <sstream>

namespace evento {

constexpr const char USER_AGENT[] = "SAST-Evento-Desktop/1";

Task<ResponseResult> HttpsAccessManager::makeReply(std::string host,
http::request<http::string_body> req) {
std::string url = host + std::string(req.target());

// Check cache first
{
std::lock_guard<std::mutex> lock(_cacheMutex);
auto it = _cache.find(url);
if (it != _cache.end()) {
co_return Ok(it->second);
}
}

auto resolver = net::use_awaitable_t<boost::asio::any_io_executor>::as_default_on(
tcp::resolver(co_await net::this_coro::executor));

Expand Down Expand Up @@ -71,12 +84,39 @@ Task<ResponseResult> HttpsAccessManager::makeReply(std::string host,

// Gracefully close the stream - do not threat every error as an exception!
auto [ec] = co_await stream.async_shutdown(net::as_tuple(net::use_awaitable));
if (!ec || ec == net::error::eof || (ignoreSslError && ec == ssl::error::stream_truncated))
if (!ec || ec == net::error::eof || (ignoreSslError && ec == ssl::error::stream_truncated)) {
// If we get here then the connection is closed gracefully
// Cache the response
{
std::lock_guard<std::mutex> lock(_cacheMutex);
_cache[url] = res;
}

// Generate a unique filename and save the response body to a file
std::string filename = generateFilename(url);
saveToFile(filename, _buffer);

co_return Ok(res);
}

debug(), ec, ec.message();
co_return Err(Error(Error::Network, ec.message()));
}

void HttpsAccessManager::saveToFile(const std::string& filename, const beast::flat_buffer& buffer) {
Serein207 marked this conversation as resolved.
Show resolved Hide resolved
Serein207 marked this conversation as resolved.
Show resolved Hide resolved
std::ofstream file(filename, std::ios::binary);
if (file.is_open()) {
file.write(static_cast<const char*>(buffer.data().data()), buffer.size());
Serein207 marked this conversation as resolved.
Show resolved Hide resolved
Serein207 marked this conversation as resolved.
Show resolved Hide resolved
file.close();
}
}

std::string HttpsAccessManager::generateFilename(const std::string& url) {
Serein207 marked this conversation as resolved.
Show resolved Hide resolved
Serein207 marked this conversation as resolved.
Show resolved Hide resolved
std::hash<std::string> hasher;
std::size_t hash = hasher(url);
std::stringstream ss;
ss << "image_" << hash << ".jpg";
return ss.str();
}

} // namespace evento
13 changes: 13 additions & 0 deletions src/Infrastructure/Network/HttpsAccessManager.h
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,10 @@
#include <boost/beast/ssl.hpp>
#include <boost/url/url.hpp>
#include <chrono>
#include <functional> // for std::hash
#include <mutex>
#include <string>
#include <unordered_map>

namespace evento {

Expand Down Expand Up @@ -49,6 +52,16 @@ class HttpsAccessManager {
net::ssl::context& _ctx;
std::chrono::seconds _timeout; // respective timeout of ssl handshake & http
beast::flat_buffer _buffer; // http buffer is used for reading and must be persisted

// Cache for storing downloaded images
std::unordered_map<std::string, http::response<http::dynamic_body>> _cache;
std::mutex _cacheMutex;

// Helper function to save response body to a file
void saveToFile(const std::string& filename, const beast::flat_buffer& buffer);

// Helper function to generate a unique filename based on URL
std::string generateFilename(const std::string& url);
};

} // namespace evento
12 changes: 12 additions & 0 deletions src/Infrastructure/Network/NetworkClient.cc
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include <Infrastructure/Utils/Debug.h>
#include <boost/url/param.hpp>
#include <boost/url/params_view.hpp>
#include <initializer_list>
#include <memory>
#include <nlohmann/json.hpp>
#include <string>
Expand Down Expand Up @@ -347,6 +348,17 @@ Task<Result<DepartmentEntityList>> NetworkClient::getDepartmentList() {
co_return Ok(entity);
}

std::string NetworkClient::generateCacheKey(http::verb verb,
const urls::url_view& url,
const std::initializer_list<urls::param>& params) {
std::string key = std::string(url.data(), url.size()) + "|"
+ std::to_string(static_cast<int>(verb));
for (const auto& param : params) {
key += "|" + std::string(param.key) + "=" + std::string(param.value);
}
return key;
}

urls::url NetworkClient::endpoint(std::string_view endpoint) {
return urls::url(EVENTO_API_GATEWAY + endpoint.data());
}
Expand Down
82 changes: 74 additions & 8 deletions src/Infrastructure/Network/NetworkClient.h
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,15 @@
#include <boost/beast/http/verb.hpp>
#include <boost/beast/ssl.hpp>
#include <boost/url.hpp>
#include <chrono>
#include <concepts>
#include <initializer_list>
#include <iostream>
#include <list>
#include <nlohmann/json.hpp>
#include <optional>
#include <string>
#include <unordered_map>

namespace evento {

Expand All @@ -33,6 +37,12 @@ using ContributorList = std::vector<ContributorEntity>;
template<typename T>
using Task = net::awaitable<T>;

struct CacheEntry {
JsonResult result;
std::chrono::steady_clock::time_point insertTime;
std::size_t size; //cache size
};

class NetworkClient {
public:
NetworkClient(const NetworkClient&) = delete;
Expand Down Expand Up @@ -89,23 +99,76 @@ class NetworkClient {
private:
NetworkClient(net::ssl::context& ctx);
static NetworkClient* getInstance();

//cache data processing
static std::string generateCacheKey(
http::verb verb,
const urls::url_view& url,
const std::initializer_list<urls::param>& params); //auxiliary function to generate cache key

static bool isCacheEntryExpired(const CacheEntry& entry) {
auto now = std::chrono::steady_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::hours>(now - entry.insertTime);
return duration.count() >= 3;
}
void updateCache(const std::string& key, CacheEntry entry) {
try {
auto it = cacheMap.find(key);
if (it != cacheMap.end()) {
cacheList.erase(it->second);
currentCacheSize -= it->second->second.size;
}
cacheList.emplace_front(key, std::move(entry));
cacheMap[key] = cacheList.begin();
currentCacheSize += cacheList.begin()->second.size;

while (currentCacheSize > maxCacheSize) {
auto last = cacheList.end();
--last;
currentCacheSize -= last->second.size;
cacheMap.erase(last->first);
cacheList.pop_back();
}
} catch (const std::exception& e) {
std::cerr << "Error occurred: " << e.what() << std::endl;
}
}
// - success => return the `data` field from response json
// maybe json object or json array
// - error => return error message
template<std::same_as<api::Evento> Api>
Task<JsonResult> request(http::verb verb,
urls::url_view url,
std::initializer_list<urls::param> const& params = {}) {
debug(), url;
auto req = Api::makeRequest(verb, url, tokenBytes, params);
try {
//Generate cache
std::string cacheKey = generateCacheKey(verb, url, params);

auto reply = co_await _manager->makeReply(url.host(), req);
//Check cache
auto it = cacheMap.find(cacheKey);
if (it != cacheMap.end() && !isCacheEntryExpired(it->second->second)) {
cacheList.splice(cacheList.begin(), cacheList, it->second);
co_return it->second->second.result;
}
debug(), url;
auto req = Api::makeRequest(verb, url, tokenBytes, params);

if (reply.isErr())
co_return reply.unwrapErr();
auto reply = co_await _manager->makeReply(url.host(), req);

if (reply.isErr())
co_return reply.unwrapErr();

auto result = handleResponse(reply.unwrap());

co_return handleResponse(reply.unwrap());
// Update cache
size_t entrySize = result.unwrap().dump().size();
updateCache(cacheKey,
CacheEntry{std::move(result), std::chrono::steady_clock::now(), entrySize});

co_return result;
} catch (const std::exception& e) {
std::cerr << "Request error occurred: " << e.what() << std::endl;
Serein207 marked this conversation as resolved.
Show resolved Hide resolved
co_return Err(Error(Error::JsonDes, e.what()));
}
}

template<std::same_as<api::Github> Api>
Expand Down Expand Up @@ -142,7 +205,10 @@ class NetworkClient {
private:
net::ssl::context& _ctx;
std::unique_ptr<HttpsAccessManager> _manager;

std::list<std::pair<std::string, CacheEntry>> cacheList; // LRU 缓存列表
std::unordered_map<std::string, decltype(cacheList.begin())> cacheMap; // 缓存映射(changed,TBD)
size_t currentCacheSize = 0; // 当前缓存大小
const size_t maxCacheSize = 64 * 1024 * 1024;
friend NetworkClient* networkClient();
};

Expand Down