From 2973865d3e0a66c19209623d3409e5c32450b70f Mon Sep 17 00:00:00 2001 From: deanlee Date: Sun, 20 Oct 2024 00:20:36 +0800 Subject: [PATCH] Replay: Replace HttpRequest with libcurl for accessing comma api --- tools/replay/SConscript | 2 +- tools/replay/api.cc | 162 ++++++++++++++++++++++++++++++++++++++++ tools/replay/api.h | 15 ++++ tools/replay/route.cc | 62 ++++++++------- tools/replay/route.h | 5 +- 5 files changed, 213 insertions(+), 33 deletions(-) create mode 100644 tools/replay/api.cc create mode 100644 tools/replay/api.h diff --git a/tools/replay/SConscript b/tools/replay/SConscript index 179af69d42ace7..4a907849cbba28 100644 --- a/tools/replay/SConscript +++ b/tools/replay/SConscript @@ -10,7 +10,7 @@ else: base_libs.append('OpenCL') replay_lib_src = ["replay.cc", "consoleui.cc", "camera.cc", "filereader.cc", "logreader.cc", "framereader.cc", - "route.cc", "util.cc", "timeline.cc"] + "route.cc", "util.cc", "timeline.cc", "api.cc"] replay_lib = qt_env.Library("qt_replay", replay_lib_src, LIBS=base_libs, FRAMEWORKS=base_frameworks) Export('replay_lib') replay_libs = [replay_lib, 'avutil', 'avcodec', 'avformat', 'bz2', 'zstd', 'curl', 'yuv', 'ncurses'] + base_libs diff --git a/tools/replay/api.cc b/tools/replay/api.cc new file mode 100644 index 00000000000000..85e4e52b282b4b --- /dev/null +++ b/tools/replay/api.cc @@ -0,0 +1,162 @@ + +#include "tools/replay/api.h" + +#include +#include +#include +#include + +#include +#include +#include + +#include "common/params.h" +#include "common/version.h" +#include "system/hardware/hw.h" + +namespace CommaApi2 { + +// Base64 URL-safe character set (uses '-' and '_' instead of '+' and '/') +static const std::string base64url_chars = + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789-_"; + +std::string base64url_encode(const std::string &in) { + std::string out; + int val = 0, valb = -6; + for (unsigned char c : in) { + val = (val << 8) + c; + valb += 8; + while (valb >= 0) { + out.push_back(base64url_chars[(val >> valb) & 0x3F]); + valb -= 6; + } + } + if (valb > -6) { + out.push_back(base64url_chars[((val << 8) >> (valb + 8)) & 0x3F]); + } + + return out; +} + +EVP_PKEY *get_rsa_private_key() { + static std::unique_ptr rsa_private(nullptr, EVP_PKEY_free); + if (!rsa_private) { + FILE *fp = fopen(Path::rsa_file().c_str(), "rb"); + if (!fp) { + std::cerr << "No RSA private key found, please run manager.py or registration.py" << std::endl; + return nullptr; + } + rsa_private.reset(PEM_read_PrivateKey(fp, NULL, NULL, NULL)); + fclose(fp); + } + return rsa_private.get(); +} + +std::string rsa_sign(const std::string &data) { + EVP_PKEY *private_key = get_rsa_private_key(); + if (!private_key) return {}; + + EVP_MD_CTX *mdctx = EVP_MD_CTX_new(); + assert(mdctx != nullptr); + + std::vector sig(EVP_PKEY_size(private_key)); + uint32_t sig_len; + + EVP_SignInit(mdctx, EVP_sha256()); + EVP_SignUpdate(mdctx, data.data(), data.size()); + int ret = EVP_SignFinal(mdctx, sig.data(), &sig_len, private_key); + + EVP_MD_CTX_free(mdctx); + + assert(ret == 1); + assert(sig.size() == sig_len); + return std::string(sig.begin(), sig.begin() + sig_len); +} + +std::string create_jwt(const json11::Json &extra, int exp_time) { + int now = std::chrono::seconds(std::time(nullptr)).count(); + std::string dongle_id = Params().get("DongleId"); + + // Create header and payload + json11::Json header = json11::Json::object{{"alg", "RS256"}}; + auto payload = json11::Json::object{ + {"identity", dongle_id}, + {"iat", now}, + {"nbf", now}, + {"exp", now + exp_time}, + }; + // Merge extra payload + for (const auto &item : extra.object_items()) { + payload[item.first] = item.second; + } + + // JWT construction + std::string jwt = base64url_encode(header.dump()) + '.' + + base64url_encode(json11::Json(payload).dump()); + + // Hash and sign + std::string hash(SHA256_DIGEST_LENGTH, '\0'); + SHA256((uint8_t *)jwt.data(), jwt.size(), (uint8_t *)hash.data()); + std::string signature = rsa_sign(hash); + + return jwt + "." + base64url_encode(signature); +} + +std::string create_token(bool use_jwt, const json11::Json &payloads, int expiry) { + if (use_jwt) { + return create_jwt(payloads, expiry); + } + + std::string token_json = util::read_file(util::getenv("HOME") + "/.comma/auth.json"); + std::string err; + auto json = json11::Json::parse(token_json, err); + if (!err.empty()) { + std::cerr << "Error parsing auth.json " << err << std::endl; + return ""; + } + return json["access_token"].string_value(); +} + +std::string httpGet(const std::string &url, long *response_code) { + CURL *curl = curl_easy_init(); + assert(curl); + + std::string readBuffer; + const std::string token = CommaApi2::create_token(!Hardware::PC()); + + // Set up the lambda for the write callback + // The '+' makes the lambda non-capturing, allowing it to be used as a C function pointer + auto writeCallback = +[](char *contents, size_t size, size_t nmemb, std::string *userp) ->size_t{ + size_t totalSize = size * nmemb; + userp->append((char *)contents, totalSize); + return totalSize; + }; + + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + + // Handle headers + struct curl_slist *headers = nullptr; + headers = curl_slist_append(headers, "User-Agent: openpilot-" COMMA_VERSION); + if (!token.empty()) { + headers = curl_slist_append(headers, ("Authorization: JWT " + token).c_str()); + } + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + CURLcode res = curl_easy_perform(curl); + + if (response_code) { + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, response_code); + } + + curl_slist_free_all(headers); + curl_easy_cleanup(curl); + + return res == CURLE_OK ? readBuffer : std::string{}; +} + +} // namespace CommaApi diff --git a/tools/replay/api.h b/tools/replay/api.h new file mode 100644 index 00000000000000..dff59c065905fd --- /dev/null +++ b/tools/replay/api.h @@ -0,0 +1,15 @@ +#pragma once + +#include +#include + +#include "common/util.h" +#include "third_party/json11/json11.hpp" + +namespace CommaApi2 { + +const std::string BASE_URL = util::getenv("API_HOST", "https://api.commadotai.com").c_str(); +std::string create_token(bool use_jwt, const json11::Json& payloads = {}, int expiry = 3600); +std::string httpGet(const std::string &url, long *response_code = nullptr); + +} // namespace CommaApi2 diff --git a/tools/replay/route.cc b/tools/replay/route.cc index 0d8d6d8fb7741d..9306b9fb07d289 100644 --- a/tools/replay/route.cc +++ b/tools/replay/route.cc @@ -1,14 +1,12 @@ #include "tools/replay/route.h" -#include -#include -#include #include #include #include -#include "selfdrive/ui/qt/api.h" +#include "third_party/json11/json11.hpp" #include "system/hardware/hw.h" +#include "tools/replay/api.h" #include "tools/replay/replay.h" #include "tools/replay/util.h" @@ -72,42 +70,48 @@ bool Route::load() { } bool Route::loadFromServer(int retries) { + const std::string url = CommaApi2::BASE_URL + "/v1/route/" + route_.str + "/files"; for (int i = 1; i <= retries; ++i) { - QString result; - QEventLoop loop; - HttpRequest http(nullptr, !Hardware::PC()); - QObject::connect(&http, &HttpRequest::requestDone, [&loop, &result](const QString &json, bool success, QNetworkReply::NetworkError err) { - result = json; - loop.exit((int)err); - }); - http.sendRequest(CommaApi::BASE_URL + "/v1/route/" + QString::fromStdString(route_.str) + "/files"); - auto err = (QNetworkReply::NetworkError)loop.exec(); - if (err == QNetworkReply::NoError) { + long response_code = 0; + std::string result = CommaApi2::httpGet(url, &response_code); + if (response_code == 200) { return loadFromJson(result); - } else if (err == QNetworkReply::ContentAccessDenied || err == QNetworkReply::AuthenticationRequiredError) { - rWarning(">> Unauthorized. Authenticate with tools/lib/auth.py <<"); - err_ = RouteLoadError::AccessDenied; - return false; - } else if (err == QNetworkReply::ContentNotFoundError) { + } + + if (response_code == 401 || response_code == 403) { + rWarning(">> Unauthorized. Authenticate with tools/lib/auth.py <<"); + err_ = RouteLoadError::Unauthorized; + break; + } + if (response_code == 404) { rWarning("The specified route could not be found on the server."); err_ = RouteLoadError::FileNotFound; - return false; - } else { - err_ = RouteLoadError::NetworkError; + break; } + + err_ = RouteLoadError::NetworkError; rWarning("Retrying %d/%d", i, retries); util::sleep_for(3000); } + return false; } -bool Route::loadFromJson(const QString &json) { - QRegExp rx(R"(\/(\d+)\/)"); - for (const auto &value : QJsonDocument::fromJson(json.trimmed().toUtf8()).object()) { - for (const auto &url : value.toArray()) { - QString url_str = url.toString(); - if (rx.indexIn(url_str) != -1) { - addFileToSegment(rx.cap(1).toInt(), url_str.toStdString()); +bool Route::loadFromJson(const std::string &json) { + const static std::regex rx(R"(\/(\d+)\/)"); + std::string err; + auto jsonData = json11::Json::parse(json, err); + if (!err.empty()) { + rWarning("JSON parsing error: %s", err.c_str()); + return false; + } + for (const auto &value : jsonData.object_items()) { + const auto &urlArray = value.second.array_items(); + for (const auto &url : urlArray) { + std::string url_str = url.string_value(); + std::smatch match; + if (std::regex_search(url_str, match, rx)) { + addFileToSegment(std::stoi(match[1]), url_str); } } } diff --git a/tools/replay/route.h b/tools/replay/route.h index a2a8121de7fe16..c2c7af6bc771f1 100644 --- a/tools/replay/route.h +++ b/tools/replay/route.h @@ -7,14 +7,13 @@ #include #include -#include - #include "tools/replay/framereader.h" #include "tools/replay/logreader.h" #include "tools/replay/util.h" enum class RouteLoadError { None, + Unauthorized, AccessDenied, NetworkError, FileNotFound, @@ -54,7 +53,7 @@ class Route { protected: bool loadFromLocal(); bool loadFromServer(int retries = 3); - bool loadFromJson(const QString &json); + bool loadFromJson(const std::string &json); void addFileToSegment(int seg_num, const std::string &file); RouteIdentifier route_ = {}; std::string data_dir_;