From b7d8d40b2517197ab4b639ce3eddb97393d94cbb Mon Sep 17 00:00:00 2001 From: PabloMK7 Date: Sun, 1 Oct 2023 12:27:31 +0200 Subject: [PATCH 1/3] Implement missing http:c functionality. --- externals/httplib/README.md | 2 +- externals/httplib/httplib.h | 1856 +++++++++++++++++++------- src/core/hle/service/http/http_c.cpp | 321 ++++- src/core/hle/service/http/http_c.h | 63 +- 4 files changed, 1672 insertions(+), 570 deletions(-) diff --git a/externals/httplib/README.md b/externals/httplib/README.md index f491f5200a..9dee85f145 100644 --- a/externals/httplib/README.md +++ b/externals/httplib/README.md @@ -1,4 +1,4 @@ -From https://github.com/yhirose/cpp-httplib/commit/8e10d4e8e7febafce0632810262e81e853b2065f +From https://github.com/yhirose/cpp-httplib/commit/0a629d739127dcc5d828474a5aedae1f234687d3 MIT License diff --git a/externals/httplib/httplib.h b/externals/httplib/httplib.h index 3657b47946..89449452aa 100644 --- a/externals/httplib/httplib.h +++ b/externals/httplib/httplib.h @@ -1,14 +1,14 @@ // // httplib.h // -// Copyright (c) 2022 Yuji Hirose. All rights reserved. +// Copyright (c) 2023 Yuji Hirose. All rights reserved. // MIT License // #ifndef CPPHTTPLIB_HTTPLIB_H #define CPPHTTPLIB_HTTPLIB_H -#define CPPHTTPLIB_VERSION "0.11.2" +#define CPPHTTPLIB_VERSION "0.14.0" /* * Configuration @@ -172,7 +172,15 @@ using socket_t = SOCKET; #else // not _WIN32 #include +#if !defined(_AIX) && !defined(__MVS__) #include +#endif +#ifdef __MVS__ +#include +#ifndef NI_MAXHOST +#define NI_MAXHOST 1025 +#endif +#endif #include #include #include @@ -185,6 +193,7 @@ using socket_t = SOCKET; #endif #include #include +#include #include #include #include @@ -221,6 +230,9 @@ using socket_t = int; #include #include #include +#include +#include +#include #ifdef CPPHTTPLIB_OPENSSL_SUPPORT #ifdef _WIN32 @@ -233,17 +245,17 @@ using socket_t = int; #undef X509_EXTENSIONS #undef PKCS7_SIGNER_INFO -// libressl will warn without this, which becomes an error. -#undef OCSP_REQUEST -#undef OCSP_RESPONSE -#undef PKCS7_ISSUER_AND_SERIAL -#undef __WINCRYPT_H__ - #ifdef _MSC_VER #pragma comment(lib, "crypt32.lib") #pragma comment(lib, "cryptui.lib") #endif -#endif //_WIN32 +#elif defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN) && defined(__APPLE__) +#include +#if TARGET_OS_OSX +#include +#include +#endif // TARGET_OS_OSX +#endif // _WIN32 #include #include @@ -259,14 +271,10 @@ using socket_t = int; #if OPENSSL_VERSION_NUMBER < 0x1010100fL #error Sorry, OpenSSL versions prior to 1.1.1 are not supported +#elif OPENSSL_VERSION_NUMBER < 0x30000000L +#define SSL_get1_peer_certificate SSL_get_peer_certificate #endif -#if OPENSSL_VERSION_NUMBER < 0x10100000L -#include -inline const unsigned char *ASN1_STRING_get0_data(const ASN1_STRING *asn1) { - return M_ASN1_STRING_data(asn1); -} -#endif #endif #ifdef CPPHTTPLIB_ZLIB_SUPPORT @@ -316,6 +324,34 @@ struct ci { } }; +// This is based on +// "http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2014/n4189". + +struct scope_exit { + explicit scope_exit(std::function &&f) + : exit_function(std::move(f)), execute_on_destruction{true} {} + + scope_exit(scope_exit &&rhs) + : exit_function(std::move(rhs.exit_function)), + execute_on_destruction{rhs.execute_on_destruction} { + rhs.release(); + } + + ~scope_exit() { + if (execute_on_destruction) { this->exit_function(); } + } + + void release() { this->execute_on_destruction = false; } + +private: + scope_exit(const scope_exit &) = delete; + void operator=(const scope_exit &) = delete; + scope_exit &operator=(scope_exit &&) = delete; + + std::function exit_function; + bool execute_on_destruction; +}; + } // namespace detail using Headers = std::multimap; @@ -348,7 +384,7 @@ public: std::function write; std::function done; - std::function is_writable; + std::function done_with_trailer; std::ostream os; private: @@ -377,6 +413,14 @@ using ContentProviderWithoutLength = using ContentProviderResourceReleaser = std::function; +struct MultipartFormDataProvider { + std::string name; + ContentProviderWithoutLength provider; + std::string filename; + std::string content_type; +}; +using MultipartFormDataProviderItems = std::vector; + using ContentReceiverWithProgress = std::function; @@ -421,6 +465,8 @@ struct Request { std::string remote_addr; int remote_port = -1; + std::string local_addr; + int local_port = -1; // for server std::string version; @@ -429,6 +475,7 @@ struct Request { MultipartFormDataMap files; Ranges ranges; Match matches; + std::unordered_map path_params; // for client ResponseHandler response_handler; @@ -440,8 +487,7 @@ struct Request { bool has_header(const std::string &key) const; std::string get_header_value(const std::string &key, size_t id = 0) const; - template - T get_header_value(const std::string &key, size_t id = 0) const; + uint64_t get_header_value_u64(const std::string &key, size_t id = 0) const; size_t get_header_value_count(const std::string &key) const; void set_header(const std::string &key, const std::string &val); @@ -453,6 +499,7 @@ struct Request { bool has_file(const std::string &key) const; MultipartFormData get_file_value(const std::string &key) const; + std::vector get_file_values(const std::string &key) const; // private members... size_t redirect_count_ = CPPHTTPLIB_REDIRECT_MAX_COUNT; @@ -472,8 +519,7 @@ struct Response { bool has_header(const std::string &key) const; std::string get_header_value(const std::string &key, size_t id = 0) const; - template - T get_header_value(const std::string &key, size_t id = 0) const; + uint64_t get_header_value_u64(const std::string &key, size_t id = 0) const; size_t get_header_value_count(const std::string &key) const; void set_header(const std::string &key, const std::string &val); @@ -522,6 +568,7 @@ public: virtual ssize_t read(char *ptr, size_t size) = 0; virtual ssize_t write(const char *ptr, size_t size) = 0; virtual void get_remote_ip_and_port(std::string &ip, int &port) const = 0; + virtual void get_local_ip_and_port(std::string &ip, int &port) const = 0; virtual socket_t socket() const = 0; template @@ -554,8 +601,11 @@ public: ~ThreadPool() override = default; void enqueue(std::function fn) override { - std::unique_lock lock(mutex_); - jobs_.push_back(std::move(fn)); + { + std::unique_lock lock(mutex_); + jobs_.push_back(std::move(fn)); + } + cond_.notify_one(); } @@ -589,7 +639,7 @@ private: if (pool_.shutdown_ && pool_.jobs_.empty()) { break; } - fn = pool_.jobs_.front(); + fn = std::move(pool_.jobs_.front()); pool_.jobs_.pop_front(); } @@ -617,6 +667,80 @@ using SocketOptions = std::function; void default_socket_options(socket_t sock); +const char *status_message(int status); + +namespace detail { + +class MatcherBase { +public: + virtual ~MatcherBase() = default; + + // Match request path and populate its matches and + virtual bool match(Request &request) const = 0; +}; + +/** + * Captures parameters in request path and stores them in Request::path_params + * + * Capture name is a substring of a pattern from : to /. + * The rest of the pattern is matched agains the request path directly + * Parameters are captured starting from the next character after + * the end of the last matched static pattern fragment until the next /. + * + * Example pattern: + * "/path/fragments/:capture/more/fragments/:second_capture" + * Static fragments: + * "/path/fragments/", "more/fragments/" + * + * Given the following request path: + * "/path/fragments/:1/more/fragments/:2" + * the resulting capture will be + * {{"capture", "1"}, {"second_capture", "2"}} + */ +class PathParamsMatcher : public MatcherBase { +public: + PathParamsMatcher(const std::string &pattern); + + bool match(Request &request) const override; + +private: + static constexpr char marker = ':'; + // Treat segment separators as the end of path parameter capture + // Does not need to handle query parameters as they are parsed before path + // matching + static constexpr char separator = '/'; + + // Contains static path fragments to match against, excluding the '/' after + // path params + // Fragments are separated by path params + std::vector static_fragments_; + // Stores the names of the path parameters to be used as keys in the + // Request::path_params map + std::vector param_names_; +}; + +/** + * Performs std::regex_match on request path + * and stores the result in Request::matches + * + * Note that regex match is performed directly on the whole request. + * This means that wildcard patterns may match multiple path segments with /: + * "/begin/(.*)/end" will match both "/begin/middle/end" and "/begin/1/2/end". + */ +class RegexMatcher : public MatcherBase { +public: + RegexMatcher(const std::string &pattern) : regex_(pattern) {} + + bool match(Request &request) const override; + +private: + std::regex regex_; +}; + +ssize_t write_headers(Stream &strm, const Headers &headers); + +} // namespace detail + class Server { public: using Handler = std::function; @@ -661,6 +785,7 @@ public: bool remove_mount_point(const std::string &mount_point); Server &set_file_extension_and_mimetype_mapping(const std::string &ext, const std::string &mime); + Server &set_default_file_mimetype(const std::string &mime); Server &set_file_request_handler(Handler handler); Server &set_error_handler(HandlerWithResponse handler); @@ -677,6 +802,8 @@ public: Server &set_socket_options(SocketOptions socket_options); Server &set_default_headers(Headers headers); + Server & + set_header_writer(std::function const &writer); Server &set_keep_alive_max_count(size_t count); Server &set_keep_alive_timeout(time_t sec); @@ -702,6 +829,7 @@ public: bool listen(const std::string &host, int port, int socket_flags = 0); bool is_running() const; + void wait_until_ready() const; void stop(); std::function new_task_queue; @@ -711,7 +839,7 @@ protected: bool &connection_closed, const std::function &setup_request); - std::atomic svr_sock_; + std::atomic svr_sock_{INVALID_SOCKET}; size_t keep_alive_max_count_ = CPPHTTPLIB_KEEPALIVE_MAX_COUNT; time_t keep_alive_timeout_sec_ = CPPHTTPLIB_KEEPALIVE_TIMEOUT_SECOND; time_t read_timeout_sec_ = CPPHTTPLIB_READ_TIMEOUT_SECOND; @@ -723,9 +851,14 @@ protected: size_t payload_max_length_ = CPPHTTPLIB_PAYLOAD_MAX_LENGTH; private: - using Handlers = std::vector>; + using Handlers = + std::vector, Handler>>; using HandlersForContentReader = - std::vector>; + std::vector, + HandlerWithContentReader>>; + + static std::unique_ptr + make_matcher(const std::string &pattern); socket_t create_server_socket(const std::string &host, int port, int socket_flags, @@ -763,21 +896,24 @@ private: ContentReceiver multipart_receiver); bool read_content_core(Stream &strm, Request &req, Response &res, ContentReceiver receiver, - MultipartContentHeader mulitpart_header, + MultipartContentHeader multipart_header, ContentReceiver multipart_receiver); virtual bool process_and_close_socket(socket_t sock); + std::atomic is_running_{false}; + std::atomic done_{false}; + struct MountPointEntry { std::string mount_point; std::string base_dir; Headers headers; }; std::vector base_dirs_; - - std::atomic is_running_; std::map file_extension_and_mimetype_map_; + std::string default_file_mimetype_ = "application/octet-stream"; Handler file_request_handler_; + Handlers get_handlers_; Handlers post_handlers_; HandlersForContentReader post_handlers_for_content_reader_; @@ -788,18 +924,22 @@ private: Handlers delete_handlers_; HandlersForContentReader delete_handlers_for_content_reader_; Handlers options_handlers_; + HandlerWithResponse error_handler_; ExceptionHandler exception_handler_; HandlerWithResponse pre_routing_handler_; Handler post_routing_handler_; - Logger logger_; Expect100ContinueHandler expect_100_continue_handler_; + Logger logger_; + int address_family_ = AF_UNSPEC; bool tcp_nodelay_ = CPPHTTPLIB_TCP_NODELAY; SocketOptions socket_options_ = default_socket_options; Headers default_headers_; + std::function header_writer_ = + detail::write_headers; }; enum class Error { @@ -817,6 +957,10 @@ enum class Error { UnsupportedMultipartBoundaryChars, Compression, ConnectionTimeout, + ProxyConnection, + + // For internal use only + SSLPeerCouldBeClosed_, }; std::string to_string(const Error error); @@ -825,6 +969,7 @@ std::ostream &operator<<(std::ostream &os, const Error &obj); class Result { public: + Result() = default; Result(std::unique_ptr &&res, Error err, Headers &&request_headers = Headers{}) : res_(std::move(res)), err_(err), @@ -847,13 +992,13 @@ public: bool has_request_header(const std::string &key) const; std::string get_request_header_value(const std::string &key, size_t id = 0) const; - template - T get_request_header_value(const std::string &key, size_t id = 0) const; + uint64_t get_request_header_value_u64(const std::string &key, + size_t id = 0) const; size_t get_request_header_value_count(const std::string &key) const; private: std::unique_ptr res_; - Error err_; + Error err_ = Error::Unknown; Headers request_headers_; }; @@ -907,6 +1052,7 @@ public: Result Head(const std::string &path, const Headers &headers); Result Post(const std::string &path); + Result Post(const std::string &path, const Headers &headers); Result Post(const std::string &path, const char *body, size_t content_length, const std::string &content_type); Result Post(const std::string &path, const Headers &headers, const char *body, @@ -935,6 +1081,9 @@ public: const MultipartFormDataItems &items); Result Post(const std::string &path, const Headers &headers, const MultipartFormDataItems &items, const std::string &boundary); + Result Post(const std::string &path, const Headers &headers, + const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items); Result Put(const std::string &path); Result Put(const std::string &path, const char *body, size_t content_length, @@ -964,6 +1113,9 @@ public: const MultipartFormDataItems &items); Result Put(const std::string &path, const Headers &headers, const MultipartFormDataItems &items, const std::string &boundary); + Result Put(const std::string &path, const Headers &headers, + const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items); Result Patch(const std::string &path); Result Patch(const std::string &path, const char *body, size_t content_length, @@ -1006,16 +1158,21 @@ public: bool send(Request &req, Response &res, Error &error); Result send(const Request &req); - size_t is_socket_open() const; - - socket_t socket() const; - void stop(); + std::string host() const; + int port() const; + + size_t is_socket_open() const; + socket_t socket() const; + void set_hostname_addr_map(std::map addr_map); void set_default_headers(Headers headers); + void + set_header_writer(std::function const &writer); + void set_address_family(int family); void set_tcp_nodelay(bool on); void set_socket_options(SocketOptions socket_options); @@ -1064,6 +1221,7 @@ public: void set_ca_cert_path(const std::string &ca_cert_file_path, const std::string &ca_cert_dir_path = std::string()); void set_ca_cert_store(X509_STORE *ca_cert_store); + X509_STORE *create_ca_cert_store(const char *ca_cert, std::size_t size); #endif #ifdef CPPHTTPLIB_OPENSSL_SUPPORT @@ -1082,8 +1240,6 @@ protected: bool is_open() const { return sock != INVALID_SOCKET; } }; - Result send_(Request &&req); - virtual bool create_and_connect_socket(Socket &socket, Error &error); // All of: @@ -1105,7 +1261,7 @@ protected: void copy_settings(const ClientImpl &rhs); - // Socket endoint information + // Socket endpoint information const std::string host_; const int port_; const std::string host_and_port_; @@ -1126,6 +1282,10 @@ protected: // Default headers Headers default_headers_; + // Header writer + std::function header_writer_ = + detail::write_headers; + // Settings std::string client_cert_path_; std::string client_key_path_; @@ -1184,6 +1344,9 @@ protected: Logger logger_; private: + bool send_(Request &req, Response &res, Error &error); + Result send_(Request &&req); + socket_t create_client_socket(Error &error) const; bool read_response_line(Stream &strm, const Request &req, Response &res); bool write_request(Stream &strm, Request &req, bool close_connection, @@ -1202,6 +1365,9 @@ private: ContentProvider content_provider, ContentProviderWithoutLength content_provider_without_length, const std::string &content_type); + ContentProviderWithoutLength get_multipart_content_provider( + const std::string &boundary, const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items); std::string adjust_host_string(const std::string &host) const; @@ -1268,6 +1434,7 @@ public: Result Head(const std::string &path, const Headers &headers); Result Post(const std::string &path); + Result Post(const std::string &path, const Headers &headers); Result Post(const std::string &path, const char *body, size_t content_length, const std::string &content_type); Result Post(const std::string &path, const Headers &headers, const char *body, @@ -1296,6 +1463,10 @@ public: const MultipartFormDataItems &items); Result Post(const std::string &path, const Headers &headers, const MultipartFormDataItems &items, const std::string &boundary); + Result Post(const std::string &path, const Headers &headers, + const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items); + Result Put(const std::string &path); Result Put(const std::string &path, const char *body, size_t content_length, const std::string &content_type); @@ -1324,6 +1495,10 @@ public: const MultipartFormDataItems &items); Result Put(const std::string &path, const Headers &headers, const MultipartFormDataItems &items, const std::string &boundary); + Result Put(const std::string &path, const Headers &headers, + const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items); + Result Patch(const std::string &path); Result Patch(const std::string &path, const char *body, size_t content_length, const std::string &content_type); @@ -1365,16 +1540,21 @@ public: bool send(Request &req, Response &res, Error &error); Result send(const Request &req); - size_t is_socket_open() const; - - socket_t socket() const; - void stop(); + std::string host() const; + int port() const; + + size_t is_socket_open() const; + socket_t socket() const; + void set_hostname_addr_map(std::map addr_map); void set_default_headers(Headers headers); + void + set_header_writer(std::function const &writer); + void set_address_family(int family); void set_tcp_nodelay(bool on); void set_socket_options(SocketOptions socket_options); @@ -1431,6 +1611,7 @@ public: const std::string &ca_cert_dir_path = std::string()); void set_ca_cert_store(X509_STORE *ca_cert_store); + void load_ca_cert_store(const char *ca_cert, std::size_t size); long get_openssl_verify_result() const; @@ -1490,6 +1671,7 @@ public: bool is_valid() const override; void set_ca_cert_store(X509_STORE *ca_cert_store); + void load_ca_cert_store(const char *ca_cert, std::size_t size); long get_openssl_verify_result() const; @@ -1539,18 +1721,12 @@ inline void duration_to_sec_and_usec(const T &duration, U callback) { auto usec = std::chrono::duration_cast( duration - std::chrono::seconds(sec)) .count(); - callback(sec, usec); + callback(static_cast(sec), static_cast(usec)); } -template -inline T get_header_value(const Headers & /*headers*/, - const std::string & /*key*/, size_t /*id*/ = 0, - uint64_t /*def*/ = 0) {} - -template <> -inline uint64_t get_header_value(const Headers &headers, - const std::string &key, size_t id, - uint64_t def) { +inline uint64_t get_header_value_u64(const Headers &headers, + const std::string &key, size_t id, + uint64_t def) { auto rng = headers.equal_range(key); auto it = rng.first; std::advance(it, static_cast(id)); @@ -1562,14 +1738,14 @@ inline uint64_t get_header_value(const Headers &headers, } // namespace detail -template -inline T Request::get_header_value(const std::string &key, size_t id) const { - return detail::get_header_value(headers, key, id, 0); +inline uint64_t Request::get_header_value_u64(const std::string &key, + size_t id) const { + return detail::get_header_value_u64(headers, key, id, 0); } -template -inline T Response::get_header_value(const std::string &key, size_t id) const { - return detail::get_header_value(headers, key, id, 0); +inline uint64_t Response::get_header_value_u64(const std::string &key, + size_t id) const { + return detail::get_header_value_u64(headers, key, id, 0); } template @@ -1599,21 +1775,91 @@ inline ssize_t Stream::write_format(const char *fmt, const Args &...args) { inline void default_socket_options(socket_t sock) { int yes = 1; #ifdef _WIN32 - setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast(&yes), - sizeof(yes)); + setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, + reinterpret_cast(&yes), sizeof(yes)); setsockopt(sock, SOL_SOCKET, SO_EXCLUSIVEADDRUSE, - reinterpret_cast(&yes), sizeof(yes)); + reinterpret_cast(&yes), sizeof(yes)); #else #ifdef SO_REUSEPORT - setsockopt(sock, SOL_SOCKET, SO_REUSEPORT, reinterpret_cast(&yes), - sizeof(yes)); + setsockopt(sock, SOL_SOCKET, SO_REUSEPORT, + reinterpret_cast(&yes), sizeof(yes)); #else - setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast(&yes), - sizeof(yes)); + setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, + reinterpret_cast(&yes), sizeof(yes)); #endif #endif } +inline const char *status_message(int status) { + switch (status) { + case 100: return "Continue"; + case 101: return "Switching Protocol"; + case 102: return "Processing"; + case 103: return "Early Hints"; + case 200: return "OK"; + case 201: return "Created"; + case 202: return "Accepted"; + case 203: return "Non-Authoritative Information"; + case 204: return "No Content"; + case 205: return "Reset Content"; + case 206: return "Partial Content"; + case 207: return "Multi-Status"; + case 208: return "Already Reported"; + case 226: return "IM Used"; + case 300: return "Multiple Choice"; + case 301: return "Moved Permanently"; + case 302: return "Found"; + case 303: return "See Other"; + case 304: return "Not Modified"; + case 305: return "Use Proxy"; + case 306: return "unused"; + case 307: return "Temporary Redirect"; + case 308: return "Permanent Redirect"; + case 400: return "Bad Request"; + case 401: return "Unauthorized"; + case 402: return "Payment Required"; + case 403: return "Forbidden"; + case 404: return "Not Found"; + case 405: return "Method Not Allowed"; + case 406: return "Not Acceptable"; + case 407: return "Proxy Authentication Required"; + case 408: return "Request Timeout"; + case 409: return "Conflict"; + case 410: return "Gone"; + case 411: return "Length Required"; + case 412: return "Precondition Failed"; + case 413: return "Payload Too Large"; + case 414: return "URI Too Long"; + case 415: return "Unsupported Media Type"; + case 416: return "Range Not Satisfiable"; + case 417: return "Expectation Failed"; + case 418: return "I'm a teapot"; + case 421: return "Misdirected Request"; + case 422: return "Unprocessable Entity"; + case 423: return "Locked"; + case 424: return "Failed Dependency"; + case 425: return "Too Early"; + case 426: return "Upgrade Required"; + case 428: return "Precondition Required"; + case 429: return "Too Many Requests"; + case 431: return "Request Header Fields Too Large"; + case 451: return "Unavailable For Legal Reasons"; + case 501: return "Not Implemented"; + case 502: return "Bad Gateway"; + case 503: return "Service Unavailable"; + case 504: return "Gateway Timeout"; + case 505: return "HTTP Version Not Supported"; + case 506: return "Variant Also Negotiates"; + case 507: return "Insufficient Storage"; + case 508: return "Loop Detected"; + case 510: return "Not Extended"; + case 511: return "Network Authentication Required"; + + default: + case 500: return "Internal Server Error"; + } +} + template inline Server & Server::set_read_timeout(const std::chrono::duration &duration) { @@ -1640,20 +1886,21 @@ Server::set_idle_interval(const std::chrono::duration &duration) { inline std::string to_string(const Error error) { switch (error) { - case Error::Success: return "Success"; - case Error::Connection: return "Connection"; - case Error::BindIPAddress: return "BindIPAddress"; - case Error::Read: return "Read"; - case Error::Write: return "Write"; - case Error::ExceedRedirectCount: return "ExceedRedirectCount"; - case Error::Canceled: return "Canceled"; - case Error::SSLConnection: return "SSLConnection"; - case Error::SSLLoadingCerts: return "SSLLoadingCerts"; - case Error::SSLServerVerification: return "SSLServerVerification"; + case Error::Success: return "Success (no error)"; + case Error::Connection: return "Could not establish connection"; + case Error::BindIPAddress: return "Failed to bind IP address"; + case Error::Read: return "Failed to read connection"; + case Error::Write: return "Failed to write connection"; + case Error::ExceedRedirectCount: return "Maximum redirect count exceeded"; + case Error::Canceled: return "Connection handling canceled"; + case Error::SSLConnection: return "SSL connection failed"; + case Error::SSLLoadingCerts: return "SSL certificate loading failed"; + case Error::SSLServerVerification: return "SSL server verification failed"; case Error::UnsupportedMultipartBoundaryChars: - return "UnsupportedMultipartBoundaryChars"; - case Error::Compression: return "Compression"; - case Error::ConnectionTimeout: return "ConnectionTimeout"; + return "Unsupported HTTP multipart boundary characters"; + case Error::Compression: return "Compression failed"; + case Error::ConnectionTimeout: return "Connection timed out"; + case Error::ProxyConnection: return "Proxy connection failed"; case Error::Unknown: return "Unknown"; default: break; } @@ -1667,10 +1914,9 @@ inline std::ostream &operator<<(std::ostream &os, const Error &obj) { return os; } -template -inline T Result::get_request_header_value(const std::string &key, - size_t id) const { - return detail::get_header_value(request_headers_, key, id, 0); +inline uint64_t Result::get_request_header_value_u64(const std::string &key, + size_t id) const { + return detail::get_header_value_u64(request_headers_, key, id, 0); } template @@ -1763,6 +2009,9 @@ std::string params_to_query_str(const Params ¶ms); void parse_query_text(const std::string &s, Params ¶ms); +bool parse_multipart_boundary(const std::string &content_type, + std::string &boundary); + bool parse_range_header(const std::string &s, Ranges &ranges); int close_socket(socket_t sock); @@ -1785,6 +2034,7 @@ public: ssize_t read(char *ptr, size_t size) override; ssize_t write(const char *ptr, size_t size) override; void get_remote_ip_and_port(std::string &ip, int &port) const override; + void get_local_ip_and_port(std::string &ip, int &port) const override; socket_t socket() const override; const std::string &get_buffer() const; @@ -1902,6 +2152,29 @@ private: std::string glowable_buffer_; }; +class mmap { +public: + mmap(const char *path); + ~mmap(); + + bool open(const char *path); + void close(); + + bool is_open() const; + size_t size() const; + const char *data() const; + +private: +#if defined(_WIN32) + HANDLE hFile_; + HANDLE hMapping_; +#else + int fd_; +#endif + size_t size_; + void *addr_; +}; + } // namespace detail // ---------------------------------------------------------------------------- @@ -1933,7 +2206,7 @@ inline bool from_hex_to_i(const std::string &s, size_t i, size_t cnt, val = 0; for (; cnt; i++, cnt--) { if (!s[i]) { return false; } - int v = 0; + auto v = 0; if (is_hex(s[i], v)) { val = val * 16 + v; } else { @@ -1944,7 +2217,7 @@ inline bool from_hex_to_i(const std::string &s, size_t i, size_t cnt, } inline std::string from_i_to_hex(size_t n) { - const char *charset = "0123456789abcdef"; + static const auto charset = "0123456789abcdef"; std::string ret; do { ret = charset[n & 15] + ret; @@ -1994,8 +2267,8 @@ inline std::string base64_encode(const std::string &in) { std::string out; out.reserve(in.size()); - int val = 0; - int valb = -6; + auto val = 0; + auto valb = -6; for (auto c : in) { val = (val << 8) + static_cast(c); @@ -2126,7 +2399,7 @@ inline std::string decode_url(const std::string &s, for (size_t i = 0; i < s.size(); i++) { if (s[i] == '%' && i + 1 < s.size()) { if (s[i + 1] == 'u') { - int val = 0; + auto val = 0; if (from_hex_to_i(s, i + 2, 4, val)) { // 4 digits Unicode codes char buff[4]; @@ -2137,7 +2410,7 @@ inline std::string decode_url(const std::string &s, result += s[i]; } } else { - int val = 0; + auto val = 0; if (from_hex_to_i(s, i + 1, 2, val)) { // 2 digits hex codes result += static_cast(val); @@ -2190,6 +2463,13 @@ inline std::string trim_copy(const std::string &s) { return s.substr(r.first, r.second - r.first); } +inline std::string trim_double_quotes_copy(const std::string &s) { + if (s.length() >= 2 && s.front() == '"' && s.back() == '"') { + return s.substr(1, s.size() - 2); + } + return s; +} + inline void split(const char *b, const char *e, char d, std::function fn) { size_t i = 0; @@ -2275,6 +2555,95 @@ inline void stream_line_reader::append(char c) { } } +inline mmap::mmap(const char *path) +#if defined(_WIN32) + : hFile_(NULL), hMapping_(NULL) +#else + : fd_(-1) +#endif + , + size_(0), addr_(nullptr) { + if (!open(path)) { std::runtime_error(""); } +} + +inline mmap::~mmap() { close(); } + +inline bool mmap::open(const char *path) { + close(); + +#if defined(_WIN32) + hFile_ = ::CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, NULL, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + + if (hFile_ == INVALID_HANDLE_VALUE) { return false; } + + size_ = ::GetFileSize(hFile_, NULL); + + hMapping_ = ::CreateFileMapping(hFile_, NULL, PAGE_READONLY, 0, 0, NULL); + + if (hMapping_ == NULL) { + close(); + return false; + } + + addr_ = ::MapViewOfFile(hMapping_, FILE_MAP_READ, 0, 0, 0); +#else + fd_ = ::open(path, O_RDONLY); + if (fd_ == -1) { return false; } + + struct stat sb; + if (fstat(fd_, &sb) == -1) { + close(); + return false; + } + size_ = static_cast(sb.st_size); + + addr_ = ::mmap(NULL, size_, PROT_READ, MAP_PRIVATE, fd_, 0); +#endif + + if (addr_ == nullptr) { + close(); + return false; + } + + return true; +} + +inline bool mmap::is_open() const { return addr_ != nullptr; } + +inline size_t mmap::size() const { return size_; } + +inline const char *mmap::data() const { return (const char *)addr_; } + +inline void mmap::close() { +#if defined(_WIN32) + if (addr_) { + ::UnmapViewOfFile(addr_); + addr_ = nullptr; + } + + if (hMapping_) { + ::CloseHandle(hMapping_); + hMapping_ = NULL; + } + + if (hFile_ != INVALID_HANDLE_VALUE) { + ::CloseHandle(hFile_); + hFile_ = INVALID_HANDLE_VALUE; + } +#else + if (addr_ != nullptr) { + munmap(addr_, size_); + addr_ = nullptr; + } + + if (fd_ != -1) { + ::close(fd_); + fd_ = -1; + } +#endif + size_ = 0; +} inline int close_socket(socket_t sock) { #ifdef _WIN32 return closesocket(sock); @@ -2284,7 +2653,7 @@ inline int close_socket(socket_t sock) { } template inline ssize_t handle_EINTR(T fn) { - ssize_t res = false; + ssize_t res = 0; while (true) { res = fn(); if (res < 0 && errno == EINTR) { continue; } @@ -2388,7 +2757,7 @@ inline Error wait_until_socket_is_ready(socket_t sock, time_t sec, if (poll_res == 0) { return Error::ConnectionTimeout; } if (poll_res > 0 && pfd_read.revents & (POLLIN | POLLOUT)) { - int error = 0; + auto error = 0; socklen_t len = sizeof(error); auto res = getsockopt(sock, SOL_SOCKET, SO_ERROR, reinterpret_cast(&error), &len); @@ -2420,7 +2789,7 @@ inline Error wait_until_socket_is_ready(socket_t sock, time_t sec, if (ret == 0) { return Error::ConnectionTimeout; } if (ret > 0 && (FD_ISSET(sock, &fdsr) || FD_ISSET(sock, &fdsw))) { - int error = 0; + auto error = 0; socklen_t len = sizeof(error); auto res = getsockopt(sock, SOL_SOCKET, SO_ERROR, reinterpret_cast(&error), &len); @@ -2453,6 +2822,7 @@ public: ssize_t read(char *ptr, size_t size) override; ssize_t write(const char *ptr, size_t size) override; void get_remote_ip_and_port(std::string &ip, int &port) const override; + void get_local_ip_and_port(std::string &ip, int &port) const override; socket_t socket() const override; private: @@ -2482,6 +2852,7 @@ public: ssize_t read(char *ptr, size_t size) override; ssize_t write(const char *ptr, size_t size) override; void get_remote_ip_and_port(std::string &ip, int &port) const override; + void get_local_ip_and_port(std::string &ip, int &port) const override; socket_t socket() const override; private: @@ -2598,7 +2969,7 @@ socket_t create_socket(const std::string &host, const std::string &ip, int port, auto sock = socket(hints.ai_family, hints.ai_socktype, hints.ai_protocol); if (sock != INVALID_SOCKET) { - sockaddr_un addr; + sockaddr_un addr{}; addr.sun_family = AF_UNIX; std::copy(host.begin(), host.end(), addr.sun_path); @@ -2606,6 +2977,9 @@ socket_t create_socket(const std::string &host, const std::string &ip, int port, hints.ai_addrlen = static_cast( sizeof(addr) - sizeof(addr.sun_path) + addrlen); + fcntl(sock, F_SETFD, FD_CLOEXEC); + if (socket_options) { socket_options(sock); } + if (!bind_or_connect(sock, hints)) { close_socket(sock); sock = INVALID_SOCKET; @@ -2653,21 +3027,34 @@ socket_t create_socket(const std::string &host, const std::string &ip, int port, if (sock == INVALID_SOCKET) { continue; } #ifndef _WIN32 - if (fcntl(sock, F_SETFD, FD_CLOEXEC) == -1) { continue; } + if (fcntl(sock, F_SETFD, FD_CLOEXEC) == -1) { + close_socket(sock); + continue; + } #endif if (tcp_nodelay) { - int yes = 1; - setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, reinterpret_cast(&yes), - sizeof(yes)); + auto yes = 1; +#ifdef _WIN32 + setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, + reinterpret_cast(&yes), sizeof(yes)); +#else + setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, + reinterpret_cast(&yes), sizeof(yes)); +#endif } if (socket_options) { socket_options(sock); } if (rp->ai_family == AF_INET6) { - int no = 0; - setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, reinterpret_cast(&no), - sizeof(no)); + auto no = 0; +#ifdef _WIN32 + setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, + reinterpret_cast(&no), sizeof(no)); +#else + setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, + reinterpret_cast(&no), sizeof(no)); +#endif } // bind or connect @@ -2726,7 +3113,7 @@ inline bool bind_ip_address(socket_t sock, const std::string &host) { return ret; } -#if !defined _WIN32 && !defined ANDROID +#if !defined _WIN32 && !defined ANDROID && !defined _AIX && !defined __MVS__ #define USE_IF2IP #endif @@ -2810,13 +3197,14 @@ inline socket_t create_client_socket( #ifdef _WIN32 auto timeout = static_cast(read_timeout_sec * 1000 + read_timeout_usec / 1000); - setsockopt(sock2, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout, - sizeof(timeout)); + setsockopt(sock2, SOL_SOCKET, SO_RCVTIMEO, + reinterpret_cast(&timeout), sizeof(timeout)); #else timeval tv; tv.tv_sec = static_cast(read_timeout_sec); tv.tv_usec = static_cast(read_timeout_usec); - setsockopt(sock2, SOL_SOCKET, SO_RCVTIMEO, (char *)&tv, sizeof(tv)); + setsockopt(sock2, SOL_SOCKET, SO_RCVTIMEO, + reinterpret_cast(&tv), sizeof(tv)); #endif } { @@ -2824,13 +3212,14 @@ inline socket_t create_client_socket( #ifdef _WIN32 auto timeout = static_cast(write_timeout_sec * 1000 + write_timeout_usec / 1000); - setsockopt(sock2, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout, - sizeof(timeout)); + setsockopt(sock2, SOL_SOCKET, SO_SNDTIMEO, + reinterpret_cast(&timeout), sizeof(timeout)); #else timeval tv; tv.tv_sec = static_cast(write_timeout_sec); tv.tv_usec = static_cast(write_timeout_usec); - setsockopt(sock2, SOL_SOCKET, SO_SNDTIMEO, (char *)&tv, sizeof(tv)); + setsockopt(sock2, SOL_SOCKET, SO_SNDTIMEO, + reinterpret_cast(&tv), sizeof(tv)); #endif } @@ -2847,9 +3236,8 @@ inline socket_t create_client_socket( return sock; } -inline bool get_remote_ip_and_port(const struct sockaddr_storage &addr, - socklen_t addr_len, std::string &ip, - int &port) { +inline bool get_ip_and_port(const struct sockaddr_storage &addr, + socklen_t addr_len, std::string &ip, int &port) { if (addr.ss_family == AF_INET) { port = ntohs(reinterpret_cast(&addr)->sin_port); } else if (addr.ss_family == AF_INET6) { @@ -2870,21 +3258,53 @@ inline bool get_remote_ip_and_port(const struct sockaddr_storage &addr, return true; } +inline void get_local_ip_and_port(socket_t sock, std::string &ip, int &port) { + struct sockaddr_storage addr; + socklen_t addr_len = sizeof(addr); + if (!getsockname(sock, reinterpret_cast(&addr), + &addr_len)) { + get_ip_and_port(addr, addr_len, ip, port); + } +} + inline void get_remote_ip_and_port(socket_t sock, std::string &ip, int &port) { struct sockaddr_storage addr; socklen_t addr_len = sizeof(addr); if (!getpeername(sock, reinterpret_cast(&addr), &addr_len)) { - get_remote_ip_and_port(addr, addr_len, ip, port); +#ifndef _WIN32 + if (addr.ss_family == AF_UNIX) { +#if defined(__linux__) + struct ucred ucred; + socklen_t len = sizeof(ucred); + if (getsockopt(sock, SOL_SOCKET, SO_PEERCRED, &ucred, &len) == 0) { + port = ucred.pid; + } +#elif defined(SOL_LOCAL) && defined(SO_PEERPID) // __APPLE__ + pid_t pid; + socklen_t len = sizeof(pid); + if (getsockopt(sock, SOL_LOCAL, SO_PEERPID, &pid, &len) == 0) { + port = pid; + } +#endif + return; + } +#endif + get_ip_and_port(addr, addr_len, ip, port); } } inline constexpr unsigned int str2tag_core(const char *s, size_t l, unsigned int h) { - return (l == 0) ? h - : str2tag_core(s + 1, l - 1, - (h * 33) ^ static_cast(*s)); + return (l == 0) + ? h + : str2tag_core( + s + 1, l - 1, + // Unsets the 6 high bits of h, therefore no overflow happens + (((std::numeric_limits::max)() >> 6) & + h * 33) ^ + static_cast(*s)); } inline unsigned int str2tag(const std::string &s) { @@ -2899,9 +3319,10 @@ inline constexpr unsigned int operator"" _t(const char *s, size_t l) { } // namespace udl -inline const char * +inline std::string find_content_type(const std::string &path, - const std::map &user_data) { + const std::map &user_data, + const std::string &default_content_type) { auto ext = file_extension(path); auto it = user_data.find(ext); @@ -2910,7 +3331,8 @@ find_content_type(const std::string &path, using udl::operator""_t; switch (str2tag(ext)) { - default: return nullptr; + default: return default_content_type; + case "css"_t: return "text/css"; case "csv"_t: return "text/csv"; case "htm"_t: @@ -2963,76 +3385,6 @@ find_content_type(const std::string &path, } } -inline const char *status_message(int status) { - switch (status) { - case 100: return "Continue"; - case 101: return "Switching Protocol"; - case 102: return "Processing"; - case 103: return "Early Hints"; - case 200: return "OK"; - case 201: return "Created"; - case 202: return "Accepted"; - case 203: return "Non-Authoritative Information"; - case 204: return "No Content"; - case 205: return "Reset Content"; - case 206: return "Partial Content"; - case 207: return "Multi-Status"; - case 208: return "Already Reported"; - case 226: return "IM Used"; - case 300: return "Multiple Choice"; - case 301: return "Moved Permanently"; - case 302: return "Found"; - case 303: return "See Other"; - case 304: return "Not Modified"; - case 305: return "Use Proxy"; - case 306: return "unused"; - case 307: return "Temporary Redirect"; - case 308: return "Permanent Redirect"; - case 400: return "Bad Request"; - case 401: return "Unauthorized"; - case 402: return "Payment Required"; - case 403: return "Forbidden"; - case 404: return "Not Found"; - case 405: return "Method Not Allowed"; - case 406: return "Not Acceptable"; - case 407: return "Proxy Authentication Required"; - case 408: return "Request Timeout"; - case 409: return "Conflict"; - case 410: return "Gone"; - case 411: return "Length Required"; - case 412: return "Precondition Failed"; - case 413: return "Payload Too Large"; - case 414: return "URI Too Long"; - case 415: return "Unsupported Media Type"; - case 416: return "Range Not Satisfiable"; - case 417: return "Expectation Failed"; - case 418: return "I'm a teapot"; - case 421: return "Misdirected Request"; - case 422: return "Unprocessable Entity"; - case 423: return "Locked"; - case 424: return "Failed Dependency"; - case 425: return "Too Early"; - case 426: return "Upgrade Required"; - case 428: return "Precondition Required"; - case 429: return "Too Many Requests"; - case 431: return "Request Header Fields Too Large"; - case 451: return "Unavailable For Legal Reasons"; - case 501: return "Not Implemented"; - case 502: return "Bad Gateway"; - case 503: return "Service Unavailable"; - case 504: return "Gateway Timeout"; - case 505: return "HTTP Version Not Supported"; - case 506: return "Variant Also Negotiates"; - case 507: return "Insufficient Storage"; - case 508: return "Loop Detected"; - case 510: return "Not Extended"; - case 511: return "Network Authentication Required"; - - default: - case 500: return "Internal Server Error"; - } -} - inline bool can_compress_content_type(const std::string &content_type) { using udl::operator""_t; @@ -3109,7 +3461,7 @@ inline bool gzip_compressor::compress(const char *data, size_t data_length, data += strm_.avail_in; auto flush = (last && data_length == 0) ? Z_FINISH : Z_NO_FLUSH; - int ret = Z_OK; + auto ret = Z_OK; std::array buff{}; do { @@ -3153,7 +3505,7 @@ inline bool gzip_decompressor::decompress(const char *data, size_t data_length, Callback callback) { assert(is_valid_); - int ret = Z_OK; + auto ret = Z_OK; do { constexpr size_t max_avail_in = @@ -3167,16 +3519,12 @@ inline bool gzip_decompressor::decompress(const char *data, size_t data_length, data += strm_.avail_in; std::array buff{}; - while (strm_.avail_in > 0) { + while (strm_.avail_in > 0 && ret == Z_OK) { strm_.avail_out = static_cast(buff.size()); strm_.next_out = reinterpret_cast(buff.data()); - auto prev_avail_in = strm_.avail_in; - ret = inflate(&strm_, Z_NO_FLUSH); - if (prev_avail_in - strm_.avail_in == 0) { return false; } - assert(ret != Z_STREAM_ERROR); switch (ret) { case Z_NEED_DICT: @@ -3258,7 +3606,7 @@ inline bool brotli_decompressor::decompress(const char *data, return 0; } - const uint8_t *next_in = (const uint8_t *)data; + auto next_in = reinterpret_cast(data); size_t avail_in = data_length; size_t total_out; @@ -3297,6 +3645,14 @@ inline const char *get_header_value(const Headers &headers, return def; } +inline bool compare_case_ignore(const std::string &a, const std::string &b) { + if (a.size() != b.size()) { return false; } + for (size_t i = 0; i < b.size(); i++) { + if (::tolower(a[i]) != ::tolower(b[i])) { return false; } + } + return true; +} + template inline bool parse_header(const char *beg, const char *end, T fn) { // Skip trailing spaces and tabs. @@ -3320,7 +3676,11 @@ inline bool parse_header(const char *beg, const char *end, T fn) { } if (p < end) { - fn(std::string(beg, key_end), decode_url(std::string(p, end), false)); + auto key = std::string(beg, key_end); + auto val = compare_case_ignore(key, "Location") + ? std::string(p, end) + : decode_url(std::string(p, end), false); + fn(std::move(key), std::move(val)); return true; } @@ -3418,7 +3778,8 @@ inline bool read_content_without_length(Stream &strm, return true; } -inline bool read_content_chunked(Stream &strm, +template +inline bool read_content_chunked(Stream &strm, T &x, ContentReceiverWithProgress out) { const auto bufsiz = 16; char buf[bufsiz]; @@ -3444,15 +3805,29 @@ inline bool read_content_chunked(Stream &strm, if (!line_reader.getline()) { return false; } - if (strcmp(line_reader.ptr(), "\r\n")) { break; } + if (strcmp(line_reader.ptr(), "\r\n")) { return false; } if (!line_reader.getline()) { return false; } } - if (chunk_len == 0) { - // Reader terminator after chunks - if (!line_reader.getline() || strcmp(line_reader.ptr(), "\r\n")) - return false; + assert(chunk_len == 0); + + // Trailer + if (!line_reader.getline()) { return false; } + + while (strcmp(line_reader.ptr(), "\r\n")) { + if (line_reader.size() > CPPHTTPLIB_HEADER_MAX_LENGTH) { return false; } + + // Exclude line terminator + constexpr auto line_terminator_len = 2; + auto end = line_reader.ptr() + line_reader.size() - line_terminator_len; + + parse_header(line_reader.ptr(), end, + [&](std::string &&key, std::string &&val) { + x.headers.emplace(std::move(key), std::move(val)); + }); + + if (!line_reader.getline()) { return false; } } return true; @@ -3522,11 +3897,11 @@ bool read_content(Stream &strm, T &x, size_t payload_max_length, int &status, auto exceed_payload_max_length = false; if (is_chunked_transfer_encoding(x.headers)) { - ret = read_content_chunked(strm, out); + ret = read_content_chunked(strm, x, out); } else if (!has_header(x.headers, "Content-Length")) { ret = read_content_without_length(strm, out); } else { - auto len = get_header_value(x.headers, "Content-Length"); + auto len = get_header_value_u64(x.headers, "Content-Length", 0, 0); if (len > payload_max_length) { exceed_payload_max_length = true; skip_content_with_length(strm, len); @@ -3575,7 +3950,7 @@ inline bool write_content(Stream &strm, const ContentProvider &content_provider, data_sink.write = [&](const char *d, size_t l) -> bool { if (ok) { - if (write_data(strm, d, l)) { + if (strm.is_writable() && write_data(strm, d, l)) { offset += l; } else { ok = false; @@ -3584,14 +3959,14 @@ inline bool write_content(Stream &strm, const ContentProvider &content_provider, return ok; }; - data_sink.is_writable = [&](void) { return ok && strm.is_writable(); }; - while (offset < end_offset && !is_shutting_down()) { - if (!content_provider(offset, end_offset - offset, data_sink)) { + if (!strm.is_writable()) { + error = Error::Write; + return false; + } else if (!content_provider(offset, end_offset - offset, data_sink)) { error = Error::Canceled; return false; - } - if (!ok) { + } else if (!ok) { error = Error::Write; return false; } @@ -3623,18 +3998,21 @@ write_content_without_length(Stream &strm, data_sink.write = [&](const char *d, size_t l) -> bool { if (ok) { offset += l; - if (!write_data(strm, d, l)) { ok = false; } + if (!strm.is_writable() || !write_data(strm, d, l)) { ok = false; } } return ok; }; data_sink.done = [&](void) { data_available = false; }; - data_sink.is_writable = [&](void) { return ok && strm.is_writable(); }; - while (data_available && !is_shutting_down()) { - if (!content_provider(offset, 0, data_sink)) { return false; } - if (!ok) { return false; } + if (!strm.is_writable()) { + return false; + } else if (!content_provider(offset, 0, data_sink)) { + return false; + } else if (!ok) { + return false; + } } return true; } @@ -3663,7 +4041,10 @@ write_content_chunked(Stream &strm, const ContentProvider &content_provider, // Emit chunked response header and footer for each chunk auto chunk = from_i_to_hex(payload.size()) + "\r\n" + payload + "\r\n"; - if (!write_data(strm, chunk.data(), chunk.size())) { ok = false; } + if (!strm.is_writable() || + !write_data(strm, chunk.data(), chunk.size())) { + ok = false; + } } } else { ok = false; @@ -3672,7 +4053,7 @@ write_content_chunked(Stream &strm, const ContentProvider &content_provider, return ok; }; - data_sink.done = [&](void) { + auto done_with_trailer = [&](const Headers *trailer) { if (!ok) { return; } data_available = false; @@ -3690,26 +4071,46 @@ write_content_chunked(Stream &strm, const ContentProvider &content_provider, if (!payload.empty()) { // Emit chunked response header and footer for each chunk auto chunk = from_i_to_hex(payload.size()) + "\r\n" + payload + "\r\n"; - if (!write_data(strm, chunk.data(), chunk.size())) { + if (!strm.is_writable() || + !write_data(strm, chunk.data(), chunk.size())) { ok = false; return; } } - static const std::string done_marker("0\r\n\r\n"); + static const std::string done_marker("0\r\n"); if (!write_data(strm, done_marker.data(), done_marker.size())) { ok = false; } + + // Trailer + if (trailer) { + for (const auto &kv : *trailer) { + std::string field_line = kv.first + ": " + kv.second + "\r\n"; + if (!write_data(strm, field_line.data(), field_line.size())) { + ok = false; + } + } + } + + static const std::string crlf("\r\n"); + if (!write_data(strm, crlf.data(), crlf.size())) { ok = false; } }; - data_sink.is_writable = [&](void) { return ok && strm.is_writable(); }; + data_sink.done = [&](void) { done_with_trailer(nullptr); }; + + data_sink.done_with_trailer = [&](const Headers &trailer) { + done_with_trailer(&trailer); + }; while (data_available && !is_shutting_down()) { - if (!content_provider(offset, 0, data_sink)) { + if (!strm.is_writable()) { + error = Error::Write; + return false; + } else if (!content_provider(offset, 0, data_sink)) { error = Error::Canceled; return false; - } - if (!ok) { + } else if (!ok) { error = Error::Write; return false; } @@ -3748,7 +4149,8 @@ inline bool redirect(T &cli, Request &req, Response &res, if (ret) { req = new_req; res = new_res; - res.location = location; + + if (res.location.empty()) res.location = location; } return ret; } @@ -3790,16 +4192,39 @@ inline void parse_query_text(const std::string &s, Params ¶ms) { inline bool parse_multipart_boundary(const std::string &content_type, std::string &boundary) { - auto pos = content_type.find("boundary="); + auto boundary_keyword = "boundary="; + auto pos = content_type.find(boundary_keyword); if (pos == std::string::npos) { return false; } - boundary = content_type.substr(pos + 9); - if (boundary.length() >= 2 && boundary.front() == '"' && - boundary.back() == '"') { - boundary = boundary.substr(1, boundary.size() - 2); - } + auto end = content_type.find(';', pos); + auto beg = pos + strlen(boundary_keyword); + boundary = trim_double_quotes_copy(content_type.substr(beg, end - beg)); return !boundary.empty(); } +inline void parse_disposition_params(const std::string &s, Params ¶ms) { + std::set cache; + split(s.data(), s.data() + s.size(), ';', [&](const char *b, const char *e) { + std::string kv(b, e); + if (cache.find(kv) != cache.end()) { return; } + cache.insert(kv); + + std::string key; + std::string val; + split(b, e, '=', [&](const char *b2, const char *e2) { + if (key.empty()) { + key.assign(b2, e2); + } else { + val.assign(b2, e2); + } + }); + + if (!key.empty()) { + params.emplace(trim_double_quotes_copy((key)), + trim_double_quotes_copy((val))); + } + }); +} + #ifdef CPPHTTPLIB_NO_EXCEPTIONS inline bool parse_range_header(const std::string &s, Ranges &ranges) { #else @@ -3810,7 +4235,7 @@ inline bool parse_range_header(const std::string &s, Ranges &ranges) try { if (std::regex_match(s, m, re_first_range)) { auto pos = static_cast(m.position(1)); auto len = static_cast(m.length(1)); - bool all_valid_ranges = true; + auto all_valid_ranges = true; split(&s[pos], &s[pos + len], ',', [&](const char *b, const char *e) { if (!all_valid_ranges) return; static auto re_another_range = std::regex(R"(\s*(\d*)-(\d*))"); @@ -3857,11 +4282,6 @@ public: bool parse(const char *buf, size_t n, const ContentReceiver &content_callback, const MultipartContentHeader &header_callback) { - // TODO: support 'filename*' - static const std::regex re_content_disposition( - R"~(^Content-Disposition:\s*form-data;\s*name="(.*?)"(?:;\s*filename="(.*?)")?(?:;\s*filename\*=\S+)?\s*$)~", - std::regex_constants::icase); - buf_append(buf, n); while (buf_size() > 0) { @@ -3899,10 +4319,43 @@ public: if (start_with_case_ignore(header, header_name)) { file_.content_type = trim_copy(header.substr(header_name.size())); } else { + static const std::regex re_content_disposition( + R"~(^Content-Disposition:\s*form-data;\s*(.*)$)~", + std::regex_constants::icase); + std::smatch m; if (std::regex_match(header, m, re_content_disposition)) { - file_.name = m[1]; - file_.filename = m[2]; + Params params; + parse_disposition_params(m[1], params); + + auto it = params.find("name"); + if (it != params.end()) { + file_.name = it->second; + } else { + is_valid_ = false; + return false; + } + + it = params.find("filename"); + if (it != params.end()) { file_.filename = it->second; } + + it = params.find("filename*"); + if (it != params.end()) { + // Only allow UTF-8 enconnding... + static const std::regex re_rfc5987_encoding( + R"~(^UTF-8''(.+?)$)~", std::regex_constants::icase); + + std::smatch m2; + if (std::regex_match(it->second, m2, re_rfc5987_encoding)) { + file_.filename = decode_url(m2[1], false); // override... + } else { + is_valid_ = false; + return false; + } + } + } else { + is_valid_ = false; + return false; } } buf_erase(pos + crlf_.size()); @@ -3940,9 +4393,9 @@ public: buf_erase(crlf_.size()); state_ = 1; } else { - if (dash_crlf_.size() > buf_size()) { return true; } - if (buf_start_with(dash_crlf_)) { - buf_erase(dash_crlf_.size()); + if (dash_.size() > buf_size()) { return true; } + if (buf_start_with(dash_)) { + buf_erase(dash_.size()); is_valid_ = true; buf_erase(buf_size()); // Remove epilogue } else { @@ -3975,7 +4428,6 @@ private: const std::string dash_ = "--"; const std::string crlf_ = "\r\n"; - const std::string dash_crlf_ = "--\r\n"; std::string boundary_; std::string dash_boundary_crlf_; std::string crlf_dash_boundary_; @@ -4096,29 +4548,48 @@ inline bool is_multipart_boundary_chars_valid(const std::string &boundary) { return valid; } +template +inline std::string +serialize_multipart_formdata_item_begin(const T &item, + const std::string &boundary) { + std::string body = "--" + boundary + "\r\n"; + body += "Content-Disposition: form-data; name=\"" + item.name + "\""; + if (!item.filename.empty()) { + body += "; filename=\"" + item.filename + "\""; + } + body += "\r\n"; + if (!item.content_type.empty()) { + body += "Content-Type: " + item.content_type + "\r\n"; + } + body += "\r\n"; + + return body; +} + +inline std::string serialize_multipart_formdata_item_end() { return "\r\n"; } + +inline std::string +serialize_multipart_formdata_finish(const std::string &boundary) { + return "--" + boundary + "--\r\n"; +} + +inline std::string +serialize_multipart_formdata_get_content_type(const std::string &boundary) { + return "multipart/form-data; boundary=" + boundary; +} + inline std::string serialize_multipart_formdata(const MultipartFormDataItems &items, - const std::string &boundary, - std::string &content_type) { + const std::string &boundary, bool finish = true) { std::string body; for (const auto &item : items) { - body += "--" + boundary + "\r\n"; - body += "Content-Disposition: form-data; name=\"" + item.name + "\""; - if (!item.filename.empty()) { - body += "; filename=\"" + item.filename + "\""; - } - body += "\r\n"; - if (!item.content_type.empty()) { - body += "Content-Type: " + item.content_type + "\r\n"; - } - body += "\r\n"; - body += item.content + "\r\n"; + body += serialize_multipart_formdata_item_begin(item, boundary); + body += item.content + serialize_multipart_formdata_item_end(); } - body += "--" + boundary + "--\r\n"; + if (finish) body += serialize_multipart_formdata_finish(boundary); - content_type = "multipart/form-data; boundary=" + boundary; return body; } @@ -4142,12 +4613,13 @@ get_range_offset_and_length(const Request &req, size_t content_length, return std::make_pair(r.first, static_cast(r.second - r.first) + 1); } -inline std::string make_content_range_header_field(size_t offset, size_t length, - size_t content_length) { +inline std::string +make_content_range_header_field(const std::pair &range, + size_t content_length) { std::string field = "bytes "; - field += std::to_string(offset); + if (range.first != -1) { field += std::to_string(range.first); } field += "-"; - field += std::to_string(offset + length - 1); + if (range.second != -1) { field += std::to_string(range.second); } field += "/"; field += std::to_string(content_length); return field; @@ -4169,21 +4641,22 @@ bool process_multipart_ranges_data(const Request &req, Response &res, ctoken("\r\n"); } - auto offsets = get_range_offset_and_length(req, res.body.size(), i); + ctoken("Content-Range: "); + const auto &range = req.ranges[i]; + stoken(make_content_range_header_field(range, res.content_length_)); + ctoken("\r\n"); + ctoken("\r\n"); + + auto offsets = get_range_offset_and_length(req, res.content_length_, i); auto offset = offsets.first; auto length = offsets.second; - - ctoken("Content-Range: "); - stoken(make_content_range_header_field(offset, length, res.body.size())); - ctoken("\r\n"); - ctoken("\r\n"); if (!content(offset, length)) { return false; } ctoken("\r\n"); } ctoken("--"); stoken(boundary); - ctoken("--\r\n"); + ctoken("--"); return true; } @@ -4284,7 +4757,7 @@ inline std::string message_digest(const std::string &s, const EVP_MD *algo) { std::stringstream ss; for (auto i = 0u; i < hash_length; ++i) { ss << std::hex << std::setw(2) << std::setfill('0') - << (unsigned int)hash[i]; + << static_cast(hash[i]); } return ss.str(); @@ -4303,15 +4776,15 @@ inline std::string SHA_512(const std::string &s) { } #endif -#ifdef _WIN32 #ifdef CPPHTTPLIB_OPENSSL_SUPPORT +#ifdef _WIN32 // NOTE: This code came up with the following stackoverflow post: // https://stackoverflow.com/questions/9507184/can-openssl-on-windows-use-the-system-certificate-store inline bool load_system_certs_on_windows(X509_STORE *store) { auto hStore = CertOpenSystemStoreW((HCRYPTPROV_LEGACY)NULL, L"ROOT"); - if (!hStore) { return false; } + auto result = false; PCCERT_CONTEXT pContext = NULL; while ((pContext = CertEnumCertificatesInStore(hStore, pContext)) != nullptr) { @@ -4322,16 +4795,109 @@ inline bool load_system_certs_on_windows(X509_STORE *store) { if (x509) { X509_STORE_add_cert(store, x509); X509_free(x509); + result = true; } } CertFreeCertificateContext(pContext); CertCloseStore(hStore, 0); + return result; +} +#elif defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN) && defined(__APPLE__) +#if TARGET_OS_OSX +template +using CFObjectPtr = + std::unique_ptr::type, void (*)(CFTypeRef)>; + +inline void cf_object_ptr_deleter(CFTypeRef obj) { + if (obj) { CFRelease(obj); } +} + +inline bool retrieve_certs_from_keychain(CFObjectPtr &certs) { + CFStringRef keys[] = {kSecClass, kSecMatchLimit, kSecReturnRef}; + CFTypeRef values[] = {kSecClassCertificate, kSecMatchLimitAll, + kCFBooleanTrue}; + + CFObjectPtr query( + CFDictionaryCreate(nullptr, reinterpret_cast(keys), values, + sizeof(keys) / sizeof(keys[0]), + &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks), + cf_object_ptr_deleter); + + if (!query) { return false; } + + CFTypeRef security_items = nullptr; + if (SecItemCopyMatching(query.get(), &security_items) != errSecSuccess || + CFArrayGetTypeID() != CFGetTypeID(security_items)) { + return false; + } + + certs.reset(reinterpret_cast(security_items)); return true; } -#endif +inline bool retrieve_root_certs_from_keychain(CFObjectPtr &certs) { + CFArrayRef root_security_items = nullptr; + if (SecTrustCopyAnchorCertificates(&root_security_items) != errSecSuccess) { + return false; + } + + certs.reset(root_security_items); + return true; +} + +inline bool add_certs_to_x509_store(CFArrayRef certs, X509_STORE *store) { + auto result = false; + for (auto i = 0; i < CFArrayGetCount(certs); ++i) { + const auto cert = reinterpret_cast( + CFArrayGetValueAtIndex(certs, i)); + + if (SecCertificateGetTypeID() != CFGetTypeID(cert)) { continue; } + + CFDataRef cert_data = nullptr; + if (SecItemExport(cert, kSecFormatX509Cert, 0, nullptr, &cert_data) != + errSecSuccess) { + continue; + } + + CFObjectPtr cert_data_ptr(cert_data, cf_object_ptr_deleter); + + auto encoded_cert = static_cast( + CFDataGetBytePtr(cert_data_ptr.get())); + + auto x509 = + d2i_X509(NULL, &encoded_cert, CFDataGetLength(cert_data_ptr.get())); + + if (x509) { + X509_STORE_add_cert(store, x509); + X509_free(x509); + result = true; + } + } + + return result; +} + +inline bool load_system_certs_on_macos(X509_STORE *store) { + auto result = false; + CFObjectPtr certs(nullptr, cf_object_ptr_deleter); + if (retrieve_certs_from_keychain(certs) && certs) { + result = add_certs_to_x509_store(certs.get(), store); + } + + if (retrieve_root_certs_from_keychain(certs) && certs) { + result = add_certs_to_x509_store(certs.get(), store) || result; + } + + return result; +} +#endif // TARGET_OS_OSX +#endif // _WIN32 +#endif // CPPHTTPLIB_OPENSSL_SUPPORT + +#ifdef _WIN32 class WSInit { public: WSInit() { @@ -4427,7 +4993,7 @@ inline bool parse_www_authenticate(const Response &res, s = s.substr(pos + 1); auto beg = std::sregex_iterator(s.begin(), s.end(), re); for (auto i = beg; i != std::sregex_iterator(); ++i) { - auto m = *i; + const auto &m = *i; auto key = s.substr(static_cast(m.position(1)), static_cast(m.length(1))); auto val = m.length(2) > 0 @@ -4502,9 +5068,9 @@ inline void hosted_at(const std::string &hostname, const auto &addr = *reinterpret_cast(rp->ai_addr); std::string ip; - int dummy = -1; - if (detail::get_remote_ip_and_port(addr, sizeof(struct sockaddr_storage), - ip, dummy)) { + auto dummy = -1; + if (detail::get_ip_and_port(addr, sizeof(struct sockaddr_storage), ip, + dummy)) { addrs.push_back(ip); } } @@ -4606,6 +5172,16 @@ inline MultipartFormData Request::get_file_value(const std::string &key) const { return MultipartFormData(); } +inline std::vector +Request::get_file_values(const std::string &key) const { + std::vector values; + auto rng = files.equal_range(key); + for (auto it = rng.first; it != rng.second; it++) { + values.push_back(it->second); + } + return values; +} + // Response implementation inline bool Response::has_header(const std::string &key) const { return headers.find(key) != headers.end(); @@ -4656,10 +5232,9 @@ inline void Response::set_content(const std::string &s, inline void Response::set_content_provider( size_t in_length, const std::string &content_type, ContentProvider provider, ContentProviderResourceReleaser resource_releaser) { - assert(in_length > 0); set_header("Content-Type", content_type); content_length_ = in_length; - content_provider_ = std::move(provider); + if (in_length > 0) { content_provider_ = std::move(provider); } content_provider_resource_releaser_ = resource_releaser; is_chunked_content_provider_ = false; } @@ -4729,7 +5304,7 @@ inline bool SocketStream::is_readable() const { inline bool SocketStream::is_writable() const { return select_write(sock_, write_timeout_sec_, write_timeout_usec_) > 0 && - is_socket_alive(sock_); + is_socket_alive(sock_); } inline ssize_t SocketStream::read(char *ptr, size_t size) { @@ -4794,6 +5369,11 @@ inline void SocketStream::get_remote_ip_and_port(std::string &ip, return detail::get_remote_ip_and_port(sock_, ip, port); } +inline void SocketStream::get_local_ip_and_port(std::string &ip, + int &port) const { + return detail::get_local_ip_and_port(sock_, ip, port); +} + inline socket_t SocketStream::socket() const { return sock_; } // Buffer stream implementation @@ -4819,17 +5399,112 @@ inline ssize_t BufferStream::write(const char *ptr, size_t size) { inline void BufferStream::get_remote_ip_and_port(std::string & /*ip*/, int & /*port*/) const {} +inline void BufferStream::get_local_ip_and_port(std::string & /*ip*/, + int & /*port*/) const {} + inline socket_t BufferStream::socket() const { return 0; } inline const std::string &BufferStream::get_buffer() const { return buffer; } +inline PathParamsMatcher::PathParamsMatcher(const std::string &pattern) { + // One past the last ending position of a path param substring + std::size_t last_param_end = 0; + +#ifndef CPPHTTPLIB_NO_EXCEPTIONS + // Needed to ensure that parameter names are unique during matcher + // construction + // If exceptions are disabled, only last duplicate path + // parameter will be set + std::unordered_set param_name_set; +#endif + + while (true) { + const auto marker_pos = pattern.find(marker, last_param_end); + if (marker_pos == std::string::npos) { break; } + + static_fragments_.push_back( + pattern.substr(last_param_end, marker_pos - last_param_end)); + + const auto param_name_start = marker_pos + 1; + + auto sep_pos = pattern.find(separator, param_name_start); + if (sep_pos == std::string::npos) { sep_pos = pattern.length(); } + + auto param_name = + pattern.substr(param_name_start, sep_pos - param_name_start); + +#ifndef CPPHTTPLIB_NO_EXCEPTIONS + if (param_name_set.find(param_name) != param_name_set.cend()) { + std::string msg = "Encountered path parameter '" + param_name + + "' multiple times in route pattern '" + pattern + "'."; + throw std::invalid_argument(msg); + } +#endif + + param_names_.push_back(std::move(param_name)); + + last_param_end = sep_pos + 1; + } + + if (last_param_end < pattern.length()) { + static_fragments_.push_back(pattern.substr(last_param_end)); + } +} + +inline bool PathParamsMatcher::match(Request &request) const { + request.matches = std::smatch(); + request.path_params.clear(); + request.path_params.reserve(param_names_.size()); + + // One past the position at which the path matched the pattern last time + std::size_t starting_pos = 0; + for (size_t i = 0; i < static_fragments_.size(); ++i) { + const auto &fragment = static_fragments_[i]; + + if (starting_pos + fragment.length() > request.path.length()) { + return false; + } + + // Avoid unnecessary allocation by using strncmp instead of substr + + // comparison + if (std::strncmp(request.path.c_str() + starting_pos, fragment.c_str(), + fragment.length()) != 0) { + return false; + } + + starting_pos += fragment.length(); + + // Should only happen when we have a static fragment after a param + // Example: '/users/:id/subscriptions' + // The 'subscriptions' fragment here does not have a corresponding param + if (i >= param_names_.size()) { continue; } + + auto sep_pos = request.path.find(separator, starting_pos); + if (sep_pos == std::string::npos) { sep_pos = request.path.length(); } + + const auto ¶m_name = param_names_[i]; + + request.path_params.emplace( + param_name, request.path.substr(starting_pos, sep_pos - starting_pos)); + + // Mark everythin up to '/' as matched + starting_pos = sep_pos + 1; + } + // Returns false if the path is longer than the pattern + return starting_pos >= request.path.length(); +} + +inline bool RegexMatcher::match(Request &request) const { + request.path_params.clear(); + return std::regex_match(request.path, request.matches, regex_); +} + } // namespace detail // HTTP server implementation inline Server::Server() : new_task_queue( - [] { return new ThreadPool(CPPHTTPLIB_THREAD_POOL_COUNT); }), - svr_sock_(INVALID_SOCKET), is_running_(false) { + [] { return new ThreadPool(CPPHTTPLIB_THREAD_POOL_COUNT); }) { #ifndef _WIN32 signal(SIGPIPE, SIG_IGN); #endif @@ -4837,67 +5512,76 @@ inline Server::Server() inline Server::~Server() {} +inline std::unique_ptr +Server::make_matcher(const std::string &pattern) { + if (pattern.find("/:") != std::string::npos) { + return detail::make_unique(pattern); + } else { + return detail::make_unique(pattern); + } +} + inline Server &Server::Get(const std::string &pattern, Handler handler) { get_handlers_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + std::make_pair(make_matcher(pattern), std::move(handler))); return *this; } inline Server &Server::Post(const std::string &pattern, Handler handler) { post_handlers_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + std::make_pair(make_matcher(pattern), std::move(handler))); return *this; } inline Server &Server::Post(const std::string &pattern, HandlerWithContentReader handler) { post_handlers_for_content_reader_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + std::make_pair(make_matcher(pattern), std::move(handler))); return *this; } inline Server &Server::Put(const std::string &pattern, Handler handler) { put_handlers_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + std::make_pair(make_matcher(pattern), std::move(handler))); return *this; } inline Server &Server::Put(const std::string &pattern, HandlerWithContentReader handler) { put_handlers_for_content_reader_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + std::make_pair(make_matcher(pattern), std::move(handler))); return *this; } inline Server &Server::Patch(const std::string &pattern, Handler handler) { patch_handlers_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + std::make_pair(make_matcher(pattern), std::move(handler))); return *this; } inline Server &Server::Patch(const std::string &pattern, HandlerWithContentReader handler) { patch_handlers_for_content_reader_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + std::make_pair(make_matcher(pattern), std::move(handler))); return *this; } inline Server &Server::Delete(const std::string &pattern, Handler handler) { delete_handlers_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + std::make_pair(make_matcher(pattern), std::move(handler))); return *this; } inline Server &Server::Delete(const std::string &pattern, HandlerWithContentReader handler) { delete_handlers_for_content_reader_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + std::make_pair(make_matcher(pattern), std::move(handler))); return *this; } inline Server &Server::Options(const std::string &pattern, Handler handler) { options_handlers_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + std::make_pair(make_matcher(pattern), std::move(handler))); return *this; } @@ -4935,6 +5619,11 @@ Server::set_file_extension_and_mimetype_mapping(const std::string &ext, return *this; } +inline Server &Server::set_default_file_mimetype(const std::string &mime) { + default_file_mimetype_ = mime; + return *this; +} + inline Server &Server::set_file_request_handler(Handler handler) { file_request_handler_ = std::move(handler); return *this; @@ -4976,7 +5665,6 @@ inline Server &Server::set_logger(Logger logger) { inline Server & Server::set_expect_100_continue_handler(Expect100ContinueHandler handler) { expect_100_continue_handler_ = std::move(handler); - return *this; } @@ -5000,6 +5688,12 @@ inline Server &Server::set_default_headers(Headers headers) { return *this; } +inline Server &Server::set_header_writer( + std::function const &writer) { + header_writer_ = writer; + return *this; +} + inline Server &Server::set_keep_alive_max_count(size_t count) { keep_alive_max_count_ = count; return *this; @@ -5042,15 +5736,25 @@ inline int Server::bind_to_any_port(const std::string &host, int socket_flags) { return bind_internal(host, 0, socket_flags); } -inline bool Server::listen_after_bind() { return listen_internal(); } +inline bool Server::listen_after_bind() { + auto se = detail::scope_exit([&]() { done_ = true; }); + return listen_internal(); +} inline bool Server::listen(const std::string &host, int port, int socket_flags) { + auto se = detail::scope_exit([&]() { done_ = true; }); return bind_to_port(host, port, socket_flags) && listen_internal(); } inline bool Server::is_running() const { return is_running_; } +inline void Server::wait_until_ready() const { + while (!is_running() && !done_) { + std::this_thread::sleep_for(std::chrono::milliseconds{1}); + } +} + inline void Server::stop() { if (is_running_) { assert(svr_sock_ != INVALID_SOCKET); @@ -5180,11 +5884,11 @@ inline bool Server::write_response_core(Stream &strm, bool close_connection, detail::BufferStream bstrm; if (!bstrm.write_format("HTTP/1.1 %d %s\r\n", res.status, - detail::status_message(res.status))) { + status_message(res.status))) { return false; } - if (!detail::write_headers(bstrm, res.headers)) { return false; } + if (!header_writer_(bstrm, res.headers)) { return false; } // Flush buffer auto &data = bstrm.get_buffer(); @@ -5313,7 +6017,7 @@ inline bool Server::read_content_with_content_receiver( inline bool Server::read_content_core(Stream &strm, Request &req, Response &res, ContentReceiver receiver, - MultipartContentHeader mulitpart_header, + MultipartContentHeader multipart_header, ContentReceiver multipart_receiver) { detail::MultipartFormDataParser multipart_form_data_parser; ContentReceiverWithProgress out; @@ -5333,14 +6037,14 @@ inline bool Server::read_content_core(Stream &strm, Request &req, Response &res, while (pos < n) { auto read_size = (std::min)(1, n - pos); auto ret = multipart_form_data_parser.parse( - buf + pos, read_size, multipart_receiver, mulitpart_header); + buf + pos, read_size, multipart_receiver, multipart_header); if (!ret) { return false; } pos += read_size; } return true; */ return multipart_form_data_parser.parse(buf, n, multipart_receiver, - mulitpart_header); + multipart_header); }; } else { out = [receiver](const char *buf, size_t n, uint64_t /*off*/, @@ -5377,17 +6081,26 @@ inline bool Server::handle_file_request(const Request &req, Response &res, if (path.back() == '/') { path += "index.html"; } if (detail::is_file(path)) { - detail::read_file(path, res.body); - auto type = - detail::find_content_type(path, file_extension_and_mimetype_map_); - if (type) { res.set_header("Content-Type", type); } for (const auto &kv : entry.headers) { res.set_header(kv.first.c_str(), kv.second); } - res.status = req.has_header("Range") ? 206 : 200; + + auto mm = std::make_shared(path.c_str()); + if (!mm->is_open()) { return false; } + + res.set_content_provider( + mm->size(), + detail::find_content_type(path, file_extension_and_mimetype_map_, + default_file_mimetype_), + [mm](size_t offset, size_t length, DataSink &sink) -> bool { + sink.write(mm->data() + offset, length); + return true; + }); + if (!head && file_request_handler_) { file_request_handler_(req, res); } + return true; } } @@ -5441,6 +6154,7 @@ inline int Server::bind_internal(const std::string &host, int port, inline bool Server::listen_internal() { auto ret = true; is_running_ = true; + auto se = detail::scope_exit([&]() { is_running_ = false; }); { std::unique_ptr task_queue(new_task_queue()); @@ -5466,6 +6180,8 @@ inline bool Server::listen_internal() { // Try to accept new connections after a short sleep. std::this_thread::sleep_for(std::chrono::milliseconds(1)); continue; + } else if (errno == EINTR || errno == EAGAIN) { + continue; } if (svr_sock_ != INVALID_SOCKET) { detail::close_socket(svr_sock_); @@ -5480,13 +6196,14 @@ inline bool Server::listen_internal() { #ifdef _WIN32 auto timeout = static_cast(read_timeout_sec_ * 1000 + read_timeout_usec_ / 1000); - setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout, - sizeof(timeout)); + setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, + reinterpret_cast(&timeout), sizeof(timeout)); #else timeval tv; tv.tv_sec = static_cast(read_timeout_sec_); tv.tv_usec = static_cast(read_timeout_usec_); - setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *)&tv, sizeof(tv)); + setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, + reinterpret_cast(&tv), sizeof(tv)); #endif } { @@ -5494,27 +6211,23 @@ inline bool Server::listen_internal() { #ifdef _WIN32 auto timeout = static_cast(write_timeout_sec_ * 1000 + write_timeout_usec_ / 1000); - setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout, - sizeof(timeout)); + setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, + reinterpret_cast(&timeout), sizeof(timeout)); #else timeval tv; tv.tv_sec = static_cast(write_timeout_sec_); tv.tv_usec = static_cast(write_timeout_usec_); - setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (char *)&tv, sizeof(tv)); + setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, + reinterpret_cast(&tv), sizeof(tv)); #endif } -#if __cplusplus > 201703L - task_queue->enqueue([=, this]() { process_and_close_socket(sock); }); -#else - task_queue->enqueue([=]() { process_and_close_socket(sock); }); -#endif + task_queue->enqueue([this, sock]() { process_and_close_socket(sock); }); } task_queue->shutdown(); } - is_running_ = false; return ret; } @@ -5525,7 +6238,7 @@ inline bool Server::routing(Request &req, Response &res, Stream &strm) { } // File handler - bool is_head_request = req.method == "HEAD"; + auto is_head_request = req.method == "HEAD"; if ((req.method == "GET" || is_head_request) && handle_file_request(req, res, is_head_request)) { return true; @@ -5598,10 +6311,10 @@ inline bool Server::routing(Request &req, Response &res, Stream &strm) { inline bool Server::dispatch_request(Request &req, Response &res, const Handlers &handlers) { for (const auto &x : handlers) { - const auto &pattern = x.first; + const auto &matcher = x.first; const auto &handler = x.second; - if (std::regex_match(req.path, req.matches, pattern)) { + if (matcher->match(req)) { handler(req, res); return true; } @@ -5621,8 +6334,8 @@ inline void Server::apply_ranges(const Request &req, Response &res, res.headers.erase(it); } - res.headers.emplace("Content-Type", - "multipart/byteranges; boundary=" + boundary); + res.set_header("Content-Type", + "multipart/byteranges; boundary=" + boundary); } auto type = detail::encoding_type(req, res); @@ -5635,10 +6348,10 @@ inline void Server::apply_ranges(const Request &req, Response &res, } else if (req.ranges.size() == 1) { auto offsets = detail::get_range_offset_and_length(req, res.content_length_, 0); - auto offset = offsets.first; length = offsets.second; + auto content_range = detail::make_content_range_header_field( - offset, length, res.content_length_); + req.ranges[0], res.content_length_); res.set_header("Content-Range", content_range); } else { length = detail::get_multipart_ranges_data_length(req, res, boundary, @@ -5661,13 +6374,15 @@ inline void Server::apply_ranges(const Request &req, Response &res, if (req.ranges.empty()) { ; } else if (req.ranges.size() == 1) { + auto content_range = detail::make_content_range_header_field( + req.ranges[0], res.body.size()); + res.set_header("Content-Range", content_range); + auto offsets = detail::get_range_offset_and_length(req, res.body.size(), 0); auto offset = offsets.first; auto length = offsets.second; - auto content_range = detail::make_content_range_header_field( - offset, length, res.body.size()); - res.set_header("Content-Range", content_range); + if (offset < res.body.size()) { res.body = res.body.substr(offset, length); } else { @@ -5723,10 +6438,10 @@ inline bool Server::dispatch_request_for_content_reader( Request &req, Response &res, ContentReader content_reader, const HandlersForContentReader &handlers) { for (const auto &x : handlers) { - const auto &pattern = x.first; + const auto &matcher = x.first; const auto &handler = x.second; - if (std::regex_match(req.path, req.matches, pattern)) { + if (matcher->match(req)) { handler(req, res, content_reader); return true; } @@ -5746,15 +6461,10 @@ Server::process_request(Stream &strm, bool close_connection, if (!line_reader.getline()) { return false; } Request req; + Response res; - res.version = "HTTP/1.1"; - - for (const auto &header : default_headers_) { - if (res.headers.find(header.first) == res.headers.end()) { - res.headers.insert(header); - } - } + res.headers = default_headers_; #ifdef _WIN32 // TODO: Increase FD_SETSIZE statically (libzmq), dynamically (MySQL). @@ -5798,6 +6508,10 @@ Server::process_request(Stream &strm, bool close_connection, req.set_header("REMOTE_ADDR", req.remote_addr); req.set_header("REMOTE_PORT", std::to_string(req.remote_port)); + strm.get_local_ip_and_port(req.local_addr, req.local_port); + req.set_header("LOCAL_ADDR", req.local_addr); + req.set_header("LOCAL_PORT", std::to_string(req.local_port)); + if (req.has_header("Range")) { const auto &range_header_value = req.get_header_value("Range"); if (!detail::parse_range_header(range_header_value, req.ranges)) { @@ -5817,14 +6531,14 @@ Server::process_request(Stream &strm, bool close_connection, case 100: case 417: strm.write_format("HTTP/1.1 %d %s\r\n\r\n", status, - detail::status_message(status)); + status_message(status)); break; default: return write_response(strm, close_connection, req, res); } } // Rounting - bool routed = false; + auto routed = false; #ifdef CPPHTTPLIB_NO_EXCEPTIONS routed = routing(req, res, strm); #else @@ -5837,7 +6551,16 @@ Server::process_request(Stream &strm, bool close_connection, routed = true; } else { res.status = 500; - res.set_header("EXCEPTION_WHAT", e.what()); + std::string val; + auto s = e.what(); + for (size_t i = 0; s[i]; i++) { + switch (s[i]) { + case '\r': val += "\\r"; break; + case '\n': val += "\\n"; break; + default: val += s[i]; break; + } + } + res.set_header("EXCEPTION_WHAT", val); } } catch (...) { if (exception_handler_) { @@ -6013,9 +6736,9 @@ inline bool ClientImpl::read_response_line(Stream &strm, const Request &req, if (!line_reader.getline()) { return false; } #ifdef CPPHTTPLIB_ALLOW_LF_AS_LINE_TERMINATOR - const static std::regex re("(HTTP/1\\.[01]) (\\d{3})(?: (.*?))?\r\n"); -#else const static std::regex re("(HTTP/1\\.[01]) (\\d{3})(?: (.*?))?\r?\n"); +#else + const static std::regex re("(HTTP/1\\.[01]) (\\d{3})(?: (.*?))?\r\n"); #endif std::cmatch m; @@ -6042,7 +6765,15 @@ inline bool ClientImpl::read_response_line(Stream &strm, const Request &req, inline bool ClientImpl::send(Request &req, Response &res, Error &error) { std::lock_guard request_mutex_guard(request_mutex_); + auto ret = send_(req, res, error); + if (error == Error::SSLPeerCouldBeClosed_) { + assert(!ret); + ret = send_(req, res, error); + } + return ret; +} +inline bool ClientImpl::send_(Request &req, Response &res, Error &error) { { std::lock_guard guard(socket_mutex_); @@ -6073,7 +6804,7 @@ inline bool ClientImpl::send(Request &req, Response &res, Error &error) { if (is_ssl()) { auto &scli = static_cast(*this); if (!proxy_host_.empty() && proxy_port_ != -1) { - bool success = false; + auto success = false; if (!scli.connect_with_proxy(socket_, res, success, error)) { return success; } @@ -6100,13 +6831,11 @@ inline bool ClientImpl::send(Request &req, Response &res, Error &error) { } } + auto ret = false; auto close_connection = !keep_alive_; - auto ret = process_socket(socket_, [&](Stream &strm) { - return handle_request(strm, req, res, close_connection, error); - }); - // Briefly lock mutex in order to mark that a request is no longer ongoing - { + auto se = detail::scope_exit([&]() { + // Briefly lock mutex in order to mark that a request is no longer ongoing std::lock_guard guard(socket_mutex_); socket_requests_in_flight_ -= 1; if (socket_requests_in_flight_ <= 0) { @@ -6120,7 +6849,11 @@ inline bool ClientImpl::send(Request &req, Response &res, Error &error) { shutdown_socket(socket_); close_socket(socket_); } - } + }); + + ret = process_socket(socket_, [&](Stream &strm) { + return handle_request(strm, req, res, close_connection, error); + }); if (!ret) { if (error == Error::Success) { error = Error::Unknown; } @@ -6165,6 +6898,21 @@ inline bool ClientImpl::handle_request(Stream &strm, Request &req, if (!ret) { return false; } + if (res.get_header_value("Connection") == "close" || + (res.version == "HTTP/1.0" && res.reason != "Connection established")) { + // TODO this requires a not-entirely-obvious chain of calls to be correct + // for this to be safe. + + // This is safe to call because handle_request is only called by send_ + // which locks the request mutex during the process. It would be a bug + // to call it from a different thread since it's a thread-safety issue + // to do these things to the socket if another thread is using the socket. + std::lock_guard guard(socket_mutex_); + shutdown_ssl(socket_, true); + shutdown_socket(socket_); + close_socket(socket_); + } + if (300 < res.status && res.status < 400 && follow_location_) { req = req_save; ret = redirect(req, res, error); @@ -6208,11 +6956,11 @@ inline bool ClientImpl::redirect(Request &req, Response &res, Error &error) { return false; } - auto location = detail::decode_url(res.get_header_value("location"), true); + auto location = res.get_header_value("location"); if (location.empty()) { return false; } const static std::regex re( - R"((?:(https?):)?(?://(?:\[([\d:]+)\]|([^:/?#]+))(?::(\d+))?)?([^?#]*(?:\?[^#]*)?)(?:#.*)?)"); + R"((?:(https?):)?(?://(?:\[([\d:]+)\]|([^:/?#]+))(?::(\d+))?)?([^?#]*)(\?[^#]*)?(?:#.*)?)"); std::smatch m; if (!std::regex_match(location, m, re)) { return false; } @@ -6224,6 +6972,7 @@ inline bool ClientImpl::redirect(Request &req, Response &res, Error &error) { if (next_host.empty()) { next_host = m[3].str(); } auto port_str = m[4].str(); auto next_path = m[5].str(); + auto next_query = m[6].str(); auto next_port = port_; if (!port_str.empty()) { @@ -6236,22 +6985,24 @@ inline bool ClientImpl::redirect(Request &req, Response &res, Error &error) { if (next_host.empty()) { next_host = host_; } if (next_path.empty()) { next_path = "/"; } + auto path = detail::decode_url(next_path, true) + next_query; + if (next_scheme == scheme && next_host == host_ && next_port == port_) { - return detail::redirect(*this, req, res, next_path, location, error); + return detail::redirect(*this, req, res, path, location, error); } else { if (next_scheme == "https") { #ifdef CPPHTTPLIB_OPENSSL_SUPPORT SSLClient cli(next_host.c_str(), next_port); cli.copy_settings(*this); if (ca_cert_store_) { cli.set_ca_cert_store(ca_cert_store_); } - return detail::redirect(cli, req, res, next_path, location, error); + return detail::redirect(cli, req, res, path, location, error); #else return false; #endif } else { ClientImpl cli(next_host.c_str(), next_port); cli.copy_settings(*this); - return detail::redirect(cli, req, res, next_path, location, error); + return detail::redirect(cli, req, res, path, location, error); } } } @@ -6262,7 +7013,7 @@ inline bool ClientImpl::write_content_with_provider(Stream &strm, auto is_shutting_down = []() { return false; }; if (req.is_chunked_content_provider_) { - // TODO: Brotli suport + // TODO: Brotli support std::unique_ptr compressor; #ifdef CPPHTTPLIB_ZLIB_SUPPORT if (compress_) { @@ -6279,39 +7030,39 @@ inline bool ClientImpl::write_content_with_provider(Stream &strm, return detail::write_content(strm, req.content_provider_, 0, req.content_length_, is_shutting_down, error); } -} // namespace httplib +} inline bool ClientImpl::write_request(Stream &strm, Request &req, bool close_connection, Error &error) { // Prepare additional headers if (close_connection) { if (!req.has_header("Connection")) { - req.headers.emplace("Connection", "close"); + req.set_header("Connection", "close"); } } if (!req.has_header("Host")) { if (is_ssl()) { if (port_ == 443) { - req.headers.emplace("Host", host_); + req.set_header("Host", host_); } else { - req.headers.emplace("Host", host_and_port_); + req.set_header("Host", host_and_port_); } } else { if (port_ == 80) { - req.headers.emplace("Host", host_); + req.set_header("Host", host_); } else { - req.headers.emplace("Host", host_and_port_); + req.set_header("Host", host_and_port_); } } } - if (!req.has_header("Accept")) { req.headers.emplace("Accept", "*/*"); } + if (!req.has_header("Accept")) { req.set_header("Accept", "*/*"); } #ifndef CPPHTTPLIB_NO_DEFAULT_USER_AGENT if (!req.has_header("User-Agent")) { auto agent = std::string("cpp-httplib/") + CPPHTTPLIB_VERSION; - req.headers.emplace("User-Agent", agent); + req.set_header("User-Agent", agent); } #endif @@ -6320,23 +7071,23 @@ inline bool ClientImpl::write_request(Stream &strm, Request &req, if (!req.is_chunked_content_provider_) { if (!req.has_header("Content-Length")) { auto length = std::to_string(req.content_length_); - req.headers.emplace("Content-Length", length); + req.set_header("Content-Length", length); } } } else { if (req.method == "POST" || req.method == "PUT" || req.method == "PATCH") { - req.headers.emplace("Content-Length", "0"); + req.set_header("Content-Length", "0"); } } } else { if (!req.has_header("Content-Type")) { - req.headers.emplace("Content-Type", "text/plain"); + req.set_header("Content-Type", "text/plain"); } if (!req.has_header("Content-Length")) { auto length = std::to_string(req.body.size()); - req.headers.emplace("Content-Length", length); + req.set_header("Content-Length", length); } } @@ -6376,7 +7127,7 @@ inline bool ClientImpl::write_request(Stream &strm, Request &req, const auto &path = url_encode_ ? detail::encode_url(req.path) : req.path; bstrm.write_format("%s %s HTTP/1.1\r\n", req.method.c_str(), path.c_str()); - detail::write_headers(bstrm, req.headers); + header_writer_(bstrm, req.headers); // Flush buffer auto &data = bstrm.get_buffer(); @@ -6404,12 +7155,10 @@ inline std::unique_ptr ClientImpl::send_with_content_provider( ContentProvider content_provider, ContentProviderWithoutLength content_provider_without_length, const std::string &content_type, Error &error) { - if (!content_type.empty()) { - req.headers.emplace("Content-Type", content_type); - } + if (!content_type.empty()) { req.set_header("Content-Type", content_type); } #ifdef CPPHTTPLIB_ZLIB_SUPPORT - if (compress_) { req.headers.emplace("Content-Encoding", "gzip"); } + if (compress_) { req.set_header("Content-Encoding", "gzip"); } #endif #ifdef CPPHTTPLIB_ZLIB_SUPPORT @@ -6442,8 +7191,6 @@ inline std::unique_ptr ClientImpl::send_with_content_provider( return ok; }; - data_sink.is_writable = [&](void) { return ok && true; }; - while (ok && offset < content_length) { if (!content_provider(offset, content_length - offset, data_sink)) { error = Error::Canceled; @@ -6472,10 +7219,9 @@ inline std::unique_ptr ClientImpl::send_with_content_provider( req.content_provider_ = detail::ContentProviderAdapter( std::move(content_provider_without_length)); req.is_chunked_content_provider_ = true; - req.headers.emplace("Transfer-Encoding", "chunked"); + req.set_header("Transfer-Encoding", "chunked"); } else { req.body.assign(body, content_length); - ; } } @@ -6514,6 +7260,20 @@ inline bool ClientImpl::process_request(Stream &strm, Request &req, // Send request if (!write_request(strm, req, close_connection, error)) { return false; } +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + if (is_ssl()) { + auto is_proxy_enabled = !proxy_host_.empty() && proxy_port_ != -1; + if (!is_proxy_enabled) { + char buf[1]; + if (SSL_peek(socket_.ssl, buf, 1) == 0 && + SSL_get_error(socket_.ssl, 0) == SSL_ERROR_ZERO_RETURN) { + error = Error::SSLPeerCouldBeClosed_; + return false; + } + } + } +#endif + // Receive response and headers if (!read_response_line(strm, req, res) || !detail::read_headers(strm, res.headers)) { @@ -6567,30 +7327,54 @@ inline bool ClientImpl::process_request(Stream &strm, Request &req, } } - if (res.get_header_value("Connection") == "close" || - (res.version == "HTTP/1.0" && res.reason != "Connection established")) { - // TODO this requires a not-entirely-obvious chain of calls to be correct - // for this to be safe. Maybe a code refactor (such as moving this out to - // the send function and getting rid of the recursiveness of the mutex) - // could make this more obvious. - - // This is safe to call because process_request is only called by - // handle_request which is only called by send, which locks the request - // mutex during the process. It would be a bug to call it from a different - // thread since it's a thread-safety issue to do these things to the socket - // if another thread is using the socket. - std::lock_guard guard(socket_mutex_); - shutdown_ssl(socket_, true); - shutdown_socket(socket_); - close_socket(socket_); - } - // Log if (logger_) { logger_(req, res); } return true; } +inline ContentProviderWithoutLength ClientImpl::get_multipart_content_provider( + const std::string &boundary, const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items) { + size_t cur_item = 0, cur_start = 0; + // cur_item and cur_start are copied to within the std::function and maintain + // state between successive calls + return [&, cur_item, cur_start](size_t offset, + DataSink &sink) mutable -> bool { + if (!offset && items.size()) { + sink.os << detail::serialize_multipart_formdata(items, boundary, false); + return true; + } else if (cur_item < provider_items.size()) { + if (!cur_start) { + const auto &begin = detail::serialize_multipart_formdata_item_begin( + provider_items[cur_item], boundary); + offset += begin.size(); + cur_start = offset; + sink.os << begin; + } + + DataSink cur_sink; + auto has_data = true; + cur_sink.write = sink.write; + cur_sink.done = [&]() { has_data = false; }; + + if (!provider_items[cur_item].provider(offset - cur_start, cur_sink)) + return false; + + if (!has_data) { + sink.os << detail::serialize_multipart_formdata_item_end(); + cur_item++; + cur_start = 0; + } + return true; + } else { + sink.os << detail::serialize_multipart_formdata_finish(boundary); + sink.done(); + return true; + } + }; +} + inline bool ClientImpl::process_socket(const Socket &socket, std::function callback) { @@ -6736,6 +7520,11 @@ inline Result ClientImpl::Post(const std::string &path) { return Post(path, std::string(), std::string()); } +inline Result ClientImpl::Post(const std::string &path, + const Headers &headers) { + return Post(path, headers, nullptr, 0, std::string()); +} + inline Result ClientImpl::Post(const std::string &path, const char *body, size_t content_length, const std::string &content_type) { @@ -6808,9 +7597,10 @@ inline Result ClientImpl::Post(const std::string &path, inline Result ClientImpl::Post(const std::string &path, const Headers &headers, const MultipartFormDataItems &items) { - std::string content_type; - const auto &body = detail::serialize_multipart_formdata( - items, detail::make_multipart_data_boundary(), content_type); + const auto &boundary = detail::make_multipart_data_boundary(); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + const auto &body = detail::serialize_multipart_formdata(items, boundary); return Post(path, headers, body, content_type.c_str()); } @@ -6821,12 +7611,25 @@ inline Result ClientImpl::Post(const std::string &path, const Headers &headers, return Result{nullptr, Error::UnsupportedMultipartBoundaryChars}; } - std::string content_type; - const auto &body = - detail::serialize_multipart_formdata(items, boundary, content_type); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + const auto &body = detail::serialize_multipart_formdata(items, boundary); return Post(path, headers, body, content_type.c_str()); } +inline Result +ClientImpl::Post(const std::string &path, const Headers &headers, + const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items) { + const auto &boundary = detail::make_multipart_data_boundary(); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + return send_with_content_provider( + "POST", path, headers, nullptr, 0, nullptr, + get_multipart_content_provider(boundary, items, provider_items), + content_type); +} + inline Result ClientImpl::Put(const std::string &path) { return Put(path, std::string(), std::string()); } @@ -6903,9 +7706,10 @@ inline Result ClientImpl::Put(const std::string &path, inline Result ClientImpl::Put(const std::string &path, const Headers &headers, const MultipartFormDataItems &items) { - std::string content_type; - const auto &body = detail::serialize_multipart_formdata( - items, detail::make_multipart_data_boundary(), content_type); + const auto &boundary = detail::make_multipart_data_boundary(); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + const auto &body = detail::serialize_multipart_formdata(items, boundary); return Put(path, headers, body, content_type); } @@ -6916,12 +7720,24 @@ inline Result ClientImpl::Put(const std::string &path, const Headers &headers, return Result{nullptr, Error::UnsupportedMultipartBoundaryChars}; } - std::string content_type; - const auto &body = - detail::serialize_multipart_formdata(items, boundary, content_type); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + const auto &body = detail::serialize_multipart_formdata(items, boundary); return Put(path, headers, body, content_type); } +inline Result +ClientImpl::Put(const std::string &path, const Headers &headers, + const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items) { + const auto &boundary = detail::make_multipart_data_boundary(); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + return send_with_content_provider( + "PUT", path, headers, nullptr, 0, nullptr, + get_multipart_content_provider(boundary, items, provider_items), + content_type); +} inline Result ClientImpl::Patch(const std::string &path) { return Patch(path, std::string(), std::string()); } @@ -7007,9 +7823,7 @@ inline Result ClientImpl::Delete(const std::string &path, req.headers = headers; req.path = path; - if (!content_type.empty()) { - req.headers.emplace("Content-Type", content_type); - } + if (!content_type.empty()) { req.set_header("Content-Type", content_type); } req.body.assign(body, content_length); return send_(std::move(req)); @@ -7042,13 +7856,6 @@ inline Result ClientImpl::Options(const std::string &path, return send_(std::move(req)); } -inline size_t ClientImpl::is_socket_open() const { - std::lock_guard guard(socket_mutex_); - return socket_.is_open(); -} - -inline socket_t ClientImpl::socket() const { return socket_.sock; } - inline void ClientImpl::stop() { std::lock_guard guard(socket_mutex_); @@ -7066,12 +7873,23 @@ inline void ClientImpl::stop() { return; } - // Otherwise, sitll holding the mutex, we can shut everything down ourselves + // Otherwise, still holding the mutex, we can shut everything down ourselves shutdown_ssl(socket_, true); shutdown_socket(socket_); close_socket(socket_); } +inline std::string ClientImpl::host() const { return host_; } + +inline int ClientImpl::port() const { return port_; } + +inline size_t ClientImpl::is_socket_open() const { + std::lock_guard guard(socket_mutex_); + return socket_.is_open(); +} + +inline socket_t ClientImpl::socket() const { return socket_.sock; } + inline void ClientImpl::set_connection_timeout(time_t sec, time_t usec) { connection_timeout_sec_ = sec; connection_timeout_usec_ = usec; @@ -7120,6 +7938,11 @@ inline void ClientImpl::set_default_headers(Headers headers) { default_headers_ = std::move(headers); } +inline void ClientImpl::set_header_writer( + std::function const &writer) { + header_writer_ = writer; +} + inline void ClientImpl::set_address_family(int family) { address_family_ = family; } @@ -7159,9 +7982,7 @@ inline void ClientImpl::set_proxy_digest_auth(const std::string &username, proxy_digest_auth_username_ = username; proxy_digest_auth_password_ = password; } -#endif -#ifdef CPPHTTPLIB_OPENSSL_SUPPORT inline void ClientImpl::set_ca_cert_path(const std::string &ca_cert_file_path, const std::string &ca_cert_dir_path) { ca_cert_file_path_ = ca_cert_file_path; @@ -7173,9 +7994,34 @@ inline void ClientImpl::set_ca_cert_store(X509_STORE *ca_cert_store) { ca_cert_store_ = ca_cert_store; } } -#endif -#ifdef CPPHTTPLIB_OPENSSL_SUPPORT +inline X509_STORE *ClientImpl::create_ca_cert_store(const char *ca_cert, + std::size_t size) { + auto mem = BIO_new_mem_buf(ca_cert, static_cast(size)); + if (!mem) return nullptr; + + auto inf = PEM_X509_INFO_read_bio(mem, nullptr, nullptr, nullptr); + if (!inf) { + BIO_free_all(mem); + return nullptr; + } + + auto cts = X509_STORE_new(); + if (cts) { + for (auto i = 0; i < static_cast(sk_X509_INFO_num(inf)); i++) { + auto itmp = sk_X509_INFO_value(inf, i); + if (!itmp) { continue; } + + if (itmp->x509) { X509_STORE_add_cert(cts, itmp->x509); } + if (itmp->crl) { X509_STORE_add_crl(cts, itmp->crl); } + } + } + + sk_X509_INFO_pop_free(inf, X509_INFO_free); + BIO_free_all(mem); + return cts; +} + inline void ClientImpl::enable_server_certificate_verification(bool enabled) { server_certificate_verification_ = enabled; } @@ -7239,7 +8085,7 @@ bool ssl_connect_or_accept_nonblocking(socket_t sock, SSL *ssl, U ssl_connect_or_accept, time_t timeout_sec, time_t timeout_usec) { - int res = 0; + auto res = 0; while ((res = ssl_connect_or_accept(ssl)) != 1) { auto err = SSL_get_error(ssl, res); switch (err) { @@ -7281,55 +8127,12 @@ process_client_socket_ssl(SSL *ssl, socket_t sock, time_t read_timeout_sec, return callback(strm); } -#if OPENSSL_VERSION_NUMBER < 0x10100000L -static std::shared_ptr> openSSL_locks_; - -class SSLThreadLocks { -public: - SSLThreadLocks() { - openSSL_locks_ = - std::make_shared>(CRYPTO_num_locks()); - CRYPTO_set_locking_callback(locking_callback); - } - - ~SSLThreadLocks() { CRYPTO_set_locking_callback(nullptr); } - -private: - static void locking_callback(int mode, int type, const char * /*file*/, - int /*line*/) { - auto &lk = (*openSSL_locks_)[static_cast(type)]; - if (mode & CRYPTO_LOCK) { - lk.lock(); - } else { - lk.unlock(); - } - } -}; - -#endif - class SSLInit { public: SSLInit() { -#if OPENSSL_VERSION_NUMBER < 0x1010001fL - SSL_load_error_strings(); - SSL_library_init(); -#else OPENSSL_init_ssl( OPENSSL_INIT_LOAD_SSL_STRINGS | OPENSSL_INIT_LOAD_CRYPTO_STRINGS, NULL); -#endif } - - ~SSLInit() { -#if OPENSSL_VERSION_NUMBER < 0x1010001fL - ERR_free_strings(); -#endif - } - -private: -#if OPENSSL_VERSION_NUMBER < 0x10100000L - SSLThreadLocks thread_init_; -#endif }; // SSL socket stream implementation @@ -7353,7 +8156,7 @@ inline bool SSLSocketStream::is_readable() const { inline bool SSLSocketStream::is_writable() const { return select_write(sock_, write_timeout_sec_, write_timeout_usec_) > 0 && - is_socket_alive(sock_); + is_socket_alive(sock_); } inline ssize_t SSLSocketStream::read(char *ptr, size_t size) { @@ -7363,7 +8166,7 @@ inline ssize_t SSLSocketStream::read(char *ptr, size_t size) { auto ret = SSL_read(ssl_, ptr, static_cast(size)); if (ret < 0) { auto err = SSL_get_error(ssl_, ret); - int n = 1000; + auto n = 1000; #ifdef _WIN32 while (--n >= 0 && (err == SSL_ERROR_WANT_READ || (err == SSL_ERROR_SYSCALL && @@ -7396,7 +8199,7 @@ inline ssize_t SSLSocketStream::write(const char *ptr, size_t size) { auto ret = SSL_write(ssl_, ptr, static_cast(handle_size)); if (ret < 0) { auto err = SSL_get_error(ssl_, ret); - int n = 1000; + auto n = 1000; #ifdef _WIN32 while (--n >= 0 && (err == SSL_ERROR_WANT_WRITE || (err == SSL_ERROR_SYSCALL && @@ -7424,6 +8227,11 @@ inline void SSLSocketStream::get_remote_ip_and_port(std::string &ip, detail::get_remote_ip_and_port(sock_, ip, port); } +inline void SSLSocketStream::get_local_ip_and_port(std::string &ip, + int &port) const { + detail::get_local_ip_and_port(sock_, ip, port); +} + inline socket_t SSLSocketStream::socket() const { return sock_; } static SSLInit sslinit_; @@ -7446,8 +8254,9 @@ inline SSLServer::SSLServer(const char *cert_path, const char *private_key_path, // add default password callback before opening encrypted private key if (private_key_password != nullptr && (private_key_password[0] != '\0')) { - SSL_CTX_set_default_passwd_cb_userdata(ctx_, - (char *)private_key_password); + SSL_CTX_set_default_passwd_cb_userdata( + ctx_, + reinterpret_cast(const_cast(private_key_password))); } if (SSL_CTX_use_certificate_chain_file(ctx_, cert_path) != 1 || @@ -7517,7 +8326,7 @@ inline bool SSLServer::process_and_close_socket(socket_t sock) { }, [](SSL * /*ssl2*/) { return true; }); - bool ret = false; + auto ret = false; if (ssl) { ret = detail::process_server_socket_ssl( svr_sock_, ssl, sock, keep_alive_max_count_, keep_alive_timeout_sec_, @@ -7611,6 +8420,11 @@ inline void SSLClient::set_ca_cert_store(X509_STORE *ca_cert_store) { } } +inline void SSLClient::load_ca_cert_store(const char *ca_cert, + std::size_t size) { + set_ca_cert_store(ClientImpl::create_ca_cert_store(ca_cert, size)); +} + inline long SSLClient::get_openssl_verify_result() const { return verify_result_; } @@ -7625,14 +8439,14 @@ inline bool SSLClient::create_and_connect_socket(Socket &socket, Error &error) { inline bool SSLClient::connect_with_proxy(Socket &socket, Response &res, bool &success, Error &error) { success = true; - Response res2; + Response proxy_res; if (!detail::process_client_socket( socket.sock, read_timeout_sec_, read_timeout_usec_, write_timeout_sec_, write_timeout_usec_, [&](Stream &strm) { Request req2; req2.method = "CONNECT"; req2.path = host_and_port_; - return process_request(strm, req2, res2, false, error); + return process_request(strm, req2, proxy_res, false, error); })) { // Thread-safe to close everything because we are assuming there are no // requests in flight @@ -7643,12 +8457,12 @@ inline bool SSLClient::connect_with_proxy(Socket &socket, Response &res, return false; } - if (res2.status == 407) { + if (proxy_res.status == 407) { if (!proxy_digest_auth_username_.empty() && !proxy_digest_auth_password_.empty()) { std::map auth; - if (detail::parse_www_authenticate(res2, auth, true)) { - Response res3; + if (detail::parse_www_authenticate(proxy_res, auth, true)) { + proxy_res = Response(); if (!detail::process_client_socket( socket.sock, read_timeout_sec_, read_timeout_usec_, write_timeout_sec_, write_timeout_usec_, [&](Stream &strm) { @@ -7659,7 +8473,7 @@ inline bool SSLClient::connect_with_proxy(Socket &socket, Response &res, req3, auth, 1, detail::random_string(10), proxy_digest_auth_username_, proxy_digest_auth_password_, true)); - return process_request(strm, req3, res3, false, error); + return process_request(strm, req3, proxy_res, false, error); })) { // Thread-safe to close everything because we are assuming there are // no requests in flight @@ -7670,17 +8484,28 @@ inline bool SSLClient::connect_with_proxy(Socket &socket, Response &res, return false; } } - } else { - res = res2; - return false; } } + // If status code is not 200, proxy request is failed. + // Set error to ProxyConnection and return proxy response + // as the response of the request + if (proxy_res.status != 200) { + error = Error::ProxyConnection; + res = std::move(proxy_res); + // Thread-safe to close everything because we are assuming there are + // no requests in flight + shutdown_ssl(socket, true); + shutdown_socket(socket); + close_socket(socket); + return false; + } + return true; } inline bool SSLClient::load_certs() { - bool ret = true; + auto ret = true; std::call_once(initialize_cert_, [&]() { std::lock_guard guard(ctx_mutex_); @@ -7695,11 +8520,16 @@ inline bool SSLClient::load_certs() { ret = false; } } else { + auto loaded = false; #ifdef _WIN32 - detail::load_system_certs_on_windows(SSL_CTX_get_cert_store(ctx_)); -#else - SSL_CTX_set_default_verify_paths(ctx_); -#endif + loaded = + detail::load_system_certs_on_windows(SSL_CTX_get_cert_store(ctx_)); +#elif defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN) && defined(__APPLE__) +#if TARGET_OS_OSX + loaded = detail::load_system_certs_on_macos(SSL_CTX_get_cert_store(ctx_)); +#endif // TARGET_OS_OSX +#endif // _WIN32 + if (!loaded) { SSL_CTX_set_default_verify_paths(ctx_); } } }); @@ -7733,7 +8563,7 @@ inline bool SSLClient::initialize_ssl(Socket &socket, Error &error) { return false; } - auto server_cert = SSL_get_peer_certificate(ssl2); + auto server_cert = SSL_get1_peer_certificate(ssl2); if (server_cert == nullptr) { error = Error::SSLServerVerification; @@ -7751,6 +8581,10 @@ inline bool SSLClient::initialize_ssl(Socket &socket, Error &error) { return true; }, [&](SSL *ssl2) { + // NOTE: With -Wold-style-cast, this can produce a warning, since + // SSL_set_tlsext_host_name is a macro (in OpenSSL), which contains + // an old style cast. Short of doing compiler specific pragma's + // here, we can't get rid of this warning. :'( SSL_set_tlsext_host_name(ssl2, host_.c_str()); return true; }); @@ -7844,15 +8678,16 @@ SSLClient::verify_host_with_subject_alt_name(X509 *server_cert) const { if (alt_names) { auto dsn_matched = false; - auto ip_mached = false; + auto ip_matched = false; auto count = sk_GENERAL_NAME_num(alt_names); for (decltype(count) i = 0; i < count && !dsn_matched; i++) { auto val = sk_GENERAL_NAME_value(alt_names, i); if (val->type == type) { - auto name = (const char *)ASN1_STRING_get0_data(val->d.ia5); - auto name_len = (size_t)ASN1_STRING_length(val->d.ia5); + auto name = + reinterpret_cast(ASN1_STRING_get0_data(val->d.ia5)); + auto name_len = static_cast(ASN1_STRING_length(val->d.ia5)); switch (type) { case GEN_DNS: dsn_matched = check_host_name(name, name_len); break; @@ -7860,17 +8695,18 @@ SSLClient::verify_host_with_subject_alt_name(X509 *server_cert) const { case GEN_IPADD: if (!memcmp(&addr6, name, addr_len) || !memcmp(&addr, name, addr_len)) { - ip_mached = true; + ip_matched = true; } break; } } } - if (dsn_matched || ip_mached) { ret = true; } + if (dsn_matched || ip_matched) { ret = true; } } - GENERAL_NAMES_free((STACK_OF(GENERAL_NAME) *)alt_names); + GENERAL_NAMES_free(const_cast( + reinterpret_cast(alt_names))); return ret; } @@ -8059,6 +8895,9 @@ inline Result Client::Head(const std::string &path, const Headers &headers) { } inline Result Client::Post(const std::string &path) { return cli_->Post(path); } +inline Result Client::Post(const std::string &path, const Headers &headers) { + return cli_->Post(path, headers); +} inline Result Client::Post(const std::string &path, const char *body, size_t content_length, const std::string &content_type) { @@ -8121,6 +8960,12 @@ inline Result Client::Post(const std::string &path, const Headers &headers, const std::string &boundary) { return cli_->Post(path, headers, items, boundary); } +inline Result +Client::Post(const std::string &path, const Headers &headers, + const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items) { + return cli_->Post(path, headers, items, provider_items); +} inline Result Client::Put(const std::string &path) { return cli_->Put(path); } inline Result Client::Put(const std::string &path, const char *body, size_t content_length, @@ -8184,6 +9029,12 @@ inline Result Client::Put(const std::string &path, const Headers &headers, const std::string &boundary) { return cli_->Put(path, headers, items, boundary); } +inline Result +Client::Put(const std::string &path, const Headers &headers, + const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items) { + return cli_->Put(path, headers, items, provider_items); +} inline Result Client::Patch(const std::string &path) { return cli_->Patch(path); } @@ -8267,12 +9118,16 @@ inline bool Client::send(Request &req, Response &res, Error &error) { inline Result Client::send(const Request &req) { return cli_->send(req); } +inline void Client::stop() { cli_->stop(); } + +inline std::string Client::host() const { return cli_->host(); } + +inline int Client::port() const { return cli_->port(); } + inline size_t Client::is_socket_open() const { return cli_->is_socket_open(); } inline socket_t Client::socket() const { return cli_->socket(); } -inline void Client::stop() { cli_->stop(); } - inline void Client::set_hostname_addr_map(std::map addr_map) { cli_->set_hostname_addr_map(std::move(addr_map)); @@ -8282,6 +9137,11 @@ inline void Client::set_default_headers(Headers headers) { cli_->set_default_headers(std::move(headers)); } +inline void Client::set_header_writer( + std::function const &writer) { + cli_->set_header_writer(writer); +} + inline void Client::set_address_family(int family) { cli_->set_address_family(family); } @@ -8356,7 +9216,9 @@ inline void Client::enable_server_certificate_verification(bool enabled) { } #endif -inline void Client::set_logger(Logger logger) { cli_->set_logger(logger); } +inline void Client::set_logger(Logger logger) { + cli_->set_logger(std::move(logger)); +} #ifdef CPPHTTPLIB_OPENSSL_SUPPORT inline void Client::set_ca_cert_path(const std::string &ca_cert_file_path, @@ -8372,6 +9234,10 @@ inline void Client::set_ca_cert_store(X509_STORE *ca_cert_store) { } } +inline void Client::load_ca_cert_store(const char *ca_cert, std::size_t size) { + set_ca_cert_store(cli_->create_ca_cert_store(ca_cert, size)); +} + inline long Client::get_openssl_verify_result() const { if (is_ssl_) { return static_cast(*cli_).get_openssl_verify_result(); @@ -8389,4 +9255,8 @@ inline SSL_CTX *Client::ssl_context() const { } // namespace httplib +#if defined(_WIN32) && defined(CPPHTTPLIB_USE_POLL) +#undef poll +#endif + #endif // CPPHTTPLIB_HTTPLIB_H diff --git a/src/core/hle/service/http/http_c.cpp b/src/core/hle/service/http/http_c.cpp index fb6b6d07c9..0ca2f7c428 100644 --- a/src/core/hle/service/http/http_c.cpp +++ b/src/core/hle/service/http/http_c.cpp @@ -3,6 +3,7 @@ // Refer to the license.txt file included. #include +#include #include #include #include @@ -65,10 +66,10 @@ static std::pair SplitUrl(const std::string& url) { std::string path; if (path_index == std::string::npos) { // If no path is specified after the host, set it to "/" - host = url; + host = url.substr(prefix_end); path = "/"; } else { - host = url.substr(0, path_index); + host = url.substr(prefix_end, path_index - prefix_end); path = url.substr(path_index); } return std::make_pair(host, path); @@ -77,25 +78,6 @@ static std::pair SplitUrl(const std::string& url) { void Context::MakeRequest() { ASSERT(state == RequestState::NotStarted); - const auto& [host, path] = SplitUrl(url); - const auto client = std::make_unique(host); - SSL_CTX* ctx = client->ssl_context(); - if (ctx) { - if (auto client_cert = ssl_config.client_cert_ctx.lock()) { - SSL_CTX_use_certificate_ASN1(ctx, static_cast(client_cert->certificate.size()), - client_cert->certificate.data()); - SSL_CTX_use_PrivateKey_ASN1(EVP_PKEY_RSA, ctx, client_cert->private_key.data(), - static_cast(client_cert->private_key.size())); - } - - // TODO(B3N30): Check for SSLOptions-Bits and set the verify method accordingly - // https://www.3dbrew.org/wiki/SSL_Services#SSLOpt - // Hack: Since for now RootCerts are not implemented we set the VerifyMode to None. - SSL_CTX_set_verify(ctx, SSL_VERIFY_NONE, NULL); - } - - state = RequestState::InProgress; - static const std::unordered_map request_method_strings{ {RequestMethod::Get, "GET"}, {RequestMethod::Post, "POST"}, {RequestMethod::Head, "HEAD"}, {RequestMethod::Put, "PUT"}, @@ -103,11 +85,13 @@ void Context::MakeRequest() { {RequestMethod::PutEmpty, "PUT"}, }; + const auto& [host, path] = SplitUrl(url.c_str()); + httplib::Request request; - httplib::Error error; + httplib::Error error{-1}; request.method = request_method_strings.at(method); request.path = path; - // TODO(B3N30): Add post data body + request.progress = [this](u64 current, u64 total) -> bool { // TODO(B3N30): Is there a state that shows response header are available current_download_size_bytes = current; @@ -119,14 +103,70 @@ void Context::MakeRequest() { request.headers.emplace(header.name, header.value); } - if (!client->send(request, response, error)) { - LOG_ERROR(Service_HTTP, "Request failed: {}: {}", error, httplib::to_string(error)); - state = RequestState::TimedOut; - } else { - LOG_DEBUG(Service_HTTP, "Request successful"); - // TODO(B3N30): Verify this state on HW - state = RequestState::ReadyToDownloadContent; + if (!post_data.empty()) { + request.headers.emplace("Content-Type", "application/x-www-form-urlencoded"); + request.body = httplib::detail::params_to_query_str(post_data); + boost::replace_all(request.body, "*", "%2A"); } + + if (!post_data_raw.empty()) { + request.body = post_data_raw; + } + + state = RequestState::InProgress; + + const unsigned char* tmpCertPtr = clcert_data.certificate.data(); + const unsigned char* tmpKeyPtr = clcert_data.private_key.data(); + X509* cert = d2i_X509(nullptr, &tmpCertPtr, (long)clcert_data.certificate.size()); + EVP_PKEY* key = + d2i_PrivateKey(EVP_PKEY_RSA, nullptr, &tmpKeyPtr, (long)clcert_data.private_key.size()); + // Sadly, we have to duplicate code, the class hierarchy in httplib is not very useful... + if (uses_default_client_cert) { + + std::unique_ptr client = + std::make_unique(host, 443, cert, key); + + // TODO(B3N30): Check for SSLOptions-Bits and set the verify method accordingly + // https://www.3dbrew.org/wiki/SSL_Services#SSLOpt + // Hack: Since for now RootCerts are not implemented we set the VerifyMode to None. + client->enable_server_certificate_verification(false); + + if (!client->send(request, response, error)) { + LOG_ERROR(Service_HTTP, "Request failed: {}: {}", error, httplib::to_string(error)); + state = RequestState::TimedOut; + } else { + LOG_DEBUG(Service_HTTP, "Request successful"); + // TODO(B3N30): Verify this state on HW + state = RequestState::ReadyToDownloadContent; + } + } else { + std::unique_ptr client = std::make_unique(host); + SSL_CTX* ctx = client->ssl_context(); + if (ctx) { + if (auto client_cert = ssl_config.client_cert_ctx.lock()) { + SSL_CTX_use_certificate_ASN1(ctx, static_cast(client_cert->certificate.size()), + client_cert->certificate.data()); + SSL_CTX_use_PrivateKey_ASN1(EVP_PKEY_RSA, ctx, client_cert->private_key.data(), + static_cast(client_cert->private_key.size())); + } + + // TODO(B3N30): Check for SSLOptions-Bits and set the verify method accordingly + // https://www.3dbrew.org/wiki/SSL_Services#SSLOpt + // Hack: Since for now RootCerts are not implemented we set the VerifyMode to None. + client->enable_server_certificate_verification(false); + } + + if (!client->send(request, response, error)) { + LOG_ERROR(Service_HTTP, "Request failed: {}: {}", error, httplib::to_string(error)); + state = RequestState::TimedOut; + } else { + LOG_DEBUG(Service_HTTP, "Request successful"); + // TODO(B3N30): Verify this state on HW + state = RequestState::ReadyToDownloadContent; + } + } + X509_free(cert); + EVP_PKEY_free(key); } void HTTP_C::Initialize(Kernel::HLERequestContext& ctx) { @@ -216,6 +256,7 @@ void HTTP_C::BeginRequest(Kernel::HLERequestContext& ctx) { itr->second.request_future = std::async(std::launch::async, &Context::MakeRequest, std::ref(itr->second)); + itr->second.current_copied_data = 0; IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); rb.Push(RESULT_SUCCESS); @@ -243,6 +284,7 @@ void HTTP_C::BeginRequestAsync(Kernel::HLERequestContext& ctx) { itr->second.request_future = std::async(std::launch::async, &Context::MakeRequest, std::ref(itr->second)); + itr->second.current_copied_data = 0; IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); rb.Push(RESULT_SUCCESS); @@ -259,7 +301,7 @@ void HTTP_C::ReceiveDataTimeout(Kernel::HLERequestContext& ctx) { void HTTP_C::ReceiveDataImpl(Kernel::HLERequestContext& ctx, bool timeout) { IPC::RequestParser rp(ctx); const Context::Handle context_handle = rp.Pop(); - [[maybe_unused]] const u32 buffer_size = rp.Pop(); + const u32 buffer_size = rp.Pop(); u64 timeout_nanos = 0; if (timeout) { timeout_nanos = rp.Pop(); @@ -267,7 +309,7 @@ void HTTP_C::ReceiveDataImpl(Kernel::HLERequestContext& ctx, bool timeout) { } else { LOG_WARNING(Service_HTTP, "(STUBBED) called"); } - [[maybe_unused]] Kernel::MappedBuffer& buffer = rp.PopMappedBuffer(); + Kernel::MappedBuffer& buffer = rp.PopMappedBuffer(); if (!PerformStateChecks(ctx, rp, context_handle)) { return; @@ -276,13 +318,45 @@ void HTTP_C::ReceiveDataImpl(Kernel::HLERequestContext& ctx, bool timeout) { auto itr = contexts.find(context_handle); ASSERT(itr != contexts.end()); + Context& http_context = itr->second; + IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); + if (timeout) { - itr->second.request_future.wait_for(std::chrono::nanoseconds(timeout_nanos)); - // TODO (flTobi): Return error on timeout + auto wait_res = + http_context.request_future.wait_for(std::chrono::nanoseconds(timeout_nanos)); + if (wait_res == std::future_status::timeout) { + rb.Push(ResultCode(105, ErrorModule::HTTP, ErrorSummary::NothingHappened, + ErrorLevel::Permanent)); + return; + } } else { - itr->second.request_future.wait(); + http_context.request_future.wait(); } + size_t remaining_data = http_context.response.body.size() - http_context.current_copied_data; + + if (buffer_size >= remaining_data) { + buffer.Write(http_context.response.body.data() + http_context.current_copied_data, 0, + remaining_data); + http_context.current_copied_data += remaining_data; + rb.Push(RESULT_SUCCESS); + } else { + buffer.Write(http_context.response.body.data() + http_context.current_copied_data, 0, + buffer_size); + http_context.current_copied_data += buffer_size; + rb.Push(ResultCode(43, ErrorModule::HTTP, ErrorSummary::WouldBlock, ErrorLevel::Permanent)); + } + LOG_DEBUG(Service_HTTP, "Receive: buffer_size= {}, total_copied={}, total_body={}", buffer_size, + http_context.current_copied_data, http_context.response.body.size()); + ctx.SleepClientThread("http_data", std::chrono::nanoseconds(100000), nullptr); +} + +void HTTP_C::SetProxyDefault(Kernel::HLERequestContext& ctx) { + IPC::RequestParser rp(ctx); + const Context::Handle context_handle = rp.Pop(); + + LOG_WARNING(Service_HTTP, "(STUBBED) called, handle={}", context_handle); + IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); rb.Push(RESULT_SUCCESS); } @@ -389,6 +463,24 @@ void HTTP_C::CloseContext(Kernel::HLERequestContext& ctx) { rb.Push(RESULT_SUCCESS); } +void HTTP_C::CancelConnection(Kernel::HLERequestContext& ctx) { + IPC::RequestParser rp(ctx); + const u32 context_handle = rp.Pop(); + + LOG_WARNING(Service_HTTP, "(STUBBED) called, handle={}", context_handle); + + const auto* session_data = EnsureSessionInitialized(ctx, rp); + if (!session_data) { + return; + } + + auto itr = contexts.find(context_handle); + ASSERT(itr != contexts.end()); + + IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); + rb.Push(RESULT_SUCCESS); +} + void HTTP_C::AddRequestHeader(Kernel::HLERequestContext& ctx) { IPC::RequestParser rp(ctx); const u32 context_handle = rp.Pop(); @@ -424,11 +516,6 @@ void HTTP_C::AddRequestHeader(Kernel::HLERequestContext& ctx) { return; } - ASSERT(std::find_if(itr->second.headers.begin(), itr->second.headers.end(), - [&name](const Context::RequestHeader& m) -> bool { - return m.name == name; - }) == itr->second.headers.end()); - itr->second.headers.emplace_back(name, value); IPC::RequestBuilder rb = rp.MakeBuilder(1, 2); @@ -471,17 +558,81 @@ void HTTP_C::AddPostDataAscii(Kernel::HLERequestContext& ctx) { return; } - ASSERT(std::find_if(itr->second.post_data.begin(), itr->second.post_data.end(), - [&name](const Context::PostData& m) -> bool { return m.name == name; }) == - itr->second.post_data.end()); - - itr->second.post_data.emplace_back(name, value); + itr->second.post_data.emplace(name, value); IPC::RequestBuilder rb = rp.MakeBuilder(1, 2); rb.Push(RESULT_SUCCESS); rb.PushMappedBuffer(value_buffer); } +void HTTP_C::AddPostDataRaw(Kernel::HLERequestContext& ctx) { + IPC::RequestParser rp(ctx); + const u32 context_handle = rp.Pop(); + const u32 post_data_len = rp.Pop(); + auto buffer = rp.PopMappedBuffer(); + + LOG_DEBUG(Service_HTTP, "context_handle={}, post_data_len={}", context_handle, post_data_len); + + if (!PerformStateChecks(ctx, rp, context_handle)) { + return; + } + + auto itr = contexts.find(context_handle); + ASSERT(itr != contexts.end()); + + itr->second.post_data_raw.resize(buffer.GetSize()); + buffer.Read(itr->second.post_data_raw.data(), 0, buffer.GetSize()); + + IPC::RequestBuilder rb = rp.MakeBuilder(1, 2); + rb.Push(RESULT_SUCCESS); + rb.PushMappedBuffer(buffer); +} + +void HTTP_C::GetResponseHeader(Kernel::HLERequestContext& ctx) { + IPC::RequestParser rp(ctx); + const u32 context_handle = rp.Pop(); + const u32 name_len = rp.Pop(); + [[maybe_unused]] const u32 value_max_len = rp.Pop(); + auto& header_name = rp.PopStaticBuffer(); + Kernel::MappedBuffer& value_buffer = rp.PopMappedBuffer(); + std::string header_name_str(reinterpret_cast(header_name.data()), name_len); + while (header_name_str.size() && header_name_str.back() == '\0') { + header_name_str.pop_back(); + } + + if (!PerformStateChecks(ctx, rp, context_handle)) { + return; + } + + auto itr = contexts.find(context_handle); + ASSERT(itr != contexts.end()); + + itr->second.request_future.wait(); + + auto& headers = itr->second.response.headers; + u32 copied_size = 0; + + LOG_DEBUG(Service_HTTP, "header={}, max_len={}", header_name_str, value_buffer.GetSize()); + + auto header = headers.find(header_name_str); + if (header != headers.end()) { + std::string header_value = header->second; + copied_size = (u32)header_value.size(); + if (header_value.size() + 1 > value_buffer.GetSize()) { + header_value.resize(value_buffer.GetSize() - 1); + } + header_value.push_back('\0'); + value_buffer.Write(header_value.data(), 0, header_value.size()); + } else { + LOG_WARNING(Service_HTTP, "header={} not found", header_name_str); + } + + IPC::RequestBuilder rb = rp.MakeBuilder(2, 2); + rb.Push(RESULT_SUCCESS); + rb.Push(copied_size); + rb.PushMappedBuffer(value_buffer); +} + void HTTP_C::GetResponseStatusCode(Kernel::HLERequestContext& ctx) { GetResponseStatusCodeImpl(ctx, false); } @@ -509,19 +660,74 @@ void HTTP_C::GetResponseStatusCodeImpl(Kernel::HLERequestContext& ctx, bool time ASSERT(itr != contexts.end()); if (timeout) { - itr->second.request_future.wait_for(std::chrono::nanoseconds(timeout)); - // TODO (flTobi): Return error on timeout + auto wait_res = itr->second.request_future.wait_for(std::chrono::nanoseconds(timeout)); + if (wait_res == std::future_status::timeout) { + LOG_DEBUG(Service_HTTP, "Status code: {}", "timeout"); + IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); + rb.Push(ResultCode(105, ErrorModule::HTTP, ErrorSummary::NothingHappened, + ErrorLevel::Permanent)); + return; + } } else { itr->second.request_future.wait(); } const u32 response_code = itr->second.response.status; + LOG_DEBUG(Service_HTTP, "Status code: {}, response_code={}", "good", response_code); IPC::RequestBuilder rb = rp.MakeBuilder(2, 0); rb.Push(RESULT_SUCCESS); rb.Push(response_code); } +void HTTP_C::AddTrustedRootCA(Kernel::HLERequestContext& ctx) { + IPC::RequestParser rp(ctx); + const Context::Handle context_handle = rp.Pop(); + [[maybe_unused]] const u32 root_ca_len = rp.Pop(); + auto root_ca_data = rp.PopMappedBuffer(); + + LOG_WARNING(Service_HTTP, "(STUBBED) called, handle={}", context_handle); + + IPC::RequestBuilder rb = rp.MakeBuilder(1, 2); + rb.Push(RESULT_SUCCESS); + rb.PushMappedBuffer(root_ca_data); +} + +void HTTP_C::AddDefaultCert(Kernel::HLERequestContext& ctx) { + IPC::RequestParser rp(ctx); + const Context::Handle context_handle = rp.Pop(); + const u32 certificate_id = rp.Pop(); + + LOG_WARNING(Service_HTTP, "(STUBBED) called, handle={}, certificate_id={}", context_handle, + certificate_id); + + IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); + rb.Push(RESULT_SUCCESS); +} + +void HTTP_C::SetDefaultClientCert(Kernel::HLERequestContext& ctx) { + IPC::RequestParser rp(ctx); + const Context::Handle context_handle = rp.Pop(); + // TODO(PabloMK7): There is only a single cert ID with value 64. Check it is valid and return + // error if not. + [[maybe_unused]] const u32 client_cert_id = rp.Pop(); + + LOG_DEBUG(Service_HTTP, "client_cert_id={}", client_cert_id); + + if (!PerformStateChecks(ctx, rp, context_handle)) { + return; + } + + auto itr = contexts.find(context_handle); + ASSERT(itr != contexts.end()); + + itr->second.uses_default_client_cert = true; + itr->second.clcert_data = GetClCertA(); + + IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); + rb.Push(RESULT_SUCCESS); +} + void HTTP_C::SetClientCertContext(Kernel::HLERequestContext& ctx) { IPC::RequestParser rp(ctx); const u32 context_handle = rp.Pop(); @@ -763,9 +969,11 @@ void HTTP_C::GetDownloadSizeState(Kernel::HLERequestContext& ctx) { } } + LOG_DEBUG(Service_HTTP, "current={}, total={}", itr->second.current_copied_data, + content_length); IPC::RequestBuilder rb = rp.MakeBuilder(3, 0); rb.Push(RESULT_SUCCESS); - rb.Push(content_length); + rb.Push(itr->second.current_copied_data); rb.Push(content_length); } @@ -895,7 +1103,7 @@ HTTP_C::HTTP_C() : ServiceFramework("http:C", 32) { {0x0001, &HTTP_C::Initialize, "Initialize"}, {0x0002, &HTTP_C::CreateContext, "CreateContext"}, {0x0003, &HTTP_C::CloseContext, "CloseContext"}, - {0x0004, nullptr, "CancelConnection"}, + {0x0004, &HTTP_C::CancelConnection, "CancelConnection"}, {0x0005, nullptr, "GetRequestState"}, {0x0006, &HTTP_C::GetDownloadSizeState, "GetDownloadSizeState"}, {0x0007, nullptr, "GetRequestError"}, @@ -905,13 +1113,13 @@ HTTP_C::HTTP_C() : ServiceFramework("http:C", 32) { {0x000B, &HTTP_C::ReceiveData, "ReceiveData"}, {0x000C, &HTTP_C::ReceiveDataTimeout, "ReceiveDataTimeout"}, {0x000D, nullptr, "SetProxy"}, - {0x000E, nullptr, "SetProxyDefault"}, + {0x000E, &HTTP_C::SetProxyDefault, "SetProxyDefault"}, {0x000F, nullptr, "SetBasicAuthorization"}, {0x0010, nullptr, "SetSocketBufferSize"}, {0x0011, &HTTP_C::AddRequestHeader, "AddRequestHeader"}, {0x0012, &HTTP_C::AddPostDataAscii, "AddPostDataAscii"}, {0x0013, nullptr, "AddPostDataBinary"}, - {0x0014, nullptr, "AddPostDataRaw"}, + {0x0014, &HTTP_C::AddPostDataRaw, "AddPostDataRaw"}, {0x0015, nullptr, "SetPostDataType"}, {0x0016, nullptr, "SendPostDataAscii"}, {0x0017, nullptr, "SendPostDataAsciiTimeout"}, @@ -921,16 +1129,17 @@ HTTP_C::HTTP_C() : ServiceFramework("http:C", 32) { {0x001B, nullptr, "SendPOSTDataRawTimeout"}, {0x001C, nullptr, "SetPostDataEncoding"}, {0x001D, nullptr, "NotifyFinishSendPostData"}, - {0x001E, nullptr, "GetResponseHeader"}, + {0x001E, &HTTP_C::GetResponseHeader, "GetResponseHeader"}, {0x001F, nullptr, "GetResponseHeaderTimeout"}, {0x0020, nullptr, "GetResponseData"}, {0x0021, nullptr, "GetResponseDataTimeout"}, {0x0022, &HTTP_C::GetResponseStatusCode, "GetResponseStatusCode"}, {0x0023, &HTTP_C::GetResponseStatusCodeTimeout, "GetResponseStatusCodeTimeout"}, - {0x0024, nullptr, "AddTrustedRootCA"}, - {0x0025, nullptr, "AddDefaultCert"}, + {0x0024, &HTTP_C::AddTrustedRootCA, "AddTrustedRootCA"}, + {0x0025, &HTTP_C::AddDefaultCert, "AddDefaultCert"}, {0x0026, nullptr, "SelectRootCertChain"}, {0x0027, nullptr, "SetClientCert"}, + {0x0028, &HTTP_C::SetDefaultClientCert, "SetDefaultClientCert"}, {0x0029, &HTTP_C::SetClientCertContext, "SetClientCertContext"}, {0x002A, &HTTP_C::GetSSLError, "GetSSLError"}, {0x002B, nullptr, "SetSSLOpt"}, @@ -955,6 +1164,10 @@ HTTP_C::HTTP_C() : ServiceFramework("http:C", 32) { DecryptClCertA(); } +std::shared_ptr GetService(Core::System& system) { + return system.ServiceManager().GetService("http:C"); +} + void InstallInterfaces(Core::System& system) { auto& service_manager = system.ServiceManager(); std::make_shared()->InstallAsService(service_manager); diff --git a/src/core/hle/service/http/http_c.h b/src/core/hle/service/http/http_c.h index 4ba7d2576c..99c0be0494 100644 --- a/src/core/hle/service/http/http_c.h +++ b/src/core/hle/service/http/http_c.h @@ -21,6 +21,7 @@ #include #endif #include +#include "core/hle/ipc_helpers.h" #include "core/hle/kernel/shared_memory.h" #include "core/hle/service/service.h" @@ -114,6 +115,12 @@ private: friend class boost::serialization::access; }; +struct ClCertAData { + std::vector certificate; + std::vector private_key; + bool init = false; +}; + /// Represents an HTTP context. class Context final { public: @@ -169,22 +176,6 @@ public: friend class boost::serialization::access; }; - struct PostData { - // TODO(Subv): Support Binary and Raw POST elements. - PostData(std::string name, std::string value) : name(name), value(value){}; - PostData() = default; - std::string name; - std::string value; - - private: - template - void serialize(Archive& ar, const unsigned int) { - ar& name; - ar& value; - } - friend class boost::serialization::access; - }; - struct SSLConfig { u32 options; std::weak_ptr client_cert_ctx; @@ -210,11 +201,15 @@ public: SSLConfig ssl_config{}; u32 socket_buffer_size; std::vector headers; - std::vector post_data; + ClCertAData clcert_data; + httplib::Params post_data; + std::string post_data_raw; std::future request_future; std::atomic current_download_size_bytes; std::atomic total_download_size_bytes; + size_t current_copied_data; + bool uses_default_client_cert{}; httplib::Response response; }; @@ -252,6 +247,10 @@ class HTTP_C final : public ServiceFramework { public: HTTP_C(); + const ClCertAData& GetClCertA() const { + return ClCertA; + } + private: /** * HTTP_C::Initialize service function @@ -288,6 +287,8 @@ private: */ void CloseContext(Kernel::HLERequestContext& ctx); + void CancelConnection(Kernel::HLERequestContext& ctx); + /** * HTTP_C::GetDownloadSizeState service function * Inputs: @@ -328,6 +329,8 @@ private: */ void BeginRequestAsync(Kernel::HLERequestContext& ctx); + void SetProxyDefault(Kernel::HLERequestContext& ctx); + /** * HTTP_C::ReceiveData service function * Inputs: @@ -389,6 +392,10 @@ private: */ void AddPostDataAscii(Kernel::HLERequestContext& ctx); + void AddPostDataRaw(Kernel::HLERequestContext& ctx); + + void GetResponseHeader(Kernel::HLERequestContext& ctx); + /** * HTTP_C::GetResponseStatusCode service function * Inputs: @@ -410,12 +417,26 @@ private: */ void GetResponseStatusCodeTimeout(Kernel::HLERequestContext& ctx); + void AddTrustedRootCA(Kernel::HLERequestContext& ctx); + + /** + * HTTP_C::AddDefaultCert service function + * Inputs: + * 1 : Context handle + * 2 : Cert ID + * Outputs: + * 1 : Result of function, 0 on success, otherwise error code + */ + void AddDefaultCert(Kernel::HLERequestContext& ctx); + /** * GetResponseStatusCodeImpl: * Implements GetResponseStatusCode and GetResponseStatusCodeTimeout service functions */ void GetResponseStatusCodeImpl(Kernel::HLERequestContext& ctx, bool timeout); + void SetDefaultClientCert(Kernel::HLERequestContext& ctx); + /** * HTTP_C::SetClientCertContext service function * Inputs: @@ -502,11 +523,7 @@ private: /// Global list of ClientCert contexts currently opened. std::unordered_map> client_certs; - struct { - std::vector certificate; - std::vector private_key; - bool init = false; - } ClCertA; + ClCertAData ClCertA; private: template @@ -527,6 +544,8 @@ private: friend class boost::serialization::access; }; +std::shared_ptr GetService(Core::System& system); + void InstallInterfaces(Core::System& system); } // namespace Service::HTTP From 5bf960c9efdcfdd6b1b23362ddb7afc56b1f755e Mon Sep 17 00:00:00 2001 From: PabloMK7 Date: Mon, 2 Oct 2023 20:23:57 +0200 Subject: [PATCH 2/3] More implementation details and cleanup. --- src/core/hle/service/http/http_c.cpp | 217 ++++++++++++++++++++++----- src/core/hle/service/http/http_c.h | 6 +- 2 files changed, 182 insertions(+), 41 deletions(-) diff --git a/src/core/hle/service/http/http_c.cpp b/src/core/hle/service/http/http_c.cpp index 0ca2f7c428..a0d1a2c15f 100644 --- a/src/core/hle/service/http/http_c.cpp +++ b/src/core/hle/service/http/http_c.cpp @@ -3,8 +3,8 @@ // Refer to the license.txt file included. #include -#include #include +#include #include #include #include "common/archives.h" @@ -29,6 +29,8 @@ enum { InvalidRequestState = 22, TooManyContexts = 26, InvalidRequestMethod = 32, + HeaderNotFound = 40, + BufferTooSmall = 43, ContextNotFound = 100, /// This error is returned in multiple situations: when trying to initialize an @@ -49,6 +51,10 @@ const ResultCode ERROR_NOT_IMPLEMENTED = // 0xD960A3F4 const ResultCode ERROR_TOO_MANY_CLIENT_CERTS = // 0xD8A0A0CB ResultCode(ErrCodes::TooManyClientCerts, ErrorModule::HTTP, ErrorSummary::InvalidState, ErrorLevel::Permanent); +const ResultCode ERROR_HEADER_NOT_FOUND = ResultCode( + ErrCodes::HeaderNotFound, ErrorModule::HTTP, ErrorSummary::InvalidState, ErrorLevel::Permanent); +const ResultCode ERROR_BUFFER_SMALL = ResultCode(ErrCodes::BufferTooSmall, ErrorModule::HTTP, + ErrorSummary::WouldBlock, ErrorLevel::Permanent); const ResultCode ERROR_WRONG_CERT_ID = // 0xD8E0B839 ResultCode(57, ErrorModule::SSL, ErrorSummary::InvalidArgument, ErrorLevel::Permanent); const ResultCode ERROR_WRONG_CERT_HANDLE = // 0xD8A0A0C9 @@ -56,6 +62,24 @@ const ResultCode ERROR_WRONG_CERT_HANDLE = // 0xD8A0A0C9 const ResultCode ERROR_CERT_ALREADY_SET = // 0xD8A0A03D ResultCode(61, ErrorModule::HTTP, ErrorSummary::InvalidState, ErrorLevel::Permanent); +static ssize_t write_headers(httplib::Stream& strm, + const std::vector& headers) { + ssize_t write_len = 0; + for (const auto& x : headers) { + auto len = strm.write_format("%s: %s\r\n", x.name.c_str(), x.value.c_str()); + if (len < 0) { + return len; + } + write_len += len; + } + auto len = strm.write("\r\n"); + if (len < 0) { + return len; + } + write_len += len; + return write_len; +} + static std::pair SplitUrl(const std::string& url) { const std::string prefix = "://"; const auto scheme_end = url.find(prefix); @@ -89,6 +113,7 @@ void Context::MakeRequest() { httplib::Request request; httplib::Error error{-1}; + std::vector pending_headers; request.method = request_method_strings.at(method); request.path = path; @@ -99,46 +124,103 @@ void Context::MakeRequest() { return true; }; - for (const auto& header : headers) { - request.headers.emplace(header.name, header.value); - } + // Watch out for header ordering! + auto header_writter = [&pending_headers](httplib::Stream& strm, + httplib::Headers& httplib_headers) { + std::vector final_headers; + std::vector::iterator it_p; + httplib::Headers::iterator it_h; + + auto find_pending_header = [&pending_headers](const std::string& str) { + return std::find_if(pending_headers.begin(), pending_headers.end(), + [&str](Context::RequestHeader& rh) { return rh.name == str; }); + }; + + // First: Host + it_p = find_pending_header("Host"); + if (it_p != pending_headers.end()) { + final_headers.push_back(Context::RequestHeader(it_p->name, it_p->value)); + pending_headers.erase(it_p); + } else { + it_h = httplib_headers.find("Host"); + if (it_h != httplib_headers.end()) { + final_headers.push_back(Context::RequestHeader(it_h->first, it_h->second)); + } + } + + // Second: Content-Type (optional, ignore httplib) + it_p = find_pending_header("Content-Type"); + if (it_p != pending_headers.end()) { + final_headers.push_back(Context::RequestHeader(it_p->name, it_p->value)); + pending_headers.erase(it_p); + } + + // Third: Content-Length + it_p = find_pending_header("Content-Length"); + if (it_p != pending_headers.end()) { + final_headers.push_back(Context::RequestHeader(it_p->name, it_p->value)); + pending_headers.erase(it_p); + } else { + it_h = httplib_headers.find("Content-Length"); + if (it_h != httplib_headers.end()) { + final_headers.push_back(Context::RequestHeader(it_h->first, it_h->second)); + } + } + + // Finally, user defined headers + for (const auto& header : pending_headers) { + final_headers.push_back(header); + } + + return write_headers(strm, final_headers); + }; if (!post_data.empty()) { - request.headers.emplace("Content-Type", "application/x-www-form-urlencoded"); + pending_headers.push_back( + Context::RequestHeader("Content-Type", "application/x-www-form-urlencoded")); request.body = httplib::detail::params_to_query_str(post_data); boost::replace_all(request.body, "*", "%2A"); } + for (const auto& header : headers) { + pending_headers.push_back(header); + } + if (!post_data_raw.empty()) { request.body = post_data_raw; } state = RequestState::InProgress; - const unsigned char* tmpCertPtr = clcert_data.certificate.data(); - const unsigned char* tmpKeyPtr = clcert_data.private_key.data(); - X509* cert = d2i_X509(nullptr, &tmpCertPtr, (long)clcert_data.certificate.size()); - EVP_PKEY* key = - d2i_PrivateKey(EVP_PKEY_RSA, nullptr, &tmpKeyPtr, (long)clcert_data.private_key.size()); // Sadly, we have to duplicate code, the class hierarchy in httplib is not very useful... if (uses_default_client_cert) { + const unsigned char* tmpCertPtr = clcert_data->certificate.data(); + const unsigned char* tmpKeyPtr = clcert_data->private_key.data(); + X509* cert = d2i_X509(nullptr, &tmpCertPtr, (long)clcert_data->certificate.size()); + EVP_PKEY* key = d2i_PrivateKey(EVP_PKEY_RSA, nullptr, &tmpKeyPtr, + (long)clcert_data->private_key.size()); + { + std::unique_ptr client = + std::make_unique(host, 443, cert, key); - std::unique_ptr client = - std::make_unique(host, 443, cert, key); + // TODO(B3N30): Check for SSLOptions-Bits and set the verify method accordingly + // https://www.3dbrew.org/wiki/SSL_Services#SSLOpt + // Hack: Since for now RootCerts are not implemented we set the VerifyMode to None. + client->enable_server_certificate_verification(false); - // TODO(B3N30): Check for SSLOptions-Bits and set the verify method accordingly - // https://www.3dbrew.org/wiki/SSL_Services#SSLOpt - // Hack: Since for now RootCerts are not implemented we set the VerifyMode to None. - client->enable_server_certificate_verification(false); + client->set_header_writer(header_writter); - if (!client->send(request, response, error)) { - LOG_ERROR(Service_HTTP, "Request failed: {}: {}", error, httplib::to_string(error)); - state = RequestState::TimedOut; - } else { - LOG_DEBUG(Service_HTTP, "Request successful"); - // TODO(B3N30): Verify this state on HW - state = RequestState::ReadyToDownloadContent; + if (!client->send(request, response, error)) { + LOG_ERROR(Service_HTTP, "Request failed: {}: {}", error, httplib::to_string(error)); + state = RequestState::TimedOut; + } else { + LOG_DEBUG(Service_HTTP, "Request successful"); + // TODO(B3N30): Verify this state on HW + state = RequestState::ReadyToDownloadContent; + } } + X509_free(cert); + EVP_PKEY_free(key); } else { std::unique_ptr client = std::make_unique(host); SSL_CTX* ctx = client->ssl_context(); @@ -156,6 +238,8 @@ void Context::MakeRequest() { client->enable_server_certificate_verification(false); } + client->set_header_writer(header_writter); + if (!client->send(request, response, error)) { LOG_ERROR(Service_HTTP, "Request failed: {}: {}", error, httplib::to_string(error)); state = RequestState::TimedOut; @@ -165,8 +249,6 @@ void Context::MakeRequest() { state = RequestState::ReadyToDownloadContent; } } - X509_free(cert); - EVP_PKEY_free(key); } void HTTP_C::Initialize(Kernel::HLERequestContext& ctx) { @@ -178,7 +260,7 @@ void HTTP_C::Initialize(Kernel::HLERequestContext& ctx) { shared_memory->SetName("HTTP_C:shared_memory"); } - LOG_WARNING(Service_HTTP, "(STUBBED) called, shared memory size: {} pid: {}", shmem_size, pid); + LOG_DEBUG(Service_HTTP, "shared memory size: {} pid: {}", shmem_size, pid); auto* session_data = GetSessionData(ctx.Session()); ASSERT(session_data); @@ -238,7 +320,7 @@ void HTTP_C::BeginRequest(Kernel::HLERequestContext& ctx) { IPC::RequestParser rp(ctx); const Context::Handle context_handle = rp.Pop(); - LOG_WARNING(Service_HTTP, "(STUBBED) called, context_id={}", context_handle); + LOG_DEBUG(Service_HTTP, "context_id={}", context_handle); if (!PerformStateChecks(ctx, rp, context_handle)) { return; @@ -247,6 +329,13 @@ void HTTP_C::BeginRequest(Kernel::HLERequestContext& ctx) { auto itr = contexts.find(context_handle); ASSERT(itr != contexts.end()); + // This should never happen in real hardware, but can happen on citra. + if (itr->second.uses_default_client_cert && !itr->second.clcert_data->init) { + IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); + rb.Push(ERROR_STATE_ERROR); + return; + } + // On a 3DS BeginRequest and BeginRequestAsync will push the Request to a worker queue. // You can only enqueue 8 requests at the same time. // trying to enqueue any more will either fail (BeginRequestAsync), or block (BeginRequest) @@ -266,7 +355,7 @@ void HTTP_C::BeginRequestAsync(Kernel::HLERequestContext& ctx) { IPC::RequestParser rp(ctx); const Context::Handle context_handle = rp.Pop(); - LOG_WARNING(Service_HTTP, "(STUBBED) called, context_id={}", context_handle); + LOG_WARNING(Service_HTTP, "context_id={}", context_handle); if (!PerformStateChecks(ctx, rp, context_handle)) { return; @@ -275,6 +364,13 @@ void HTTP_C::BeginRequestAsync(Kernel::HLERequestContext& ctx) { auto itr = contexts.find(context_handle); ASSERT(itr != contexts.end()); + // This should never happen in real hardware, but can happen on citra. + if (itr->second.uses_default_client_cert && !itr->second.clcert_data->init) { + IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); + rb.Push(ERROR_STATE_ERROR); + return; + } + // On a 3DS BeginRequest and BeginRequestAsync will push the Request to a worker queue. // You can only enqueue 8 requests at the same time. // trying to enqueue any more will either fail (BeginRequestAsync), or block (BeginRequest) @@ -305,9 +401,9 @@ void HTTP_C::ReceiveDataImpl(Kernel::HLERequestContext& ctx, bool timeout) { u64 timeout_nanos = 0; if (timeout) { timeout_nanos = rp.Pop(); - LOG_WARNING(Service_HTTP, "(STUBBED) called, timeout={}", timeout_nanos); + LOG_DEBUG(Service_HTTP, "timeout={}", timeout_nanos); } else { - LOG_WARNING(Service_HTTP, "(STUBBED) called"); + LOG_DEBUG(Service_HTTP, ""); } Kernel::MappedBuffer& buffer = rp.PopMappedBuffer(); @@ -344,10 +440,11 @@ void HTTP_C::ReceiveDataImpl(Kernel::HLERequestContext& ctx, bool timeout) { buffer.Write(http_context.response.body.data() + http_context.current_copied_data, 0, buffer_size); http_context.current_copied_data += buffer_size; - rb.Push(ResultCode(43, ErrorModule::HTTP, ErrorSummary::WouldBlock, ErrorLevel::Permanent)); + rb.Push(ERROR_BUFFER_SMALL); } LOG_DEBUG(Service_HTTP, "Receive: buffer_size= {}, total_copied={}, total_body={}", buffer_size, http_context.current_copied_data, http_context.response.body.size()); + // Simulate small delay from HTTP receive. ctx.SleepClientThread("http_data", std::chrono::nanoseconds(100000), nullptr); } @@ -580,6 +677,16 @@ void HTTP_C::AddPostDataRaw(Kernel::HLERequestContext& ctx) { auto itr = contexts.find(context_handle); ASSERT(itr != contexts.end()); + if (itr->second.state != RequestState::NotStarted) { + LOG_ERROR(Service_HTTP, + "Tried to add post data on a context that has already been started."); + IPC::RequestBuilder rb = rp.MakeBuilder(1, 2); + rb.Push(ResultCode(ErrCodes::InvalidRequestState, ErrorModule::HTTP, + ErrorSummary::InvalidState, ErrorLevel::Permanent)); + rb.PushMappedBuffer(buffer); + return; + } + itr->second.post_data_raw.resize(buffer.GetSize()); buffer.Read(itr->second.post_data_raw.data(), 0, buffer.GetSize()); @@ -624,7 +731,10 @@ void HTTP_C::GetResponseHeader(Kernel::HLERequestContext& ctx) { header_value.push_back('\0'); value_buffer.Write(header_value.data(), 0, header_value.size()); } else { - LOG_WARNING(Service_HTTP, "header={} not found", header_name_str); + LOG_DEBUG(Service_HTTP, "header={} not found", header_name_str); + IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); + rb.Push(ERROR_HEADER_NOT_FOUND); + return; } IPC::RequestBuilder rb = rp.MakeBuilder(2, 2); @@ -660,7 +770,8 @@ void HTTP_C::GetResponseStatusCodeImpl(Kernel::HLERequestContext& ctx, bool time ASSERT(itr != contexts.end()); if (timeout) { - auto wait_res = itr->second.request_future.wait_for(std::chrono::nanoseconds(timeout)); + auto wait_res = + itr->second.request_future.wait_for(std::chrono::nanoseconds(timeout_nanos)); if (wait_res == std::future_status::timeout) { LOG_DEBUG(Service_HTTP, "Status code: {}", "timeout"); IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); @@ -708,9 +819,7 @@ void HTTP_C::AddDefaultCert(Kernel::HLERequestContext& ctx) { void HTTP_C::SetDefaultClientCert(Kernel::HLERequestContext& ctx) { IPC::RequestParser rp(ctx); const Context::Handle context_handle = rp.Pop(); - // TODO(PabloMK7): There is only a single cert ID with value 64. Check it is valid and return - // error if not. - [[maybe_unused]] const u32 client_cert_id = rp.Pop(); + const u32 client_cert_id = rp.Pop(); LOG_DEBUG(Service_HTTP, "client_cert_id={}", client_cert_id); @@ -721,8 +830,14 @@ void HTTP_C::SetDefaultClientCert(Kernel::HLERequestContext& ctx) { auto itr = contexts.find(context_handle); ASSERT(itr != contexts.end()); + if (client_cert_id != 0x40) { + IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); + rb.Push(ERROR_WRONG_CERT_ID); + return; + } + itr->second.uses_default_client_cert = true; - itr->second.clcert_data = GetClCertA(); + itr->second.clcert_data = &GetClCertA(); IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); rb.Push(RESULT_SUCCESS); @@ -790,6 +905,17 @@ void HTTP_C::GetSSLError(Kernel::HLERequestContext& ctx) { rb.Push(0); } +void HTTP_C::SetSSLOpt(Kernel::HLERequestContext& ctx) { + IPC::RequestParser rp(ctx); + const u32 context_handle = rp.Pop(); + const u32 opts = rp.Pop(); + + LOG_WARNING(Service_HTTP, "(STUBBED) called, context_handle={}, opts={}", context_handle, opts); + + IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); + rb.Push(RESULT_SUCCESS); +} + void HTTP_C::OpenClientCertContext(Kernel::HLERequestContext& ctx) { IPC::RequestParser rp(ctx); u32 cert_size = rp.Pop(); @@ -930,6 +1056,17 @@ void HTTP_C::CloseClientCertContext(Kernel::HLERequestContext& ctx) { rb.Push(RESULT_SUCCESS); } +void HTTP_C::SetKeepAlive(Kernel::HLERequestContext& ctx) { + IPC::RequestParser rp(ctx); + const u32 context_handle = rp.Pop(); + const u32 option = rp.Pop(); + + LOG_WARNING(Service_HTTP, "(STUBBED) called, handle={}, option={}", context_handle, option); + + IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); + rb.Push(RESULT_SUCCESS); +} + void HTTP_C::Finalize(Kernel::HLERequestContext& ctx) { IPC::RequestParser rp(ctx); @@ -973,7 +1110,7 @@ void HTTP_C::GetDownloadSizeState(Kernel::HLERequestContext& ctx) { content_length); IPC::RequestBuilder rb = rp.MakeBuilder(3, 0); rb.Push(RESULT_SUCCESS); - rb.Push(itr->second.current_copied_data); + rb.Push(static_cast(itr->second.current_copied_data)); rb.Push(content_length); } @@ -1142,7 +1279,7 @@ HTTP_C::HTTP_C() : ServiceFramework("http:C", 32) { {0x0028, &HTTP_C::SetDefaultClientCert, "SetDefaultClientCert"}, {0x0029, &HTTP_C::SetClientCertContext, "SetClientCertContext"}, {0x002A, &HTTP_C::GetSSLError, "GetSSLError"}, - {0x002B, nullptr, "SetSSLOpt"}, + {0x002B, &HTTP_C::SetSSLOpt, "SetSSLOpt"}, {0x002C, nullptr, "SetSSLClearOpt"}, {0x002D, nullptr, "CreateRootCertChain"}, {0x002E, nullptr, "DestroyRootCertChain"}, @@ -1154,7 +1291,7 @@ HTTP_C::HTTP_C() : ServiceFramework("http:C", 32) { {0x0034, &HTTP_C::CloseClientCertContext, "CloseClientCertContext"}, {0x0035, nullptr, "SetDefaultProxy"}, {0x0036, nullptr, "ClearDNSCache"}, - {0x0037, nullptr, "SetKeepAlive"}, + {0x0037, &HTTP_C::SetKeepAlive, "SetKeepAlive"}, {0x0038, nullptr, "SetPostDataTypeSize"}, {0x0039, &HTTP_C::Finalize, "Finalize"}, // clang-format on diff --git a/src/core/hle/service/http/http_c.h b/src/core/hle/service/http/http_c.h index 99c0be0494..0ba5762ff6 100644 --- a/src/core/hle/service/http/http_c.h +++ b/src/core/hle/service/http/http_c.h @@ -201,7 +201,7 @@ public: SSLConfig ssl_config{}; u32 socket_buffer_size; std::vector headers; - ClCertAData clcert_data; + const ClCertAData* clcert_data; httplib::Params post_data; std::string post_data_raw; @@ -458,6 +458,8 @@ private: */ void GetSSLError(Kernel::HLERequestContext& ctx); + void SetSSLOpt(Kernel::HLERequestContext& ctx); + /** * HTTP_C::OpenClientCertContext service function * Inputs: @@ -491,6 +493,8 @@ private: */ void CloseClientCertContext(Kernel::HLERequestContext& ctx); + void SetKeepAlive(Kernel::HLERequestContext& ctx); + /** * HTTP_C::Finalize service function * Outputs: From 415d0c2abdbef594179c990429e43388b69e2edf Mon Sep 17 00:00:00 2001 From: PabloMK7 Date: Tue, 3 Oct 2023 13:31:11 +0200 Subject: [PATCH 3/3] Organize code --- src/core/hle/service/http/http_c.cpp | 375 ++++++++++++++++++--------- src/core/hle/service/http/http_c.h | 71 +++++ 2 files changed, 319 insertions(+), 127 deletions(-) diff --git a/src/core/hle/service/http/http_c.cpp b/src/core/hle/service/http/http_c.cpp index a0d1a2c15f..45861d4d5d 100644 --- a/src/core/hle/service/http/http_c.cpp +++ b/src/core/hle/service/http/http_c.cpp @@ -3,6 +3,7 @@ // Refer to the license.txt file included. #include +#include #include #include #include @@ -80,23 +81,49 @@ static ssize_t write_headers(httplib::Stream& strm, return write_len; } -static std::pair SplitUrl(const std::string& url) { +static std::tuple SplitUrl(const std::string& url) { const std::string prefix = "://"; const auto scheme_end = url.find(prefix); const auto prefix_end = scheme_end == std::string::npos ? 0 : scheme_end + prefix.length(); + bool is_https = scheme_end != std::string::npos && url.starts_with("https"); + const auto path_index = url.find("/", prefix_end); std::string host; + int port = -1; std::string path; if (path_index == std::string::npos) { // If no path is specified after the host, set it to "/" host = url.substr(prefix_end); + const auto port_start = host.find(":"); + if (port_start != std::string::npos) { + std::string port_str = host.substr(port_start + 1); + host = host.substr(0, port_start); + char* pEnd = NULL; + port = std::strtol(port_str.c_str(), &pEnd, 10); + if (*pEnd) { + port = -1; + } + } path = "/"; } else { host = url.substr(prefix_end, path_index - prefix_end); + const auto port_start = host.find(":"); + if (port_start != std::string::npos) { + std::string port_str = host.substr(port_start + 1); + host = host.substr(0, port_start); + char* pEnd = NULL; + port = std::strtol(port_str.c_str(), &pEnd, 10); + if (*pEnd) { + port = -1; + } + } path = url.substr(path_index); } - return std::make_pair(host, path); + if (port == -1) { + port = is_https ? 443 : 80; + } + return std::make_tuple(host, port, path, is_https); } void Context::MakeRequest() { @@ -109,7 +136,7 @@ void Context::MakeRequest() { {RequestMethod::PutEmpty, "PUT"}, }; - const auto& [host, path] = SplitUrl(url.c_str()); + const auto& [host, port, path, is_https] = SplitUrl(url.c_str()); httplib::Request request; httplib::Error error{-1}; @@ -193,15 +220,30 @@ void Context::MakeRequest() { state = RequestState::InProgress; // Sadly, we have to duplicate code, the class hierarchy in httplib is not very useful... - if (uses_default_client_cert) { - const unsigned char* tmpCertPtr = clcert_data->certificate.data(); - const unsigned char* tmpKeyPtr = clcert_data->private_key.data(); - X509* cert = d2i_X509(nullptr, &tmpCertPtr, (long)clcert_data->certificate.size()); - EVP_PKEY* key = d2i_PrivateKey(EVP_PKEY_RSA, nullptr, &tmpKeyPtr, - (long)clcert_data->private_key.size()); + if (is_https) { + X509* cert = nullptr; + EVP_PKEY* key = nullptr; { - std::unique_ptr client = - std::make_unique(host, 443, cert, key); + std::unique_ptr client; + if (uses_default_client_cert) { + const unsigned char* tmpCertPtr = clcert_data->certificate.data(); + const unsigned char* tmpKeyPtr = clcert_data->private_key.data(); + cert = d2i_X509(nullptr, &tmpCertPtr, (long)clcert_data->certificate.size()); + key = d2i_PrivateKey(EVP_PKEY_RSA, nullptr, &tmpKeyPtr, + (long)clcert_data->private_key.size()); + client = std::make_unique(host, port, cert, key); + } else { + if (auto client_cert = ssl_config.client_cert_ctx.lock()) { + const unsigned char* tmpCertPtr = client_cert->certificate.data(); + const unsigned char* tmpKeyPtr = client_cert->private_key.data(); + cert = d2i_X509(nullptr, &tmpCertPtr, (long)client_cert->certificate.size()); + key = d2i_PrivateKey(EVP_PKEY_RSA, nullptr, &tmpKeyPtr, + (long)client_cert->private_key.size()); + client = std::make_unique(host, port, cert, key); + } else { + client = std::make_unique(host, port); + } + } // TODO(B3N30): Check for SSLOptions-Bits and set the verify method accordingly // https://www.3dbrew.org/wiki/SSL_Services#SSLOpt @@ -219,24 +261,14 @@ void Context::MakeRequest() { state = RequestState::ReadyToDownloadContent; } } - X509_free(cert); - EVP_PKEY_free(key); - } else { - std::unique_ptr client = std::make_unique(host); - SSL_CTX* ctx = client->ssl_context(); - if (ctx) { - if (auto client_cert = ssl_config.client_cert_ctx.lock()) { - SSL_CTX_use_certificate_ASN1(ctx, static_cast(client_cert->certificate.size()), - client_cert->certificate.data()); - SSL_CTX_use_PrivateKey_ASN1(EVP_PKEY_RSA, ctx, client_cert->private_key.data(), - static_cast(client_cert->private_key.size())); - } - - // TODO(B3N30): Check for SSLOptions-Bits and set the verify method accordingly - // https://www.3dbrew.org/wiki/SSL_Services#SSLOpt - // Hack: Since for now RootCerts are not implemented we set the VerifyMode to None. - client->enable_server_certificate_verification(false); + if (cert) { + X509_free(cert); } + if (key) { + EVP_PKEY_free(key); + } + } else { + std::unique_ptr client = std::make_unique(host, port); client->set_header_writer(header_writter); @@ -396,56 +428,91 @@ void HTTP_C::ReceiveDataTimeout(Kernel::HLERequestContext& ctx) { void HTTP_C::ReceiveDataImpl(Kernel::HLERequestContext& ctx, bool timeout) { IPC::RequestParser rp(ctx); - const Context::Handle context_handle = rp.Pop(); - const u32 buffer_size = rp.Pop(); - u64 timeout_nanos = 0; + + struct AsyncData { + // Input + HTTP_C* own; + u64 timeout_nanos = 0; + bool timeout; + Context::Handle context_handle; + u32 buffer_size; + Kernel::MappedBuffer* buffer; + bool is_complete; + // Output + ResultCode async_res = RESULT_SUCCESS; + }; + std::shared_ptr async_data = std::make_shared(); + async_data->own = this; + async_data->timeout = timeout; + + async_data->context_handle = rp.Pop(); + async_data->buffer_size = rp.Pop(); + if (timeout) { - timeout_nanos = rp.Pop(); - LOG_DEBUG(Service_HTTP, "timeout={}", timeout_nanos); + async_data->timeout_nanos = rp.Pop(); + LOG_DEBUG(Service_HTTP, "timeout={}", async_data->timeout_nanos); } else { LOG_DEBUG(Service_HTTP, ""); } - Kernel::MappedBuffer& buffer = rp.PopMappedBuffer(); + async_data->buffer = &rp.PopMappedBuffer(); - if (!PerformStateChecks(ctx, rp, context_handle)) { + if (!PerformStateChecks(ctx, rp, async_data->context_handle)) { return; } - auto itr = contexts.find(context_handle); - ASSERT(itr != contexts.end()); + ctx.RunAsync( + [async_data](Kernel::HLERequestContext& ctx) { + auto itr = async_data->own->contexts.find(async_data->context_handle); + ASSERT(itr != async_data->own->contexts.end()); - Context& http_context = itr->second; - IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); + Context& http_context = itr->second; - if (timeout) { - auto wait_res = - http_context.request_future.wait_for(std::chrono::nanoseconds(timeout_nanos)); - if (wait_res == std::future_status::timeout) { - rb.Push(ResultCode(105, ErrorModule::HTTP, ErrorSummary::NothingHappened, - ErrorLevel::Permanent)); - return; - } - } else { - http_context.request_future.wait(); - } + if (async_data->timeout) { + auto wait_res = http_context.request_future.wait_for( + std::chrono::nanoseconds(async_data->timeout_nanos)); + if (wait_res == std::future_status::timeout) { + async_data->async_res = + ResultCode(105, ErrorModule::HTTP, ErrorSummary::NothingHappened, + ErrorLevel::Permanent); + } + } else { + http_context.request_future.wait(); + } + // Simulate small delay from HTTP receive. + return 1'000'000; + }, + [async_data](Kernel::HLERequestContext& ctx) { + IPC::RequestBuilder rb(ctx, static_cast(ctx.CommandHeader().command_id.Value()), 1, + 0); + if (async_data->async_res != RESULT_SUCCESS) { + rb.Push(async_data->async_res); + return; + } + auto itr = async_data->own->contexts.find(async_data->context_handle); + ASSERT(itr != async_data->own->contexts.end()); - size_t remaining_data = http_context.response.body.size() - http_context.current_copied_data; + Context& http_context = itr->second; - if (buffer_size >= remaining_data) { - buffer.Write(http_context.response.body.data() + http_context.current_copied_data, 0, - remaining_data); - http_context.current_copied_data += remaining_data; - rb.Push(RESULT_SUCCESS); - } else { - buffer.Write(http_context.response.body.data() + http_context.current_copied_data, 0, - buffer_size); - http_context.current_copied_data += buffer_size; - rb.Push(ERROR_BUFFER_SMALL); - } - LOG_DEBUG(Service_HTTP, "Receive: buffer_size= {}, total_copied={}, total_body={}", buffer_size, - http_context.current_copied_data, http_context.response.body.size()); - // Simulate small delay from HTTP receive. - ctx.SleepClientThread("http_data", std::chrono::nanoseconds(100000), nullptr); + size_t remaining_data = + http_context.response.body.size() - http_context.current_copied_data; + + if (async_data->buffer_size >= remaining_data) { + async_data->buffer->Write(http_context.response.body.data() + + http_context.current_copied_data, + 0, remaining_data); + http_context.current_copied_data += remaining_data; + rb.Push(RESULT_SUCCESS); + } else { + async_data->buffer->Write(http_context.response.body.data() + + http_context.current_copied_data, + 0, async_data->buffer_size); + http_context.current_copied_data += async_data->buffer_size; + rb.Push(ERROR_BUFFER_SMALL); + } + LOG_DEBUG(Service_HTTP, "Receive: buffer_size= {}, total_copied={}, total_body={}", + async_data->buffer_size, http_context.current_copied_data, + http_context.response.body.size()); + }); } void HTTP_C::SetProxyDefault(Kernel::HLERequestContext& ctx) { @@ -530,7 +597,7 @@ void HTTP_C::CloseContext(Kernel::HLERequestContext& ctx) { u32 context_handle = rp.Pop(); - LOG_WARNING(Service_HTTP, "(STUBBED) called, handle={}", context_handle); + LOG_DEBUG(Service_HTTP, "handle={}", context_handle); auto* session_data = EnsureSessionInitialized(ctx, rp); if (!session_data) { @@ -697,50 +764,75 @@ void HTTP_C::AddPostDataRaw(Kernel::HLERequestContext& ctx) { void HTTP_C::GetResponseHeader(Kernel::HLERequestContext& ctx) { IPC::RequestParser rp(ctx); - const u32 context_handle = rp.Pop(); - const u32 name_len = rp.Pop(); - [[maybe_unused]] const u32 value_max_len = rp.Pop(); - auto& header_name = rp.PopStaticBuffer(); - Kernel::MappedBuffer& value_buffer = rp.PopMappedBuffer(); - std::string header_name_str(reinterpret_cast(header_name.data()), name_len); - while (header_name_str.size() && header_name_str.back() == '\0') { - header_name_str.pop_back(); - } - if (!PerformStateChecks(ctx, rp, context_handle)) { + struct AsyncData { + HTTP_C* own; + u32 context_handle; + u32 name_len; + u32 value_max_len; + const std::vector* header_name; + Kernel::MappedBuffer* value_buffer; + }; + std::shared_ptr async_data = std::make_shared(); + + async_data->own = this; + async_data->context_handle = rp.Pop(); + async_data->name_len = rp.Pop(); + async_data->value_max_len = rp.Pop(); + async_data->header_name = &rp.PopStaticBuffer(); + async_data->value_buffer = &rp.PopMappedBuffer(); + + if (!PerformStateChecks(ctx, rp, async_data->context_handle)) { return; } - auto itr = contexts.find(context_handle); - ASSERT(itr != contexts.end()); + ctx.RunAsync( + [async_data](Kernel::HLERequestContext& ctx) { + auto itr = async_data->own->contexts.find(async_data->context_handle); + ASSERT(itr != async_data->own->contexts.end()); - itr->second.request_future.wait(); + itr->second.request_future.wait(); + return 0; + }, + [async_data](Kernel::HLERequestContext& ctx) { + std::string header_name_str( + reinterpret_cast(async_data->header_name->data()), + async_data->name_len); + while (header_name_str.size() && header_name_str.back() == '\0') { + header_name_str.pop_back(); + } - auto& headers = itr->second.response.headers; - u32 copied_size = 0; + auto itr = async_data->own->contexts.find(async_data->context_handle); + ASSERT(itr != async_data->own->contexts.end()); - LOG_DEBUG(Service_HTTP, "header={}, max_len={}", header_name_str, value_buffer.GetSize()); + auto& headers = itr->second.response.headers; + u32 copied_size = 0; - auto header = headers.find(header_name_str); - if (header != headers.end()) { - std::string header_value = header->second; - copied_size = (u32)header_value.size(); - if (header_value.size() + 1 > value_buffer.GetSize()) { - header_value.resize(value_buffer.GetSize() - 1); - } - header_value.push_back('\0'); - value_buffer.Write(header_value.data(), 0, header_value.size()); - } else { - LOG_DEBUG(Service_HTTP, "header={} not found", header_name_str); - IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); - rb.Push(ERROR_HEADER_NOT_FOUND); - return; - } + LOG_DEBUG(Service_HTTP, "header={}, max_len={}", header_name_str, + async_data->value_buffer->GetSize()); - IPC::RequestBuilder rb = rp.MakeBuilder(2, 2); - rb.Push(RESULT_SUCCESS); - rb.Push(copied_size); - rb.PushMappedBuffer(value_buffer); + auto header = headers.find(header_name_str); + if (header != headers.end()) { + std::string header_value = header->second; + copied_size = (u32)header_value.size(); + if (header_value.size() + 1 > async_data->value_buffer->GetSize()) { + header_value.resize(async_data->value_buffer->GetSize() - 1); + } + header_value.push_back('\0'); + async_data->value_buffer->Write(header_value.data(), 0, header_value.size()); + } else { + LOG_DEBUG(Service_HTTP, "header={} not found", header_name_str); + IPC::RequestBuilder rb(ctx, 0x001E, 1, 2); + rb.Push(ERROR_HEADER_NOT_FOUND); + rb.PushMappedBuffer(*async_data->value_buffer); + return; + } + + IPC::RequestBuilder rb(ctx, 0x001E, 2, 2); + rb.Push(RESULT_SUCCESS); + rb.Push(copied_size); + rb.PushMappedBuffer(*async_data->value_buffer); + }); } void HTTP_C::GetResponseStatusCode(Kernel::HLERequestContext& ctx) { @@ -753,42 +845,71 @@ void HTTP_C::GetResponseStatusCodeTimeout(Kernel::HLERequestContext& ctx) { void HTTP_C::GetResponseStatusCodeImpl(Kernel::HLERequestContext& ctx, bool timeout) { IPC::RequestParser rp(ctx); - const Context::Handle context_handle = rp.Pop(); - u64 timeout_nanos = 0; + + struct AsyncData { + // Input + HTTP_C* own; + Context::Handle context_handle; + bool timeout; + u64 timeout_nanos = 0; + // Output + ResultCode async_res = RESULT_SUCCESS; + }; + std::shared_ptr async_data = std::make_shared(); + + async_data->own = this; + async_data->context_handle = rp.Pop(); + async_data->timeout = timeout; + if (timeout) { - timeout_nanos = rp.Pop(); - LOG_INFO(Service_HTTP, "called, timeout={}", timeout_nanos); + async_data->timeout_nanos = rp.Pop(); + LOG_INFO(Service_HTTP, "called, timeout={}", async_data->timeout_nanos); } else { LOG_INFO(Service_HTTP, "called"); } - if (!PerformStateChecks(ctx, rp, context_handle)) { + if (!PerformStateChecks(ctx, rp, async_data->context_handle)) { return; } - auto itr = contexts.find(context_handle); - ASSERT(itr != contexts.end()); + ctx.RunAsync( + [async_data](Kernel::HLERequestContext& ctx) { + auto itr = async_data->own->contexts.find(async_data->context_handle); + ASSERT(itr != async_data->own->contexts.end()); - if (timeout) { - auto wait_res = - itr->second.request_future.wait_for(std::chrono::nanoseconds(timeout_nanos)); - if (wait_res == std::future_status::timeout) { - LOG_DEBUG(Service_HTTP, "Status code: {}", "timeout"); - IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); - rb.Push(ResultCode(105, ErrorModule::HTTP, ErrorSummary::NothingHappened, - ErrorLevel::Permanent)); - return; - } - } else { - itr->second.request_future.wait(); - } + if (async_data->timeout) { + auto wait_res = itr->second.request_future.wait_for( + std::chrono::nanoseconds(async_data->timeout_nanos)); + if (wait_res == std::future_status::timeout) { + LOG_DEBUG(Service_HTTP, "Status code: {}", "timeout"); + async_data->async_res = + ResultCode(105, ErrorModule::HTTP, ErrorSummary::NothingHappened, + ErrorLevel::Permanent); + } + } else { + itr->second.request_future.wait(); + } + return 0; + }, + [async_data](Kernel::HLERequestContext& ctx) { + if (async_data->async_res != RESULT_SUCCESS) { + IPC::RequestBuilder rb( + ctx, static_cast(ctx.CommandHeader().command_id.Value()), 1, 0); + rb.Push(async_data->async_res); + return; + } - const u32 response_code = itr->second.response.status; - LOG_DEBUG(Service_HTTP, "Status code: {}, response_code={}", "good", response_code); + auto itr = async_data->own->contexts.find(async_data->context_handle); + ASSERT(itr != async_data->own->contexts.end()); - IPC::RequestBuilder rb = rp.MakeBuilder(2, 0); - rb.Push(RESULT_SUCCESS); - rb.Push(response_code); + const u32 response_code = itr->second.response.status; + LOG_DEBUG(Service_HTTP, "Status code: {}, response_code={}", "good", response_code); + + IPC::RequestBuilder rb(ctx, static_cast(ctx.CommandHeader().command_id.Value()), 2, + 0); + rb.Push(RESULT_SUCCESS); + rb.Push(response_code); + }); } void HTTP_C::AddTrustedRootCA(Kernel::HLERequestContext& ctx) { diff --git a/src/core/hle/service/http/http_c.h b/src/core/hle/service/http/http_c.h index 0ba5762ff6..722e5e93ae 100644 --- a/src/core/hle/service/http/http_c.h +++ b/src/core/hle/service/http/http_c.h @@ -287,6 +287,13 @@ private: */ void CloseContext(Kernel::HLERequestContext& ctx); + /** + * HTTP_C::CancelConnection service function + * Inputs: + * 1 : Context handle + * Outputs: + * 1 : Result of function, 0 on success, otherwise error code + */ void CancelConnection(Kernel::HLERequestContext& ctx); /** @@ -329,6 +336,13 @@ private: */ void BeginRequestAsync(Kernel::HLERequestContext& ctx); + /** + * HTTP_C::SetProxyDefault service function + * Inputs: + * 1 : Context handle + * Outputs: + * 1 : Result of function, 0 on success, otherwise error code + */ void SetProxyDefault(Kernel::HLERequestContext& ctx); /** @@ -392,8 +406,31 @@ private: */ void AddPostDataAscii(Kernel::HLERequestContext& ctx); + /** + * HTTP_C::AddPostDataRaw service function + * Inputs: + * 1 : Context handle + * 2 : Post data length + * 3-4: (Mapped buffer) Post data + * Outputs: + * 1 : Result of function, 0 on success, otherwise error code + * 2-3: (Mapped buffer) Post data + */ void AddPostDataRaw(Kernel::HLERequestContext& ctx); + /** + * HTTP_C::GetResponseHeader service function + * Inputs: + * 1 : Context handle + * 2 : Header name length + * 3 : Return value length + * 4-5 : (Static buffer) Header name + * 6-7 : (Mapped buffer) Header value + * Outputs: + * 1 : Result of function, 0 on success, otherwise error code + * 2 : Header value copied size + * 3-4: (Mapped buffer) Header value + */ void GetResponseHeader(Kernel::HLERequestContext& ctx); /** @@ -417,6 +454,16 @@ private: */ void GetResponseStatusCodeTimeout(Kernel::HLERequestContext& ctx); + /** + * HTTP_C::AddTrustedRootCA service function + * Inputs: + * 1 : Context handle + * 2 : CA data length + * 3-4: (Mapped buffer) CA data + * Outputs: + * 1 : Result of function, 0 on success, otherwise error code + * 2-3: (Mapped buffer) CA data + */ void AddTrustedRootCA(Kernel::HLERequestContext& ctx); /** @@ -435,6 +482,14 @@ private: */ void GetResponseStatusCodeImpl(Kernel::HLERequestContext& ctx, bool timeout); + /** + * HTTP_C::SetDefaultClientCert service function + * Inputs: + * 1 : Context handle + * 2 : Client cert ID + * Outputs: + * 1 : Result of function, 0 on success, otherwise error code + */ void SetDefaultClientCert(Kernel::HLERequestContext& ctx); /** @@ -458,6 +513,14 @@ private: */ void GetSSLError(Kernel::HLERequestContext& ctx); + /** + * HTTP_C::SetSSLOpt service function + * Inputs: + * 1 : Context handle + * 2 : SSL Option + * Outputs: + * 1 : Result of function, 0 on success, otherwise error code + */ void SetSSLOpt(Kernel::HLERequestContext& ctx); /** @@ -493,6 +556,14 @@ private: */ void CloseClientCertContext(Kernel::HLERequestContext& ctx); + /** + * HTTP_C::SetKeepAlive service function + * Inputs: + * 1 : Context handle + * 2 : Keep Alive Option + * Outputs: + * 1 : Result of function, 0 on success, otherwise error code + */ void SetKeepAlive(Kernel::HLERequestContext& ctx); /**