From 897d1fa957dc110f4b21290cedad167a594ab683 Mon Sep 17 00:00:00 2001 From: PabloMK7 Date: Wed, 11 Oct 2023 19:09:16 +0200 Subject: [PATCH] Implement more HTTP:C functionality (#7035) * Implement missing http:c functionality. * More implementation details and cleanup. * Organize code * Disable treat errors as warnings for httplib * Fix defines * Remove pragmas that do nothing and mark as SYSTEM * Make httplib system * Try to fix issue from httplib * Apply suggestions * Fix header ordering * Fix compilation issue * Create and use ctx.CommandID() * Add and use Common::TruncateString * Apply more suggestions * Apply suggestions * Fix compilation * Apply suggestions * Fix format * Revert SplitURL to previous version * Apply suggestions --- externals/CMakeLists.txt | 8 +- externals/httplib/README.md | 2 +- externals/httplib/httplib.h | 1850 +++++++++++++++++++------- src/common/string_util.h | 10 + src/core/hle/service/http/http_c.cpp | 697 ++++++++-- src/core/hle/service/http/http_c.h | 164 ++- 6 files changed, 2102 insertions(+), 629 deletions(-) diff --git a/externals/CMakeLists.txt b/externals/CMakeLists.txt index 1f69d4fa70..e678facb1b 100644 --- a/externals/CMakeLists.txt +++ b/externals/CMakeLists.txt @@ -218,7 +218,7 @@ if (NOT OPENSSL_FOUND) set(LIBRESSL_SKIP_INSTALL ON CACHE BOOL "") set(OPENSSLDIR "/etc/ssl/") add_subdirectory(libressl EXCLUDE_FROM_ALL) - target_include_directories(ssl INTERFACE ./libressl/include) + target_include_directories(ssl SYSTEM INTERFACE ./libressl/include) target_compile_definitions(ssl PRIVATE -DHAVE_INET_NTOP) get_directory_property(OPENSSL_LIBRARIES DIRECTORY libressl @@ -230,13 +230,13 @@ add_library(httplib INTERFACE) if(USE_SYSTEM_CPP_HTTPLIB) find_package(CppHttp 0.14.1) if(CppHttp_FOUND) - target_link_libraries(httplib INTERFACE cpp-httplib::cpp-httplib) + target_link_libraries(httplib SYSTEM INTERFACE cpp-httplib::cpp-httplib) else() message(STATUS "Cpp-httplib not found or not suitable version! Falling back to bundled...") - target_include_directories(httplib INTERFACE ./httplib) + target_include_directories(httplib SYSTEM INTERFACE ./httplib) endif() else() - target_include_directories(httplib INTERFACE ./httplib) + target_include_directories(httplib SYSTEM INTERFACE ./httplib) endif() target_compile_options(httplib INTERFACE -DCPPHTTPLIB_OPENSSL_SUPPORT) target_link_libraries(httplib INTERFACE ${OPENSSL_LIBRARIES}) diff --git a/externals/httplib/README.md b/externals/httplib/README.md index f491f5200a..9dee85f145 100644 --- a/externals/httplib/README.md +++ b/externals/httplib/README.md @@ -1,4 +1,4 @@ -From https://github.com/yhirose/cpp-httplib/commit/8e10d4e8e7febafce0632810262e81e853b2065f +From https://github.com/yhirose/cpp-httplib/commit/0a629d739127dcc5d828474a5aedae1f234687d3 MIT License diff --git a/externals/httplib/httplib.h b/externals/httplib/httplib.h index 3657b47946..c49e50e695 100644 --- a/externals/httplib/httplib.h +++ b/externals/httplib/httplib.h @@ -1,14 +1,14 @@ // // httplib.h // -// Copyright (c) 2022 Yuji Hirose. All rights reserved. +// Copyright (c) 2023 Yuji Hirose. All rights reserved. // MIT License // #ifndef CPPHTTPLIB_HTTPLIB_H #define CPPHTTPLIB_HTTPLIB_H -#define CPPHTTPLIB_VERSION "0.11.2" +#define CPPHTTPLIB_VERSION "0.14.0" /* * Configuration @@ -172,7 +172,15 @@ using socket_t = SOCKET; #else // not _WIN32 #include +#if !defined(_AIX) && !defined(__MVS__) #include +#endif +#ifdef __MVS__ +#include +#ifndef NI_MAXHOST +#define NI_MAXHOST 1025 +#endif +#endif #include #include #include @@ -185,6 +193,7 @@ using socket_t = SOCKET; #endif #include #include +#include #include #include #include @@ -221,6 +230,9 @@ using socket_t = int; #include #include #include +#include +#include +#include #ifdef CPPHTTPLIB_OPENSSL_SUPPORT #ifdef _WIN32 @@ -243,7 +255,13 @@ using socket_t = int; #pragma comment(lib, "crypt32.lib") #pragma comment(lib, "cryptui.lib") #endif -#endif //_WIN32 +#elif defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN) && defined(__APPLE__) +#include +#if TARGET_OS_OSX +#include +#include +#endif // TARGET_OS_OSX +#endif // _WIN32 #include #include @@ -259,14 +277,10 @@ using socket_t = int; #if OPENSSL_VERSION_NUMBER < 0x1010100fL #error Sorry, OpenSSL versions prior to 1.1.1 are not supported +#elif OPENSSL_VERSION_NUMBER < 0x30000000L +#define SSL_get1_peer_certificate SSL_get_peer_certificate #endif -#if OPENSSL_VERSION_NUMBER < 0x10100000L -#include -inline const unsigned char *ASN1_STRING_get0_data(const ASN1_STRING *asn1) { - return M_ASN1_STRING_data(asn1); -} -#endif #endif #ifdef CPPHTTPLIB_ZLIB_SUPPORT @@ -316,6 +330,34 @@ struct ci { } }; +// This is based on +// "http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2014/n4189". + +struct scope_exit { + explicit scope_exit(std::function &&f) + : exit_function(std::move(f)), execute_on_destruction{true} {} + + scope_exit(scope_exit &&rhs) + : exit_function(std::move(rhs.exit_function)), + execute_on_destruction{rhs.execute_on_destruction} { + rhs.release(); + } + + ~scope_exit() { + if (execute_on_destruction) { this->exit_function(); } + } + + void release() { this->execute_on_destruction = false; } + +private: + scope_exit(const scope_exit &) = delete; + void operator=(const scope_exit &) = delete; + scope_exit &operator=(scope_exit &&) = delete; + + std::function exit_function; + bool execute_on_destruction; +}; + } // namespace detail using Headers = std::multimap; @@ -348,7 +390,7 @@ public: std::function write; std::function done; - std::function is_writable; + std::function done_with_trailer; std::ostream os; private: @@ -377,6 +419,14 @@ using ContentProviderWithoutLength = using ContentProviderResourceReleaser = std::function; +struct MultipartFormDataProvider { + std::string name; + ContentProviderWithoutLength provider; + std::string filename; + std::string content_type; +}; +using MultipartFormDataProviderItems = std::vector; + using ContentReceiverWithProgress = std::function; @@ -421,6 +471,8 @@ struct Request { std::string remote_addr; int remote_port = -1; + std::string local_addr; + int local_port = -1; // for server std::string version; @@ -429,6 +481,7 @@ struct Request { MultipartFormDataMap files; Ranges ranges; Match matches; + std::unordered_map path_params; // for client ResponseHandler response_handler; @@ -440,8 +493,7 @@ struct Request { bool has_header(const std::string &key) const; std::string get_header_value(const std::string &key, size_t id = 0) const; - template - T get_header_value(const std::string &key, size_t id = 0) const; + uint64_t get_header_value_u64(const std::string &key, size_t id = 0) const; size_t get_header_value_count(const std::string &key) const; void set_header(const std::string &key, const std::string &val); @@ -453,6 +505,7 @@ struct Request { bool has_file(const std::string &key) const; MultipartFormData get_file_value(const std::string &key) const; + std::vector get_file_values(const std::string &key) const; // private members... size_t redirect_count_ = CPPHTTPLIB_REDIRECT_MAX_COUNT; @@ -472,8 +525,7 @@ struct Response { bool has_header(const std::string &key) const; std::string get_header_value(const std::string &key, size_t id = 0) const; - template - T get_header_value(const std::string &key, size_t id = 0) const; + uint64_t get_header_value_u64(const std::string &key, size_t id = 0) const; size_t get_header_value_count(const std::string &key) const; void set_header(const std::string &key, const std::string &val); @@ -522,6 +574,7 @@ public: virtual ssize_t read(char *ptr, size_t size) = 0; virtual ssize_t write(const char *ptr, size_t size) = 0; virtual void get_remote_ip_and_port(std::string &ip, int &port) const = 0; + virtual void get_local_ip_and_port(std::string &ip, int &port) const = 0; virtual socket_t socket() const = 0; template @@ -554,8 +607,11 @@ public: ~ThreadPool() override = default; void enqueue(std::function fn) override { - std::unique_lock lock(mutex_); - jobs_.push_back(std::move(fn)); + { + std::unique_lock lock(mutex_); + jobs_.push_back(std::move(fn)); + } + cond_.notify_one(); } @@ -589,7 +645,7 @@ private: if (pool_.shutdown_ && pool_.jobs_.empty()) { break; } - fn = pool_.jobs_.front(); + fn = std::move(pool_.jobs_.front()); pool_.jobs_.pop_front(); } @@ -617,6 +673,80 @@ using SocketOptions = std::function; void default_socket_options(socket_t sock); +const char *status_message(int status); + +namespace detail { + +class MatcherBase { +public: + virtual ~MatcherBase() = default; + + // Match request path and populate its matches and + virtual bool match(Request &request) const = 0; +}; + +/** + * Captures parameters in request path and stores them in Request::path_params + * + * Capture name is a substring of a pattern from : to /. + * The rest of the pattern is matched agains the request path directly + * Parameters are captured starting from the next character after + * the end of the last matched static pattern fragment until the next /. + * + * Example pattern: + * "/path/fragments/:capture/more/fragments/:second_capture" + * Static fragments: + * "/path/fragments/", "more/fragments/" + * + * Given the following request path: + * "/path/fragments/:1/more/fragments/:2" + * the resulting capture will be + * {{"capture", "1"}, {"second_capture", "2"}} + */ +class PathParamsMatcher : public MatcherBase { +public: + PathParamsMatcher(const std::string &pattern); + + bool match(Request &request) const override; + +private: + static constexpr char marker = ':'; + // Treat segment separators as the end of path parameter capture + // Does not need to handle query parameters as they are parsed before path + // matching + static constexpr char separator = '/'; + + // Contains static path fragments to match against, excluding the '/' after + // path params + // Fragments are separated by path params + std::vector static_fragments_; + // Stores the names of the path parameters to be used as keys in the + // Request::path_params map + std::vector param_names_; +}; + +/** + * Performs std::regex_match on request path + * and stores the result in Request::matches + * + * Note that regex match is performed directly on the whole request. + * This means that wildcard patterns may match multiple path segments with /: + * "/begin/(.*)/end" will match both "/begin/middle/end" and "/begin/1/2/end". + */ +class RegexMatcher : public MatcherBase { +public: + RegexMatcher(const std::string &pattern) : regex_(pattern) {} + + bool match(Request &request) const override; + +private: + std::regex regex_; +}; + +ssize_t write_headers(Stream &strm, const Headers &headers); + +} // namespace detail + class Server { public: using Handler = std::function; @@ -661,6 +791,7 @@ public: bool remove_mount_point(const std::string &mount_point); Server &set_file_extension_and_mimetype_mapping(const std::string &ext, const std::string &mime); + Server &set_default_file_mimetype(const std::string &mime); Server &set_file_request_handler(Handler handler); Server &set_error_handler(HandlerWithResponse handler); @@ -677,6 +808,8 @@ public: Server &set_socket_options(SocketOptions socket_options); Server &set_default_headers(Headers headers); + Server & + set_header_writer(std::function const &writer); Server &set_keep_alive_max_count(size_t count); Server &set_keep_alive_timeout(time_t sec); @@ -702,6 +835,7 @@ public: bool listen(const std::string &host, int port, int socket_flags = 0); bool is_running() const; + void wait_until_ready() const; void stop(); std::function new_task_queue; @@ -711,7 +845,7 @@ protected: bool &connection_closed, const std::function &setup_request); - std::atomic svr_sock_; + std::atomic svr_sock_{INVALID_SOCKET}; size_t keep_alive_max_count_ = CPPHTTPLIB_KEEPALIVE_MAX_COUNT; time_t keep_alive_timeout_sec_ = CPPHTTPLIB_KEEPALIVE_TIMEOUT_SECOND; time_t read_timeout_sec_ = CPPHTTPLIB_READ_TIMEOUT_SECOND; @@ -723,9 +857,14 @@ protected: size_t payload_max_length_ = CPPHTTPLIB_PAYLOAD_MAX_LENGTH; private: - using Handlers = std::vector>; + using Handlers = + std::vector, Handler>>; using HandlersForContentReader = - std::vector>; + std::vector, + HandlerWithContentReader>>; + + static std::unique_ptr + make_matcher(const std::string &pattern); socket_t create_server_socket(const std::string &host, int port, int socket_flags, @@ -763,21 +902,24 @@ private: ContentReceiver multipart_receiver); bool read_content_core(Stream &strm, Request &req, Response &res, ContentReceiver receiver, - MultipartContentHeader mulitpart_header, + MultipartContentHeader multipart_header, ContentReceiver multipart_receiver); virtual bool process_and_close_socket(socket_t sock); + std::atomic is_running_{false}; + std::atomic done_{false}; + struct MountPointEntry { std::string mount_point; std::string base_dir; Headers headers; }; std::vector base_dirs_; - - std::atomic is_running_; std::map file_extension_and_mimetype_map_; + std::string default_file_mimetype_ = "application/octet-stream"; Handler file_request_handler_; + Handlers get_handlers_; Handlers post_handlers_; HandlersForContentReader post_handlers_for_content_reader_; @@ -788,18 +930,22 @@ private: Handlers delete_handlers_; HandlersForContentReader delete_handlers_for_content_reader_; Handlers options_handlers_; + HandlerWithResponse error_handler_; ExceptionHandler exception_handler_; HandlerWithResponse pre_routing_handler_; Handler post_routing_handler_; - Logger logger_; Expect100ContinueHandler expect_100_continue_handler_; + Logger logger_; + int address_family_ = AF_UNSPEC; bool tcp_nodelay_ = CPPHTTPLIB_TCP_NODELAY; SocketOptions socket_options_ = default_socket_options; Headers default_headers_; + std::function header_writer_ = + detail::write_headers; }; enum class Error { @@ -817,6 +963,10 @@ enum class Error { UnsupportedMultipartBoundaryChars, Compression, ConnectionTimeout, + ProxyConnection, + + // For internal use only + SSLPeerCouldBeClosed_, }; std::string to_string(const Error error); @@ -825,6 +975,7 @@ std::ostream &operator<<(std::ostream &os, const Error &obj); class Result { public: + Result() = default; Result(std::unique_ptr &&res, Error err, Headers &&request_headers = Headers{}) : res_(std::move(res)), err_(err), @@ -847,13 +998,13 @@ public: bool has_request_header(const std::string &key) const; std::string get_request_header_value(const std::string &key, size_t id = 0) const; - template - T get_request_header_value(const std::string &key, size_t id = 0) const; + uint64_t get_request_header_value_u64(const std::string &key, + size_t id = 0) const; size_t get_request_header_value_count(const std::string &key) const; private: std::unique_ptr res_; - Error err_; + Error err_ = Error::Unknown; Headers request_headers_; }; @@ -907,6 +1058,7 @@ public: Result Head(const std::string &path, const Headers &headers); Result Post(const std::string &path); + Result Post(const std::string &path, const Headers &headers); Result Post(const std::string &path, const char *body, size_t content_length, const std::string &content_type); Result Post(const std::string &path, const Headers &headers, const char *body, @@ -935,6 +1087,9 @@ public: const MultipartFormDataItems &items); Result Post(const std::string &path, const Headers &headers, const MultipartFormDataItems &items, const std::string &boundary); + Result Post(const std::string &path, const Headers &headers, + const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items); Result Put(const std::string &path); Result Put(const std::string &path, const char *body, size_t content_length, @@ -964,6 +1119,9 @@ public: const MultipartFormDataItems &items); Result Put(const std::string &path, const Headers &headers, const MultipartFormDataItems &items, const std::string &boundary); + Result Put(const std::string &path, const Headers &headers, + const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items); Result Patch(const std::string &path); Result Patch(const std::string &path, const char *body, size_t content_length, @@ -1006,16 +1164,21 @@ public: bool send(Request &req, Response &res, Error &error); Result send(const Request &req); - size_t is_socket_open() const; - - socket_t socket() const; - void stop(); + std::string host() const; + int port() const; + + size_t is_socket_open() const; + socket_t socket() const; + void set_hostname_addr_map(std::map addr_map); void set_default_headers(Headers headers); + void + set_header_writer(std::function const &writer); + void set_address_family(int family); void set_tcp_nodelay(bool on); void set_socket_options(SocketOptions socket_options); @@ -1064,6 +1227,7 @@ public: void set_ca_cert_path(const std::string &ca_cert_file_path, const std::string &ca_cert_dir_path = std::string()); void set_ca_cert_store(X509_STORE *ca_cert_store); + X509_STORE *create_ca_cert_store(const char *ca_cert, std::size_t size); #endif #ifdef CPPHTTPLIB_OPENSSL_SUPPORT @@ -1082,8 +1246,6 @@ protected: bool is_open() const { return sock != INVALID_SOCKET; } }; - Result send_(Request &&req); - virtual bool create_and_connect_socket(Socket &socket, Error &error); // All of: @@ -1105,7 +1267,7 @@ protected: void copy_settings(const ClientImpl &rhs); - // Socket endoint information + // Socket endpoint information const std::string host_; const int port_; const std::string host_and_port_; @@ -1126,6 +1288,10 @@ protected: // Default headers Headers default_headers_; + // Header writer + std::function header_writer_ = + detail::write_headers; + // Settings std::string client_cert_path_; std::string client_key_path_; @@ -1184,6 +1350,9 @@ protected: Logger logger_; private: + bool send_(Request &req, Response &res, Error &error); + Result send_(Request &&req); + socket_t create_client_socket(Error &error) const; bool read_response_line(Stream &strm, const Request &req, Response &res); bool write_request(Stream &strm, Request &req, bool close_connection, @@ -1202,6 +1371,9 @@ private: ContentProvider content_provider, ContentProviderWithoutLength content_provider_without_length, const std::string &content_type); + ContentProviderWithoutLength get_multipart_content_provider( + const std::string &boundary, const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items); std::string adjust_host_string(const std::string &host) const; @@ -1268,6 +1440,7 @@ public: Result Head(const std::string &path, const Headers &headers); Result Post(const std::string &path); + Result Post(const std::string &path, const Headers &headers); Result Post(const std::string &path, const char *body, size_t content_length, const std::string &content_type); Result Post(const std::string &path, const Headers &headers, const char *body, @@ -1296,6 +1469,10 @@ public: const MultipartFormDataItems &items); Result Post(const std::string &path, const Headers &headers, const MultipartFormDataItems &items, const std::string &boundary); + Result Post(const std::string &path, const Headers &headers, + const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items); + Result Put(const std::string &path); Result Put(const std::string &path, const char *body, size_t content_length, const std::string &content_type); @@ -1324,6 +1501,10 @@ public: const MultipartFormDataItems &items); Result Put(const std::string &path, const Headers &headers, const MultipartFormDataItems &items, const std::string &boundary); + Result Put(const std::string &path, const Headers &headers, + const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items); + Result Patch(const std::string &path); Result Patch(const std::string &path, const char *body, size_t content_length, const std::string &content_type); @@ -1365,16 +1546,21 @@ public: bool send(Request &req, Response &res, Error &error); Result send(const Request &req); - size_t is_socket_open() const; - - socket_t socket() const; - void stop(); + std::string host() const; + int port() const; + + size_t is_socket_open() const; + socket_t socket() const; + void set_hostname_addr_map(std::map addr_map); void set_default_headers(Headers headers); + void + set_header_writer(std::function const &writer); + void set_address_family(int family); void set_tcp_nodelay(bool on); void set_socket_options(SocketOptions socket_options); @@ -1431,6 +1617,7 @@ public: const std::string &ca_cert_dir_path = std::string()); void set_ca_cert_store(X509_STORE *ca_cert_store); + void load_ca_cert_store(const char *ca_cert, std::size_t size); long get_openssl_verify_result() const; @@ -1490,6 +1677,7 @@ public: bool is_valid() const override; void set_ca_cert_store(X509_STORE *ca_cert_store); + void load_ca_cert_store(const char *ca_cert, std::size_t size); long get_openssl_verify_result() const; @@ -1539,18 +1727,12 @@ inline void duration_to_sec_and_usec(const T &duration, U callback) { auto usec = std::chrono::duration_cast( duration - std::chrono::seconds(sec)) .count(); - callback(sec, usec); + callback(static_cast(sec), static_cast(usec)); } -template -inline T get_header_value(const Headers & /*headers*/, - const std::string & /*key*/, size_t /*id*/ = 0, - uint64_t /*def*/ = 0) {} - -template <> -inline uint64_t get_header_value(const Headers &headers, - const std::string &key, size_t id, - uint64_t def) { +inline uint64_t get_header_value_u64(const Headers &headers, + const std::string &key, size_t id, + uint64_t def) { auto rng = headers.equal_range(key); auto it = rng.first; std::advance(it, static_cast(id)); @@ -1562,14 +1744,14 @@ inline uint64_t get_header_value(const Headers &headers, } // namespace detail -template -inline T Request::get_header_value(const std::string &key, size_t id) const { - return detail::get_header_value(headers, key, id, 0); +inline uint64_t Request::get_header_value_u64(const std::string &key, + size_t id) const { + return detail::get_header_value_u64(headers, key, id, 0); } -template -inline T Response::get_header_value(const std::string &key, size_t id) const { - return detail::get_header_value(headers, key, id, 0); +inline uint64_t Response::get_header_value_u64(const std::string &key, + size_t id) const { + return detail::get_header_value_u64(headers, key, id, 0); } template @@ -1599,21 +1781,91 @@ inline ssize_t Stream::write_format(const char *fmt, const Args &...args) { inline void default_socket_options(socket_t sock) { int yes = 1; #ifdef _WIN32 - setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast(&yes), - sizeof(yes)); + setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, + reinterpret_cast(&yes), sizeof(yes)); setsockopt(sock, SOL_SOCKET, SO_EXCLUSIVEADDRUSE, - reinterpret_cast(&yes), sizeof(yes)); + reinterpret_cast(&yes), sizeof(yes)); #else #ifdef SO_REUSEPORT - setsockopt(sock, SOL_SOCKET, SO_REUSEPORT, reinterpret_cast(&yes), - sizeof(yes)); + setsockopt(sock, SOL_SOCKET, SO_REUSEPORT, + reinterpret_cast(&yes), sizeof(yes)); #else - setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast(&yes), - sizeof(yes)); + setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, + reinterpret_cast(&yes), sizeof(yes)); #endif #endif } +inline const char *status_message(int status) { + switch (status) { + case 100: return "Continue"; + case 101: return "Switching Protocol"; + case 102: return "Processing"; + case 103: return "Early Hints"; + case 200: return "OK"; + case 201: return "Created"; + case 202: return "Accepted"; + case 203: return "Non-Authoritative Information"; + case 204: return "No Content"; + case 205: return "Reset Content"; + case 206: return "Partial Content"; + case 207: return "Multi-Status"; + case 208: return "Already Reported"; + case 226: return "IM Used"; + case 300: return "Multiple Choice"; + case 301: return "Moved Permanently"; + case 302: return "Found"; + case 303: return "See Other"; + case 304: return "Not Modified"; + case 305: return "Use Proxy"; + case 306: return "unused"; + case 307: return "Temporary Redirect"; + case 308: return "Permanent Redirect"; + case 400: return "Bad Request"; + case 401: return "Unauthorized"; + case 402: return "Payment Required"; + case 403: return "Forbidden"; + case 404: return "Not Found"; + case 405: return "Method Not Allowed"; + case 406: return "Not Acceptable"; + case 407: return "Proxy Authentication Required"; + case 408: return "Request Timeout"; + case 409: return "Conflict"; + case 410: return "Gone"; + case 411: return "Length Required"; + case 412: return "Precondition Failed"; + case 413: return "Payload Too Large"; + case 414: return "URI Too Long"; + case 415: return "Unsupported Media Type"; + case 416: return "Range Not Satisfiable"; + case 417: return "Expectation Failed"; + case 418: return "I'm a teapot"; + case 421: return "Misdirected Request"; + case 422: return "Unprocessable Entity"; + case 423: return "Locked"; + case 424: return "Failed Dependency"; + case 425: return "Too Early"; + case 426: return "Upgrade Required"; + case 428: return "Precondition Required"; + case 429: return "Too Many Requests"; + case 431: return "Request Header Fields Too Large"; + case 451: return "Unavailable For Legal Reasons"; + case 501: return "Not Implemented"; + case 502: return "Bad Gateway"; + case 503: return "Service Unavailable"; + case 504: return "Gateway Timeout"; + case 505: return "HTTP Version Not Supported"; + case 506: return "Variant Also Negotiates"; + case 507: return "Insufficient Storage"; + case 508: return "Loop Detected"; + case 510: return "Not Extended"; + case 511: return "Network Authentication Required"; + + default: + case 500: return "Internal Server Error"; + } +} + template inline Server & Server::set_read_timeout(const std::chrono::duration &duration) { @@ -1640,20 +1892,21 @@ Server::set_idle_interval(const std::chrono::duration &duration) { inline std::string to_string(const Error error) { switch (error) { - case Error::Success: return "Success"; - case Error::Connection: return "Connection"; - case Error::BindIPAddress: return "BindIPAddress"; - case Error::Read: return "Read"; - case Error::Write: return "Write"; - case Error::ExceedRedirectCount: return "ExceedRedirectCount"; - case Error::Canceled: return "Canceled"; - case Error::SSLConnection: return "SSLConnection"; - case Error::SSLLoadingCerts: return "SSLLoadingCerts"; - case Error::SSLServerVerification: return "SSLServerVerification"; + case Error::Success: return "Success (no error)"; + case Error::Connection: return "Could not establish connection"; + case Error::BindIPAddress: return "Failed to bind IP address"; + case Error::Read: return "Failed to read connection"; + case Error::Write: return "Failed to write connection"; + case Error::ExceedRedirectCount: return "Maximum redirect count exceeded"; + case Error::Canceled: return "Connection handling canceled"; + case Error::SSLConnection: return "SSL connection failed"; + case Error::SSLLoadingCerts: return "SSL certificate loading failed"; + case Error::SSLServerVerification: return "SSL server verification failed"; case Error::UnsupportedMultipartBoundaryChars: - return "UnsupportedMultipartBoundaryChars"; - case Error::Compression: return "Compression"; - case Error::ConnectionTimeout: return "ConnectionTimeout"; + return "Unsupported HTTP multipart boundary characters"; + case Error::Compression: return "Compression failed"; + case Error::ConnectionTimeout: return "Connection timed out"; + case Error::ProxyConnection: return "Proxy connection failed"; case Error::Unknown: return "Unknown"; default: break; } @@ -1667,10 +1920,9 @@ inline std::ostream &operator<<(std::ostream &os, const Error &obj) { return os; } -template -inline T Result::get_request_header_value(const std::string &key, - size_t id) const { - return detail::get_header_value(request_headers_, key, id, 0); +inline uint64_t Result::get_request_header_value_u64(const std::string &key, + size_t id) const { + return detail::get_header_value_u64(request_headers_, key, id, 0); } template @@ -1763,6 +2015,9 @@ std::string params_to_query_str(const Params ¶ms); void parse_query_text(const std::string &s, Params ¶ms); +bool parse_multipart_boundary(const std::string &content_type, + std::string &boundary); + bool parse_range_header(const std::string &s, Ranges &ranges); int close_socket(socket_t sock); @@ -1785,6 +2040,7 @@ public: ssize_t read(char *ptr, size_t size) override; ssize_t write(const char *ptr, size_t size) override; void get_remote_ip_and_port(std::string &ip, int &port) const override; + void get_local_ip_and_port(std::string &ip, int &port) const override; socket_t socket() const override; const std::string &get_buffer() const; @@ -1902,6 +2158,29 @@ private: std::string glowable_buffer_; }; +class mmap { +public: + mmap(const char *path); + ~mmap(); + + bool open(const char *path); + void close(); + + bool is_open() const; + size_t size() const; + const char *data() const; + +private: +#if defined(_WIN32) + HANDLE hFile_; + HANDLE hMapping_; +#else + int fd_; +#endif + size_t size_; + void *addr_; +}; + } // namespace detail // ---------------------------------------------------------------------------- @@ -1933,7 +2212,7 @@ inline bool from_hex_to_i(const std::string &s, size_t i, size_t cnt, val = 0; for (; cnt; i++, cnt--) { if (!s[i]) { return false; } - int v = 0; + auto v = 0; if (is_hex(s[i], v)) { val = val * 16 + v; } else { @@ -1944,7 +2223,7 @@ inline bool from_hex_to_i(const std::string &s, size_t i, size_t cnt, } inline std::string from_i_to_hex(size_t n) { - const char *charset = "0123456789abcdef"; + static const auto charset = "0123456789abcdef"; std::string ret; do { ret = charset[n & 15] + ret; @@ -1994,8 +2273,8 @@ inline std::string base64_encode(const std::string &in) { std::string out; out.reserve(in.size()); - int val = 0; - int valb = -6; + auto val = 0; + auto valb = -6; for (auto c : in) { val = (val << 8) + static_cast(c); @@ -2126,7 +2405,7 @@ inline std::string decode_url(const std::string &s, for (size_t i = 0; i < s.size(); i++) { if (s[i] == '%' && i + 1 < s.size()) { if (s[i + 1] == 'u') { - int val = 0; + auto val = 0; if (from_hex_to_i(s, i + 2, 4, val)) { // 4 digits Unicode codes char buff[4]; @@ -2137,7 +2416,7 @@ inline std::string decode_url(const std::string &s, result += s[i]; } } else { - int val = 0; + auto val = 0; if (from_hex_to_i(s, i + 1, 2, val)) { // 2 digits hex codes result += static_cast(val); @@ -2190,6 +2469,13 @@ inline std::string trim_copy(const std::string &s) { return s.substr(r.first, r.second - r.first); } +inline std::string trim_double_quotes_copy(const std::string &s) { + if (s.length() >= 2 && s.front() == '"' && s.back() == '"') { + return s.substr(1, s.size() - 2); + } + return s; +} + inline void split(const char *b, const char *e, char d, std::function fn) { size_t i = 0; @@ -2275,6 +2561,95 @@ inline void stream_line_reader::append(char c) { } } +inline mmap::mmap(const char *path) +#if defined(_WIN32) + : hFile_(NULL), hMapping_(NULL) +#else + : fd_(-1) +#endif + , + size_(0), addr_(nullptr) { + if (!open(path)) { std::runtime_error(""); } +} + +inline mmap::~mmap() { close(); } + +inline bool mmap::open(const char *path) { + close(); + +#if defined(_WIN32) + hFile_ = ::CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, NULL, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + + if (hFile_ == INVALID_HANDLE_VALUE) { return false; } + + size_ = ::GetFileSize(hFile_, NULL); + + hMapping_ = ::CreateFileMapping(hFile_, NULL, PAGE_READONLY, 0, 0, NULL); + + if (hMapping_ == NULL) { + close(); + return false; + } + + addr_ = ::MapViewOfFile(hMapping_, FILE_MAP_READ, 0, 0, 0); +#else + fd_ = ::open(path, O_RDONLY); + if (fd_ == -1) { return false; } + + struct stat sb; + if (fstat(fd_, &sb) == -1) { + close(); + return false; + } + size_ = static_cast(sb.st_size); + + addr_ = ::mmap(NULL, size_, PROT_READ, MAP_PRIVATE, fd_, 0); +#endif + + if (addr_ == nullptr) { + close(); + return false; + } + + return true; +} + +inline bool mmap::is_open() const { return addr_ != nullptr; } + +inline size_t mmap::size() const { return size_; } + +inline const char *mmap::data() const { return (const char *)addr_; } + +inline void mmap::close() { +#if defined(_WIN32) + if (addr_) { + ::UnmapViewOfFile(addr_); + addr_ = nullptr; + } + + if (hMapping_) { + ::CloseHandle(hMapping_); + hMapping_ = NULL; + } + + if (hFile_ != INVALID_HANDLE_VALUE) { + ::CloseHandle(hFile_); + hFile_ = INVALID_HANDLE_VALUE; + } +#else + if (addr_ != nullptr) { + munmap(addr_, size_); + addr_ = nullptr; + } + + if (fd_ != -1) { + ::close(fd_); + fd_ = -1; + } +#endif + size_ = 0; +} inline int close_socket(socket_t sock) { #ifdef _WIN32 return closesocket(sock); @@ -2284,7 +2659,7 @@ inline int close_socket(socket_t sock) { } template inline ssize_t handle_EINTR(T fn) { - ssize_t res = false; + ssize_t res = 0; while (true) { res = fn(); if (res < 0 && errno == EINTR) { continue; } @@ -2388,7 +2763,7 @@ inline Error wait_until_socket_is_ready(socket_t sock, time_t sec, if (poll_res == 0) { return Error::ConnectionTimeout; } if (poll_res > 0 && pfd_read.revents & (POLLIN | POLLOUT)) { - int error = 0; + auto error = 0; socklen_t len = sizeof(error); auto res = getsockopt(sock, SOL_SOCKET, SO_ERROR, reinterpret_cast(&error), &len); @@ -2420,7 +2795,7 @@ inline Error wait_until_socket_is_ready(socket_t sock, time_t sec, if (ret == 0) { return Error::ConnectionTimeout; } if (ret > 0 && (FD_ISSET(sock, &fdsr) || FD_ISSET(sock, &fdsw))) { - int error = 0; + auto error = 0; socklen_t len = sizeof(error); auto res = getsockopt(sock, SOL_SOCKET, SO_ERROR, reinterpret_cast(&error), &len); @@ -2453,6 +2828,7 @@ public: ssize_t read(char *ptr, size_t size) override; ssize_t write(const char *ptr, size_t size) override; void get_remote_ip_and_port(std::string &ip, int &port) const override; + void get_local_ip_and_port(std::string &ip, int &port) const override; socket_t socket() const override; private: @@ -2482,6 +2858,7 @@ public: ssize_t read(char *ptr, size_t size) override; ssize_t write(const char *ptr, size_t size) override; void get_remote_ip_and_port(std::string &ip, int &port) const override; + void get_local_ip_and_port(std::string &ip, int &port) const override; socket_t socket() const override; private: @@ -2598,7 +2975,7 @@ socket_t create_socket(const std::string &host, const std::string &ip, int port, auto sock = socket(hints.ai_family, hints.ai_socktype, hints.ai_protocol); if (sock != INVALID_SOCKET) { - sockaddr_un addr; + sockaddr_un addr{}; addr.sun_family = AF_UNIX; std::copy(host.begin(), host.end(), addr.sun_path); @@ -2606,6 +2983,9 @@ socket_t create_socket(const std::string &host, const std::string &ip, int port, hints.ai_addrlen = static_cast( sizeof(addr) - sizeof(addr.sun_path) + addrlen); + fcntl(sock, F_SETFD, FD_CLOEXEC); + if (socket_options) { socket_options(sock); } + if (!bind_or_connect(sock, hints)) { close_socket(sock); sock = INVALID_SOCKET; @@ -2653,21 +3033,34 @@ socket_t create_socket(const std::string &host, const std::string &ip, int port, if (sock == INVALID_SOCKET) { continue; } #ifndef _WIN32 - if (fcntl(sock, F_SETFD, FD_CLOEXEC) == -1) { continue; } + if (fcntl(sock, F_SETFD, FD_CLOEXEC) == -1) { + close_socket(sock); + continue; + } #endif if (tcp_nodelay) { - int yes = 1; - setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, reinterpret_cast(&yes), - sizeof(yes)); + auto yes = 1; +#ifdef _WIN32 + setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, + reinterpret_cast(&yes), sizeof(yes)); +#else + setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, + reinterpret_cast(&yes), sizeof(yes)); +#endif } if (socket_options) { socket_options(sock); } if (rp->ai_family == AF_INET6) { - int no = 0; - setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, reinterpret_cast(&no), - sizeof(no)); + auto no = 0; +#ifdef _WIN32 + setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, + reinterpret_cast(&no), sizeof(no)); +#else + setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, + reinterpret_cast(&no), sizeof(no)); +#endif } // bind or connect @@ -2726,7 +3119,7 @@ inline bool bind_ip_address(socket_t sock, const std::string &host) { return ret; } -#if !defined _WIN32 && !defined ANDROID +#if !defined _WIN32 && !defined ANDROID && !defined _AIX && !defined __MVS__ #define USE_IF2IP #endif @@ -2810,13 +3203,14 @@ inline socket_t create_client_socket( #ifdef _WIN32 auto timeout = static_cast(read_timeout_sec * 1000 + read_timeout_usec / 1000); - setsockopt(sock2, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout, - sizeof(timeout)); + setsockopt(sock2, SOL_SOCKET, SO_RCVTIMEO, + reinterpret_cast(&timeout), sizeof(timeout)); #else timeval tv; tv.tv_sec = static_cast(read_timeout_sec); tv.tv_usec = static_cast(read_timeout_usec); - setsockopt(sock2, SOL_SOCKET, SO_RCVTIMEO, (char *)&tv, sizeof(tv)); + setsockopt(sock2, SOL_SOCKET, SO_RCVTIMEO, + reinterpret_cast(&tv), sizeof(tv)); #endif } { @@ -2824,13 +3218,14 @@ inline socket_t create_client_socket( #ifdef _WIN32 auto timeout = static_cast(write_timeout_sec * 1000 + write_timeout_usec / 1000); - setsockopt(sock2, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout, - sizeof(timeout)); + setsockopt(sock2, SOL_SOCKET, SO_SNDTIMEO, + reinterpret_cast(&timeout), sizeof(timeout)); #else timeval tv; tv.tv_sec = static_cast(write_timeout_sec); tv.tv_usec = static_cast(write_timeout_usec); - setsockopt(sock2, SOL_SOCKET, SO_SNDTIMEO, (char *)&tv, sizeof(tv)); + setsockopt(sock2, SOL_SOCKET, SO_SNDTIMEO, + reinterpret_cast(&tv), sizeof(tv)); #endif } @@ -2847,9 +3242,8 @@ inline socket_t create_client_socket( return sock; } -inline bool get_remote_ip_and_port(const struct sockaddr_storage &addr, - socklen_t addr_len, std::string &ip, - int &port) { +inline bool get_ip_and_port(const struct sockaddr_storage &addr, + socklen_t addr_len, std::string &ip, int &port) { if (addr.ss_family == AF_INET) { port = ntohs(reinterpret_cast(&addr)->sin_port); } else if (addr.ss_family == AF_INET6) { @@ -2870,21 +3264,53 @@ inline bool get_remote_ip_and_port(const struct sockaddr_storage &addr, return true; } +inline void get_local_ip_and_port(socket_t sock, std::string &ip, int &port) { + struct sockaddr_storage addr; + socklen_t addr_len = sizeof(addr); + if (!getsockname(sock, reinterpret_cast(&addr), + &addr_len)) { + get_ip_and_port(addr, addr_len, ip, port); + } +} + inline void get_remote_ip_and_port(socket_t sock, std::string &ip, int &port) { struct sockaddr_storage addr; socklen_t addr_len = sizeof(addr); if (!getpeername(sock, reinterpret_cast(&addr), &addr_len)) { - get_remote_ip_and_port(addr, addr_len, ip, port); +#ifndef _WIN32 + if (addr.ss_family == AF_UNIX) { +#if defined(__linux__) + struct ucred ucred; + socklen_t len = sizeof(ucred); + if (getsockopt(sock, SOL_SOCKET, SO_PEERCRED, &ucred, &len) == 0) { + port = ucred.pid; + } +#elif defined(SOL_LOCAL) && defined(SO_PEERPID) // __APPLE__ + pid_t pid; + socklen_t len = sizeof(pid); + if (getsockopt(sock, SOL_LOCAL, SO_PEERPID, &pid, &len) == 0) { + port = pid; + } +#endif + return; + } +#endif + get_ip_and_port(addr, addr_len, ip, port); } } inline constexpr unsigned int str2tag_core(const char *s, size_t l, unsigned int h) { - return (l == 0) ? h - : str2tag_core(s + 1, l - 1, - (h * 33) ^ static_cast(*s)); + return (l == 0) + ? h + : str2tag_core( + s + 1, l - 1, + // Unsets the 6 high bits of h, therefore no overflow happens + (((std::numeric_limits::max)() >> 6) & + h * 33) ^ + static_cast(*s)); } inline unsigned int str2tag(const std::string &s) { @@ -2899,9 +3325,10 @@ inline constexpr unsigned int operator"" _t(const char *s, size_t l) { } // namespace udl -inline const char * +inline std::string find_content_type(const std::string &path, - const std::map &user_data) { + const std::map &user_data, + const std::string &default_content_type) { auto ext = file_extension(path); auto it = user_data.find(ext); @@ -2910,7 +3337,8 @@ find_content_type(const std::string &path, using udl::operator""_t; switch (str2tag(ext)) { - default: return nullptr; + default: return default_content_type; + case "css"_t: return "text/css"; case "csv"_t: return "text/csv"; case "htm"_t: @@ -2963,76 +3391,6 @@ find_content_type(const std::string &path, } } -inline const char *status_message(int status) { - switch (status) { - case 100: return "Continue"; - case 101: return "Switching Protocol"; - case 102: return "Processing"; - case 103: return "Early Hints"; - case 200: return "OK"; - case 201: return "Created"; - case 202: return "Accepted"; - case 203: return "Non-Authoritative Information"; - case 204: return "No Content"; - case 205: return "Reset Content"; - case 206: return "Partial Content"; - case 207: return "Multi-Status"; - case 208: return "Already Reported"; - case 226: return "IM Used"; - case 300: return "Multiple Choice"; - case 301: return "Moved Permanently"; - case 302: return "Found"; - case 303: return "See Other"; - case 304: return "Not Modified"; - case 305: return "Use Proxy"; - case 306: return "unused"; - case 307: return "Temporary Redirect"; - case 308: return "Permanent Redirect"; - case 400: return "Bad Request"; - case 401: return "Unauthorized"; - case 402: return "Payment Required"; - case 403: return "Forbidden"; - case 404: return "Not Found"; - case 405: return "Method Not Allowed"; - case 406: return "Not Acceptable"; - case 407: return "Proxy Authentication Required"; - case 408: return "Request Timeout"; - case 409: return "Conflict"; - case 410: return "Gone"; - case 411: return "Length Required"; - case 412: return "Precondition Failed"; - case 413: return "Payload Too Large"; - case 414: return "URI Too Long"; - case 415: return "Unsupported Media Type"; - case 416: return "Range Not Satisfiable"; - case 417: return "Expectation Failed"; - case 418: return "I'm a teapot"; - case 421: return "Misdirected Request"; - case 422: return "Unprocessable Entity"; - case 423: return "Locked"; - case 424: return "Failed Dependency"; - case 425: return "Too Early"; - case 426: return "Upgrade Required"; - case 428: return "Precondition Required"; - case 429: return "Too Many Requests"; - case 431: return "Request Header Fields Too Large"; - case 451: return "Unavailable For Legal Reasons"; - case 501: return "Not Implemented"; - case 502: return "Bad Gateway"; - case 503: return "Service Unavailable"; - case 504: return "Gateway Timeout"; - case 505: return "HTTP Version Not Supported"; - case 506: return "Variant Also Negotiates"; - case 507: return "Insufficient Storage"; - case 508: return "Loop Detected"; - case 510: return "Not Extended"; - case 511: return "Network Authentication Required"; - - default: - case 500: return "Internal Server Error"; - } -} - inline bool can_compress_content_type(const std::string &content_type) { using udl::operator""_t; @@ -3109,7 +3467,7 @@ inline bool gzip_compressor::compress(const char *data, size_t data_length, data += strm_.avail_in; auto flush = (last && data_length == 0) ? Z_FINISH : Z_NO_FLUSH; - int ret = Z_OK; + auto ret = Z_OK; std::array buff{}; do { @@ -3153,7 +3511,7 @@ inline bool gzip_decompressor::decompress(const char *data, size_t data_length, Callback callback) { assert(is_valid_); - int ret = Z_OK; + auto ret = Z_OK; do { constexpr size_t max_avail_in = @@ -3167,16 +3525,12 @@ inline bool gzip_decompressor::decompress(const char *data, size_t data_length, data += strm_.avail_in; std::array buff{}; - while (strm_.avail_in > 0) { + while (strm_.avail_in > 0 && ret == Z_OK) { strm_.avail_out = static_cast(buff.size()); strm_.next_out = reinterpret_cast(buff.data()); - auto prev_avail_in = strm_.avail_in; - ret = inflate(&strm_, Z_NO_FLUSH); - if (prev_avail_in - strm_.avail_in == 0) { return false; } - assert(ret != Z_STREAM_ERROR); switch (ret) { case Z_NEED_DICT: @@ -3258,7 +3612,7 @@ inline bool brotli_decompressor::decompress(const char *data, return 0; } - const uint8_t *next_in = (const uint8_t *)data; + auto next_in = reinterpret_cast(data); size_t avail_in = data_length; size_t total_out; @@ -3297,6 +3651,14 @@ inline const char *get_header_value(const Headers &headers, return def; } +inline bool compare_case_ignore(const std::string &a, const std::string &b) { + if (a.size() != b.size()) { return false; } + for (size_t i = 0; i < b.size(); i++) { + if (::tolower(a[i]) != ::tolower(b[i])) { return false; } + } + return true; +} + template inline bool parse_header(const char *beg, const char *end, T fn) { // Skip trailing spaces and tabs. @@ -3320,7 +3682,11 @@ inline bool parse_header(const char *beg, const char *end, T fn) { } if (p < end) { - fn(std::string(beg, key_end), decode_url(std::string(p, end), false)); + auto key = std::string(beg, key_end); + auto val = compare_case_ignore(key, "Location") + ? std::string(p, end) + : decode_url(std::string(p, end), false); + fn(std::move(key), std::move(val)); return true; } @@ -3418,7 +3784,8 @@ inline bool read_content_without_length(Stream &strm, return true; } -inline bool read_content_chunked(Stream &strm, +template +inline bool read_content_chunked(Stream &strm, T &x, ContentReceiverWithProgress out) { const auto bufsiz = 16; char buf[bufsiz]; @@ -3444,15 +3811,29 @@ inline bool read_content_chunked(Stream &strm, if (!line_reader.getline()) { return false; } - if (strcmp(line_reader.ptr(), "\r\n")) { break; } + if (strcmp(line_reader.ptr(), "\r\n")) { return false; } if (!line_reader.getline()) { return false; } } - if (chunk_len == 0) { - // Reader terminator after chunks - if (!line_reader.getline() || strcmp(line_reader.ptr(), "\r\n")) - return false; + assert(chunk_len == 0); + + // Trailer + if (!line_reader.getline()) { return false; } + + while (strcmp(line_reader.ptr(), "\r\n")) { + if (line_reader.size() > CPPHTTPLIB_HEADER_MAX_LENGTH) { return false; } + + // Exclude line terminator + constexpr auto line_terminator_len = 2; + auto end = line_reader.ptr() + line_reader.size() - line_terminator_len; + + parse_header(line_reader.ptr(), end, + [&](std::string &&key, std::string &&val) { + x.headers.emplace(std::move(key), std::move(val)); + }); + + if (!line_reader.getline()) { return false; } } return true; @@ -3522,11 +3903,11 @@ bool read_content(Stream &strm, T &x, size_t payload_max_length, int &status, auto exceed_payload_max_length = false; if (is_chunked_transfer_encoding(x.headers)) { - ret = read_content_chunked(strm, out); + ret = read_content_chunked(strm, x, out); } else if (!has_header(x.headers, "Content-Length")) { ret = read_content_without_length(strm, out); } else { - auto len = get_header_value(x.headers, "Content-Length"); + auto len = get_header_value_u64(x.headers, "Content-Length", 0, 0); if (len > payload_max_length) { exceed_payload_max_length = true; skip_content_with_length(strm, len); @@ -3575,7 +3956,7 @@ inline bool write_content(Stream &strm, const ContentProvider &content_provider, data_sink.write = [&](const char *d, size_t l) -> bool { if (ok) { - if (write_data(strm, d, l)) { + if (strm.is_writable() && write_data(strm, d, l)) { offset += l; } else { ok = false; @@ -3584,14 +3965,14 @@ inline bool write_content(Stream &strm, const ContentProvider &content_provider, return ok; }; - data_sink.is_writable = [&](void) { return ok && strm.is_writable(); }; - while (offset < end_offset && !is_shutting_down()) { - if (!content_provider(offset, end_offset - offset, data_sink)) { + if (!strm.is_writable()) { + error = Error::Write; + return false; + } else if (!content_provider(offset, end_offset - offset, data_sink)) { error = Error::Canceled; return false; - } - if (!ok) { + } else if (!ok) { error = Error::Write; return false; } @@ -3623,18 +4004,21 @@ write_content_without_length(Stream &strm, data_sink.write = [&](const char *d, size_t l) -> bool { if (ok) { offset += l; - if (!write_data(strm, d, l)) { ok = false; } + if (!strm.is_writable() || !write_data(strm, d, l)) { ok = false; } } return ok; }; data_sink.done = [&](void) { data_available = false; }; - data_sink.is_writable = [&](void) { return ok && strm.is_writable(); }; - while (data_available && !is_shutting_down()) { - if (!content_provider(offset, 0, data_sink)) { return false; } - if (!ok) { return false; } + if (!strm.is_writable()) { + return false; + } else if (!content_provider(offset, 0, data_sink)) { + return false; + } else if (!ok) { + return false; + } } return true; } @@ -3663,7 +4047,10 @@ write_content_chunked(Stream &strm, const ContentProvider &content_provider, // Emit chunked response header and footer for each chunk auto chunk = from_i_to_hex(payload.size()) + "\r\n" + payload + "\r\n"; - if (!write_data(strm, chunk.data(), chunk.size())) { ok = false; } + if (!strm.is_writable() || + !write_data(strm, chunk.data(), chunk.size())) { + ok = false; + } } } else { ok = false; @@ -3672,7 +4059,7 @@ write_content_chunked(Stream &strm, const ContentProvider &content_provider, return ok; }; - data_sink.done = [&](void) { + auto done_with_trailer = [&](const Headers *trailer) { if (!ok) { return; } data_available = false; @@ -3690,26 +4077,46 @@ write_content_chunked(Stream &strm, const ContentProvider &content_provider, if (!payload.empty()) { // Emit chunked response header and footer for each chunk auto chunk = from_i_to_hex(payload.size()) + "\r\n" + payload + "\r\n"; - if (!write_data(strm, chunk.data(), chunk.size())) { + if (!strm.is_writable() || + !write_data(strm, chunk.data(), chunk.size())) { ok = false; return; } } - static const std::string done_marker("0\r\n\r\n"); + static const std::string done_marker("0\r\n"); if (!write_data(strm, done_marker.data(), done_marker.size())) { ok = false; } + + // Trailer + if (trailer) { + for (const auto &kv : *trailer) { + std::string field_line = kv.first + ": " + kv.second + "\r\n"; + if (!write_data(strm, field_line.data(), field_line.size())) { + ok = false; + } + } + } + + static const std::string crlf("\r\n"); + if (!write_data(strm, crlf.data(), crlf.size())) { ok = false; } }; - data_sink.is_writable = [&](void) { return ok && strm.is_writable(); }; + data_sink.done = [&](void) { done_with_trailer(nullptr); }; + + data_sink.done_with_trailer = [&](const Headers &trailer) { + done_with_trailer(&trailer); + }; while (data_available && !is_shutting_down()) { - if (!content_provider(offset, 0, data_sink)) { + if (!strm.is_writable()) { + error = Error::Write; + return false; + } else if (!content_provider(offset, 0, data_sink)) { error = Error::Canceled; return false; - } - if (!ok) { + } else if (!ok) { error = Error::Write; return false; } @@ -3748,7 +4155,8 @@ inline bool redirect(T &cli, Request &req, Response &res, if (ret) { req = new_req; res = new_res; - res.location = location; + + if (res.location.empty()) res.location = location; } return ret; } @@ -3790,16 +4198,39 @@ inline void parse_query_text(const std::string &s, Params ¶ms) { inline bool parse_multipart_boundary(const std::string &content_type, std::string &boundary) { - auto pos = content_type.find("boundary="); + auto boundary_keyword = "boundary="; + auto pos = content_type.find(boundary_keyword); if (pos == std::string::npos) { return false; } - boundary = content_type.substr(pos + 9); - if (boundary.length() >= 2 && boundary.front() == '"' && - boundary.back() == '"') { - boundary = boundary.substr(1, boundary.size() - 2); - } + auto end = content_type.find(';', pos); + auto beg = pos + strlen(boundary_keyword); + boundary = trim_double_quotes_copy(content_type.substr(beg, end - beg)); return !boundary.empty(); } +inline void parse_disposition_params(const std::string &s, Params ¶ms) { + std::set cache; + split(s.data(), s.data() + s.size(), ';', [&](const char *b, const char *e) { + std::string kv(b, e); + if (cache.find(kv) != cache.end()) { return; } + cache.insert(kv); + + std::string key; + std::string val; + split(b, e, '=', [&](const char *b2, const char *e2) { + if (key.empty()) { + key.assign(b2, e2); + } else { + val.assign(b2, e2); + } + }); + + if (!key.empty()) { + params.emplace(trim_double_quotes_copy((key)), + trim_double_quotes_copy((val))); + } + }); +} + #ifdef CPPHTTPLIB_NO_EXCEPTIONS inline bool parse_range_header(const std::string &s, Ranges &ranges) { #else @@ -3810,7 +4241,7 @@ inline bool parse_range_header(const std::string &s, Ranges &ranges) try { if (std::regex_match(s, m, re_first_range)) { auto pos = static_cast(m.position(1)); auto len = static_cast(m.length(1)); - bool all_valid_ranges = true; + auto all_valid_ranges = true; split(&s[pos], &s[pos + len], ',', [&](const char *b, const char *e) { if (!all_valid_ranges) return; static auto re_another_range = std::regex(R"(\s*(\d*)-(\d*))"); @@ -3857,11 +4288,6 @@ public: bool parse(const char *buf, size_t n, const ContentReceiver &content_callback, const MultipartContentHeader &header_callback) { - // TODO: support 'filename*' - static const std::regex re_content_disposition( - R"~(^Content-Disposition:\s*form-data;\s*name="(.*?)"(?:;\s*filename="(.*?)")?(?:;\s*filename\*=\S+)?\s*$)~", - std::regex_constants::icase); - buf_append(buf, n); while (buf_size() > 0) { @@ -3899,10 +4325,43 @@ public: if (start_with_case_ignore(header, header_name)) { file_.content_type = trim_copy(header.substr(header_name.size())); } else { + static const std::regex re_content_disposition( + R"~(^Content-Disposition:\s*form-data;\s*(.*)$)~", + std::regex_constants::icase); + std::smatch m; if (std::regex_match(header, m, re_content_disposition)) { - file_.name = m[1]; - file_.filename = m[2]; + Params params; + parse_disposition_params(m[1], params); + + auto it = params.find("name"); + if (it != params.end()) { + file_.name = it->second; + } else { + is_valid_ = false; + return false; + } + + it = params.find("filename"); + if (it != params.end()) { file_.filename = it->second; } + + it = params.find("filename*"); + if (it != params.end()) { + // Only allow UTF-8 enconnding... + static const std::regex re_rfc5987_encoding( + R"~(^UTF-8''(.+?)$)~", std::regex_constants::icase); + + std::smatch m2; + if (std::regex_match(it->second, m2, re_rfc5987_encoding)) { + file_.filename = decode_url(m2[1], false); // override... + } else { + is_valid_ = false; + return false; + } + } + } else { + is_valid_ = false; + return false; } } buf_erase(pos + crlf_.size()); @@ -3940,9 +4399,9 @@ public: buf_erase(crlf_.size()); state_ = 1; } else { - if (dash_crlf_.size() > buf_size()) { return true; } - if (buf_start_with(dash_crlf_)) { - buf_erase(dash_crlf_.size()); + if (dash_.size() > buf_size()) { return true; } + if (buf_start_with(dash_)) { + buf_erase(dash_.size()); is_valid_ = true; buf_erase(buf_size()); // Remove epilogue } else { @@ -3975,7 +4434,6 @@ private: const std::string dash_ = "--"; const std::string crlf_ = "\r\n"; - const std::string dash_crlf_ = "--\r\n"; std::string boundary_; std::string dash_boundary_crlf_; std::string crlf_dash_boundary_; @@ -4096,29 +4554,48 @@ inline bool is_multipart_boundary_chars_valid(const std::string &boundary) { return valid; } +template +inline std::string +serialize_multipart_formdata_item_begin(const T &item, + const std::string &boundary) { + std::string body = "--" + boundary + "\r\n"; + body += "Content-Disposition: form-data; name=\"" + item.name + "\""; + if (!item.filename.empty()) { + body += "; filename=\"" + item.filename + "\""; + } + body += "\r\n"; + if (!item.content_type.empty()) { + body += "Content-Type: " + item.content_type + "\r\n"; + } + body += "\r\n"; + + return body; +} + +inline std::string serialize_multipart_formdata_item_end() { return "\r\n"; } + +inline std::string +serialize_multipart_formdata_finish(const std::string &boundary) { + return "--" + boundary + "--\r\n"; +} + +inline std::string +serialize_multipart_formdata_get_content_type(const std::string &boundary) { + return "multipart/form-data; boundary=" + boundary; +} + inline std::string serialize_multipart_formdata(const MultipartFormDataItems &items, - const std::string &boundary, - std::string &content_type) { + const std::string &boundary, bool finish = true) { std::string body; for (const auto &item : items) { - body += "--" + boundary + "\r\n"; - body += "Content-Disposition: form-data; name=\"" + item.name + "\""; - if (!item.filename.empty()) { - body += "; filename=\"" + item.filename + "\""; - } - body += "\r\n"; - if (!item.content_type.empty()) { - body += "Content-Type: " + item.content_type + "\r\n"; - } - body += "\r\n"; - body += item.content + "\r\n"; + body += serialize_multipart_formdata_item_begin(item, boundary); + body += item.content + serialize_multipart_formdata_item_end(); } - body += "--" + boundary + "--\r\n"; + if (finish) body += serialize_multipart_formdata_finish(boundary); - content_type = "multipart/form-data; boundary=" + boundary; return body; } @@ -4142,12 +4619,13 @@ get_range_offset_and_length(const Request &req, size_t content_length, return std::make_pair(r.first, static_cast(r.second - r.first) + 1); } -inline std::string make_content_range_header_field(size_t offset, size_t length, - size_t content_length) { +inline std::string +make_content_range_header_field(const std::pair &range, + size_t content_length) { std::string field = "bytes "; - field += std::to_string(offset); + if (range.first != -1) { field += std::to_string(range.first); } field += "-"; - field += std::to_string(offset + length - 1); + if (range.second != -1) { field += std::to_string(range.second); } field += "/"; field += std::to_string(content_length); return field; @@ -4169,21 +4647,22 @@ bool process_multipart_ranges_data(const Request &req, Response &res, ctoken("\r\n"); } - auto offsets = get_range_offset_and_length(req, res.body.size(), i); + ctoken("Content-Range: "); + const auto &range = req.ranges[i]; + stoken(make_content_range_header_field(range, res.content_length_)); + ctoken("\r\n"); + ctoken("\r\n"); + + auto offsets = get_range_offset_and_length(req, res.content_length_, i); auto offset = offsets.first; auto length = offsets.second; - - ctoken("Content-Range: "); - stoken(make_content_range_header_field(offset, length, res.body.size())); - ctoken("\r\n"); - ctoken("\r\n"); if (!content(offset, length)) { return false; } ctoken("\r\n"); } ctoken("--"); stoken(boundary); - ctoken("--\r\n"); + ctoken("--"); return true; } @@ -4284,7 +4763,7 @@ inline std::string message_digest(const std::string &s, const EVP_MD *algo) { std::stringstream ss; for (auto i = 0u; i < hash_length; ++i) { ss << std::hex << std::setw(2) << std::setfill('0') - << (unsigned int)hash[i]; + << static_cast(hash[i]); } return ss.str(); @@ -4303,15 +4782,15 @@ inline std::string SHA_512(const std::string &s) { } #endif -#ifdef _WIN32 #ifdef CPPHTTPLIB_OPENSSL_SUPPORT +#ifdef _WIN32 // NOTE: This code came up with the following stackoverflow post: // https://stackoverflow.com/questions/9507184/can-openssl-on-windows-use-the-system-certificate-store inline bool load_system_certs_on_windows(X509_STORE *store) { auto hStore = CertOpenSystemStoreW((HCRYPTPROV_LEGACY)NULL, L"ROOT"); - if (!hStore) { return false; } + auto result = false; PCCERT_CONTEXT pContext = NULL; while ((pContext = CertEnumCertificatesInStore(hStore, pContext)) != nullptr) { @@ -4322,16 +4801,109 @@ inline bool load_system_certs_on_windows(X509_STORE *store) { if (x509) { X509_STORE_add_cert(store, x509); X509_free(x509); + result = true; } } CertFreeCertificateContext(pContext); CertCloseStore(hStore, 0); + return result; +} +#elif defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN) && defined(__APPLE__) +#if TARGET_OS_OSX +template +using CFObjectPtr = + std::unique_ptr::type, void (*)(CFTypeRef)>; + +inline void cf_object_ptr_deleter(CFTypeRef obj) { + if (obj) { CFRelease(obj); } +} + +inline bool retrieve_certs_from_keychain(CFObjectPtr &certs) { + CFStringRef keys[] = {kSecClass, kSecMatchLimit, kSecReturnRef}; + CFTypeRef values[] = {kSecClassCertificate, kSecMatchLimitAll, + kCFBooleanTrue}; + + CFObjectPtr query( + CFDictionaryCreate(nullptr, reinterpret_cast(keys), values, + sizeof(keys) / sizeof(keys[0]), + &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks), + cf_object_ptr_deleter); + + if (!query) { return false; } + + CFTypeRef security_items = nullptr; + if (SecItemCopyMatching(query.get(), &security_items) != errSecSuccess || + CFArrayGetTypeID() != CFGetTypeID(security_items)) { + return false; + } + + certs.reset(reinterpret_cast(security_items)); return true; } -#endif +inline bool retrieve_root_certs_from_keychain(CFObjectPtr &certs) { + CFArrayRef root_security_items = nullptr; + if (SecTrustCopyAnchorCertificates(&root_security_items) != errSecSuccess) { + return false; + } + + certs.reset(root_security_items); + return true; +} + +inline bool add_certs_to_x509_store(CFArrayRef certs, X509_STORE *store) { + auto result = false; + for (auto i = 0; i < CFArrayGetCount(certs); ++i) { + const auto cert = reinterpret_cast( + CFArrayGetValueAtIndex(certs, i)); + + if (SecCertificateGetTypeID() != CFGetTypeID(cert)) { continue; } + + CFDataRef cert_data = nullptr; + if (SecItemExport(cert, kSecFormatX509Cert, 0, nullptr, &cert_data) != + errSecSuccess) { + continue; + } + + CFObjectPtr cert_data_ptr(cert_data, cf_object_ptr_deleter); + + auto encoded_cert = static_cast( + CFDataGetBytePtr(cert_data_ptr.get())); + + auto x509 = + d2i_X509(NULL, &encoded_cert, CFDataGetLength(cert_data_ptr.get())); + + if (x509) { + X509_STORE_add_cert(store, x509); + X509_free(x509); + result = true; + } + } + + return result; +} + +inline bool load_system_certs_on_macos(X509_STORE *store) { + auto result = false; + CFObjectPtr certs(nullptr, cf_object_ptr_deleter); + if (retrieve_certs_from_keychain(certs) && certs) { + result = add_certs_to_x509_store(certs.get(), store); + } + + if (retrieve_root_certs_from_keychain(certs) && certs) { + result = add_certs_to_x509_store(certs.get(), store) || result; + } + + return result; +} +#endif // TARGET_OS_OSX +#endif // _WIN32 +#endif // CPPHTTPLIB_OPENSSL_SUPPORT + +#ifdef _WIN32 class WSInit { public: WSInit() { @@ -4427,7 +4999,7 @@ inline bool parse_www_authenticate(const Response &res, s = s.substr(pos + 1); auto beg = std::sregex_iterator(s.begin(), s.end(), re); for (auto i = beg; i != std::sregex_iterator(); ++i) { - auto m = *i; + const auto &m = *i; auto key = s.substr(static_cast(m.position(1)), static_cast(m.length(1))); auto val = m.length(2) > 0 @@ -4502,9 +5074,9 @@ inline void hosted_at(const std::string &hostname, const auto &addr = *reinterpret_cast(rp->ai_addr); std::string ip; - int dummy = -1; - if (detail::get_remote_ip_and_port(addr, sizeof(struct sockaddr_storage), - ip, dummy)) { + auto dummy = -1; + if (detail::get_ip_and_port(addr, sizeof(struct sockaddr_storage), ip, + dummy)) { addrs.push_back(ip); } } @@ -4606,6 +5178,16 @@ inline MultipartFormData Request::get_file_value(const std::string &key) const { return MultipartFormData(); } +inline std::vector +Request::get_file_values(const std::string &key) const { + std::vector values; + auto rng = files.equal_range(key); + for (auto it = rng.first; it != rng.second; it++) { + values.push_back(it->second); + } + return values; +} + // Response implementation inline bool Response::has_header(const std::string &key) const { return headers.find(key) != headers.end(); @@ -4656,10 +5238,9 @@ inline void Response::set_content(const std::string &s, inline void Response::set_content_provider( size_t in_length, const std::string &content_type, ContentProvider provider, ContentProviderResourceReleaser resource_releaser) { - assert(in_length > 0); set_header("Content-Type", content_type); content_length_ = in_length; - content_provider_ = std::move(provider); + if (in_length > 0) { content_provider_ = std::move(provider); } content_provider_resource_releaser_ = resource_releaser; is_chunked_content_provider_ = false; } @@ -4729,7 +5310,7 @@ inline bool SocketStream::is_readable() const { inline bool SocketStream::is_writable() const { return select_write(sock_, write_timeout_sec_, write_timeout_usec_) > 0 && - is_socket_alive(sock_); + is_socket_alive(sock_); } inline ssize_t SocketStream::read(char *ptr, size_t size) { @@ -4794,6 +5375,11 @@ inline void SocketStream::get_remote_ip_and_port(std::string &ip, return detail::get_remote_ip_and_port(sock_, ip, port); } +inline void SocketStream::get_local_ip_and_port(std::string &ip, + int &port) const { + return detail::get_local_ip_and_port(sock_, ip, port); +} + inline socket_t SocketStream::socket() const { return sock_; } // Buffer stream implementation @@ -4819,17 +5405,112 @@ inline ssize_t BufferStream::write(const char *ptr, size_t size) { inline void BufferStream::get_remote_ip_and_port(std::string & /*ip*/, int & /*port*/) const {} +inline void BufferStream::get_local_ip_and_port(std::string & /*ip*/, + int & /*port*/) const {} + inline socket_t BufferStream::socket() const { return 0; } inline const std::string &BufferStream::get_buffer() const { return buffer; } +inline PathParamsMatcher::PathParamsMatcher(const std::string &pattern) { + // One past the last ending position of a path param substring + std::size_t last_param_end = 0; + +#ifndef CPPHTTPLIB_NO_EXCEPTIONS + // Needed to ensure that parameter names are unique during matcher + // construction + // If exceptions are disabled, only last duplicate path + // parameter will be set + std::unordered_set param_name_set; +#endif + + while (true) { + const auto marker_pos = pattern.find(marker, last_param_end); + if (marker_pos == std::string::npos) { break; } + + static_fragments_.push_back( + pattern.substr(last_param_end, marker_pos - last_param_end)); + + const auto param_name_start = marker_pos + 1; + + auto sep_pos = pattern.find(separator, param_name_start); + if (sep_pos == std::string::npos) { sep_pos = pattern.length(); } + + auto param_name = + pattern.substr(param_name_start, sep_pos - param_name_start); + +#ifndef CPPHTTPLIB_NO_EXCEPTIONS + if (param_name_set.find(param_name) != param_name_set.cend()) { + std::string msg = "Encountered path parameter '" + param_name + + "' multiple times in route pattern '" + pattern + "'."; + throw std::invalid_argument(msg); + } +#endif + + param_names_.push_back(std::move(param_name)); + + last_param_end = sep_pos + 1; + } + + if (last_param_end < pattern.length()) { + static_fragments_.push_back(pattern.substr(last_param_end)); + } +} + +inline bool PathParamsMatcher::match(Request &request) const { + request.matches = std::smatch(); + request.path_params.clear(); + request.path_params.reserve(param_names_.size()); + + // One past the position at which the path matched the pattern last time + std::size_t starting_pos = 0; + for (size_t i = 0; i < static_fragments_.size(); ++i) { + const auto &fragment = static_fragments_[i]; + + if (starting_pos + fragment.length() > request.path.length()) { + return false; + } + + // Avoid unnecessary allocation by using strncmp instead of substr + + // comparison + if (std::strncmp(request.path.c_str() + starting_pos, fragment.c_str(), + fragment.length()) != 0) { + return false; + } + + starting_pos += fragment.length(); + + // Should only happen when we have a static fragment after a param + // Example: '/users/:id/subscriptions' + // The 'subscriptions' fragment here does not have a corresponding param + if (i >= param_names_.size()) { continue; } + + auto sep_pos = request.path.find(separator, starting_pos); + if (sep_pos == std::string::npos) { sep_pos = request.path.length(); } + + const auto ¶m_name = param_names_[i]; + + request.path_params.emplace( + param_name, request.path.substr(starting_pos, sep_pos - starting_pos)); + + // Mark everythin up to '/' as matched + starting_pos = sep_pos + 1; + } + // Returns false if the path is longer than the pattern + return starting_pos >= request.path.length(); +} + +inline bool RegexMatcher::match(Request &request) const { + request.path_params.clear(); + return std::regex_match(request.path, request.matches, regex_); +} + } // namespace detail // HTTP server implementation inline Server::Server() : new_task_queue( - [] { return new ThreadPool(CPPHTTPLIB_THREAD_POOL_COUNT); }), - svr_sock_(INVALID_SOCKET), is_running_(false) { + [] { return new ThreadPool(CPPHTTPLIB_THREAD_POOL_COUNT); }) { #ifndef _WIN32 signal(SIGPIPE, SIG_IGN); #endif @@ -4837,67 +5518,76 @@ inline Server::Server() inline Server::~Server() {} +inline std::unique_ptr +Server::make_matcher(const std::string &pattern) { + if (pattern.find("/:") != std::string::npos) { + return detail::make_unique(pattern); + } else { + return detail::make_unique(pattern); + } +} + inline Server &Server::Get(const std::string &pattern, Handler handler) { get_handlers_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + std::make_pair(make_matcher(pattern), std::move(handler))); return *this; } inline Server &Server::Post(const std::string &pattern, Handler handler) { post_handlers_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + std::make_pair(make_matcher(pattern), std::move(handler))); return *this; } inline Server &Server::Post(const std::string &pattern, HandlerWithContentReader handler) { post_handlers_for_content_reader_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + std::make_pair(make_matcher(pattern), std::move(handler))); return *this; } inline Server &Server::Put(const std::string &pattern, Handler handler) { put_handlers_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + std::make_pair(make_matcher(pattern), std::move(handler))); return *this; } inline Server &Server::Put(const std::string &pattern, HandlerWithContentReader handler) { put_handlers_for_content_reader_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + std::make_pair(make_matcher(pattern), std::move(handler))); return *this; } inline Server &Server::Patch(const std::string &pattern, Handler handler) { patch_handlers_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + std::make_pair(make_matcher(pattern), std::move(handler))); return *this; } inline Server &Server::Patch(const std::string &pattern, HandlerWithContentReader handler) { patch_handlers_for_content_reader_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + std::make_pair(make_matcher(pattern), std::move(handler))); return *this; } inline Server &Server::Delete(const std::string &pattern, Handler handler) { delete_handlers_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + std::make_pair(make_matcher(pattern), std::move(handler))); return *this; } inline Server &Server::Delete(const std::string &pattern, HandlerWithContentReader handler) { delete_handlers_for_content_reader_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + std::make_pair(make_matcher(pattern), std::move(handler))); return *this; } inline Server &Server::Options(const std::string &pattern, Handler handler) { options_handlers_.push_back( - std::make_pair(std::regex(pattern), std::move(handler))); + std::make_pair(make_matcher(pattern), std::move(handler))); return *this; } @@ -4935,6 +5625,11 @@ Server::set_file_extension_and_mimetype_mapping(const std::string &ext, return *this; } +inline Server &Server::set_default_file_mimetype(const std::string &mime) { + default_file_mimetype_ = mime; + return *this; +} + inline Server &Server::set_file_request_handler(Handler handler) { file_request_handler_ = std::move(handler); return *this; @@ -4976,7 +5671,6 @@ inline Server &Server::set_logger(Logger logger) { inline Server & Server::set_expect_100_continue_handler(Expect100ContinueHandler handler) { expect_100_continue_handler_ = std::move(handler); - return *this; } @@ -5000,6 +5694,12 @@ inline Server &Server::set_default_headers(Headers headers) { return *this; } +inline Server &Server::set_header_writer( + std::function const &writer) { + header_writer_ = writer; + return *this; +} + inline Server &Server::set_keep_alive_max_count(size_t count) { keep_alive_max_count_ = count; return *this; @@ -5042,15 +5742,25 @@ inline int Server::bind_to_any_port(const std::string &host, int socket_flags) { return bind_internal(host, 0, socket_flags); } -inline bool Server::listen_after_bind() { return listen_internal(); } +inline bool Server::listen_after_bind() { + auto se = detail::scope_exit([&]() { done_ = true; }); + return listen_internal(); +} inline bool Server::listen(const std::string &host, int port, int socket_flags) { + auto se = detail::scope_exit([&]() { done_ = true; }); return bind_to_port(host, port, socket_flags) && listen_internal(); } inline bool Server::is_running() const { return is_running_; } +inline void Server::wait_until_ready() const { + while (!is_running() && !done_) { + std::this_thread::sleep_for(std::chrono::milliseconds{1}); + } +} + inline void Server::stop() { if (is_running_) { assert(svr_sock_ != INVALID_SOCKET); @@ -5180,11 +5890,11 @@ inline bool Server::write_response_core(Stream &strm, bool close_connection, detail::BufferStream bstrm; if (!bstrm.write_format("HTTP/1.1 %d %s\r\n", res.status, - detail::status_message(res.status))) { + status_message(res.status))) { return false; } - if (!detail::write_headers(bstrm, res.headers)) { return false; } + if (!header_writer_(bstrm, res.headers)) { return false; } // Flush buffer auto &data = bstrm.get_buffer(); @@ -5313,7 +6023,7 @@ inline bool Server::read_content_with_content_receiver( inline bool Server::read_content_core(Stream &strm, Request &req, Response &res, ContentReceiver receiver, - MultipartContentHeader mulitpart_header, + MultipartContentHeader multipart_header, ContentReceiver multipart_receiver) { detail::MultipartFormDataParser multipart_form_data_parser; ContentReceiverWithProgress out; @@ -5333,14 +6043,14 @@ inline bool Server::read_content_core(Stream &strm, Request &req, Response &res, while (pos < n) { auto read_size = (std::min)(1, n - pos); auto ret = multipart_form_data_parser.parse( - buf + pos, read_size, multipart_receiver, mulitpart_header); + buf + pos, read_size, multipart_receiver, multipart_header); if (!ret) { return false; } pos += read_size; } return true; */ return multipart_form_data_parser.parse(buf, n, multipart_receiver, - mulitpart_header); + multipart_header); }; } else { out = [receiver](const char *buf, size_t n, uint64_t /*off*/, @@ -5377,17 +6087,26 @@ inline bool Server::handle_file_request(const Request &req, Response &res, if (path.back() == '/') { path += "index.html"; } if (detail::is_file(path)) { - detail::read_file(path, res.body); - auto type = - detail::find_content_type(path, file_extension_and_mimetype_map_); - if (type) { res.set_header("Content-Type", type); } for (const auto &kv : entry.headers) { res.set_header(kv.first.c_str(), kv.second); } - res.status = req.has_header("Range") ? 206 : 200; + + auto mm = std::make_shared(path.c_str()); + if (!mm->is_open()) { return false; } + + res.set_content_provider( + mm->size(), + detail::find_content_type(path, file_extension_and_mimetype_map_, + default_file_mimetype_), + [mm](size_t offset, size_t length, DataSink &sink) -> bool { + sink.write(mm->data() + offset, length); + return true; + }); + if (!head && file_request_handler_) { file_request_handler_(req, res); } + return true; } } @@ -5441,6 +6160,7 @@ inline int Server::bind_internal(const std::string &host, int port, inline bool Server::listen_internal() { auto ret = true; is_running_ = true; + auto se = detail::scope_exit([&]() { is_running_ = false; }); { std::unique_ptr task_queue(new_task_queue()); @@ -5466,6 +6186,8 @@ inline bool Server::listen_internal() { // Try to accept new connections after a short sleep. std::this_thread::sleep_for(std::chrono::milliseconds(1)); continue; + } else if (errno == EINTR || errno == EAGAIN) { + continue; } if (svr_sock_ != INVALID_SOCKET) { detail::close_socket(svr_sock_); @@ -5480,13 +6202,14 @@ inline bool Server::listen_internal() { #ifdef _WIN32 auto timeout = static_cast(read_timeout_sec_ * 1000 + read_timeout_usec_ / 1000); - setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout, - sizeof(timeout)); + setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, + reinterpret_cast(&timeout), sizeof(timeout)); #else timeval tv; tv.tv_sec = static_cast(read_timeout_sec_); tv.tv_usec = static_cast(read_timeout_usec_); - setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *)&tv, sizeof(tv)); + setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, + reinterpret_cast(&tv), sizeof(tv)); #endif } { @@ -5494,27 +6217,23 @@ inline bool Server::listen_internal() { #ifdef _WIN32 auto timeout = static_cast(write_timeout_sec_ * 1000 + write_timeout_usec_ / 1000); - setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout, - sizeof(timeout)); + setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, + reinterpret_cast(&timeout), sizeof(timeout)); #else timeval tv; tv.tv_sec = static_cast(write_timeout_sec_); tv.tv_usec = static_cast(write_timeout_usec_); - setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (char *)&tv, sizeof(tv)); + setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, + reinterpret_cast(&tv), sizeof(tv)); #endif } -#if __cplusplus > 201703L - task_queue->enqueue([=, this]() { process_and_close_socket(sock); }); -#else - task_queue->enqueue([=]() { process_and_close_socket(sock); }); -#endif + task_queue->enqueue([this, sock]() { process_and_close_socket(sock); }); } task_queue->shutdown(); } - is_running_ = false; return ret; } @@ -5525,7 +6244,7 @@ inline bool Server::routing(Request &req, Response &res, Stream &strm) { } // File handler - bool is_head_request = req.method == "HEAD"; + auto is_head_request = req.method == "HEAD"; if ((req.method == "GET" || is_head_request) && handle_file_request(req, res, is_head_request)) { return true; @@ -5598,10 +6317,10 @@ inline bool Server::routing(Request &req, Response &res, Stream &strm) { inline bool Server::dispatch_request(Request &req, Response &res, const Handlers &handlers) { for (const auto &x : handlers) { - const auto &pattern = x.first; + const auto &matcher = x.first; const auto &handler = x.second; - if (std::regex_match(req.path, req.matches, pattern)) { + if (matcher->match(req)) { handler(req, res); return true; } @@ -5621,8 +6340,8 @@ inline void Server::apply_ranges(const Request &req, Response &res, res.headers.erase(it); } - res.headers.emplace("Content-Type", - "multipart/byteranges; boundary=" + boundary); + res.set_header("Content-Type", + "multipart/byteranges; boundary=" + boundary); } auto type = detail::encoding_type(req, res); @@ -5635,10 +6354,10 @@ inline void Server::apply_ranges(const Request &req, Response &res, } else if (req.ranges.size() == 1) { auto offsets = detail::get_range_offset_and_length(req, res.content_length_, 0); - auto offset = offsets.first; length = offsets.second; + auto content_range = detail::make_content_range_header_field( - offset, length, res.content_length_); + req.ranges[0], res.content_length_); res.set_header("Content-Range", content_range); } else { length = detail::get_multipart_ranges_data_length(req, res, boundary, @@ -5661,13 +6380,15 @@ inline void Server::apply_ranges(const Request &req, Response &res, if (req.ranges.empty()) { ; } else if (req.ranges.size() == 1) { + auto content_range = detail::make_content_range_header_field( + req.ranges[0], res.body.size()); + res.set_header("Content-Range", content_range); + auto offsets = detail::get_range_offset_and_length(req, res.body.size(), 0); auto offset = offsets.first; auto length = offsets.second; - auto content_range = detail::make_content_range_header_field( - offset, length, res.body.size()); - res.set_header("Content-Range", content_range); + if (offset < res.body.size()) { res.body = res.body.substr(offset, length); } else { @@ -5723,10 +6444,10 @@ inline bool Server::dispatch_request_for_content_reader( Request &req, Response &res, ContentReader content_reader, const HandlersForContentReader &handlers) { for (const auto &x : handlers) { - const auto &pattern = x.first; + const auto &matcher = x.first; const auto &handler = x.second; - if (std::regex_match(req.path, req.matches, pattern)) { + if (matcher->match(req)) { handler(req, res, content_reader); return true; } @@ -5746,15 +6467,10 @@ Server::process_request(Stream &strm, bool close_connection, if (!line_reader.getline()) { return false; } Request req; + Response res; - res.version = "HTTP/1.1"; - - for (const auto &header : default_headers_) { - if (res.headers.find(header.first) == res.headers.end()) { - res.headers.insert(header); - } - } + res.headers = default_headers_; #ifdef _WIN32 // TODO: Increase FD_SETSIZE statically (libzmq), dynamically (MySQL). @@ -5798,6 +6514,10 @@ Server::process_request(Stream &strm, bool close_connection, req.set_header("REMOTE_ADDR", req.remote_addr); req.set_header("REMOTE_PORT", std::to_string(req.remote_port)); + strm.get_local_ip_and_port(req.local_addr, req.local_port); + req.set_header("LOCAL_ADDR", req.local_addr); + req.set_header("LOCAL_PORT", std::to_string(req.local_port)); + if (req.has_header("Range")) { const auto &range_header_value = req.get_header_value("Range"); if (!detail::parse_range_header(range_header_value, req.ranges)) { @@ -5817,14 +6537,14 @@ Server::process_request(Stream &strm, bool close_connection, case 100: case 417: strm.write_format("HTTP/1.1 %d %s\r\n\r\n", status, - detail::status_message(status)); + status_message(status)); break; default: return write_response(strm, close_connection, req, res); } } // Rounting - bool routed = false; + auto routed = false; #ifdef CPPHTTPLIB_NO_EXCEPTIONS routed = routing(req, res, strm); #else @@ -5837,7 +6557,16 @@ Server::process_request(Stream &strm, bool close_connection, routed = true; } else { res.status = 500; - res.set_header("EXCEPTION_WHAT", e.what()); + std::string val; + auto s = e.what(); + for (size_t i = 0; s[i]; i++) { + switch (s[i]) { + case '\r': val += "\\r"; break; + case '\n': val += "\\n"; break; + default: val += s[i]; break; + } + } + res.set_header("EXCEPTION_WHAT", val); } } catch (...) { if (exception_handler_) { @@ -6013,9 +6742,9 @@ inline bool ClientImpl::read_response_line(Stream &strm, const Request &req, if (!line_reader.getline()) { return false; } #ifdef CPPHTTPLIB_ALLOW_LF_AS_LINE_TERMINATOR - const static std::regex re("(HTTP/1\\.[01]) (\\d{3})(?: (.*?))?\r\n"); -#else const static std::regex re("(HTTP/1\\.[01]) (\\d{3})(?: (.*?))?\r?\n"); +#else + const static std::regex re("(HTTP/1\\.[01]) (\\d{3})(?: (.*?))?\r\n"); #endif std::cmatch m; @@ -6042,7 +6771,15 @@ inline bool ClientImpl::read_response_line(Stream &strm, const Request &req, inline bool ClientImpl::send(Request &req, Response &res, Error &error) { std::lock_guard request_mutex_guard(request_mutex_); + auto ret = send_(req, res, error); + if (error == Error::SSLPeerCouldBeClosed_) { + assert(!ret); + ret = send_(req, res, error); + } + return ret; +} +inline bool ClientImpl::send_(Request &req, Response &res, Error &error) { { std::lock_guard guard(socket_mutex_); @@ -6073,7 +6810,7 @@ inline bool ClientImpl::send(Request &req, Response &res, Error &error) { if (is_ssl()) { auto &scli = static_cast(*this); if (!proxy_host_.empty() && proxy_port_ != -1) { - bool success = false; + auto success = false; if (!scli.connect_with_proxy(socket_, res, success, error)) { return success; } @@ -6100,13 +6837,11 @@ inline bool ClientImpl::send(Request &req, Response &res, Error &error) { } } + auto ret = false; auto close_connection = !keep_alive_; - auto ret = process_socket(socket_, [&](Stream &strm) { - return handle_request(strm, req, res, close_connection, error); - }); - // Briefly lock mutex in order to mark that a request is no longer ongoing - { + auto se = detail::scope_exit([&]() { + // Briefly lock mutex in order to mark that a request is no longer ongoing std::lock_guard guard(socket_mutex_); socket_requests_in_flight_ -= 1; if (socket_requests_in_flight_ <= 0) { @@ -6120,7 +6855,11 @@ inline bool ClientImpl::send(Request &req, Response &res, Error &error) { shutdown_socket(socket_); close_socket(socket_); } - } + }); + + ret = process_socket(socket_, [&](Stream &strm) { + return handle_request(strm, req, res, close_connection, error); + }); if (!ret) { if (error == Error::Success) { error = Error::Unknown; } @@ -6165,6 +6904,21 @@ inline bool ClientImpl::handle_request(Stream &strm, Request &req, if (!ret) { return false; } + if (res.get_header_value("Connection") == "close" || + (res.version == "HTTP/1.0" && res.reason != "Connection established")) { + // TODO this requires a not-entirely-obvious chain of calls to be correct + // for this to be safe. + + // This is safe to call because handle_request is only called by send_ + // which locks the request mutex during the process. It would be a bug + // to call it from a different thread since it's a thread-safety issue + // to do these things to the socket if another thread is using the socket. + std::lock_guard guard(socket_mutex_); + shutdown_ssl(socket_, true); + shutdown_socket(socket_); + close_socket(socket_); + } + if (300 < res.status && res.status < 400 && follow_location_) { req = req_save; ret = redirect(req, res, error); @@ -6208,11 +6962,11 @@ inline bool ClientImpl::redirect(Request &req, Response &res, Error &error) { return false; } - auto location = detail::decode_url(res.get_header_value("location"), true); + auto location = res.get_header_value("location"); if (location.empty()) { return false; } const static std::regex re( - R"((?:(https?):)?(?://(?:\[([\d:]+)\]|([^:/?#]+))(?::(\d+))?)?([^?#]*(?:\?[^#]*)?)(?:#.*)?)"); + R"((?:(https?):)?(?://(?:\[([\d:]+)\]|([^:/?#]+))(?::(\d+))?)?([^?#]*)(\?[^#]*)?(?:#.*)?)"); std::smatch m; if (!std::regex_match(location, m, re)) { return false; } @@ -6224,6 +6978,7 @@ inline bool ClientImpl::redirect(Request &req, Response &res, Error &error) { if (next_host.empty()) { next_host = m[3].str(); } auto port_str = m[4].str(); auto next_path = m[5].str(); + auto next_query = m[6].str(); auto next_port = port_; if (!port_str.empty()) { @@ -6236,22 +6991,24 @@ inline bool ClientImpl::redirect(Request &req, Response &res, Error &error) { if (next_host.empty()) { next_host = host_; } if (next_path.empty()) { next_path = "/"; } + auto path = detail::decode_url(next_path, true) + next_query; + if (next_scheme == scheme && next_host == host_ && next_port == port_) { - return detail::redirect(*this, req, res, next_path, location, error); + return detail::redirect(*this, req, res, path, location, error); } else { if (next_scheme == "https") { #ifdef CPPHTTPLIB_OPENSSL_SUPPORT SSLClient cli(next_host.c_str(), next_port); cli.copy_settings(*this); if (ca_cert_store_) { cli.set_ca_cert_store(ca_cert_store_); } - return detail::redirect(cli, req, res, next_path, location, error); + return detail::redirect(cli, req, res, path, location, error); #else return false; #endif } else { ClientImpl cli(next_host.c_str(), next_port); cli.copy_settings(*this); - return detail::redirect(cli, req, res, next_path, location, error); + return detail::redirect(cli, req, res, path, location, error); } } } @@ -6262,7 +7019,7 @@ inline bool ClientImpl::write_content_with_provider(Stream &strm, auto is_shutting_down = []() { return false; }; if (req.is_chunked_content_provider_) { - // TODO: Brotli suport + // TODO: Brotli support std::unique_ptr compressor; #ifdef CPPHTTPLIB_ZLIB_SUPPORT if (compress_) { @@ -6279,39 +7036,39 @@ inline bool ClientImpl::write_content_with_provider(Stream &strm, return detail::write_content(strm, req.content_provider_, 0, req.content_length_, is_shutting_down, error); } -} // namespace httplib +} inline bool ClientImpl::write_request(Stream &strm, Request &req, bool close_connection, Error &error) { // Prepare additional headers if (close_connection) { if (!req.has_header("Connection")) { - req.headers.emplace("Connection", "close"); + req.set_header("Connection", "close"); } } if (!req.has_header("Host")) { if (is_ssl()) { if (port_ == 443) { - req.headers.emplace("Host", host_); + req.set_header("Host", host_); } else { - req.headers.emplace("Host", host_and_port_); + req.set_header("Host", host_and_port_); } } else { if (port_ == 80) { - req.headers.emplace("Host", host_); + req.set_header("Host", host_); } else { - req.headers.emplace("Host", host_and_port_); + req.set_header("Host", host_and_port_); } } } - if (!req.has_header("Accept")) { req.headers.emplace("Accept", "*/*"); } + if (!req.has_header("Accept")) { req.set_header("Accept", "*/*"); } #ifndef CPPHTTPLIB_NO_DEFAULT_USER_AGENT if (!req.has_header("User-Agent")) { auto agent = std::string("cpp-httplib/") + CPPHTTPLIB_VERSION; - req.headers.emplace("User-Agent", agent); + req.set_header("User-Agent", agent); } #endif @@ -6320,23 +7077,23 @@ inline bool ClientImpl::write_request(Stream &strm, Request &req, if (!req.is_chunked_content_provider_) { if (!req.has_header("Content-Length")) { auto length = std::to_string(req.content_length_); - req.headers.emplace("Content-Length", length); + req.set_header("Content-Length", length); } } } else { if (req.method == "POST" || req.method == "PUT" || req.method == "PATCH") { - req.headers.emplace("Content-Length", "0"); + req.set_header("Content-Length", "0"); } } } else { if (!req.has_header("Content-Type")) { - req.headers.emplace("Content-Type", "text/plain"); + req.set_header("Content-Type", "text/plain"); } if (!req.has_header("Content-Length")) { auto length = std::to_string(req.body.size()); - req.headers.emplace("Content-Length", length); + req.set_header("Content-Length", length); } } @@ -6376,7 +7133,7 @@ inline bool ClientImpl::write_request(Stream &strm, Request &req, const auto &path = url_encode_ ? detail::encode_url(req.path) : req.path; bstrm.write_format("%s %s HTTP/1.1\r\n", req.method.c_str(), path.c_str()); - detail::write_headers(bstrm, req.headers); + header_writer_(bstrm, req.headers); // Flush buffer auto &data = bstrm.get_buffer(); @@ -6404,12 +7161,10 @@ inline std::unique_ptr ClientImpl::send_with_content_provider( ContentProvider content_provider, ContentProviderWithoutLength content_provider_without_length, const std::string &content_type, Error &error) { - if (!content_type.empty()) { - req.headers.emplace("Content-Type", content_type); - } + if (!content_type.empty()) { req.set_header("Content-Type", content_type); } #ifdef CPPHTTPLIB_ZLIB_SUPPORT - if (compress_) { req.headers.emplace("Content-Encoding", "gzip"); } + if (compress_) { req.set_header("Content-Encoding", "gzip"); } #endif #ifdef CPPHTTPLIB_ZLIB_SUPPORT @@ -6442,8 +7197,6 @@ inline std::unique_ptr ClientImpl::send_with_content_provider( return ok; }; - data_sink.is_writable = [&](void) { return ok && true; }; - while (ok && offset < content_length) { if (!content_provider(offset, content_length - offset, data_sink)) { error = Error::Canceled; @@ -6472,10 +7225,9 @@ inline std::unique_ptr ClientImpl::send_with_content_provider( req.content_provider_ = detail::ContentProviderAdapter( std::move(content_provider_without_length)); req.is_chunked_content_provider_ = true; - req.headers.emplace("Transfer-Encoding", "chunked"); + req.set_header("Transfer-Encoding", "chunked"); } else { req.body.assign(body, content_length); - ; } } @@ -6514,6 +7266,20 @@ inline bool ClientImpl::process_request(Stream &strm, Request &req, // Send request if (!write_request(strm, req, close_connection, error)) { return false; } +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + if (is_ssl()) { + auto is_proxy_enabled = !proxy_host_.empty() && proxy_port_ != -1; + if (!is_proxy_enabled) { + char buf[1]; + if (SSL_peek(socket_.ssl, buf, 1) == 0 && + SSL_get_error(socket_.ssl, 0) == SSL_ERROR_ZERO_RETURN) { + error = Error::SSLPeerCouldBeClosed_; + return false; + } + } + } +#endif + // Receive response and headers if (!read_response_line(strm, req, res) || !detail::read_headers(strm, res.headers)) { @@ -6567,30 +7333,54 @@ inline bool ClientImpl::process_request(Stream &strm, Request &req, } } - if (res.get_header_value("Connection") == "close" || - (res.version == "HTTP/1.0" && res.reason != "Connection established")) { - // TODO this requires a not-entirely-obvious chain of calls to be correct - // for this to be safe. Maybe a code refactor (such as moving this out to - // the send function and getting rid of the recursiveness of the mutex) - // could make this more obvious. - - // This is safe to call because process_request is only called by - // handle_request which is only called by send, which locks the request - // mutex during the process. It would be a bug to call it from a different - // thread since it's a thread-safety issue to do these things to the socket - // if another thread is using the socket. - std::lock_guard guard(socket_mutex_); - shutdown_ssl(socket_, true); - shutdown_socket(socket_); - close_socket(socket_); - } - // Log if (logger_) { logger_(req, res); } return true; } +inline ContentProviderWithoutLength ClientImpl::get_multipart_content_provider( + const std::string &boundary, const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items) { + size_t cur_item = 0, cur_start = 0; + // cur_item and cur_start are copied to within the std::function and maintain + // state between successive calls + return [&, cur_item, cur_start](size_t offset, + DataSink &sink) mutable -> bool { + if (!offset && items.size()) { + sink.os << detail::serialize_multipart_formdata(items, boundary, false); + return true; + } else if (cur_item < provider_items.size()) { + if (!cur_start) { + const auto &begin = detail::serialize_multipart_formdata_item_begin( + provider_items[cur_item], boundary); + offset += begin.size(); + cur_start = offset; + sink.os << begin; + } + + DataSink cur_sink; + auto has_data = true; + cur_sink.write = sink.write; + cur_sink.done = [&]() { has_data = false; }; + + if (!provider_items[cur_item].provider(offset - cur_start, cur_sink)) + return false; + + if (!has_data) { + sink.os << detail::serialize_multipart_formdata_item_end(); + cur_item++; + cur_start = 0; + } + return true; + } else { + sink.os << detail::serialize_multipart_formdata_finish(boundary); + sink.done(); + return true; + } + }; +} + inline bool ClientImpl::process_socket(const Socket &socket, std::function callback) { @@ -6736,6 +7526,11 @@ inline Result ClientImpl::Post(const std::string &path) { return Post(path, std::string(), std::string()); } +inline Result ClientImpl::Post(const std::string &path, + const Headers &headers) { + return Post(path, headers, nullptr, 0, std::string()); +} + inline Result ClientImpl::Post(const std::string &path, const char *body, size_t content_length, const std::string &content_type) { @@ -6808,9 +7603,10 @@ inline Result ClientImpl::Post(const std::string &path, inline Result ClientImpl::Post(const std::string &path, const Headers &headers, const MultipartFormDataItems &items) { - std::string content_type; - const auto &body = detail::serialize_multipart_formdata( - items, detail::make_multipart_data_boundary(), content_type); + const auto &boundary = detail::make_multipart_data_boundary(); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + const auto &body = detail::serialize_multipart_formdata(items, boundary); return Post(path, headers, body, content_type.c_str()); } @@ -6821,12 +7617,25 @@ inline Result ClientImpl::Post(const std::string &path, const Headers &headers, return Result{nullptr, Error::UnsupportedMultipartBoundaryChars}; } - std::string content_type; - const auto &body = - detail::serialize_multipart_formdata(items, boundary, content_type); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + const auto &body = detail::serialize_multipart_formdata(items, boundary); return Post(path, headers, body, content_type.c_str()); } +inline Result +ClientImpl::Post(const std::string &path, const Headers &headers, + const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items) { + const auto &boundary = detail::make_multipart_data_boundary(); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + return send_with_content_provider( + "POST", path, headers, nullptr, 0, nullptr, + get_multipart_content_provider(boundary, items, provider_items), + content_type); +} + inline Result ClientImpl::Put(const std::string &path) { return Put(path, std::string(), std::string()); } @@ -6903,9 +7712,10 @@ inline Result ClientImpl::Put(const std::string &path, inline Result ClientImpl::Put(const std::string &path, const Headers &headers, const MultipartFormDataItems &items) { - std::string content_type; - const auto &body = detail::serialize_multipart_formdata( - items, detail::make_multipart_data_boundary(), content_type); + const auto &boundary = detail::make_multipart_data_boundary(); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + const auto &body = detail::serialize_multipart_formdata(items, boundary); return Put(path, headers, body, content_type); } @@ -6916,12 +7726,24 @@ inline Result ClientImpl::Put(const std::string &path, const Headers &headers, return Result{nullptr, Error::UnsupportedMultipartBoundaryChars}; } - std::string content_type; - const auto &body = - detail::serialize_multipart_formdata(items, boundary, content_type); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + const auto &body = detail::serialize_multipart_formdata(items, boundary); return Put(path, headers, body, content_type); } +inline Result +ClientImpl::Put(const std::string &path, const Headers &headers, + const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items) { + const auto &boundary = detail::make_multipart_data_boundary(); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + return send_with_content_provider( + "PUT", path, headers, nullptr, 0, nullptr, + get_multipart_content_provider(boundary, items, provider_items), + content_type); +} inline Result ClientImpl::Patch(const std::string &path) { return Patch(path, std::string(), std::string()); } @@ -7007,9 +7829,7 @@ inline Result ClientImpl::Delete(const std::string &path, req.headers = headers; req.path = path; - if (!content_type.empty()) { - req.headers.emplace("Content-Type", content_type); - } + if (!content_type.empty()) { req.set_header("Content-Type", content_type); } req.body.assign(body, content_length); return send_(std::move(req)); @@ -7042,13 +7862,6 @@ inline Result ClientImpl::Options(const std::string &path, return send_(std::move(req)); } -inline size_t ClientImpl::is_socket_open() const { - std::lock_guard guard(socket_mutex_); - return socket_.is_open(); -} - -inline socket_t ClientImpl::socket() const { return socket_.sock; } - inline void ClientImpl::stop() { std::lock_guard guard(socket_mutex_); @@ -7066,12 +7879,23 @@ inline void ClientImpl::stop() { return; } - // Otherwise, sitll holding the mutex, we can shut everything down ourselves + // Otherwise, still holding the mutex, we can shut everything down ourselves shutdown_ssl(socket_, true); shutdown_socket(socket_); close_socket(socket_); } +inline std::string ClientImpl::host() const { return host_; } + +inline int ClientImpl::port() const { return port_; } + +inline size_t ClientImpl::is_socket_open() const { + std::lock_guard guard(socket_mutex_); + return socket_.is_open(); +} + +inline socket_t ClientImpl::socket() const { return socket_.sock; } + inline void ClientImpl::set_connection_timeout(time_t sec, time_t usec) { connection_timeout_sec_ = sec; connection_timeout_usec_ = usec; @@ -7120,6 +7944,11 @@ inline void ClientImpl::set_default_headers(Headers headers) { default_headers_ = std::move(headers); } +inline void ClientImpl::set_header_writer( + std::function const &writer) { + header_writer_ = writer; +} + inline void ClientImpl::set_address_family(int family) { address_family_ = family; } @@ -7159,9 +7988,7 @@ inline void ClientImpl::set_proxy_digest_auth(const std::string &username, proxy_digest_auth_username_ = username; proxy_digest_auth_password_ = password; } -#endif -#ifdef CPPHTTPLIB_OPENSSL_SUPPORT inline void ClientImpl::set_ca_cert_path(const std::string &ca_cert_file_path, const std::string &ca_cert_dir_path) { ca_cert_file_path_ = ca_cert_file_path; @@ -7173,9 +8000,34 @@ inline void ClientImpl::set_ca_cert_store(X509_STORE *ca_cert_store) { ca_cert_store_ = ca_cert_store; } } -#endif -#ifdef CPPHTTPLIB_OPENSSL_SUPPORT +inline X509_STORE *ClientImpl::create_ca_cert_store(const char *ca_cert, + std::size_t size) { + auto mem = BIO_new_mem_buf(ca_cert, static_cast(size)); + if (!mem) return nullptr; + + auto inf = PEM_X509_INFO_read_bio(mem, nullptr, nullptr, nullptr); + if (!inf) { + BIO_free_all(mem); + return nullptr; + } + + auto cts = X509_STORE_new(); + if (cts) { + for (auto i = 0; i < static_cast(sk_X509_INFO_num(inf)); i++) { + auto itmp = sk_X509_INFO_value(inf, i); + if (!itmp) { continue; } + + if (itmp->x509) { X509_STORE_add_cert(cts, itmp->x509); } + if (itmp->crl) { X509_STORE_add_crl(cts, itmp->crl); } + } + } + + sk_X509_INFO_pop_free(inf, X509_INFO_free); + BIO_free_all(mem); + return cts; +} + inline void ClientImpl::enable_server_certificate_verification(bool enabled) { server_certificate_verification_ = enabled; } @@ -7239,7 +8091,7 @@ bool ssl_connect_or_accept_nonblocking(socket_t sock, SSL *ssl, U ssl_connect_or_accept, time_t timeout_sec, time_t timeout_usec) { - int res = 0; + auto res = 0; while ((res = ssl_connect_or_accept(ssl)) != 1) { auto err = SSL_get_error(ssl, res); switch (err) { @@ -7281,55 +8133,12 @@ process_client_socket_ssl(SSL *ssl, socket_t sock, time_t read_timeout_sec, return callback(strm); } -#if OPENSSL_VERSION_NUMBER < 0x10100000L -static std::shared_ptr> openSSL_locks_; - -class SSLThreadLocks { -public: - SSLThreadLocks() { - openSSL_locks_ = - std::make_shared>(CRYPTO_num_locks()); - CRYPTO_set_locking_callback(locking_callback); - } - - ~SSLThreadLocks() { CRYPTO_set_locking_callback(nullptr); } - -private: - static void locking_callback(int mode, int type, const char * /*file*/, - int /*line*/) { - auto &lk = (*openSSL_locks_)[static_cast(type)]; - if (mode & CRYPTO_LOCK) { - lk.lock(); - } else { - lk.unlock(); - } - } -}; - -#endif - class SSLInit { public: SSLInit() { -#if OPENSSL_VERSION_NUMBER < 0x1010001fL - SSL_load_error_strings(); - SSL_library_init(); -#else OPENSSL_init_ssl( OPENSSL_INIT_LOAD_SSL_STRINGS | OPENSSL_INIT_LOAD_CRYPTO_STRINGS, NULL); -#endif } - - ~SSLInit() { -#if OPENSSL_VERSION_NUMBER < 0x1010001fL - ERR_free_strings(); -#endif - } - -private: -#if OPENSSL_VERSION_NUMBER < 0x10100000L - SSLThreadLocks thread_init_; -#endif }; // SSL socket stream implementation @@ -7353,7 +8162,7 @@ inline bool SSLSocketStream::is_readable() const { inline bool SSLSocketStream::is_writable() const { return select_write(sock_, write_timeout_sec_, write_timeout_usec_) > 0 && - is_socket_alive(sock_); + is_socket_alive(sock_); } inline ssize_t SSLSocketStream::read(char *ptr, size_t size) { @@ -7363,7 +8172,7 @@ inline ssize_t SSLSocketStream::read(char *ptr, size_t size) { auto ret = SSL_read(ssl_, ptr, static_cast(size)); if (ret < 0) { auto err = SSL_get_error(ssl_, ret); - int n = 1000; + auto n = 1000; #ifdef _WIN32 while (--n >= 0 && (err == SSL_ERROR_WANT_READ || (err == SSL_ERROR_SYSCALL && @@ -7396,7 +8205,7 @@ inline ssize_t SSLSocketStream::write(const char *ptr, size_t size) { auto ret = SSL_write(ssl_, ptr, static_cast(handle_size)); if (ret < 0) { auto err = SSL_get_error(ssl_, ret); - int n = 1000; + auto n = 1000; #ifdef _WIN32 while (--n >= 0 && (err == SSL_ERROR_WANT_WRITE || (err == SSL_ERROR_SYSCALL && @@ -7424,6 +8233,11 @@ inline void SSLSocketStream::get_remote_ip_and_port(std::string &ip, detail::get_remote_ip_and_port(sock_, ip, port); } +inline void SSLSocketStream::get_local_ip_and_port(std::string &ip, + int &port) const { + detail::get_local_ip_and_port(sock_, ip, port); +} + inline socket_t SSLSocketStream::socket() const { return sock_; } static SSLInit sslinit_; @@ -7446,8 +8260,9 @@ inline SSLServer::SSLServer(const char *cert_path, const char *private_key_path, // add default password callback before opening encrypted private key if (private_key_password != nullptr && (private_key_password[0] != '\0')) { - SSL_CTX_set_default_passwd_cb_userdata(ctx_, - (char *)private_key_password); + SSL_CTX_set_default_passwd_cb_userdata( + ctx_, + reinterpret_cast(const_cast(private_key_password))); } if (SSL_CTX_use_certificate_chain_file(ctx_, cert_path) != 1 || @@ -7517,7 +8332,7 @@ inline bool SSLServer::process_and_close_socket(socket_t sock) { }, [](SSL * /*ssl2*/) { return true; }); - bool ret = false; + auto ret = false; if (ssl) { ret = detail::process_server_socket_ssl( svr_sock_, ssl, sock, keep_alive_max_count_, keep_alive_timeout_sec_, @@ -7611,6 +8426,11 @@ inline void SSLClient::set_ca_cert_store(X509_STORE *ca_cert_store) { } } +inline void SSLClient::load_ca_cert_store(const char *ca_cert, + std::size_t size) { + set_ca_cert_store(ClientImpl::create_ca_cert_store(ca_cert, size)); +} + inline long SSLClient::get_openssl_verify_result() const { return verify_result_; } @@ -7625,14 +8445,14 @@ inline bool SSLClient::create_and_connect_socket(Socket &socket, Error &error) { inline bool SSLClient::connect_with_proxy(Socket &socket, Response &res, bool &success, Error &error) { success = true; - Response res2; + Response proxy_res; if (!detail::process_client_socket( socket.sock, read_timeout_sec_, read_timeout_usec_, write_timeout_sec_, write_timeout_usec_, [&](Stream &strm) { Request req2; req2.method = "CONNECT"; req2.path = host_and_port_; - return process_request(strm, req2, res2, false, error); + return process_request(strm, req2, proxy_res, false, error); })) { // Thread-safe to close everything because we are assuming there are no // requests in flight @@ -7643,12 +8463,12 @@ inline bool SSLClient::connect_with_proxy(Socket &socket, Response &res, return false; } - if (res2.status == 407) { + if (proxy_res.status == 407) { if (!proxy_digest_auth_username_.empty() && !proxy_digest_auth_password_.empty()) { std::map auth; - if (detail::parse_www_authenticate(res2, auth, true)) { - Response res3; + if (detail::parse_www_authenticate(proxy_res, auth, true)) { + proxy_res = Response(); if (!detail::process_client_socket( socket.sock, read_timeout_sec_, read_timeout_usec_, write_timeout_sec_, write_timeout_usec_, [&](Stream &strm) { @@ -7659,7 +8479,7 @@ inline bool SSLClient::connect_with_proxy(Socket &socket, Response &res, req3, auth, 1, detail::random_string(10), proxy_digest_auth_username_, proxy_digest_auth_password_, true)); - return process_request(strm, req3, res3, false, error); + return process_request(strm, req3, proxy_res, false, error); })) { // Thread-safe to close everything because we are assuming there are // no requests in flight @@ -7670,17 +8490,28 @@ inline bool SSLClient::connect_with_proxy(Socket &socket, Response &res, return false; } } - } else { - res = res2; - return false; } } + // If status code is not 200, proxy request is failed. + // Set error to ProxyConnection and return proxy response + // as the response of the request + if (proxy_res.status != 200) { + error = Error::ProxyConnection; + res = std::move(proxy_res); + // Thread-safe to close everything because we are assuming there are + // no requests in flight + shutdown_ssl(socket, true); + shutdown_socket(socket); + close_socket(socket); + return false; + } + return true; } inline bool SSLClient::load_certs() { - bool ret = true; + auto ret = true; std::call_once(initialize_cert_, [&]() { std::lock_guard guard(ctx_mutex_); @@ -7695,11 +8526,16 @@ inline bool SSLClient::load_certs() { ret = false; } } else { + auto loaded = false; #ifdef _WIN32 - detail::load_system_certs_on_windows(SSL_CTX_get_cert_store(ctx_)); -#else - SSL_CTX_set_default_verify_paths(ctx_); -#endif + loaded = + detail::load_system_certs_on_windows(SSL_CTX_get_cert_store(ctx_)); +#elif defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN) && defined(__APPLE__) +#if TARGET_OS_OSX + loaded = detail::load_system_certs_on_macos(SSL_CTX_get_cert_store(ctx_)); +#endif // TARGET_OS_OSX +#endif // _WIN32 + if (!loaded) { SSL_CTX_set_default_verify_paths(ctx_); } } }); @@ -7733,7 +8569,7 @@ inline bool SSLClient::initialize_ssl(Socket &socket, Error &error) { return false; } - auto server_cert = SSL_get_peer_certificate(ssl2); + auto server_cert = SSL_get1_peer_certificate(ssl2); if (server_cert == nullptr) { error = Error::SSLServerVerification; @@ -7751,6 +8587,10 @@ inline bool SSLClient::initialize_ssl(Socket &socket, Error &error) { return true; }, [&](SSL *ssl2) { + // NOTE: With -Wold-style-cast, this can produce a warning, since + // SSL_set_tlsext_host_name is a macro (in OpenSSL), which contains + // an old style cast. Short of doing compiler specific pragma's + // here, we can't get rid of this warning. :'( SSL_set_tlsext_host_name(ssl2, host_.c_str()); return true; }); @@ -7844,15 +8684,16 @@ SSLClient::verify_host_with_subject_alt_name(X509 *server_cert) const { if (alt_names) { auto dsn_matched = false; - auto ip_mached = false; + auto ip_matched = false; auto count = sk_GENERAL_NAME_num(alt_names); for (decltype(count) i = 0; i < count && !dsn_matched; i++) { auto val = sk_GENERAL_NAME_value(alt_names, i); if (val->type == type) { - auto name = (const char *)ASN1_STRING_get0_data(val->d.ia5); - auto name_len = (size_t)ASN1_STRING_length(val->d.ia5); + auto name = + reinterpret_cast(ASN1_STRING_get0_data(val->d.ia5)); + auto name_len = static_cast(ASN1_STRING_length(val->d.ia5)); switch (type) { case GEN_DNS: dsn_matched = check_host_name(name, name_len); break; @@ -7860,17 +8701,18 @@ SSLClient::verify_host_with_subject_alt_name(X509 *server_cert) const { case GEN_IPADD: if (!memcmp(&addr6, name, addr_len) || !memcmp(&addr, name, addr_len)) { - ip_mached = true; + ip_matched = true; } break; } } } - if (dsn_matched || ip_mached) { ret = true; } + if (dsn_matched || ip_matched) { ret = true; } } - GENERAL_NAMES_free((STACK_OF(GENERAL_NAME) *)alt_names); + GENERAL_NAMES_free(const_cast( + reinterpret_cast(alt_names))); return ret; } @@ -8059,6 +8901,9 @@ inline Result Client::Head(const std::string &path, const Headers &headers) { } inline Result Client::Post(const std::string &path) { return cli_->Post(path); } +inline Result Client::Post(const std::string &path, const Headers &headers) { + return cli_->Post(path, headers); +} inline Result Client::Post(const std::string &path, const char *body, size_t content_length, const std::string &content_type) { @@ -8121,6 +8966,12 @@ inline Result Client::Post(const std::string &path, const Headers &headers, const std::string &boundary) { return cli_->Post(path, headers, items, boundary); } +inline Result +Client::Post(const std::string &path, const Headers &headers, + const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items) { + return cli_->Post(path, headers, items, provider_items); +} inline Result Client::Put(const std::string &path) { return cli_->Put(path); } inline Result Client::Put(const std::string &path, const char *body, size_t content_length, @@ -8184,6 +9035,12 @@ inline Result Client::Put(const std::string &path, const Headers &headers, const std::string &boundary) { return cli_->Put(path, headers, items, boundary); } +inline Result +Client::Put(const std::string &path, const Headers &headers, + const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items) { + return cli_->Put(path, headers, items, provider_items); +} inline Result Client::Patch(const std::string &path) { return cli_->Patch(path); } @@ -8267,12 +9124,16 @@ inline bool Client::send(Request &req, Response &res, Error &error) { inline Result Client::send(const Request &req) { return cli_->send(req); } +inline void Client::stop() { cli_->stop(); } + +inline std::string Client::host() const { return cli_->host(); } + +inline int Client::port() const { return cli_->port(); } + inline size_t Client::is_socket_open() const { return cli_->is_socket_open(); } inline socket_t Client::socket() const { return cli_->socket(); } -inline void Client::stop() { cli_->stop(); } - inline void Client::set_hostname_addr_map(std::map addr_map) { cli_->set_hostname_addr_map(std::move(addr_map)); @@ -8282,6 +9143,11 @@ inline void Client::set_default_headers(Headers headers) { cli_->set_default_headers(std::move(headers)); } +inline void Client::set_header_writer( + std::function const &writer) { + cli_->set_header_writer(writer); +} + inline void Client::set_address_family(int family) { cli_->set_address_family(family); } @@ -8356,7 +9222,9 @@ inline void Client::enable_server_certificate_verification(bool enabled) { } #endif -inline void Client::set_logger(Logger logger) { cli_->set_logger(logger); } +inline void Client::set_logger(Logger logger) { + cli_->set_logger(std::move(logger)); +} #ifdef CPPHTTPLIB_OPENSSL_SUPPORT inline void Client::set_ca_cert_path(const std::string &ca_cert_file_path, @@ -8372,6 +9240,10 @@ inline void Client::set_ca_cert_store(X509_STORE *ca_cert_store) { } } +inline void Client::load_ca_cert_store(const char *ca_cert, std::size_t size) { + set_ca_cert_store(cli_->create_ca_cert_store(ca_cert, size)); +} + inline long Client::get_openssl_verify_result() const { if (is_ssl_) { return static_cast(*cli_).get_openssl_verify_result(); @@ -8389,4 +9261,8 @@ inline SSL_CTX *Client::ssl_context() const { } // namespace httplib +#if defined(_WIN32) && defined(CPPHTTPLIB_USE_POLL) +#undef poll +#endif + #endif // CPPHTTPLIB_HTTPLIB_H diff --git a/src/common/string_util.h b/src/common/string_util.h index 74c342e5bb..2384e9c510 100644 --- a/src/common/string_util.h +++ b/src/common/string_util.h @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -84,6 +85,15 @@ std::string UTF16BufferToUTF8(const T& text) { return UTF16ToUTF8(buffer); } +/** + * Removes trailing null bytes from the string. + */ +inline void TruncateString(std::string& str) { + while (str.size() && str.back() == '\0') { + str.pop_back(); + } +} + /** * Creates a std::string from a fixed-size NUL-terminated char buffer. If the buffer isn't * NUL-terminated then the string ends at max_len characters. diff --git a/src/core/hle/service/http/http_c.cpp b/src/core/hle/service/http/http_c.cpp index fb6b6d07c9..861f35e8b0 100644 --- a/src/core/hle/service/http/http_c.cpp +++ b/src/core/hle/service/http/http_c.cpp @@ -3,11 +3,15 @@ // Refer to the license.txt file included. #include +#include #include +#include #include #include #include "common/archives.h" #include "common/assert.h" +#include "common/scope_exit.h" +#include "common/string_util.h" #include "core/core.h" #include "core/file_sys/archive_ncch.h" #include "core/file_sys/file_backend.h" @@ -28,6 +32,8 @@ enum { InvalidRequestState = 22, TooManyContexts = 26, InvalidRequestMethod = 32, + HeaderNotFound = 40, + BufferTooSmall = 43, ContextNotFound = 100, /// This error is returned in multiple situations: when trying to initialize an @@ -48,6 +54,10 @@ const ResultCode ERROR_NOT_IMPLEMENTED = // 0xD960A3F4 const ResultCode ERROR_TOO_MANY_CLIENT_CERTS = // 0xD8A0A0CB ResultCode(ErrCodes::TooManyClientCerts, ErrorModule::HTTP, ErrorSummary::InvalidState, ErrorLevel::Permanent); +const ResultCode ERROR_HEADER_NOT_FOUND = ResultCode( + ErrCodes::HeaderNotFound, ErrorModule::HTTP, ErrorSummary::InvalidState, ErrorLevel::Permanent); +const ResultCode ERROR_BUFFER_SMALL = ResultCode(ErrCodes::BufferTooSmall, ErrorModule::HTTP, + ErrorSummary::WouldBlock, ErrorLevel::Permanent); const ResultCode ERROR_WRONG_CERT_ID = // 0xD8E0B839 ResultCode(57, ErrorModule::SSL, ErrorSummary::InvalidArgument, ErrorLevel::Permanent); const ResultCode ERROR_WRONG_CERT_HANDLE = // 0xD8A0A0C9 @@ -55,47 +65,123 @@ const ResultCode ERROR_WRONG_CERT_HANDLE = // 0xD8A0A0C9 const ResultCode ERROR_CERT_ALREADY_SET = // 0xD8A0A03D ResultCode(61, ErrorModule::HTTP, ErrorSummary::InvalidState, ErrorLevel::Permanent); -static std::pair SplitUrl(const std::string& url) { +// Splits URL into its components. Example: https://citra-emu.org:443/index.html +// is_https: true; host: citra-emu.org; port: 443; path: /index.html +static URLInfo SplitUrl(const std::string& url) { const std::string prefix = "://"; + constexpr int default_http_port = 80; + constexpr int default_https_port = 443; + + std::string host; + int port = -1; + std::string path; + const auto scheme_end = url.find(prefix); const auto prefix_end = scheme_end == std::string::npos ? 0 : scheme_end + prefix.length(); - + bool is_https = scheme_end != std::string::npos && url.starts_with("https"); const auto path_index = url.find("/", prefix_end); - std::string host; - std::string path; + if (path_index == std::string::npos) { // If no path is specified after the host, set it to "/" - host = url; + host = url.substr(prefix_end); path = "/"; } else { - host = url.substr(0, path_index); + host = url.substr(prefix_end, path_index - prefix_end); path = url.substr(path_index); } - return std::make_pair(host, path); + + const auto port_start = host.find(":"); + if (port_start != std::string::npos) { + std::string port_str = host.substr(port_start + 1); + host = host.substr(0, port_start); + char* p_end = nullptr; + port = std::strtol(port_str.c_str(), &p_end, 10); + if (*p_end) { + port = -1; + } + } + + if (port == -1) { + port = is_https ? default_https_port : default_http_port; + } + return URLInfo{ + .is_https = is_https, + .host = host, + .port = port, + .path = path, + }; } +static size_t WriteHeaders(httplib::Stream& stream, + std::span headers) { + size_t write_len = 0; + for (const auto& header : headers) { + auto len = stream.write_format("%s: %s\r\n", header.name.c_str(), header.value.c_str()); + if (len < 0) { + return len; + } + write_len += len; + } + auto len = stream.write("\r\n"); + if (len < 0) { + return len; + } + write_len += len; + return write_len; +} + +static size_t HandleHeaderWrite(std::vector& pending_headers, + httplib::Stream& strm, httplib::Headers& httplib_headers) { + std::vector final_headers; + std::vector::iterator it_pending_headers; + httplib::Headers::iterator it_httplib_headers; + + auto find_pending_header = [&pending_headers](const std::string& str) { + return std::find_if(pending_headers.begin(), pending_headers.end(), + [&str](Context::RequestHeader& rh) { return rh.name == str; }); + }; + + // Watch out for header ordering!! + // First: Host + it_pending_headers = find_pending_header("Host"); + if (it_pending_headers != pending_headers.end()) { + final_headers.push_back( + Context::RequestHeader(it_pending_headers->name, it_pending_headers->value)); + pending_headers.erase(it_pending_headers); + } else { + it_httplib_headers = httplib_headers.find("Host"); + if (it_httplib_headers != httplib_headers.end()) { + final_headers.push_back( + Context::RequestHeader(it_httplib_headers->first, it_httplib_headers->second)); + } + } + + // Second, user defined headers + // Third, Content-Type (optional, appended by MakeRequest) + for (const auto& header : pending_headers) { + final_headers.push_back(header); + } + + // Fourth: Content-Length + it_pending_headers = find_pending_header("Content-Length"); + if (it_pending_headers != pending_headers.end()) { + final_headers.push_back( + Context::RequestHeader(it_pending_headers->name, it_pending_headers->value)); + pending_headers.erase(it_pending_headers); + } else { + it_httplib_headers = httplib_headers.find("Content-Length"); + if (it_httplib_headers != httplib_headers.end()) { + final_headers.push_back( + Context::RequestHeader(it_httplib_headers->first, it_httplib_headers->second)); + } + } + + return WriteHeaders(strm, final_headers); +}; + void Context::MakeRequest() { ASSERT(state == RequestState::NotStarted); - const auto& [host, path] = SplitUrl(url); - const auto client = std::make_unique(host); - SSL_CTX* ctx = client->ssl_context(); - if (ctx) { - if (auto client_cert = ssl_config.client_cert_ctx.lock()) { - SSL_CTX_use_certificate_ASN1(ctx, static_cast(client_cert->certificate.size()), - client_cert->certificate.data()); - SSL_CTX_use_PrivateKey_ASN1(EVP_PKEY_RSA, ctx, client_cert->private_key.data(), - static_cast(client_cert->private_key.size())); - } - - // TODO(B3N30): Check for SSLOptions-Bits and set the verify method accordingly - // https://www.3dbrew.org/wiki/SSL_Services#SSLOpt - // Hack: Since for now RootCerts are not implemented we set the VerifyMode to None. - SSL_CTX_set_verify(ctx, SSL_VERIFY_NONE, NULL); - } - - state = RequestState::InProgress; - static const std::unordered_map request_method_strings{ {RequestMethod::Get, "GET"}, {RequestMethod::Post, "POST"}, {RequestMethod::Head, "HEAD"}, {RequestMethod::Put, "PUT"}, @@ -103,11 +189,13 @@ void Context::MakeRequest() { {RequestMethod::PutEmpty, "PUT"}, }; + URLInfo url_info = SplitUrl(url); + httplib::Request request; - httplib::Error error; + std::vector pending_headers; request.method = request_method_strings.at(method); - request.path = path; - // TODO(B3N30): Add post data body + request.path = url_info.path; + request.progress = [this](u64 current, u64 total) -> bool { // TODO(B3N30): Is there a state that shows response header are available current_download_size_bytes = current; @@ -116,9 +204,99 @@ void Context::MakeRequest() { }; for (const auto& header : headers) { - request.headers.emplace(header.name, header.value); + pending_headers.push_back(header); } + if (!post_data.empty()) { + pending_headers.push_back( + Context::RequestHeader("Content-Type", "application/x-www-form-urlencoded")); + request.body = httplib::detail::params_to_query_str(post_data); + boost::replace_all(request.body, "*", "%2A"); + } + + if (!post_data_raw.empty()) { + request.body = post_data_raw; + } + + state = RequestState::InProgress; + + if (url_info.is_https) { + MakeRequestSSL(request, url_info, pending_headers); + } else { + MakeRequestNonSSL(request, url_info, pending_headers); + } +} + +void Context::MakeRequestNonSSL(httplib::Request& request, const URLInfo& url_info, + std::vector& pending_headers) { + httplib::Error error{-1}; + std::unique_ptr client = + std::make_unique(url_info.host, url_info.port); + + client->set_header_writer( + [&pending_headers](httplib::Stream& strm, httplib::Headers& httplib_headers) { + return HandleHeaderWrite(pending_headers, strm, httplib_headers); + }); + + if (!client->send(request, response, error)) { + LOG_ERROR(Service_HTTP, "Request failed: {}: {}", error, httplib::to_string(error)); + state = RequestState::TimedOut; + } else { + LOG_DEBUG(Service_HTTP, "Request successful"); + // TODO(B3N30): Verify this state on HW + state = RequestState::ReadyToDownloadContent; + } +} + +void Context::MakeRequestSSL(httplib::Request& request, const URLInfo& url_info, + std::vector& pending_headers) { + httplib::Error error{-1}; + X509* cert = nullptr; + EVP_PKEY* key = nullptr; + const unsigned char* cert_data = nullptr; + const unsigned char* key_data = nullptr; + long cert_size = 0; + long key_size = 0; + SCOPE_EXIT({ + if (cert) { + X509_free(cert); + } + if (key) { + EVP_PKEY_free(key); + } + }); + + if (uses_default_client_cert) { + cert_data = clcert_data->certificate.data(); + key_data = clcert_data->private_key.data(); + cert_size = static_cast(clcert_data->certificate.size()); + key_size = static_cast(clcert_data->private_key.size()); + } else if (auto client_cert = ssl_config.client_cert_ctx.lock()) { + cert_data = client_cert->certificate.data(); + key_data = client_cert->private_key.data(); + cert_size = static_cast(client_cert->certificate.size()); + key_size = static_cast(client_cert->private_key.size()); + } + + std::unique_ptr client; + if (cert_data && key_data) { + cert = d2i_X509(nullptr, &cert_data, cert_size); + key = d2i_PrivateKey(EVP_PKEY_RSA, nullptr, &key_data, key_size); + client = std::make_unique(url_info.host, url_info.port, cert, key); + } else { + client = std::make_unique(url_info.host, url_info.port); + } + + // TODO(B3N30): Check for SSLOptions-Bits and set the verify method accordingly + // https://www.3dbrew.org/wiki/SSL_Services#SSLOpt + // Hack: Since for now RootCerts are not implemented we set the VerifyMode to None. + client->enable_server_certificate_verification(false); + + client->set_header_writer( + [&pending_headers](httplib::Stream& strm, httplib::Headers& httplib_headers) { + return HandleHeaderWrite(pending_headers, strm, httplib_headers); + }); + if (!client->send(request, response, error)) { LOG_ERROR(Service_HTTP, "Request failed: {}: {}", error, httplib::to_string(error)); state = RequestState::TimedOut; @@ -138,7 +316,7 @@ void HTTP_C::Initialize(Kernel::HLERequestContext& ctx) { shared_memory->SetName("HTTP_C:shared_memory"); } - LOG_WARNING(Service_HTTP, "(STUBBED) called, shared memory size: {} pid: {}", shmem_size, pid); + LOG_DEBUG(Service_HTTP, "called, shared memory size: {} pid: {}", shmem_size, pid); auto* session_data = GetSessionData(ctx.Session()); ASSERT(session_data); @@ -198,14 +376,21 @@ void HTTP_C::BeginRequest(Kernel::HLERequestContext& ctx) { IPC::RequestParser rp(ctx); const Context::Handle context_handle = rp.Pop(); - LOG_WARNING(Service_HTTP, "(STUBBED) called, context_id={}", context_handle); + LOG_DEBUG(Service_HTTP, "called, context_id={}", context_handle); if (!PerformStateChecks(ctx, rp, context_handle)) { return; } - auto itr = contexts.find(context_handle); - ASSERT(itr != contexts.end()); + Context& http_context = GetContext(context_handle); + + // This should never happen in real hardware, but can happen on citra. + if (http_context.uses_default_client_cert && !http_context.clcert_data->init) { + LOG_ERROR(Service_HTTP, "Failed to begin HTTP request: client cert not found."); + IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); + rb.Push(ERROR_STATE_ERROR); + return; + } // On a 3DS BeginRequest and BeginRequestAsync will push the Request to a worker queue. // You can only enqueue 8 requests at the same time. @@ -214,8 +399,9 @@ void HTTP_C::BeginRequest(Kernel::HLERequestContext& ctx) { // Then there are 3? worker threads that pop the requests from the queue and send them // For now make every request async in it's own thread. - itr->second.request_future = - std::async(std::launch::async, &Context::MakeRequest, std::ref(itr->second)); + http_context.request_future = + std::async(std::launch::async, &Context::MakeRequest, std::ref(http_context)); + http_context.current_copied_data = 0; IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); rb.Push(RESULT_SUCCESS); @@ -225,14 +411,21 @@ void HTTP_C::BeginRequestAsync(Kernel::HLERequestContext& ctx) { IPC::RequestParser rp(ctx); const Context::Handle context_handle = rp.Pop(); - LOG_WARNING(Service_HTTP, "(STUBBED) called, context_id={}", context_handle); + LOG_DEBUG(Service_HTTP, "called, context_id={}", context_handle); if (!PerformStateChecks(ctx, rp, context_handle)) { return; } - auto itr = contexts.find(context_handle); - ASSERT(itr != contexts.end()); + Context& http_context = GetContext(context_handle); + + // This should never happen in real hardware, but can happen on citra. + if (http_context.uses_default_client_cert && !http_context.clcert_data->init) { + LOG_ERROR(Service_HTTP, "Failed to begin HTTP request: client cert not found."); + IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); + rb.Push(ERROR_STATE_ERROR); + return; + } // On a 3DS BeginRequest and BeginRequestAsync will push the Request to a worker queue. // You can only enqueue 8 requests at the same time. @@ -241,8 +434,9 @@ void HTTP_C::BeginRequestAsync(Kernel::HLERequestContext& ctx) { // Then there are 3? worker threads that pop the requests from the queue and send them // For now make every request async in it's own thread. - itr->second.request_future = - std::async(std::launch::async, &Context::MakeRequest, std::ref(itr->second)); + http_context.request_future = + std::async(std::launch::async, &Context::MakeRequest, std::ref(http_context)); + http_context.current_copied_data = 0; IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); rb.Push(RESULT_SUCCESS); @@ -258,30 +452,89 @@ void HTTP_C::ReceiveDataTimeout(Kernel::HLERequestContext& ctx) { void HTTP_C::ReceiveDataImpl(Kernel::HLERequestContext& ctx, bool timeout) { IPC::RequestParser rp(ctx); - const Context::Handle context_handle = rp.Pop(); - [[maybe_unused]] const u32 buffer_size = rp.Pop(); - u64 timeout_nanos = 0; - if (timeout) { - timeout_nanos = rp.Pop(); - LOG_WARNING(Service_HTTP, "(STUBBED) called, timeout={}", timeout_nanos); - } else { - LOG_WARNING(Service_HTTP, "(STUBBED) called"); - } - [[maybe_unused]] Kernel::MappedBuffer& buffer = rp.PopMappedBuffer(); - if (!PerformStateChecks(ctx, rp, context_handle)) { + struct AsyncData { + // Input + u64 timeout_nanos = 0; + bool timeout; + Context::Handle context_handle; + u32 buffer_size; + Kernel::MappedBuffer* buffer; + bool is_complete; + // Output + ResultCode async_res = RESULT_SUCCESS; + }; + std::shared_ptr async_data = std::make_shared(); + async_data->timeout = timeout; + async_data->context_handle = rp.Pop(); + async_data->buffer_size = rp.Pop(); + + if (timeout) { + async_data->timeout_nanos = rp.Pop(); + LOG_DEBUG(Service_HTTP, "called, timeout={}", async_data->timeout_nanos); + } else { + LOG_DEBUG(Service_HTTP, "called"); + } + async_data->buffer = &rp.PopMappedBuffer(); + + if (!PerformStateChecks(ctx, rp, async_data->context_handle)) { return; } - auto itr = contexts.find(context_handle); - ASSERT(itr != contexts.end()); + ctx.RunAsync( + [this, async_data](Kernel::HLERequestContext& ctx) { + Context& http_context = GetContext(async_data->context_handle); - if (timeout) { - itr->second.request_future.wait_for(std::chrono::nanoseconds(timeout_nanos)); - // TODO (flTobi): Return error on timeout - } else { - itr->second.request_future.wait(); - } + if (async_data->timeout) { + const auto wait_res = http_context.request_future.wait_for( + std::chrono::nanoseconds(async_data->timeout_nanos)); + if (wait_res == std::future_status::timeout) { + async_data->async_res = + ResultCode(105, ErrorModule::HTTP, ErrorSummary::NothingHappened, + ErrorLevel::Permanent); + } + } else { + http_context.request_future.wait(); + } + // Simulate small delay from HTTP receive. + return 1'000'000; + }, + [this, async_data](Kernel::HLERequestContext& ctx) { + IPC::RequestBuilder rb(ctx, static_cast(ctx.CommandHeader().command_id.Value()), 1, + 0); + if (async_data->async_res != RESULT_SUCCESS) { + rb.Push(async_data->async_res); + return; + } + Context& http_context = GetContext(async_data->context_handle); + + const size_t remaining_data = + http_context.response.body.size() - http_context.current_copied_data; + + if (async_data->buffer_size >= remaining_data) { + async_data->buffer->Write(http_context.response.body.data() + + http_context.current_copied_data, + 0, remaining_data); + http_context.current_copied_data += remaining_data; + rb.Push(RESULT_SUCCESS); + } else { + async_data->buffer->Write(http_context.response.body.data() + + http_context.current_copied_data, + 0, async_data->buffer_size); + http_context.current_copied_data += async_data->buffer_size; + rb.Push(ERROR_BUFFER_SMALL); + } + LOG_DEBUG(Service_HTTP, "Receive: buffer_size= {}, total_copied={}, total_body={}", + async_data->buffer_size, http_context.current_copied_data, + http_context.response.body.size()); + }); +} + +void HTTP_C::SetProxyDefault(Kernel::HLERequestContext& ctx) { + IPC::RequestParser rp(ctx); + const Context::Handle context_handle = rp.Pop(); + + LOG_WARNING(Service_HTTP, "(STUBBED) called, handle={}", context_handle); IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); rb.Push(RESULT_SUCCESS); @@ -359,7 +612,7 @@ void HTTP_C::CloseContext(Kernel::HLERequestContext& ctx) { u32 context_handle = rp.Pop(); - LOG_WARNING(Service_HTTP, "(STUBBED) called, handle={}", context_handle); + LOG_DEBUG(Service_HTTP, "called, handle={}", context_handle); auto* session_data = EnsureSessionInitialized(ctx, rp); if (!session_data) { @@ -389,6 +642,23 @@ void HTTP_C::CloseContext(Kernel::HLERequestContext& ctx) { rb.Push(RESULT_SUCCESS); } +void HTTP_C::CancelConnection(Kernel::HLERequestContext& ctx) { + IPC::RequestParser rp(ctx); + const u32 context_handle = rp.Pop(); + + LOG_WARNING(Service_HTTP, "(STUBBED) called, handle={}", context_handle); + + const auto* session_data = EnsureSessionInitialized(ctx, rp); + if (!session_data) { + return; + } + + [[maybe_unused]] Context& http_context = GetContext(context_handle); + + IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); + rb.Push(RESULT_SUCCESS); +} + void HTTP_C::AddRequestHeader(Kernel::HLERequestContext& ctx) { IPC::RequestParser rp(ctx); const u32 context_handle = rp.Pop(); @@ -411,10 +681,9 @@ void HTTP_C::AddRequestHeader(Kernel::HLERequestContext& ctx) { return; } - auto itr = contexts.find(context_handle); - ASSERT(itr != contexts.end()); + Context& http_context = GetContext(context_handle); - if (itr->second.state != RequestState::NotStarted) { + if (http_context.state != RequestState::NotStarted) { LOG_ERROR(Service_HTTP, "Tried to add a request header on a context that has already been started."); IPC::RequestBuilder rb = rp.MakeBuilder(1, 2); @@ -424,12 +693,7 @@ void HTTP_C::AddRequestHeader(Kernel::HLERequestContext& ctx) { return; } - ASSERT(std::find_if(itr->second.headers.begin(), itr->second.headers.end(), - [&name](const Context::RequestHeader& m) -> bool { - return m.name == name; - }) == itr->second.headers.end()); - - itr->second.headers.emplace_back(name, value); + http_context.headers.emplace_back(name, value); IPC::RequestBuilder rb = rp.MakeBuilder(1, 2); rb.Push(RESULT_SUCCESS); @@ -458,10 +722,9 @@ void HTTP_C::AddPostDataAscii(Kernel::HLERequestContext& ctx) { return; } - auto itr = contexts.find(context_handle); - ASSERT(itr != contexts.end()); + Context& http_context = GetContext(context_handle); - if (itr->second.state != RequestState::NotStarted) { + if (http_context.state != RequestState::NotStarted) { LOG_ERROR(Service_HTTP, "Tried to add post data on a context that has already been started."); IPC::RequestBuilder rb = rp.MakeBuilder(1, 2); @@ -471,17 +734,114 @@ void HTTP_C::AddPostDataAscii(Kernel::HLERequestContext& ctx) { return; } - ASSERT(std::find_if(itr->second.post_data.begin(), itr->second.post_data.end(), - [&name](const Context::PostData& m) -> bool { return m.name == name; }) == - itr->second.post_data.end()); - - itr->second.post_data.emplace_back(name, value); + http_context.post_data.emplace(name, value); IPC::RequestBuilder rb = rp.MakeBuilder(1, 2); rb.Push(RESULT_SUCCESS); rb.PushMappedBuffer(value_buffer); } +void HTTP_C::AddPostDataRaw(Kernel::HLERequestContext& ctx) { + IPC::RequestParser rp(ctx); + const u32 context_handle = rp.Pop(); + const u32 post_data_len = rp.Pop(); + auto buffer = rp.PopMappedBuffer(); + + LOG_DEBUG(Service_HTTP, "called, context_handle={}, post_data_len={}", context_handle, + post_data_len); + + if (!PerformStateChecks(ctx, rp, context_handle)) { + return; + } + + Context& http_context = GetContext(context_handle); + + if (http_context.state != RequestState::NotStarted) { + LOG_ERROR(Service_HTTP, + "Tried to add post data on a context that has already been started."); + IPC::RequestBuilder rb = rp.MakeBuilder(1, 2); + rb.Push(ResultCode(ErrCodes::InvalidRequestState, ErrorModule::HTTP, + ErrorSummary::InvalidState, ErrorLevel::Permanent)); + rb.PushMappedBuffer(buffer); + return; + } + + http_context.post_data_raw.resize(buffer.GetSize()); + buffer.Read(http_context.post_data_raw.data(), 0, buffer.GetSize()); + + IPC::RequestBuilder rb = rp.MakeBuilder(1, 2); + rb.Push(RESULT_SUCCESS); + rb.PushMappedBuffer(buffer); +} + +void HTTP_C::GetResponseHeader(Kernel::HLERequestContext& ctx) { + IPC::RequestParser rp(ctx); + + struct AsyncData { + u32 context_handle; + u32 name_len; + u32 value_max_len; + std::span header_name; + Kernel::MappedBuffer* value_buffer; + }; + std::shared_ptr async_data = std::make_shared(); + + async_data->context_handle = rp.Pop(); + async_data->name_len = rp.Pop(); + async_data->value_max_len = rp.Pop(); + async_data->header_name = rp.PopStaticBuffer(); + async_data->value_buffer = &rp.PopMappedBuffer(); + + if (!PerformStateChecks(ctx, rp, async_data->context_handle)) { + return; + } + + ctx.RunAsync( + [this, async_data](Kernel::HLERequestContext& ctx) { + Context& http_context = GetContext(async_data->context_handle); + http_context.request_future.wait(); + return 0; + }, + [this, async_data](Kernel::HLERequestContext& ctx) { + std::string header_name_str( + reinterpret_cast(async_data->header_name.data()), + async_data->name_len); + Common::TruncateString(header_name_str); + + Context& http_context = GetContext(async_data->context_handle); + + auto& headers = http_context.response.headers; + u32 copied_size = 0; + + LOG_DEBUG(Service_HTTP, "header={}, max_len={}", header_name_str, + async_data->value_buffer->GetSize()); + + auto header = headers.find(header_name_str); + if (header != headers.end()) { + std::string header_value = header->second; + copied_size = static_cast(header_value.size()); + if (header_value.size() >= async_data->value_buffer->GetSize()) { + header_value.resize(async_data->value_buffer->GetSize() - 1); + } + header_value.push_back('\0'); + async_data->value_buffer->Write(header_value.data(), 0, header_value.size()); + } else { + LOG_DEBUG(Service_HTTP, "header={} not found", header_name_str); + IPC::RequestBuilder rb( + ctx, static_cast(ctx.CommandHeader().command_id.Value()), 1, 2); + rb.Push(ERROR_HEADER_NOT_FOUND); + rb.PushMappedBuffer(*async_data->value_buffer); + return; + } + + IPC::RequestBuilder rb(ctx, static_cast(ctx.CommandHeader().command_id.Value()), 2, + 2); + rb.Push(RESULT_SUCCESS); + rb.Push(copied_size); + rb.PushMappedBuffer(*async_data->value_buffer); + }); +} + void HTTP_C::GetResponseStatusCode(Kernel::HLERequestContext& ctx) { GetResponseStatusCodeImpl(ctx, false); } @@ -492,34 +852,118 @@ void HTTP_C::GetResponseStatusCodeTimeout(Kernel::HLERequestContext& ctx) { void HTTP_C::GetResponseStatusCodeImpl(Kernel::HLERequestContext& ctx, bool timeout) { IPC::RequestParser rp(ctx); - const Context::Handle context_handle = rp.Pop(); - u64 timeout_nanos = 0; + + struct AsyncData { + // Input + Context::Handle context_handle; + bool timeout; + u64 timeout_nanos = 0; + // Output + ResultCode async_res = RESULT_SUCCESS; + }; + std::shared_ptr async_data = std::make_shared(); + + async_data->context_handle = rp.Pop(); + async_data->timeout = timeout; + if (timeout) { - timeout_nanos = rp.Pop(); - LOG_INFO(Service_HTTP, "called, timeout={}", timeout_nanos); + async_data->timeout_nanos = rp.Pop(); + LOG_INFO(Service_HTTP, "called, timeout={}", async_data->timeout_nanos); } else { LOG_INFO(Service_HTTP, "called"); } + if (!PerformStateChecks(ctx, rp, async_data->context_handle)) { + return; + } + + ctx.RunAsync( + [this, async_data](Kernel::HLERequestContext& ctx) { + Context& http_context = GetContext(async_data->context_handle); + + if (async_data->timeout) { + const auto wait_res = http_context.request_future.wait_for( + std::chrono::nanoseconds(async_data->timeout_nanos)); + if (wait_res == std::future_status::timeout) { + LOG_DEBUG(Service_HTTP, "Status code: {}", "timeout"); + async_data->async_res = + ResultCode(105, ErrorModule::HTTP, ErrorSummary::NothingHappened, + ErrorLevel::Permanent); + } + } else { + http_context.request_future.wait(); + } + return 0; + }, + [this, async_data](Kernel::HLERequestContext& ctx) { + if (async_data->async_res != RESULT_SUCCESS) { + IPC::RequestBuilder rb( + ctx, static_cast(ctx.CommandHeader().command_id.Value()), 1, 0); + rb.Push(async_data->async_res); + return; + } + + Context& http_context = GetContext(async_data->context_handle); + + const u32 response_code = http_context.response.status; + LOG_DEBUG(Service_HTTP, "Status code: {}, response_code={}", "good", response_code); + + IPC::RequestBuilder rb(ctx, static_cast(ctx.CommandHeader().command_id.Value()), 2, + 0); + rb.Push(RESULT_SUCCESS); + rb.Push(response_code); + }); +} + +void HTTP_C::AddTrustedRootCA(Kernel::HLERequestContext& ctx) { + IPC::RequestParser rp(ctx); + const Context::Handle context_handle = rp.Pop(); + [[maybe_unused]] const u32 root_ca_len = rp.Pop(); + auto root_ca_data = rp.PopMappedBuffer(); + + LOG_WARNING(Service_HTTP, "(STUBBED) called, handle={}", context_handle); + + IPC::RequestBuilder rb = rp.MakeBuilder(1, 2); + rb.Push(RESULT_SUCCESS); + rb.PushMappedBuffer(root_ca_data); +} + +void HTTP_C::AddDefaultCert(Kernel::HLERequestContext& ctx) { + IPC::RequestParser rp(ctx); + const Context::Handle context_handle = rp.Pop(); + const u32 certificate_id = rp.Pop(); + + LOG_WARNING(Service_HTTP, "(STUBBED) called, handle={}, certificate_id={}", context_handle, + certificate_id); + + IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); + rb.Push(RESULT_SUCCESS); +} + +void HTTP_C::SetDefaultClientCert(Kernel::HLERequestContext& ctx) { + IPC::RequestParser rp(ctx); + const Context::Handle context_handle = rp.Pop(); + const ClientCertID client_cert_id = static_cast(rp.Pop()); + + LOG_DEBUG(Service_HTTP, "client_cert_id={}", client_cert_id); + if (!PerformStateChecks(ctx, rp, context_handle)) { return; } - auto itr = contexts.find(context_handle); - ASSERT(itr != contexts.end()); + Context& http_context = GetContext(context_handle); - if (timeout) { - itr->second.request_future.wait_for(std::chrono::nanoseconds(timeout)); - // TODO (flTobi): Return error on timeout - } else { - itr->second.request_future.wait(); + if (client_cert_id != ClientCertID::Default) { + IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); + rb.Push(ERROR_WRONG_CERT_ID); + return; } - const u32 response_code = itr->second.response.status; + http_context.uses_default_client_cert = true; + http_context.clcert_data = &GetClCertA(); - IPC::RequestBuilder rb = rp.MakeBuilder(2, 0); + IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); rb.Push(RESULT_SUCCESS); - rb.Push(response_code); } void HTTP_C::SetClientCertContext(Kernel::HLERequestContext& ctx) { @@ -534,8 +978,7 @@ void HTTP_C::SetClientCertContext(Kernel::HLERequestContext& ctx) { return; } - auto http_context_itr = contexts.find(context_handle); - ASSERT(http_context_itr != contexts.end()); + Context& http_context = GetContext(context_handle); auto cert_context_itr = client_certs.find(client_cert_handle); if (cert_context_itr == client_certs.end()) { @@ -545,7 +988,7 @@ void HTTP_C::SetClientCertContext(Kernel::HLERequestContext& ctx) { return; } - if (http_context_itr->second.ssl_config.client_cert_ctx.lock()) { + if (http_context.ssl_config.client_cert_ctx.lock()) { LOG_ERROR(Service_HTTP, "Tried to set a client cert to a context that already has a client cert"); IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); @@ -553,7 +996,7 @@ void HTTP_C::SetClientCertContext(Kernel::HLERequestContext& ctx) { return; } - if (http_context_itr->second.state != RequestState::NotStarted) { + if (http_context.state != RequestState::NotStarted) { LOG_ERROR(Service_HTTP, "Tried to set a client cert on a context that has already been started."); IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); @@ -562,7 +1005,7 @@ void HTTP_C::SetClientCertContext(Kernel::HLERequestContext& ctx) { return; } - http_context_itr->second.ssl_config.client_cert_ctx = cert_context_itr->second; + http_context.ssl_config.client_cert_ctx = cert_context_itr->second; IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); rb.Push(RESULT_SUCCESS); } @@ -574,8 +1017,7 @@ void HTTP_C::GetSSLError(Kernel::HLERequestContext& ctx) { LOG_WARNING(Service_HTTP, "(STUBBED) called, context_handle={}, unk={}", context_handle, unk); - auto http_context_itr = contexts.find(context_handle); - ASSERT(http_context_itr != contexts.end()); + [[maybe_unused]] Context& http_context = GetContext(context_handle); IPC::RequestBuilder rb = rp.MakeBuilder(2, 0); rb.Push(RESULT_SUCCESS); @@ -584,6 +1026,17 @@ void HTTP_C::GetSSLError(Kernel::HLERequestContext& ctx) { rb.Push(0); } +void HTTP_C::SetSSLOpt(Kernel::HLERequestContext& ctx) { + IPC::RequestParser rp(ctx); + const u32 context_handle = rp.Pop(); + const u32 opts = rp.Pop(); + + LOG_WARNING(Service_HTTP, "(STUBBED) called, context_handle={}, opts={}", context_handle, opts); + + IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); + rb.Push(RESULT_SUCCESS); +} + void HTTP_C::OpenClientCertContext(Kernel::HLERequestContext& ctx) { IPC::RequestParser rp(ctx); u32 cert_size = rp.Pop(); @@ -650,7 +1103,7 @@ void HTTP_C::OpenDefaultClientCertContext(Kernel::HLERequestContext& ctx) { return; } - constexpr u8 default_cert_id = 0x40; + constexpr u8 default_cert_id = static_cast(ClientCertID::Default); if (cert_id != default_cert_id) { LOG_ERROR(Service_HTTP, "called with invalid cert_id {}", cert_id); IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); @@ -724,6 +1177,17 @@ void HTTP_C::CloseClientCertContext(Kernel::HLERequestContext& ctx) { rb.Push(RESULT_SUCCESS); } +void HTTP_C::SetKeepAlive(Kernel::HLERequestContext& ctx) { + IPC::RequestParser rp(ctx); + const u32 context_handle = rp.Pop(); + const u32 option = rp.Pop(); + + LOG_WARNING(Service_HTTP, "(STUBBED) called, handle={}, option={}", context_handle, option); + + IPC::RequestBuilder rb = rp.MakeBuilder(1, 0); + rb.Push(RESULT_SUCCESS); +} + void HTTP_C::Finalize(Kernel::HLERequestContext& ctx) { IPC::RequestParser rp(ctx); @@ -746,26 +1210,28 @@ void HTTP_C::GetDownloadSizeState(Kernel::HLERequestContext& ctx) { return; } - auto itr = contexts.find(context_handle); - ASSERT(itr != contexts.end()); + Context& http_context = GetContext(context_handle); // On the real console, the current downloaded progress and the total size of the content gets // returned. Since we do not support chunked downloads on the host, always return the content // length if the download is complete and 0 otherwise. u32 content_length = 0; - const bool is_complete = itr->second.request_future.wait_for(std::chrono::milliseconds(0)) == + const bool is_complete = http_context.request_future.wait_for(std::chrono::milliseconds(0)) == std::future_status::ready; if (is_complete) { - const auto& headers = itr->second.response.headers; + const auto& headers = http_context.response.headers; const auto& it = headers.find("Content-Length"); if (it != headers.end()) { content_length = std::stoi(it->second); } } + LOG_DEBUG(Service_HTTP, "current={}, total={}", http_context.current_copied_data, + content_length); + IPC::RequestBuilder rb = rp.MakeBuilder(3, 0); rb.Push(RESULT_SUCCESS); - rb.Push(content_length); + rb.Push(static_cast(http_context.current_copied_data)); rb.Push(content_length); } @@ -895,7 +1361,7 @@ HTTP_C::HTTP_C() : ServiceFramework("http:C", 32) { {0x0001, &HTTP_C::Initialize, "Initialize"}, {0x0002, &HTTP_C::CreateContext, "CreateContext"}, {0x0003, &HTTP_C::CloseContext, "CloseContext"}, - {0x0004, nullptr, "CancelConnection"}, + {0x0004, &HTTP_C::CancelConnection, "CancelConnection"}, {0x0005, nullptr, "GetRequestState"}, {0x0006, &HTTP_C::GetDownloadSizeState, "GetDownloadSizeState"}, {0x0007, nullptr, "GetRequestError"}, @@ -905,13 +1371,13 @@ HTTP_C::HTTP_C() : ServiceFramework("http:C", 32) { {0x000B, &HTTP_C::ReceiveData, "ReceiveData"}, {0x000C, &HTTP_C::ReceiveDataTimeout, "ReceiveDataTimeout"}, {0x000D, nullptr, "SetProxy"}, - {0x000E, nullptr, "SetProxyDefault"}, + {0x000E, &HTTP_C::SetProxyDefault, "SetProxyDefault"}, {0x000F, nullptr, "SetBasicAuthorization"}, {0x0010, nullptr, "SetSocketBufferSize"}, {0x0011, &HTTP_C::AddRequestHeader, "AddRequestHeader"}, {0x0012, &HTTP_C::AddPostDataAscii, "AddPostDataAscii"}, {0x0013, nullptr, "AddPostDataBinary"}, - {0x0014, nullptr, "AddPostDataRaw"}, + {0x0014, &HTTP_C::AddPostDataRaw, "AddPostDataRaw"}, {0x0015, nullptr, "SetPostDataType"}, {0x0016, nullptr, "SendPostDataAscii"}, {0x0017, nullptr, "SendPostDataAsciiTimeout"}, @@ -921,19 +1387,20 @@ HTTP_C::HTTP_C() : ServiceFramework("http:C", 32) { {0x001B, nullptr, "SendPOSTDataRawTimeout"}, {0x001C, nullptr, "SetPostDataEncoding"}, {0x001D, nullptr, "NotifyFinishSendPostData"}, - {0x001E, nullptr, "GetResponseHeader"}, + {0x001E, &HTTP_C::GetResponseHeader, "GetResponseHeader"}, {0x001F, nullptr, "GetResponseHeaderTimeout"}, {0x0020, nullptr, "GetResponseData"}, {0x0021, nullptr, "GetResponseDataTimeout"}, {0x0022, &HTTP_C::GetResponseStatusCode, "GetResponseStatusCode"}, {0x0023, &HTTP_C::GetResponseStatusCodeTimeout, "GetResponseStatusCodeTimeout"}, - {0x0024, nullptr, "AddTrustedRootCA"}, - {0x0025, nullptr, "AddDefaultCert"}, + {0x0024, &HTTP_C::AddTrustedRootCA, "AddTrustedRootCA"}, + {0x0025, &HTTP_C::AddDefaultCert, "AddDefaultCert"}, {0x0026, nullptr, "SelectRootCertChain"}, {0x0027, nullptr, "SetClientCert"}, + {0x0028, &HTTP_C::SetDefaultClientCert, "SetDefaultClientCert"}, {0x0029, &HTTP_C::SetClientCertContext, "SetClientCertContext"}, {0x002A, &HTTP_C::GetSSLError, "GetSSLError"}, - {0x002B, nullptr, "SetSSLOpt"}, + {0x002B, &HTTP_C::SetSSLOpt, "SetSSLOpt"}, {0x002C, nullptr, "SetSSLClearOpt"}, {0x002D, nullptr, "CreateRootCertChain"}, {0x002E, nullptr, "DestroyRootCertChain"}, @@ -945,7 +1412,7 @@ HTTP_C::HTTP_C() : ServiceFramework("http:C", 32) { {0x0034, &HTTP_C::CloseClientCertContext, "CloseClientCertContext"}, {0x0035, nullptr, "SetDefaultProxy"}, {0x0036, nullptr, "ClearDNSCache"}, - {0x0037, nullptr, "SetKeepAlive"}, + {0x0037, &HTTP_C::SetKeepAlive, "SetKeepAlive"}, {0x0038, nullptr, "SetPostDataTypeSize"}, {0x0039, &HTTP_C::Finalize, "Finalize"}, // clang-format on @@ -955,6 +1422,10 @@ HTTP_C::HTTP_C() : ServiceFramework("http:C", 32) { DecryptClCertA(); } +std::shared_ptr GetService(Core::System& system) { + return system.ServiceManager().GetService("http:C"); +} + void InstallInterfaces(Core::System& system) { auto& service_manager = system.ServiceManager(); std::make_shared()->InstallAsService(service_manager); diff --git a/src/core/hle/service/http/http_c.h b/src/core/hle/service/http/http_c.h index 4ba7d2576c..07547d05ea 100644 --- a/src/core/hle/service/http/http_c.h +++ b/src/core/hle/service/http/http_c.h @@ -21,6 +21,7 @@ #include #endif #include +#include "core/hle/ipc_helpers.h" #include "core/hle/kernel/shared_memory.h" #include "core/hle/service/service.h" @@ -56,6 +57,17 @@ enum class RequestState : u8 { TimedOut = 0xA, // Request timed out? }; +enum class ClientCertID : u32 { + Default = 0x40, // Default client cert +}; + +struct URLInfo { + bool is_https; + std::string host; + int port; + std::string path; +}; + /// Represents a client certificate along with its private key, stored as a byte array of DER data. /// There can only be at most one client certificate context attached to an HTTP context at any /// given time. @@ -114,6 +126,12 @@ private: friend class boost::serialization::access; }; +struct ClCertAData { + std::vector certificate; + std::vector private_key; + bool init = false; +}; + /// Represents an HTTP context. class Context final { public: @@ -123,8 +141,6 @@ public: Context(const Context&) = delete; Context& operator=(const Context&) = delete; - void MakeRequest(); - struct Proxy { std::string url; std::string username; @@ -169,22 +185,6 @@ public: friend class boost::serialization::access; }; - struct PostData { - // TODO(Subv): Support Binary and Raw POST elements. - PostData(std::string name, std::string value) : name(name), value(value){}; - PostData() = default; - std::string name; - std::string value; - - private: - template - void serialize(Archive& ar, const unsigned int) { - ar& name; - ar& value; - } - friend class boost::serialization::access; - }; - struct SSLConfig { u32 options; std::weak_ptr client_cert_ctx; @@ -210,12 +210,22 @@ public: SSLConfig ssl_config{}; u32 socket_buffer_size; std::vector headers; - std::vector post_data; + const ClCertAData* clcert_data; + httplib::Params post_data; + std::string post_data_raw; std::future request_future; std::atomic current_download_size_bytes; std::atomic total_download_size_bytes; + size_t current_copied_data; + bool uses_default_client_cert{}; httplib::Response response; + + void MakeRequest(); + void MakeRequestNonSSL(httplib::Request& request, const URLInfo& url_info, + std::vector& pending_headers); + void MakeRequestSSL(httplib::Request& request, const URLInfo& url_info, + std::vector& pending_headers); }; struct SessionData : public Kernel::SessionRequestHandler::SessionDataBase { @@ -252,6 +262,10 @@ class HTTP_C final : public ServiceFramework { public: HTTP_C(); + const ClCertAData& GetClCertA() const { + return ClCertA; + } + private: /** * HTTP_C::Initialize service function @@ -288,6 +302,15 @@ private: */ void CloseContext(Kernel::HLERequestContext& ctx); + /** + * HTTP_C::CancelConnection service function + * Inputs: + * 1 : Context handle + * Outputs: + * 1 : Result of function, 0 on success, otherwise error code + */ + void CancelConnection(Kernel::HLERequestContext& ctx); + /** * HTTP_C::GetDownloadSizeState service function * Inputs: @@ -328,6 +351,15 @@ private: */ void BeginRequestAsync(Kernel::HLERequestContext& ctx); + /** + * HTTP_C::SetProxyDefault service function + * Inputs: + * 1 : Context handle + * Outputs: + * 1 : Result of function, 0 on success, otherwise error code + */ + void SetProxyDefault(Kernel::HLERequestContext& ctx); + /** * HTTP_C::ReceiveData service function * Inputs: @@ -389,6 +421,33 @@ private: */ void AddPostDataAscii(Kernel::HLERequestContext& ctx); + /** + * HTTP_C::AddPostDataRaw service function + * Inputs: + * 1 : Context handle + * 2 : Post data length + * 3-4: (Mapped buffer) Post data + * Outputs: + * 1 : Result of function, 0 on success, otherwise error code + * 2-3: (Mapped buffer) Post data + */ + void AddPostDataRaw(Kernel::HLERequestContext& ctx); + + /** + * HTTP_C::GetResponseHeader service function + * Inputs: + * 1 : Context handle + * 2 : Header name length + * 3 : Return value length + * 4-5 : (Static buffer) Header name + * 6-7 : (Mapped buffer) Header value + * Outputs: + * 1 : Result of function, 0 on success, otherwise error code + * 2 : Header value copied size + * 3-4: (Mapped buffer) Header value + */ + void GetResponseHeader(Kernel::HLERequestContext& ctx); + /** * HTTP_C::GetResponseStatusCode service function * Inputs: @@ -410,12 +469,44 @@ private: */ void GetResponseStatusCodeTimeout(Kernel::HLERequestContext& ctx); + /** + * HTTP_C::AddTrustedRootCA service function + * Inputs: + * 1 : Context handle + * 2 : CA data length + * 3-4: (Mapped buffer) CA data + * Outputs: + * 1 : Result of function, 0 on success, otherwise error code + * 2-3: (Mapped buffer) CA data + */ + void AddTrustedRootCA(Kernel::HLERequestContext& ctx); + + /** + * HTTP_C::AddDefaultCert service function + * Inputs: + * 1 : Context handle + * 2 : Cert ID + * Outputs: + * 1 : Result of function, 0 on success, otherwise error code + */ + void AddDefaultCert(Kernel::HLERequestContext& ctx); + /** * GetResponseStatusCodeImpl: * Implements GetResponseStatusCode and GetResponseStatusCodeTimeout service functions */ void GetResponseStatusCodeImpl(Kernel::HLERequestContext& ctx, bool timeout); + /** + * HTTP_C::SetDefaultClientCert service function + * Inputs: + * 1 : Context handle + * 2 : Client cert ID + * Outputs: + * 1 : Result of function, 0 on success, otherwise error code + */ + void SetDefaultClientCert(Kernel::HLERequestContext& ctx); + /** * HTTP_C::SetClientCertContext service function * Inputs: @@ -437,6 +528,16 @@ private: */ void GetSSLError(Kernel::HLERequestContext& ctx); + /** + * HTTP_C::SetSSLOpt service function + * Inputs: + * 1 : Context handle + * 2 : SSL Option + * Outputs: + * 1 : Result of function, 0 on success, otherwise error code + */ + void SetSSLOpt(Kernel::HLERequestContext& ctx); + /** * HTTP_C::OpenClientCertContext service function * Inputs: @@ -470,6 +571,16 @@ private: */ void CloseClientCertContext(Kernel::HLERequestContext& ctx); + /** + * HTTP_C::SetKeepAlive service function + * Inputs: + * 1 : Context handle + * 2 : Keep Alive Option + * Outputs: + * 1 : Result of function, 0 on success, otherwise error code + */ + void SetKeepAlive(Kernel::HLERequestContext& ctx); + /** * HTTP_C::Finalize service function * Outputs: @@ -499,14 +610,17 @@ private: /// Global list of HTTP contexts currently opened. std::unordered_map contexts; + // Get context from its handle + inline Context& GetContext(const Context::Handle& handle) { + auto it = contexts.find(handle); + ASSERT(it != contexts.end()); + return it->second; + } + /// Global list of ClientCert contexts currently opened. std::unordered_map> client_certs; - struct { - std::vector certificate; - std::vector private_key; - bool init = false; - } ClCertA; + ClCertAData ClCertA; private: template @@ -527,6 +641,8 @@ private: friend class boost::serialization::access; }; +std::shared_ptr GetService(Core::System& system); + void InstallInterfaces(Core::System& system); } // namespace Service::HTTP