http: share deterministic header field parser

contrib/epee/include/net/http_base.h

@@ -29,7 +29,7 @@
 #pragma once
 #include "memwipe.h"
 
-#include <boost/utility/string_ref.hpp>
+#include <boost/utility/string_view.hpp>
 
 #include <string>
 #include <utility>
@@ -58,7 +58,7 @@ namespace net_utils
 
 		typedef std::list<std::pair<std::string, std::string> > fields_list;
 
-		static inline void add_field(std::string& out, const boost::string_ref name, const boost::string_ref value)
+		static inline void add_field(std::string& out, boost::string_view name, boost::string_view value)
 		{
 			out.append(name.data(), name.size()).append(": ");
 			out.append(value.data(), value.size()).append("\r\n");
@@ -68,6 +68,42 @@ namespace net_utils
 			add_field(out, field.first, field.second);
 		}
 
+		namespace detail
+		{
+			inline bool parse_header_line(boost::string_view line, boost::string_view& name, boost::string_view& value)
+			{
+				if(!line.empty() && line.back() == '\r')
+					line.remove_suffix(1);
+				if(line.empty())
+					return false;
+				if(line.front() == ' ' || line.front() == '\t')
+					return false;
+
+				const size_t colon = line.find(':');
+				if(colon == boost::string_view::npos || colon == 0)
+					return false;
+
+				name = line.substr(0, colon);
+				value = line.substr(colon + 1);
+				while(!value.empty() && (value.front() == ' ' || value.front() == '\t'))
+					value.remove_prefix(1);
+				while(!value.empty() && (value.back() == ' ' || value.back() == '\t'))
+					value.remove_suffix(1);
+
+				for(char c : name)
+				{
+					const bool alnum = (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9');
+					const bool allowed = alnum ||
+						c == '!' || c == '#' || c == '$' || c == '%' || c == '&' ||
+						c == '\'' || c == '*' || c == '+' || c == '-' || c == '.' ||
+						c == '^' || c == '_' || c == '`' || c == '|' || c == '~';
+					if(!allowed)
+						return false;
+				}
+
+				return true;
+			}
+		}
 
 		struct http_header_info
 		{

contrib/epee/include/net/http_client.h

@@ -361,7 +361,11 @@ namespace net_utils
 					recv_buff.assign(m_header_cache.begin()+pos+4, m_header_cache.end());
 					m_header_cache.erase(m_header_cache.begin()+pos+4, m_header_cache.end());
 
-					analize_cached_header_and_invoke_state();
+					if(!analize_cached_header_and_invoke_state())
+					{
+						m_state = reciev_machine_state_error;
+						return false;
+					}
           if (!on_header(m_response_info))
           {
             MDEBUG("Connection cancelled by on_header");
@@ -594,64 +598,46 @@ namespace net_utils
 			{
 				MTRACE("http_stream_filter::parse_cached_header(*)");
 
-				const char *ptr = m_cache_to_process.c_str();
+				size_t cur = 0;
-				while (ptr[0] != '\r' || ptr[1] != '\n')
+				while(cur < m_cache_to_process.size())
 				{
-					// optional \n
+					const size_t line_end = m_cache_to_process.find('\n', cur);
-					if (*ptr == '\n')
+					CHECK_AND_ASSERT_MES(line_end != std::string::npos, false, "http_stream_filter::parse_cached_header() invalid header in: " << m_cache_to_process);
-						++ptr;
-					// an identifier composed of letters or -
-					const char *key_pos = ptr;
-					while (isalnum(*ptr) || *ptr == '_' || *ptr == '-')
-						++ptr;
-					const char *key_end = ptr;
-					// optional space (not in RFC, but in previous code)
-					if (*ptr == ' ')
-						++ptr;
-					CHECK_AND_ASSERT_MES(*ptr == ':', true, "http_stream_filter::parse_cached_header() invalid header in: " << m_cache_to_process);
-					++ptr;
-					// optional whitespace, but not newlines - line folding is obsolete, let's ignore it
-					while (isblank(*ptr))
-						++ptr;
-					const char *value_pos = ptr;
-					while (*ptr != '\r' && *ptr != '\n')
-						++ptr;
-					const char *value_end = ptr;
-					// optional trailing whitespace
-					while (value_end > value_pos && isblank(*(value_end-1)))
-						--value_end;
-					if (*ptr == '\r')
-						++ptr;
-					CHECK_AND_ASSERT_MES(*ptr == '\n', true, "http_stream_filter::parse_cached_header() invalid header in: " << m_cache_to_process);
-					++ptr;
 
-					const std::string key = std::string(key_pos, key_end - key_pos);
+					boost::string_view line(m_cache_to_process.data() + cur, line_end - cur);
-					const std::string value = std::string(value_pos, value_end - value_pos);
+					cur = line_end + 1;
-					if (!key.empty())
+
-					{
+					if(line == "\r" || line.empty())
-						if (!string_tools::compare_no_case(key, "Connection"))
+						break;
-							body_info.m_connection = value;
+
-						else if(!string_tools::compare_no_case(key, "Referrer"))
+					boost::string_view name;
-							body_info.m_referer = value;
+					boost::string_view value;
-						else if(!string_tools::compare_no_case(key, "Content-Length"))
+					CHECK_AND_ASSERT_MES(detail::parse_header_line(line, name, value), false, "http_stream_filter::parse_cached_header() invalid header in: " << m_cache_to_process);
-							body_info.m_content_length = value;
+
-						else if(!string_tools::compare_no_case(key, "Content-Type"))
+					std::string key(name.data(), name.size());
-							body_info.m_content_type = value;
+					std::string val(value.data(), value.size());
-						else if(!string_tools::compare_no_case(key, "Transfer-Encoding"))
+					if (!string_tools::compare_no_case(key, "Connection"))
-							body_info.m_transfer_encoding = value;
+						body_info.m_connection = std::move(val);
-						else if(!string_tools::compare_no_case(key, "Content-Encoding"))
+					else if(!string_tools::compare_no_case(key, "Referer"))
-							body_info.m_content_encoding = value;
+						body_info.m_referer = std::move(val);
-						else if(!string_tools::compare_no_case(key, "Host"))
+					else if(!string_tools::compare_no_case(key, "Content-Length"))
-							body_info.m_host = value;
+						body_info.m_content_length = std::move(val);
-						else if(!string_tools::compare_no_case(key, "Cookie"))
+					else if(!string_tools::compare_no_case(key, "Content-Type"))
-							body_info.m_cookie = value;
+						body_info.m_content_type = std::move(val);
-						else if(!string_tools::compare_no_case(key, "User-Agent"))
+					else if(!string_tools::compare_no_case(key, "Transfer-Encoding"))
-							body_info.m_user_agent = value;
+						body_info.m_transfer_encoding = std::move(val);
-						else if(!string_tools::compare_no_case(key, "Origin"))
+					else if(!string_tools::compare_no_case(key, "Content-Encoding"))
-							body_info.m_origin = value;
+						body_info.m_content_encoding = std::move(val);
-						else
+					else if(!string_tools::compare_no_case(key, "Host"))
-							body_info.m_etc_fields.emplace_back(key, value);
+						body_info.m_host = std::move(val);
-					}
+					else if(!string_tools::compare_no_case(key, "Cookie"))
+						body_info.m_cookie = std::move(val);
+					else if(!string_tools::compare_no_case(key, "User-Agent"))
+						body_info.m_user_agent = std::move(val);
+					else if(!string_tools::compare_no_case(key, "Origin"))
+						body_info.m_origin = std::move(val);
+					else
+						body_info.m_etc_fields.emplace_back(std::move(key), std::move(val));
 				}
 				return true;
 			}

contrib/epee/include/net/http_protocol_handler.inl

@@ -25,8 +25,9 @@
 // 
 
 
-#include <boost/regex.hpp>
+#include <boost/algorithm/string/predicate.hpp>
 #include <boost/lexical_cast.hpp>
+#include <boost/regex.hpp>
 #include "http_protocol_handler.h"
 #include "string_tools.h"
 #include "file_io_utils.h"
@@ -545,54 +546,52 @@ namespace net_utils
   template<class t_connection_context>
 	bool simple_http_connection_handler<t_connection_context>::parse_cached_header(http_header_info& body_info, const std::string& m_cache_to_process, size_t pos)
 	{ 
-		static const boost::regex rexp_mach_field(
-			"\n?((Connection)|(Referer)|(Content-Length)|(Content-Type)|(Transfer-Encoding)|(Content-Encoding)|(Host)|(Cookie)|(User-Agent)|(Origin)"
-			//  12            3         4                5              6                   7                  8      9        10           11
-			"|([\\w-]+?)) ?: ?((.*?)(\r?\n))[^\t ]",	
-			//11             1213   14 
-			boost::regex::icase | boost::regex::normal);
-
-		boost::smatch		result;
-		std::string::const_iterator it_current_bound = m_cache_to_process.begin();
-		std::string::const_iterator it_end_bound = m_cache_to_process.begin()+pos;
-
 		body_info.clear();
+		if(pos > m_cache_to_process.size() || pos > HTTP_MAX_HEADER_LEN)
+			return false;
 
-		//lookup all fields and fill well-known fields
+		size_t cur = 0;
-		while( boost::regex_search( it_current_bound, it_end_bound, result, rexp_mach_field, boost::match_default) && result[0].matched) 
+
+		while(cur < pos)
 		{
-			const size_t field_val = 14;
+			const size_t line_end = m_cache_to_process.find('\n', cur);
-			const size_t field_etc_name = 12;
+			if(line_end == std::string::npos || line_end >= pos)
+				break;
 
-			int i = 2; //start position = 2
+			boost::string_view line(m_cache_to_process.data() + cur, line_end - cur);
-			if(result[i++].matched)//"Connection"
+			cur = line_end + 1;
-				body_info.m_connection = result[field_val];
+
-			else if(result[i++].matched)//"Referer"
+			// End of header block.
-				body_info.m_referer = result[field_val];
+			if(line == "\r" || line.empty())
-			else if(result[i++].matched)//"Content-Length"
+				break;
-				body_info.m_content_length = result[field_val];
+
-			else if(result[i++].matched)//"Content-Type"
+			boost::string_view name;
-				body_info.m_content_type = result[field_val];
+			boost::string_view value;
-			else if(result[i++].matched)//"Transfer-Encoding"
+			if(!detail::parse_header_line(line, name, value))
-				body_info.m_transfer_encoding = result[field_val];
+				return false;
-			else if(result[i++].matched)//"Content-Encoding"
+
-				body_info.m_content_encoding = result[field_val];
+			if(boost::iequals(name, "Connection"))
-			else if(result[i++].matched)//"Host"
+				body_info.m_connection = std::string(value.data(), value.size());
-				body_info.m_host = result[field_val];
+			else if(boost::iequals(name, "Referer"))
-			else if(result[i++].matched)//"Cookie"
+				body_info.m_referer = std::string(value.data(), value.size());
-				body_info.m_cookie = result[field_val];
+			else if(boost::iequals(name, "Content-Length"))
-			else if(result[i++].matched)//"User-Agent"
+				body_info.m_content_length = std::string(value.data(), value.size());
-				body_info.m_user_agent = result[field_val];
+			else if(boost::iequals(name, "Content-Type"))
-			else if(result[i++].matched)//"Origin"
+				body_info.m_content_type = std::string(value.data(), value.size());
-				body_info.m_origin = result[field_val];
+			else if(boost::iequals(name, "Transfer-Encoding"))
-			else if(result[i++].matched)//e.t.c (HAVE TO BE MATCHED!)
+				body_info.m_transfer_encoding = std::string(value.data(), value.size());
-				body_info.m_etc_fields.push_back(std::pair<std::string, std::string>(result[field_etc_name], result[field_val]));
+			else if(boost::iequals(name, "Content-Encoding"))
+				body_info.m_content_encoding = std::string(value.data(), value.size());
+			else if(boost::iequals(name, "Host"))
+				body_info.m_host = std::string(value.data(), value.size());
+			else if(boost::iequals(name, "Cookie"))
+				body_info.m_cookie = std::string(value.data(), value.size());
+			else if(boost::iequals(name, "User-Agent"))
+				body_info.m_user_agent = std::string(value.data(), value.size());
+			else if(boost::iequals(name, "Origin"))
+				body_info.m_origin = std::string(value.data(), value.size());
 			else
-			{
+				body_info.m_etc_fields.push_back(std::make_pair(std::string(name.data(), name.size()), std::string(value.data(), value.size())));
-				LOG_ERROR_CC(m_conn_context, "simple_http_connection_handler<t_connection_context>::parse_cached_header() not matched last entry in:" << m_cache_to_process);
-			}
-
-			it_current_bound = result[(int)result.size()-1]. first;
 		}
 		return  true;
 	}

tests/unit_tests/http.cpp

@@ -28,6 +28,9 @@
 
 #include "gtest/gtest.h"
 #include "net/http_auth.h"
+#include "net/http_client.h"
+#include "syncobj.h"
+#include "net/http_protocol_handler.h"
 
 #include <boost/algorithm/string/predicate.hpp>
 #include <boost/algorithm/string/join.hpp>
@@ -51,6 +54,7 @@
 #include <boost/spirit/include/qi_plus.hpp>
 #include <boost/spirit/include/qi_sequence.hpp>
 #include <boost/spirit/include/qi_string.hpp>
+#include <chrono>
 #include <cstdint>
 #include <iterator>
 #include <openssl/evp.h>
@@ -67,6 +71,111 @@ namespace http = epee::net_utils::http;
 using fields = std::unordered_map<std::string, std::string>;
 using auth_responses = std::vector<fields>;
 
+class test_http_endpoint final : public epee::net_utils::i_service_endpoint
+{
+public:
+  bool do_send(epee::byte_slice message) override
+  {
+    sent.append(reinterpret_cast<const char*>(message.data()), message.size());
+    return true;
+  }
+  bool close(const bool) override { return true; }
+  bool send_done() override { return true; }
+  bool call_run_once_service_io() override { return true; }
+  bool request_callback() override { return true; }
+  boost::asio::io_context& get_io_context() override { return io_context; }
+  bool add_ref() override { return true; }
+  bool release() override { return true; }
+
+  boost::asio::io_context io_context;
+  std::string sent;
+};
+
+class dummy_client
+{
+public:
+  bool connect(const std::string&, int, std::chrono::milliseconds, bool = false, const std::string& = "0.0.0.0") { return true; }
+  bool connect(const std::string&, const std::string&, std::chrono::milliseconds, bool = false, const std::string& = "0.0.0.0") { return true; }
+  bool disconnect() { return true; }
+  bool send(const boost::string_ref, std::chrono::milliseconds) { return true; }
+  bool send(const void*, size_t) { return true; }
+  bool recv(std::string& buff, std::chrono::milliseconds)
+  {
+    buff = data;
+    data.clear();
+    return true;
+  }
+  void set_ssl(epee::net_utils::ssl_options_t) { }
+  bool is_connected(bool *ssl = NULL) { return true; }
+  uint64_t get_bytes_sent() const { return 1; }
+  uint64_t get_bytes_received() const { return 1; }
+
+  void set_test_data(const std::string& s) { data = s; }
+
+private:
+  std::string data;
+};
+
+class test_http_client final : public http::http_simple_client_template<dummy_client>
+{
+public:
+  bool on_header(const http::http_response_info& headers) override
+  {
+    ++headers_seen;
+    last_headers = headers;
+    return true;
+  }
+
+  http::http_response_info last_headers;
+  unsigned headers_seen = 0;
+};
+
+class capturing_http_handler final : public http::i_http_server_handler<epee::net_utils::connection_context_base>
+{
+public:
+  bool handle_http_request(
+    const http::http_request_info& query_info,
+    http::http_response_info& response,
+    epee::net_utils::connection_context_base&) override
+  {
+    requests.push_back(query_info);
+    response.m_response_code = 200;
+    response.m_response_comment = "OK";
+    response.m_mime_tipe = "text/plain";
+    return true;
+  }
+
+  std::vector<http::http_request_info> requests;
+};
+
+struct http_request_capture
+{
+  std::vector<bool> results;
+  std::vector<http::http_request_info> requests;
+  std::string sent;
+};
+
+http_request_capture feed_http_request(const std::vector<std::string>& chunks)
+{
+  capturing_http_handler handler;
+  test_http_endpoint endpoint;
+  epee::net_utils::connection_context_base context;
+  http::custum_handler_config<epee::net_utils::connection_context_base> config;
+  config.m_phandler = &handler;
+
+  http::http_custom_handler<epee::net_utils::connection_context_base> connection(&endpoint, config, context);
+  std::vector<bool> results;
+  for (const std::string& chunk : chunks)
+    results.push_back(connection.handle_recv(chunk.data(), chunk.size()));
+
+  return {results, handler.requests, endpoint.sent};
+}
+
+http_request_capture feed_http_request(const std::string& request)
+{
+  return feed_http_request(std::vector<std::string>{request});
+}
+
 void rng(size_t len, uint8_t *ptr)
 {
   crypto::rand(len, ptr);
@@ -701,6 +810,149 @@ TEST(HTTP_Client_Auth, MD5_auth)
 }
 
 
+TEST(HTTP, Parse_Header_Line)
+{
+  boost::string_view name;
+  boost::string_view value;
+
+  ASSERT_TRUE(http::detail::parse_header_line("Host: example.com\r", name, value));
+  EXPECT_EQ("Host", name);
+  EXPECT_EQ("example.com", value);
+
+  ASSERT_TRUE(http::detail::parse_header_line("Content-Type:\t application/json \t", name, value));
+  EXPECT_EQ("Content-Type", name);
+  EXPECT_EQ("application/json", value);
+
+  ASSERT_TRUE(http::detail::parse_header_line("Host:example.com", name, value));
+  EXPECT_EQ("Host", name);
+  EXPECT_EQ("example.com", value);
+
+  ASSERT_TRUE(http::detail::parse_header_line("X!#$%&'*+-.^_`|~: value", name, value));
+  EXPECT_EQ("X!#$%&'*+-.^_`|~", name);
+  EXPECT_EQ("value", value);
+
+  EXPECT_FALSE(http::detail::parse_header_line("", name, value));
+  EXPECT_FALSE(http::detail::parse_header_line("Host : example.com", name, value));
+  EXPECT_FALSE(http::detail::parse_header_line(" Content-Length: 1", name, value));
+  EXPECT_FALSE(http::detail::parse_header_line("\tContent-Length: 1", name, value));
+  EXPECT_FALSE(http::detail::parse_header_line("Bad Header", name, value));
+  EXPECT_FALSE(http::detail::parse_header_line("Host example.com", name, value));
+  EXPECT_FALSE(http::detail::parse_header_line("GET / HTTP/1.1", name, value));
+}
+
+TEST(HTTP, Server_Parses_Content_Length_First_Header)
+{
+  const std::string body = "0123456789";
+  const auto capture = feed_http_request(
+    "POST /json_rpc HTTP/1.1\r\n"
+    "Content-Length: 10\r\n"
+    "Host: example.com\r\n"
+    "\r\n" + body
+  );
+
+  ASSERT_EQ(1u, capture.results.size());
+  ASSERT_TRUE(capture.results.front());
+  ASSERT_EQ(1u, capture.requests.size());
+  EXPECT_STREQ("10", capture.requests.front().m_header_info.m_content_length.c_str());
+  EXPECT_EQ(body, capture.requests.front().m_body);
+}
+
+TEST(HTTP, Server_Parses_First_Header_After_Split_Request_Line)
+{
+  const std::string body = "0123456789";
+  const auto capture = feed_http_request({
+    "POST /json_rpc HTTP/1.1\r\n",
+    "Content-Length: 10\r\n"
+    "Host: example.com\r\n"
+    "\r\n" + body
+  });
+
+  ASSERT_EQ(2u, capture.results.size());
+  EXPECT_TRUE(capture.results[0]);
+  EXPECT_TRUE(capture.results[1]);
+  ASSERT_EQ(1u, capture.requests.size());
+  EXPECT_STREQ("10", capture.requests.front().m_header_info.m_content_length.c_str());
+  EXPECT_EQ(body, capture.requests.front().m_body);
+}
+
+TEST(HTTP, Server_Rejects_Malformed_First_Header)
+{
+  const auto capture = feed_http_request(
+    "GET / HTTP/1.1\r\n"
+    "Bad Header Without Colon\r\n"
+    "Host: example.com\r\n"
+    "\r\n"
+  );
+
+  ASSERT_EQ(1u, capture.results.size());
+  EXPECT_FALSE(capture.results.front());
+  EXPECT_TRUE(capture.requests.empty());
+}
+
+TEST(HTTP, Server_Rejects_Malformed_Later_Header)
+{
+  const auto capture = feed_http_request(
+    "GET / HTTP/1.1\r\n"
+    "Host: example.com\r\n"
+    "Bad Header Without Colon\r\n"
+    "\r\n"
+  );
+
+  ASSERT_EQ(1u, capture.results.size());
+  EXPECT_FALSE(capture.results.front());
+  EXPECT_TRUE(capture.requests.empty());
+}
+
+TEST(HTTP, Server_Keeps_Unknown_First_Header)
+{
+  const auto capture = feed_http_request(
+    "GET / HTTP/1.1\r\n"
+    "X-Test: abc\r\n"
+    "Host: example.com\r\n"
+    "\r\n"
+  );
+
+  ASSERT_EQ(1u, capture.results.size());
+  ASSERT_TRUE(capture.results.front());
+  ASSERT_EQ(1u, capture.requests.size());
+  ASSERT_EQ(1u, capture.requests.front().m_header_info.m_etc_fields.size());
+  EXPECT_STREQ("X-Test", capture.requests.front().m_header_info.m_etc_fields.front().first.c_str());
+  EXPECT_STREQ("abc", capture.requests.front().m_header_info.m_etc_fields.front().second.c_str());
+}
+
+TEST(HTTP, Client_Keeps_Unknown_Header)
+{
+  test_http_client client;
+  const bool result = client.test(
+    "HTTP/1.1 200 OK\r\n"
+    "X-Test: abc\r\n"
+    "Content-Length: 0\r\n"
+    "\r\n",
+    std::chrono::milliseconds(1000)
+  );
+
+  ASSERT_TRUE(result);
+  EXPECT_EQ(1u, client.headers_seen);
+  ASSERT_EQ(1u, client.last_headers.m_header_info.m_etc_fields.size());
+  EXPECT_STREQ("X-Test", client.last_headers.m_header_info.m_etc_fields.front().first.c_str());
+  EXPECT_STREQ("abc", client.last_headers.m_header_info.m_etc_fields.front().second.c_str());
+}
+
+TEST(HTTP, Client_Rejects_Malformed_Response_Header)
+{
+  test_http_client client;
+  const bool result = client.test(
+    "HTTP/1.1 200 OK\r\n"
+    "Bad Header Without Colon\r\n"
+    "Content-Length: 0\r\n"
+    "\r\n",
+    std::chrono::milliseconds(1000)
+  );
+
+  EXPECT_FALSE(result);
+  EXPECT_EQ(0u, client.headers_seen);
+}
+
 TEST(HTTP, Add_Field)
 {
   std::string str{"leading text"};