From e384008e0e0d1fd27f201979d40055a6b4f619e6 Mon Sep 17 00:00:00 2001 From: John Graham-Cumming Date: Fri, 13 May 2016 10:48:51 +0100 Subject: [PATCH 01/16] Added patch to make NGINX support HTTP/2 and SPDY simultaneously See blog.cloudflare.com/open-sourcing-our-nginx-http-2-spdy-code --- patches/nginx__http2_spdy.patch | 6263 +++++++++++++++++++++++++++++++ 1 file changed, 6263 insertions(+) create mode 100644 patches/nginx__http2_spdy.patch diff --git a/patches/nginx__http2_spdy.patch b/patches/nginx__http2_spdy.patch new file mode 100644 index 0000000..c22401e --- /dev/null +++ b/patches/nginx__http2_spdy.patch @@ -0,0 +1,6263 @@ +From 4bb4f986c696fa4ca267481a55de461980760b75 Mon Sep 17 00:00:00 2001 +From: Jiale Zhi +Date: Wed, 14 Oct 2015 17:51:06 -0700 +Subject: [PATCH] Add SPDY support back to Nginx with HTTP/2 + +--- + auto/modules | 13 + + auto/options | 3 + + auto/sources | 8 + + src/core/ngx_connection.h | 1 + + src/http/modules/ngx_http_ssl_module.c | 52 +- + src/http/ngx_http.c | 27 + + src/http/ngx_http.h | 6 + + src/http/ngx_http_core_module.c | 22 +- + src/http/ngx_http_core_module.h | 6 + + src/http/ngx_http_request.c | 67 + + src/http/ngx_http_request.h | 3 + + src/http/ngx_http_request_body.c | 13 + + src/http/ngx_http_spdy.c | 3701 ++++++++++++++++++++++++++++++++ + src/http/ngx_http_spdy.h | 261 +++ + src/http/ngx_http_spdy_filter_module.c | 1222 +++++++++++ + src/http/ngx_http_spdy_module.c | 408 ++++ + src/http/ngx_http_spdy_module.h | 41 + + src/http/ngx_http_upstream.c | 11 + + 18 files changed, 5856 insertions(+), 9 deletions(-) + create mode 100644 src/http/ngx_http_spdy.c + create mode 100644 src/http/ngx_http_spdy.h + create mode 100644 src/http/ngx_http_spdy_filter_module.c + create mode 100644 src/http/ngx_http_spdy_module.c + create mode 100644 src/http/ngx_http_spdy_module.h + +diff --git a/auto/modules b/auto/modules +index 5c734e1..ef18f20 100644 +--- a/auto/modules ++++ b/auto/modules +@@ -95,6 +95,7 @@ fi + # ngx_http_header_filter + # ngx_http_chunked_filter + # ngx_http_v2_filter ++# ngx_http_spdy_filter + # ngx_http_range_header_filter + # ngx_http_gzip_filter + # ngx_http_postpone_filter +@@ -119,6 +120,10 @@ if [ $HTTP_V2 = YES ]; then + HTTP_FILTER_MODULES="$HTTP_FILTER_MODULES $HTTP_V2_FILTER_MODULE" + fi + ++if [ $HTTP_SPDY = YES ]; then ++ HTTP_FILTER_MODULES="$HTTP_FILTER_MODULES $HTTP_SPDY_FILTER_MODULE" ++fi ++ + HTTP_FILTER_MODULES="$HTTP_FILTER_MODULES $HTTP_RANGE_HEADER_FILTER_MODULE" + + if [ $HTTP_GZIP = YES ]; then +@@ -188,6 +193,14 @@ if [ $HTTP_V2 = YES ]; then + HTTP_SRCS="$HTTP_SRCS $HTTP_V2_SRCS" + fi + ++if [ $HTTP_SPDY = YES ]; then ++ have=NGX_HTTP_SPDY . auto/have ++ USE_ZLIB=YES ++ HTTP_MODULES="$HTTP_MODULES $HTTP_SPDY_MODULE" ++ HTTP_DEPS="$HTTP_DEPS $HTTP_SPDY_DEPS" ++ HTTP_SRCS="$HTTP_SRCS $HTTP_SPDY_SRCS" ++fi ++ + HTTP_MODULES="$HTTP_MODULES $HTTP_STATIC_MODULE" + + if [ $HTTP_GZIP_STATIC = YES ]; then +diff --git a/auto/options b/auto/options +index efc8943..75e5c03 100644 +--- a/auto/options ++++ b/auto/options +@@ -59,6 +59,7 @@ HTTP_CHARSET=YES + HTTP_GZIP=YES + HTTP_SSL=NO + HTTP_V2=NO ++HTTP_SPDY=NO + HTTP_SSI=YES + HTTP_POSTPONE=NO + HTTP_REALIP=NO +@@ -211,6 +212,7 @@ do + + --with-http_ssl_module) HTTP_SSL=YES ;; + --with-http_v2_module) HTTP_V2=YES ;; ++ --with-http_spdy_module) HTTP_SPDY=YES ;; + --with-http_realip_module) HTTP_REALIP=YES ;; + --with-http_addition_module) HTTP_ADDITION=YES ;; + --with-http_xslt_module) HTTP_XSLT=YES ;; +@@ -379,6 +381,7 @@ cat << END + + --with-http_ssl_module enable ngx_http_ssl_module + --with-http_v2_module enable ngx_http_v2_module ++ --with-http_spdy_module enable ngx_http_spdy_module + --with-http_realip_module enable ngx_http_realip_module + --with-http_addition_module enable ngx_http_addition_module + --with-http_xslt_module enable ngx_http_xslt_module +diff --git a/auto/sources b/auto/sources +index 2abbc60..281bb92 100644 +--- a/auto/sources ++++ b/auto/sources +@@ -329,6 +329,14 @@ HTTP_V2_SRCS="src/http/v2/ngx_http_v2.c \ + src/http/v2/ngx_http_v2_module.c \ + src/http/v2/ngx_http_v2_filter_module.c" + ++HTTP_SPDY_MODULE=ngx_http_spdy_module ++HTTP_SPDY_FILTER_MODULE=ngx_http_spdy_filter_module ++HTTP_SPDY_DEPS="src/http/ngx_http_spdy.h \ ++ src/http/ngx_http_spdy_module.h" ++HTTP_SPDY_SRCS="src/http/ngx_http_spdy.c \ ++ src/http/ngx_http_spdy_module.c \ ++ src/http/ngx_http_spdy_filter_module.c" ++ + + HTTP_CHARSET_FILTER_MODULE=ngx_http_charset_filter_module + HTTP_CHARSET_SRCS=src/http/modules/ngx_http_charset_filter_module.c +diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h +index 977f028..ee0e62a 100644 +--- a/src/core/ngx_connection.h ++++ b/src/core/ngx_connection.h +@@ -119,6 +119,7 @@ typedef enum { + #define NGX_LOWLEVEL_BUFFERED 0x0f + #define NGX_SSL_BUFFERED 0x01 + #define NGX_HTTP_V2_BUFFERED 0x02 ++#define NGX_SPDY_BUFFERED 0x04 + + + struct ngx_connection_s { +diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c +index 7b051ea..97b71e7 100644 +--- a/src/http/modules/ngx_http_ssl_module.c ++++ b/src/http/modules/ngx_http_ssl_module.c +@@ -326,10 +326,10 @@ ngx_http_ssl_alpn_select(ngx_ssl_conn_t *ssl_conn, const unsigned char **out, + #if (NGX_DEBUG) + unsigned int i; + #endif +-#if (NGX_HTTP_V2) ++#if (NGX_HTTP_V2 || NGX_HTTP_SPDY) + ngx_http_connection_t *hc; + #endif +-#if (NGX_HTTP_V2 || NGX_DEBUG) ++#if (NGX_HTTP_V2 || NGX_HTTP_SPDY || NGX_DEBUG) + ngx_connection_t *c; + + c = ngx_ssl_get_connection(ssl_conn); +@@ -342,9 +342,20 @@ ngx_http_ssl_alpn_select(ngx_ssl_conn_t *ssl_conn, const unsigned char **out, + } + #endif + +-#if (NGX_HTTP_V2) ++#if (NGX_HTTP_V2 || NGX_HTTP_SPDY) + hc = c->data; ++#endif + ++#if (NGX_HTTP_V2 && NGX_HTTP_SPDY) ++ if (hc->addr_conf->http2 && hc->addr_conf->spdy) { ++ srv = (unsigned char *) NGX_HTTP_V2_ALPN_ADVERTISE ++ NGX_SPDY_NPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE; ++ srvlen = sizeof(NGX_HTTP_V2_ALPN_ADVERTISE NGX_SPDY_NPN_ADVERTISE ++ NGX_HTTP_NPN_ADVERTISE) - 1; ++ ++ } else ++#endif ++#if (NGX_HTTP_V2) + if (hc->addr_conf->http2) { + srv = + (unsigned char *) NGX_HTTP_V2_ALPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE; +@@ -352,6 +363,13 @@ ngx_http_ssl_alpn_select(ngx_ssl_conn_t *ssl_conn, const unsigned char **out, + + } else + #endif ++#if (NGX_HTTP_SPDY) ++ if (hc->addr_conf->spdy) { ++ srv = (unsigned char *) NGX_SPDY_NPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE; ++ srvlen = sizeof(NGX_SPDY_NPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE) - 1; ++ ++ } else ++#endif + { + srv = (unsigned char *) NGX_HTTP_NPN_ADVERTISE; + srvlen = sizeof(NGX_HTTP_NPN_ADVERTISE) - 1; +@@ -379,26 +397,50 @@ static int + ngx_http_ssl_npn_advertised(ngx_ssl_conn_t *ssl_conn, + const unsigned char **out, unsigned int *outlen, void *arg) + { +-#if (NGX_HTTP_V2 || NGX_DEBUG) ++#if (NGX_HTTP_V2 || NGX_HTTP_SPDY || NGX_DEBUG) + ngx_connection_t *c; + + c = ngx_ssl_get_connection(ssl_conn); + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "SSL NPN advertised"); + #endif + +-#if (NGX_HTTP_V2) ++#if (NGX_HTTP_V2 || NGX_HTTP_SPDY) + { + ngx_http_connection_t *hc; + + hc = c->data; ++#endif + ++#if (NGX_HTTP_V2 && NGX_HTTP_SPDY) ++ if (hc->addr_conf->http2 && hc->addr_conf->spdy) { ++ *out = (unsigned char *) NGX_HTTP_V2_NPN_ADVERTISE ++ NGX_SPDY_NPN_ADVERTISE ++ NGX_HTTP_NPN_ADVERTISE; ++ *outlen = sizeof(NGX_HTTP_V2_NPN_ADVERTISE NGX_SPDY_NPN_ADVERTISE ++ NGX_HTTP_NPN_ADVERTISE) - 1; ++ ++ return SSL_TLSEXT_ERR_OK; ++ } else ++#endif ++#if (NGX_HTTP_V2) + if (hc->addr_conf->http2) { + *out = + (unsigned char *) NGX_HTTP_V2_NPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE; + *outlen = sizeof(NGX_HTTP_V2_NPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE) - 1; + + return SSL_TLSEXT_ERR_OK; ++ } else ++#endif ++#if (NGX_HTTP_SPDY) ++ if (hc->addr_conf->spdy) { ++ *out = (unsigned char *) NGX_SPDY_NPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE; ++ *outlen = sizeof(NGX_SPDY_NPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE) - 1; ++ ++ return SSL_TLSEXT_ERR_OK; + } ++#endif ++ ++#if (NGX_HTTP_V2 || NGX_HTTP_SPDY) + } + #endif + +diff --git a/src/http/ngx_http.c b/src/http/ngx_http.c +index d09e3f0..caa5ee4 100644 +--- a/src/http/ngx_http.c ++++ b/src/http/ngx_http.c +@@ -1236,6 +1236,9 @@ ngx_http_add_addresses(ngx_conf_t *cf, ngx_http_core_srv_conf_t *cscf, + #if (NGX_HTTP_V2) + ngx_uint_t http2; + #endif ++#if (NGX_HTTP_SPDY) ++ ngx_uint_t spdy; ++#endif + + /* + * we cannot compare whole sockaddr struct's as kernel +@@ -1293,6 +1296,9 @@ ngx_http_add_addresses(ngx_conf_t *cf, ngx_http_core_srv_conf_t *cscf, + #if (NGX_HTTP_V2) + http2 = lsopt->http2 || addr[i].opt.http2; + #endif ++#if (NGX_HTTP_SPDY) ++ spdy = lsopt->spdy || addr[i].opt.spdy; ++#endif + + if (lsopt->set) { + +@@ -1327,6 +1333,9 @@ ngx_http_add_addresses(ngx_conf_t *cf, ngx_http_core_srv_conf_t *cscf, + #if (NGX_HTTP_V2) + addr[i].opt.http2 = http2; + #endif ++#if (NGX_HTTP_SPDY) ++ addr[i].opt.spdy = spdy; ++#endif + + return NGX_OK; + } +@@ -1370,6 +1379,18 @@ ngx_http_add_address(ngx_conf_t *cf, ngx_http_core_srv_conf_t *cscf, + + #endif + ++#if (NGX_HTTP_SPDY && NGX_HTTP_SSL \ ++ && !defined TLSEXT_TYPE_application_layer_protocol_negotiation \ ++ && !defined TLSEXT_TYPE_next_proto_neg) ++ if (lsopt->spdy && lsopt->ssl) { ++ ngx_conf_log_error(NGX_LOG_WARN, cf, 0, ++ "nginx was built with OpenSSL that lacks ALPN " ++ "and NPN support, SPDY is not enabled for %s", ++ lsopt->addr); ++ } ++ ++#endif ++ + addr = ngx_array_push(&port->addrs); + if (addr == NULL) { + return NGX_ERROR; +@@ -1862,6 +1883,9 @@ ngx_http_add_addrs(ngx_conf_t *cf, ngx_http_port_t *hport, + #if (NGX_HTTP_V2) + addrs[i].conf.http2 = addr[i].opt.http2; + #endif ++#if (NGX_HTTP_SPDY) ++ addrs[i].conf.spdy = addr[i].opt.spdy; ++#endif + addrs[i].conf.proxy_protocol = addr[i].opt.proxy_protocol; + + if (addr[i].hash.buckets == NULL +@@ -1927,6 +1951,9 @@ ngx_http_add_addrs6(ngx_conf_t *cf, ngx_http_port_t *hport, + #if (NGX_HTTP_V2) + addrs6[i].conf.http2 = addr[i].opt.http2; + #endif ++#if (NGX_HTTP_SPDY) ++ addrs6[i].conf.spdy = addr[i].opt.spdy; ++#endif + + if (addr[i].hash.buckets == NULL + && (addr[i].wc_head == NULL +diff --git a/src/http/ngx_http.h b/src/http/ngx_http.h +index 19cb680..7d443c1 100644 +--- a/src/http/ngx_http.h ++++ b/src/http/ngx_http.h +@@ -23,6 +23,9 @@ typedef struct ngx_http_chunked_s ngx_http_chunked_t; + #if (NGX_HTTP_V2) + typedef struct ngx_http_v2_stream_s ngx_http_v2_stream_t; + #endif ++#if (NGX_HTTP_SPDY) ++typedef struct ngx_http_spdy_stream_s ngx_http_spdy_stream_t; ++#endif + + typedef ngx_int_t (*ngx_http_header_handler_pt)(ngx_http_request_t *r, + ngx_table_elt_t *h, ngx_uint_t offset); +@@ -41,6 +44,9 @@ typedef u_char *(*ngx_http_log_handler_pt)(ngx_http_request_t *r, + #if (NGX_HTTP_V2) + #include + #endif ++#if (NGX_HTTP_SPDY) ++#include ++#endif + #if (NGX_HTTP_CACHE) + #include + #endif +diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c +index 7a29608..39de2a8 100644 +--- a/src/http/ngx_http_core_module.c ++++ b/src/http/ngx_http_core_module.c +@@ -2132,6 +2132,13 @@ ngx_http_gzip_ok(ngx_http_request_t *r) + return NGX_DECLINED; + } + ++#if (NGX_HTTP_SPDY) ++ if (r->spdy_stream) { ++ r->gzip_ok = 1; ++ return NGX_OK; ++ } ++#endif ++ + ae = r->headers_in.accept_encoding; + if (ae == NULL) { + return NGX_DECLINED; +@@ -2476,6 +2483,9 @@ ngx_http_subrequest(ngx_http_request_t *r, + #if (NGX_HTTP_V2) + sr->stream = r->stream; + #endif ++#if (NGX_HTTP_SPDY) ++ sr->spdy_stream = r->spdy_stream; ++#endif + + sr->method = NGX_HTTP_GET; + sr->http_version = r->http_version; +@@ -4209,11 +4219,15 @@ ngx_http_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) + } + + if (ngx_strcmp(value[n].data, "spdy") == 0) { +- ngx_conf_log_error(NGX_LOG_WARN, cf, 0, +- "invalid parameter \"spdy\": " +- "ngx_http_spdy_module was superseded " +- "by ngx_http_v2_module"); ++#if (NGX_HTTP_SPDY) ++ lsopt.spdy = 1; + continue; ++#else ++ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, ++ "the \"spdy\" parameter requires " ++ "ngx_http_spdy_module"); ++ return NGX_CONF_ERROR; ++#endif + } + + if (ngx_strncmp(value[n].data, "so_keepalive=", 13) == 0) { +diff --git a/src/http/ngx_http_core_module.h b/src/http/ngx_http_core_module.h +index 7dec9e1..01110d3 100644 +--- a/src/http/ngx_http_core_module.h ++++ b/src/http/ngx_http_core_module.h +@@ -82,6 +82,9 @@ typedef struct { + #if (NGX_HTTP_V2) + unsigned http2:1; + #endif ++#if (NGX_HTTP_SPDY) ++ unsigned spdy:1; ++#endif + #if (NGX_HAVE_INET6 && defined IPV6_V6ONLY) + unsigned ipv6only:1; + #endif +@@ -251,6 +254,9 @@ struct ngx_http_addr_conf_s { + #if (NGX_HTTP_V2) + unsigned http2:1; + #endif ++#if (NGX_HTTP_SPDY) ++ unsigned spdy:1; ++#endif + unsigned proxy_protocol:1; + }; + +diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c +index ab7c15f..0115630 100644 +--- a/src/http/ngx_http_request.c ++++ b/src/http/ngx_http_request.c +@@ -312,6 +312,11 @@ ngx_http_init_connection(ngx_connection_t *c) + rev->handler = ngx_http_wait_request_handler; + c->write->handler = ngx_http_empty_handler; + ++#if (NGX_HTTP_SPDY) ++ if (hc->addr_conf->spdy) { ++ rev->handler = ngx_http_spdy_init; ++ } ++#endif + #if (NGX_HTTP_V2) + if (hc->addr_conf->http2) { + rev->handler = ngx_http_v2_init; +@@ -797,6 +802,34 @@ ngx_http_ssl_handshake_handler(ngx_connection_t *c) + } + #endif + ++#if (NGX_HTTP_SPDY \ ++ && (defined TLSEXT_TYPE_application_layer_protocol_negotiation \ ++ || defined TLSEXT_TYPE_next_proto_neg)) ++ { ++ unsigned int len; ++ const unsigned char *data; ++ static const ngx_str_t spdy = ngx_string(NGX_SPDY_NPN_NEGOTIATED); ++ ++#ifdef TLSEXT_TYPE_application_layer_protocol_negotiation ++ SSL_get0_alpn_selected(c->ssl->connection, &data, &len); ++ ++#ifdef TLSEXT_TYPE_next_proto_neg ++ if (len == 0) { ++ SSL_get0_next_proto_negotiated(c->ssl->connection, &data, &len); ++ } ++#endif ++ ++#else /* TLSEXT_TYPE_next_proto_neg */ ++ SSL_get0_next_proto_negotiated(c->ssl->connection, &data, &len); ++#endif ++ ++ if (len == spdy.len && ngx_strncmp(data, spdy.data, spdy.len) == 0) { ++ ngx_http_spdy_init(c->read); ++ return; ++ } ++ } ++#endif ++ + c->log->action = "waiting for request"; + + c->read->handler = ngx_http_wait_request_handler; +@@ -2510,6 +2543,12 @@ ngx_http_finalize_connection(ngx_http_request_t *r) + return; + } + #endif ++#if (NGX_HTTP_SPDY) ++ if (r->spdy_stream) { ++ ngx_http_close_request(r, 0); ++ return; ++ } ++#endif + + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); + +@@ -2575,6 +2614,11 @@ ngx_http_set_write_handler(ngx_http_request_t *r) + return NGX_OK; + } + #endif ++#if (NGX_HTTP_SPDY) ++ if (r->spdy_stream) { ++ return NGX_OK; ++ } ++#endif + + wev = r->connection->write; + +@@ -2666,6 +2710,11 @@ ngx_http_writer(ngx_http_request_t *r) + return; + } + #endif ++#if (NGX_HTTP_SPDY) ++ if (r->spdy_stream) { ++ return; ++ } ++#endif + + if (!wev->delayed) { + ngx_add_timer(wev, clcf->send_timeout); +@@ -2741,6 +2790,18 @@ ngx_http_test_reading(ngx_http_request_t *r) + } + + #endif ++#if (NGX_HTTP_SPDY) ++ ++ if (r->spdy_stream) { ++ if (c->error) { ++ err = 0; ++ goto closed; ++ } ++ ++ return; ++ } ++ ++#endif + + #if (NGX_HAVE_KQUEUE) + +@@ -3410,6 +3471,12 @@ ngx_http_close_request(ngx_http_request_t *r, ngx_int_t rc) + return; + } + #endif ++#if (NGX_HTTP_SPDY) ++ if (r->spdy_stream) { ++ ngx_http_spdy_close_stream(r->spdy_stream, rc); ++ return; ++ } ++#endif + + ngx_http_free_request(r, rc); + ngx_http_close_connection(c); +diff --git a/src/http/ngx_http_request.h b/src/http/ngx_http_request.h +index 967032a..2cccd0f 100644 +--- a/src/http/ngx_http_request.h ++++ b/src/http/ngx_http_request.h +@@ -435,6 +435,9 @@ struct ngx_http_request_s { + #if (NGX_HTTP_V2) + ngx_http_v2_stream_t *stream; + #endif ++#if (NGX_HTTP_SPDY) ++ ngx_http_spdy_stream_t *spdy_stream; ++#endif + + ngx_http_log_handler_pt log_handler; + +diff --git a/src/http/ngx_http_request_body.c b/src/http/ngx_http_request_body.c +index 77e92e3..4be35c8 100644 +--- a/src/http/ngx_http_request_body.c ++++ b/src/http/ngx_http_request_body.c +@@ -47,6 +47,13 @@ ngx_http_read_client_request_body(ngx_http_request_t *r, + goto done; + } + #endif ++#if (NGX_HTTP_SPDY) ++ if (r->spdy_stream && r == r->main) { ++ r->request_body_no_buffering = 0; ++ rc = ngx_http_spdy_read_request_body(r, post_handler); ++ goto done; ++ } ++#endif + + if (r != r->main || r->request_body || r->discard_body) { + r->request_body_no_buffering = 0; +@@ -576,6 +583,12 @@ ngx_http_discard_request_body(ngx_http_request_t *r) + return NGX_OK; + } + #endif ++#if (NGX_HTTP_SPDY) ++ if (r->spdy_stream && r == r->main) { ++ r->spdy_stream->skip_data = NGX_SPDY_DATA_DISCARD; ++ return NGX_OK; ++ } ++#endif + + if (r != r->main || r->discard_body || r->request_body) { + return NGX_OK; +diff --git a/src/http/ngx_http_spdy.c b/src/http/ngx_http_spdy.c +new file mode 100644 +index 0000000..21c5217 +--- /dev/null ++++ b/src/http/ngx_http_spdy.c +@@ -0,0 +1,3701 @@ ++ ++/* ++ * Copyright (C) Nginx, Inc. ++ * Copyright (C) Valentin V. Bartenev ++ */ ++ ++ ++#include ++#include ++#include ++#include ++ ++#include ++ ++ ++#if (NGX_HAVE_LITTLE_ENDIAN && NGX_HAVE_NONALIGNED) ++ ++#define ngx_str5cmp(m, c0, c1, c2, c3, c4) \ ++ *(uint32_t *) m == (c3 << 24 | c2 << 16 | c1 << 8 | c0) \ ++ && m[4] == c4 ++ ++#else ++ ++#define ngx_str5cmp(m, c0, c1, c2, c3, c4) \ ++ m[0] == c0 && m[1] == c1 && m[2] == c2 && m[3] == c3 && m[4] == c4 ++ ++#endif ++ ++ ++#if (NGX_HAVE_NONALIGNED) ++ ++#define ngx_spdy_frame_parse_uint16(p) ntohs(*(uint16_t *) (p)) ++#define ngx_spdy_frame_parse_uint32(p) ntohl(*(uint32_t *) (p)) ++ ++#else ++ ++#define ngx_spdy_frame_parse_uint16(p) ((p)[0] << 8 | (p)[1]) ++#define ngx_spdy_frame_parse_uint32(p) \ ++ ((p)[0] << 24 | (p)[1] << 16 | (p)[2] << 8 | (p)[3]) ++ ++#endif ++ ++#define ngx_spdy_frame_parse_sid(p) \ ++ (ngx_spdy_frame_parse_uint32(p) & 0x7fffffff) ++#define ngx_spdy_frame_parse_delta(p) \ ++ (ngx_spdy_frame_parse_uint32(p) & 0x7fffffff) ++ ++ ++#define ngx_spdy_ctl_frame_check(h) \ ++ (((h) & 0xffff0000) == ngx_spdy_ctl_frame_head(0)) ++#define ngx_spdy_data_frame_check(h) \ ++ (!((h) & (uint32_t) NGX_SPDY_CTL_BIT << 31)) ++ ++#define ngx_spdy_ctl_frame_type(h) ((h) & 0x0000ffff) ++#define ngx_spdy_frame_flags(p) ((p) >> 24) ++#define ngx_spdy_frame_length(p) ((p) & 0x00ffffff) ++#define ngx_spdy_frame_id(p) ((p) & 0x00ffffff) ++ ++ ++#define NGX_SPDY_SKIP_HEADERS_BUFFER_SIZE 4096 ++#define NGX_SPDY_CTL_FRAME_BUFFER_SIZE 16 ++ ++#define NGX_SPDY_PROTOCOL_ERROR 1 ++#define NGX_SPDY_INVALID_STREAM 2 ++#define NGX_SPDY_REFUSED_STREAM 3 ++#define NGX_SPDY_UNSUPPORTED_VERSION 4 ++#define NGX_SPDY_CANCEL 5 ++#define NGX_SPDY_INTERNAL_ERROR 6 ++#define NGX_SPDY_FLOW_CONTROL_ERROR 7 ++#define NGX_SPDY_STREAM_IN_USE 8 ++#define NGX_SPDY_STREAM_ALREADY_CLOSED 9 ++/* deprecated 10 */ ++#define NGX_SPDY_FRAME_TOO_LARGE 11 ++ ++#define NGX_SPDY_SETTINGS_MAX_STREAMS 4 ++#define NGX_SPDY_SETTINGS_INIT_WINDOW 7 ++ ++#define NGX_SPDY_SETTINGS_FLAG_PERSIST 0x01 ++#define NGX_SPDY_SETTINGS_FLAG_PERSISTED 0x02 ++ ++#define NGX_SPDY_MAX_WINDOW NGX_MAX_INT32_VALUE ++#define NGX_SPDY_CONNECTION_WINDOW 65536 ++#define NGX_SPDY_INIT_STREAM_WINDOW 65536 ++#define NGX_SPDY_STREAM_WINDOW NGX_SPDY_MAX_WINDOW ++ ++typedef struct { ++ ngx_uint_t hash; ++ u_char len; ++ u_char header[7]; ++ ngx_int_t (*handler)(ngx_http_request_t *r); ++} ngx_http_spdy_request_header_t; ++ ++ ++static void ngx_http_spdy_read_handler(ngx_event_t *rev); ++static void ngx_http_spdy_write_handler(ngx_event_t *wev); ++static void ngx_http_spdy_handle_connection(ngx_http_spdy_connection_t *sc); ++ ++static u_char *ngx_http_spdy_proxy_protocol(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_head(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_syn_stream(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_headers(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_headers_skip(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_headers_error(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_window_update(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_data(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_read_data(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_rst_stream(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_ping(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_skip(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_settings(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_complete(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_save(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end, ngx_http_spdy_handler_pt handler); ++ ++static u_char *ngx_http_spdy_state_inflate_error( ++ ngx_http_spdy_connection_t *sc, int rc); ++static u_char *ngx_http_spdy_state_protocol_error( ++ ngx_http_spdy_connection_t *sc); ++static u_char *ngx_http_spdy_state_internal_error( ++ ngx_http_spdy_connection_t *sc); ++ ++static ngx_int_t ngx_http_spdy_send_window_update( ++ ngx_http_spdy_connection_t *sc, ngx_uint_t sid, ngx_uint_t delta); ++static ngx_int_t ngx_http_spdy_send_rst_stream(ngx_http_spdy_connection_t *sc, ++ ngx_uint_t sid, ngx_uint_t status, ngx_uint_t priority); ++static ngx_int_t ngx_http_spdy_send_settings(ngx_http_spdy_connection_t *sc); ++static ngx_int_t ngx_http_spdy_settings_frame_handler( ++ ngx_http_spdy_connection_t *sc, ngx_http_spdy_out_frame_t *frame); ++static ngx_http_spdy_out_frame_t *ngx_http_spdy_get_ctl_frame( ++ ngx_http_spdy_connection_t *sc, size_t size, ngx_uint_t priority); ++static ngx_int_t ngx_http_spdy_ctl_frame_handler( ++ ngx_http_spdy_connection_t *sc, ngx_http_spdy_out_frame_t *frame); ++ ++static ngx_http_spdy_stream_t *ngx_http_spdy_create_stream( ++ ngx_http_spdy_connection_t *sc, ngx_uint_t id, ngx_uint_t priority); ++static ngx_http_spdy_stream_t *ngx_http_spdy_get_stream_by_id( ++ ngx_http_spdy_connection_t *sc, ngx_uint_t sid); ++#define ngx_http_spdy_streams_index_size(sscf) (sscf->streams_index_mask + 1) ++#define ngx_http_spdy_stream_index(sscf, sid) \ ++ ((sid >> 1) & sscf->streams_index_mask) ++ ++static ngx_int_t ngx_http_spdy_parse_header(ngx_http_request_t *r); ++static ngx_int_t ngx_http_spdy_alloc_large_header_buffer(ngx_http_request_t *r); ++ ++static ngx_int_t ngx_http_spdy_handle_request_header(ngx_http_request_t *r); ++static ngx_int_t ngx_http_spdy_parse_method(ngx_http_request_t *r); ++static ngx_int_t ngx_http_spdy_parse_scheme(ngx_http_request_t *r); ++static ngx_int_t ngx_http_spdy_parse_host(ngx_http_request_t *r); ++static ngx_int_t ngx_http_spdy_parse_path(ngx_http_request_t *r); ++static ngx_int_t ngx_http_spdy_parse_version(ngx_http_request_t *r); ++ ++static ngx_int_t ngx_http_spdy_construct_request_line(ngx_http_request_t *r); ++static void ngx_http_spdy_run_request(ngx_http_request_t *r); ++static ngx_int_t ngx_http_spdy_init_request_body(ngx_http_request_t *r); ++ ++static ngx_int_t ngx_http_spdy_terminate_stream(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_stream_t *stream, ngx_uint_t status); ++ ++static void ngx_http_spdy_close_stream_handler(ngx_event_t *ev); ++ ++static void ngx_http_spdy_handle_connection_handler(ngx_event_t *rev); ++static void ngx_http_spdy_keepalive_handler(ngx_event_t *rev); ++static void ngx_http_spdy_finalize_connection(ngx_http_spdy_connection_t *sc, ++ ngx_int_t rc); ++ ++static ngx_int_t ngx_http_spdy_adjust_windows(ngx_http_spdy_connection_t *sc, ++ ssize_t delta); ++ ++static void ngx_http_spdy_pool_cleanup(void *data); ++ ++static void *ngx_http_spdy_zalloc(void *opaque, u_int items, u_int size); ++static void ngx_http_spdy_zfree(void *opaque, void *address); ++ ++ ++static const u_char ngx_http_spdy_dict[] = { ++ 0x00, 0x00, 0x00, 0x07, 0x6f, 0x70, 0x74, 0x69, /* - - - - o p t i */ ++ 0x6f, 0x6e, 0x73, 0x00, 0x00, 0x00, 0x04, 0x68, /* o n s - - - - h */ ++ 0x65, 0x61, 0x64, 0x00, 0x00, 0x00, 0x04, 0x70, /* e a d - - - - p */ ++ 0x6f, 0x73, 0x74, 0x00, 0x00, 0x00, 0x03, 0x70, /* o s t - - - - p */ ++ 0x75, 0x74, 0x00, 0x00, 0x00, 0x06, 0x64, 0x65, /* u t - - - - d e */ ++ 0x6c, 0x65, 0x74, 0x65, 0x00, 0x00, 0x00, 0x05, /* l e t e - - - - */ ++ 0x74, 0x72, 0x61, 0x63, 0x65, 0x00, 0x00, 0x00, /* t r a c e - - - */ ++ 0x06, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x00, /* - a c c e p t - */ ++ 0x00, 0x00, 0x0e, 0x61, 0x63, 0x63, 0x65, 0x70, /* - - - a c c e p */ ++ 0x74, 0x2d, 0x63, 0x68, 0x61, 0x72, 0x73, 0x65, /* t - c h a r s e */ ++ 0x74, 0x00, 0x00, 0x00, 0x0f, 0x61, 0x63, 0x63, /* t - - - - a c c */ ++ 0x65, 0x70, 0x74, 0x2d, 0x65, 0x6e, 0x63, 0x6f, /* e p t - e n c o */ ++ 0x64, 0x69, 0x6e, 0x67, 0x00, 0x00, 0x00, 0x0f, /* d i n g - - - - */ ++ 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x2d, 0x6c, /* a c c e p t - l */ ++ 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, 0x00, /* a n g u a g e - */ ++ 0x00, 0x00, 0x0d, 0x61, 0x63, 0x63, 0x65, 0x70, /* - - - a c c e p */ ++ 0x74, 0x2d, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x73, /* t - r a n g e s */ ++ 0x00, 0x00, 0x00, 0x03, 0x61, 0x67, 0x65, 0x00, /* - - - - a g e - */ ++ 0x00, 0x00, 0x05, 0x61, 0x6c, 0x6c, 0x6f, 0x77, /* - - - a l l o w */ ++ 0x00, 0x00, 0x00, 0x0d, 0x61, 0x75, 0x74, 0x68, /* - - - - a u t h */ ++ 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, /* o r i z a t i o */ ++ 0x6e, 0x00, 0x00, 0x00, 0x0d, 0x63, 0x61, 0x63, /* n - - - - c a c */ ++ 0x68, 0x65, 0x2d, 0x63, 0x6f, 0x6e, 0x74, 0x72, /* h e - c o n t r */ ++ 0x6f, 0x6c, 0x00, 0x00, 0x00, 0x0a, 0x63, 0x6f, /* o l - - - - c o */ ++ 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, /* n n e c t i o n */ ++ 0x00, 0x00, 0x00, 0x0c, 0x63, 0x6f, 0x6e, 0x74, /* - - - - c o n t */ ++ 0x65, 0x6e, 0x74, 0x2d, 0x62, 0x61, 0x73, 0x65, /* e n t - b a s e */ ++ 0x00, 0x00, 0x00, 0x10, 0x63, 0x6f, 0x6e, 0x74, /* - - - - c o n t */ ++ 0x65, 0x6e, 0x74, 0x2d, 0x65, 0x6e, 0x63, 0x6f, /* e n t - e n c o */ ++ 0x64, 0x69, 0x6e, 0x67, 0x00, 0x00, 0x00, 0x10, /* d i n g - - - - */ ++ 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, /* c o n t e n t - */ ++ 0x6c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, /* l a n g u a g e */ ++ 0x00, 0x00, 0x00, 0x0e, 0x63, 0x6f, 0x6e, 0x74, /* - - - - c o n t */ ++ 0x65, 0x6e, 0x74, 0x2d, 0x6c, 0x65, 0x6e, 0x67, /* e n t - l e n g */ ++ 0x74, 0x68, 0x00, 0x00, 0x00, 0x10, 0x63, 0x6f, /* t h - - - - c o */ ++ 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x6c, 0x6f, /* n t e n t - l o */ ++ 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x00, 0x00, /* c a t i o n - - */ ++ 0x00, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, /* - - c o n t e n */ ++ 0x74, 0x2d, 0x6d, 0x64, 0x35, 0x00, 0x00, 0x00, /* t - m d 5 - - - */ ++ 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, /* - c o n t e n t */ ++ 0x2d, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x00, 0x00, /* - r a n g e - - */ ++ 0x00, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, /* - - c o n t e n */ ++ 0x74, 0x2d, 0x74, 0x79, 0x70, 0x65, 0x00, 0x00, /* t - t y p e - - */ ++ 0x00, 0x04, 0x64, 0x61, 0x74, 0x65, 0x00, 0x00, /* - - d a t e - - */ ++ 0x00, 0x04, 0x65, 0x74, 0x61, 0x67, 0x00, 0x00, /* - - e t a g - - */ ++ 0x00, 0x06, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, /* - - e x p e c t */ ++ 0x00, 0x00, 0x00, 0x07, 0x65, 0x78, 0x70, 0x69, /* - - - - e x p i */ ++ 0x72, 0x65, 0x73, 0x00, 0x00, 0x00, 0x04, 0x66, /* r e s - - - - f */ ++ 0x72, 0x6f, 0x6d, 0x00, 0x00, 0x00, 0x04, 0x68, /* r o m - - - - h */ ++ 0x6f, 0x73, 0x74, 0x00, 0x00, 0x00, 0x08, 0x69, /* o s t - - - - i */ ++ 0x66, 0x2d, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x00, /* f - m a t c h - */ ++ 0x00, 0x00, 0x11, 0x69, 0x66, 0x2d, 0x6d, 0x6f, /* - - - i f - m o */ ++ 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x2d, 0x73, /* d i f i e d - s */ ++ 0x69, 0x6e, 0x63, 0x65, 0x00, 0x00, 0x00, 0x0d, /* i n c e - - - - */ ++ 0x69, 0x66, 0x2d, 0x6e, 0x6f, 0x6e, 0x65, 0x2d, /* i f - n o n e - */ ++ 0x6d, 0x61, 0x74, 0x63, 0x68, 0x00, 0x00, 0x00, /* m a t c h - - - */ ++ 0x08, 0x69, 0x66, 0x2d, 0x72, 0x61, 0x6e, 0x67, /* - i f - r a n g */ ++ 0x65, 0x00, 0x00, 0x00, 0x13, 0x69, 0x66, 0x2d, /* e - - - - i f - */ ++ 0x75, 0x6e, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, /* u n m o d i f i */ ++ 0x65, 0x64, 0x2d, 0x73, 0x69, 0x6e, 0x63, 0x65, /* e d - s i n c e */ ++ 0x00, 0x00, 0x00, 0x0d, 0x6c, 0x61, 0x73, 0x74, /* - - - - l a s t */ ++ 0x2d, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, /* - m o d i f i e */ ++ 0x64, 0x00, 0x00, 0x00, 0x08, 0x6c, 0x6f, 0x63, /* d - - - - l o c */ ++ 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x00, 0x00, 0x00, /* a t i o n - - - */ ++ 0x0c, 0x6d, 0x61, 0x78, 0x2d, 0x66, 0x6f, 0x72, /* - m a x - f o r */ ++ 0x77, 0x61, 0x72, 0x64, 0x73, 0x00, 0x00, 0x00, /* w a r d s - - - */ ++ 0x06, 0x70, 0x72, 0x61, 0x67, 0x6d, 0x61, 0x00, /* - p r a g m a - */ ++ 0x00, 0x00, 0x12, 0x70, 0x72, 0x6f, 0x78, 0x79, /* - - - p r o x y */ ++ 0x2d, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, /* - a u t h e n t */ ++ 0x69, 0x63, 0x61, 0x74, 0x65, 0x00, 0x00, 0x00, /* i c a t e - - - */ ++ 0x13, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x2d, 0x61, /* - p r o x y - a */ ++ 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, /* u t h o r i z a */ ++ 0x74, 0x69, 0x6f, 0x6e, 0x00, 0x00, 0x00, 0x05, /* t i o n - - - - */ ++ 0x72, 0x61, 0x6e, 0x67, 0x65, 0x00, 0x00, 0x00, /* r a n g e - - - */ ++ 0x07, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x72, /* - r e f e r e r */ ++ 0x00, 0x00, 0x00, 0x0b, 0x72, 0x65, 0x74, 0x72, /* - - - - r e t r */ ++ 0x79, 0x2d, 0x61, 0x66, 0x74, 0x65, 0x72, 0x00, /* y - a f t e r - */ ++ 0x00, 0x00, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, /* - - - s e r v e */ ++ 0x72, 0x00, 0x00, 0x00, 0x02, 0x74, 0x65, 0x00, /* r - - - - t e - */ ++ 0x00, 0x00, 0x07, 0x74, 0x72, 0x61, 0x69, 0x6c, /* - - - t r a i l */ ++ 0x65, 0x72, 0x00, 0x00, 0x00, 0x11, 0x74, 0x72, /* e r - - - - t r */ ++ 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x2d, 0x65, /* a n s f e r - e */ ++ 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x00, /* n c o d i n g - */ ++ 0x00, 0x00, 0x07, 0x75, 0x70, 0x67, 0x72, 0x61, /* - - - u p g r a */ ++ 0x64, 0x65, 0x00, 0x00, 0x00, 0x0a, 0x75, 0x73, /* d e - - - - u s */ ++ 0x65, 0x72, 0x2d, 0x61, 0x67, 0x65, 0x6e, 0x74, /* e r - a g e n t */ ++ 0x00, 0x00, 0x00, 0x04, 0x76, 0x61, 0x72, 0x79, /* - - - - v a r y */ ++ 0x00, 0x00, 0x00, 0x03, 0x76, 0x69, 0x61, 0x00, /* - - - - v i a - */ ++ 0x00, 0x00, 0x07, 0x77, 0x61, 0x72, 0x6e, 0x69, /* - - - w a r n i */ ++ 0x6e, 0x67, 0x00, 0x00, 0x00, 0x10, 0x77, 0x77, /* n g - - - - w w */ ++ 0x77, 0x2d, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, /* w - a u t h e n */ ++ 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x00, 0x00, /* t i c a t e - - */ ++ 0x00, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, /* - - m e t h o d */ ++ 0x00, 0x00, 0x00, 0x03, 0x67, 0x65, 0x74, 0x00, /* - - - - g e t - */ ++ 0x00, 0x00, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, /* - - - s t a t u */ ++ 0x73, 0x00, 0x00, 0x00, 0x06, 0x32, 0x30, 0x30, /* s - - - - 2 0 0 */ ++ 0x20, 0x4f, 0x4b, 0x00, 0x00, 0x00, 0x07, 0x76, /* - O K - - - - v */ ++ 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x00, 0x00, /* e r s i o n - - */ ++ 0x00, 0x08, 0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, /* - - H T T P - 1 */ ++ 0x2e, 0x31, 0x00, 0x00, 0x00, 0x03, 0x75, 0x72, /* - 1 - - - - u r */ ++ 0x6c, 0x00, 0x00, 0x00, 0x06, 0x70, 0x75, 0x62, /* l - - - - p u b */ ++ 0x6c, 0x69, 0x63, 0x00, 0x00, 0x00, 0x0a, 0x73, /* l i c - - - - s */ ++ 0x65, 0x74, 0x2d, 0x63, 0x6f, 0x6f, 0x6b, 0x69, /* e t - c o o k i */ ++ 0x65, 0x00, 0x00, 0x00, 0x0a, 0x6b, 0x65, 0x65, /* e - - - - k e e */ ++ 0x70, 0x2d, 0x61, 0x6c, 0x69, 0x76, 0x65, 0x00, /* p - a l i v e - */ ++ 0x00, 0x00, 0x06, 0x6f, 0x72, 0x69, 0x67, 0x69, /* - - - o r i g i */ ++ 0x6e, 0x31, 0x30, 0x30, 0x31, 0x30, 0x31, 0x32, /* n 1 0 0 1 0 1 2 */ ++ 0x30, 0x31, 0x32, 0x30, 0x32, 0x32, 0x30, 0x35, /* 0 1 2 0 2 2 0 5 */ ++ 0x32, 0x30, 0x36, 0x33, 0x30, 0x30, 0x33, 0x30, /* 2 0 6 3 0 0 3 0 */ ++ 0x32, 0x33, 0x30, 0x33, 0x33, 0x30, 0x34, 0x33, /* 2 3 0 3 3 0 4 3 */ ++ 0x30, 0x35, 0x33, 0x30, 0x36, 0x33, 0x30, 0x37, /* 0 5 3 0 6 3 0 7 */ ++ 0x34, 0x30, 0x32, 0x34, 0x30, 0x35, 0x34, 0x30, /* 4 0 2 4 0 5 4 0 */ ++ 0x36, 0x34, 0x30, 0x37, 0x34, 0x30, 0x38, 0x34, /* 6 4 0 7 4 0 8 4 */ ++ 0x30, 0x39, 0x34, 0x31, 0x30, 0x34, 0x31, 0x31, /* 0 9 4 1 0 4 1 1 */ ++ 0x34, 0x31, 0x32, 0x34, 0x31, 0x33, 0x34, 0x31, /* 4 1 2 4 1 3 4 1 */ ++ 0x34, 0x34, 0x31, 0x35, 0x34, 0x31, 0x36, 0x34, /* 4 4 1 5 4 1 6 4 */ ++ 0x31, 0x37, 0x35, 0x30, 0x32, 0x35, 0x30, 0x34, /* 1 7 5 0 2 5 0 4 */ ++ 0x35, 0x30, 0x35, 0x32, 0x30, 0x33, 0x20, 0x4e, /* 5 0 5 2 0 3 - N */ ++ 0x6f, 0x6e, 0x2d, 0x41, 0x75, 0x74, 0x68, 0x6f, /* o n - A u t h o */ ++ 0x72, 0x69, 0x74, 0x61, 0x74, 0x69, 0x76, 0x65, /* r i t a t i v e */ ++ 0x20, 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, /* - I n f o r m a */ ++ 0x74, 0x69, 0x6f, 0x6e, 0x32, 0x30, 0x34, 0x20, /* t i o n 2 0 4 - */ ++ 0x4e, 0x6f, 0x20, 0x43, 0x6f, 0x6e, 0x74, 0x65, /* N o - C o n t e */ ++ 0x6e, 0x74, 0x33, 0x30, 0x31, 0x20, 0x4d, 0x6f, /* n t 3 0 1 - M o */ ++ 0x76, 0x65, 0x64, 0x20, 0x50, 0x65, 0x72, 0x6d, /* v e d - P e r m */ ++ 0x61, 0x6e, 0x65, 0x6e, 0x74, 0x6c, 0x79, 0x34, /* a n e n t l y 4 */ ++ 0x30, 0x30, 0x20, 0x42, 0x61, 0x64, 0x20, 0x52, /* 0 0 - B a d - R */ ++ 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x34, 0x30, /* e q u e s t 4 0 */ ++ 0x31, 0x20, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, /* 1 - U n a u t h */ ++ 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x34, 0x30, /* o r i z e d 4 0 */ ++ 0x33, 0x20, 0x46, 0x6f, 0x72, 0x62, 0x69, 0x64, /* 3 - F o r b i d */ ++ 0x64, 0x65, 0x6e, 0x34, 0x30, 0x34, 0x20, 0x4e, /* d e n 4 0 4 - N */ ++ 0x6f, 0x74, 0x20, 0x46, 0x6f, 0x75, 0x6e, 0x64, /* o t - F o u n d */ ++ 0x35, 0x30, 0x30, 0x20, 0x49, 0x6e, 0x74, 0x65, /* 5 0 0 - I n t e */ ++ 0x72, 0x6e, 0x61, 0x6c, 0x20, 0x53, 0x65, 0x72, /* r n a l - S e r */ ++ 0x76, 0x65, 0x72, 0x20, 0x45, 0x72, 0x72, 0x6f, /* v e r - E r r o */ ++ 0x72, 0x35, 0x30, 0x31, 0x20, 0x4e, 0x6f, 0x74, /* r 5 0 1 - N o t */ ++ 0x20, 0x49, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, /* - I m p l e m e */ ++ 0x6e, 0x74, 0x65, 0x64, 0x35, 0x30, 0x33, 0x20, /* n t e d 5 0 3 - */ ++ 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x20, /* S e r v i c e - */ ++ 0x55, 0x6e, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, /* U n a v a i l a */ ++ 0x62, 0x6c, 0x65, 0x4a, 0x61, 0x6e, 0x20, 0x46, /* b l e J a n - F */ ++ 0x65, 0x62, 0x20, 0x4d, 0x61, 0x72, 0x20, 0x41, /* e b - M a r - A */ ++ 0x70, 0x72, 0x20, 0x4d, 0x61, 0x79, 0x20, 0x4a, /* p r - M a y - J */ ++ 0x75, 0x6e, 0x20, 0x4a, 0x75, 0x6c, 0x20, 0x41, /* u n - J u l - A */ ++ 0x75, 0x67, 0x20, 0x53, 0x65, 0x70, 0x74, 0x20, /* u g - S e p t - */ ++ 0x4f, 0x63, 0x74, 0x20, 0x4e, 0x6f, 0x76, 0x20, /* O c t - N o v - */ ++ 0x44, 0x65, 0x63, 0x20, 0x30, 0x30, 0x3a, 0x30, /* D e c - 0 0 - 0 */ ++ 0x30, 0x3a, 0x30, 0x30, 0x20, 0x4d, 0x6f, 0x6e, /* 0 - 0 0 - M o n */ ++ 0x2c, 0x20, 0x54, 0x75, 0x65, 0x2c, 0x20, 0x57, /* - - T u e - - W */ ++ 0x65, 0x64, 0x2c, 0x20, 0x54, 0x68, 0x75, 0x2c, /* e d - - T h u - */ ++ 0x20, 0x46, 0x72, 0x69, 0x2c, 0x20, 0x53, 0x61, /* - F r i - - S a */ ++ 0x74, 0x2c, 0x20, 0x53, 0x75, 0x6e, 0x2c, 0x20, /* t - - S u n - - */ ++ 0x47, 0x4d, 0x54, 0x63, 0x68, 0x75, 0x6e, 0x6b, /* G M T c h u n k */ ++ 0x65, 0x64, 0x2c, 0x74, 0x65, 0x78, 0x74, 0x2f, /* e d - t e x t - */ ++ 0x68, 0x74, 0x6d, 0x6c, 0x2c, 0x69, 0x6d, 0x61, /* h t m l - i m a */ ++ 0x67, 0x65, 0x2f, 0x70, 0x6e, 0x67, 0x2c, 0x69, /* g e - p n g - i */ ++ 0x6d, 0x61, 0x67, 0x65, 0x2f, 0x6a, 0x70, 0x67, /* m a g e - j p g */ ++ 0x2c, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x2f, 0x67, /* - i m a g e - g */ ++ 0x69, 0x66, 0x2c, 0x61, 0x70, 0x70, 0x6c, 0x69, /* i f - a p p l i */ ++ 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x78, /* c a t i o n - x */ ++ 0x6d, 0x6c, 0x2c, 0x61, 0x70, 0x70, 0x6c, 0x69, /* m l - a p p l i */ ++ 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x78, /* c a t i o n - x */ ++ 0x68, 0x74, 0x6d, 0x6c, 0x2b, 0x78, 0x6d, 0x6c, /* h t m l - x m l */ ++ 0x2c, 0x74, 0x65, 0x78, 0x74, 0x2f, 0x70, 0x6c, /* - t e x t - p l */ ++ 0x61, 0x69, 0x6e, 0x2c, 0x74, 0x65, 0x78, 0x74, /* a i n - t e x t */ ++ 0x2f, 0x6a, 0x61, 0x76, 0x61, 0x73, 0x63, 0x72, /* - j a v a s c r */ ++ 0x69, 0x70, 0x74, 0x2c, 0x70, 0x75, 0x62, 0x6c, /* i p t - p u b l */ ++ 0x69, 0x63, 0x70, 0x72, 0x69, 0x76, 0x61, 0x74, /* i c p r i v a t */ ++ 0x65, 0x6d, 0x61, 0x78, 0x2d, 0x61, 0x67, 0x65, /* e m a x - a g e */ ++ 0x3d, 0x67, 0x7a, 0x69, 0x70, 0x2c, 0x64, 0x65, /* - g z i p - d e */ ++ 0x66, 0x6c, 0x61, 0x74, 0x65, 0x2c, 0x73, 0x64, /* f l a t e - s d */ ++ 0x63, 0x68, 0x63, 0x68, 0x61, 0x72, 0x73, 0x65, /* c h c h a r s e */ ++ 0x74, 0x3d, 0x75, 0x74, 0x66, 0x2d, 0x38, 0x63, /* t - u t f - 8 c */ ++ 0x68, 0x61, 0x72, 0x73, 0x65, 0x74, 0x3d, 0x69, /* h a r s e t - i */ ++ 0x73, 0x6f, 0x2d, 0x38, 0x38, 0x35, 0x39, 0x2d, /* s o - 8 8 5 9 - */ ++ 0x31, 0x2c, 0x75, 0x74, 0x66, 0x2d, 0x2c, 0x2a, /* 1 - u t f - - - */ ++ 0x2c, 0x65, 0x6e, 0x71, 0x3d, 0x30, 0x2e /* - e n q - 0 - */ ++}; ++ ++ ++static ngx_http_spdy_request_header_t ngx_http_spdy_request_headers[] = { ++ { 0, 6, "method", ngx_http_spdy_parse_method }, ++ { 0, 6, "scheme", ngx_http_spdy_parse_scheme }, ++ { 0, 4, "host", ngx_http_spdy_parse_host }, ++ { 0, 4, "path", ngx_http_spdy_parse_path }, ++ { 0, 7, "version", ngx_http_spdy_parse_version }, ++}; ++ ++#define NGX_SPDY_REQUEST_HEADERS \ ++ (sizeof(ngx_http_spdy_request_headers) \ ++ / sizeof(ngx_http_spdy_request_header_t)) ++ ++ ++void ++ngx_http_spdy_init(ngx_event_t *rev) ++{ ++ int rc; ++ ngx_connection_t *c; ++ ngx_pool_cleanup_t *cln; ++ ngx_http_connection_t *hc; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ngx_http_spdy_main_conf_t *smcf; ++ ngx_http_spdy_connection_t *sc; ++ ++ c = rev->data; ++ hc = c->data; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "init spdy request"); ++ ++ c->log->action = "processing SPDY"; ++ ++ smcf = ngx_http_get_module_main_conf(hc->conf_ctx, ngx_http_spdy_module); ++ ++ if (smcf->recv_buffer == NULL) { ++ smcf->recv_buffer = ngx_palloc(ngx_cycle->pool, smcf->recv_buffer_size); ++ if (smcf->recv_buffer == NULL) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ } ++ ++ sc = ngx_pcalloc(c->pool, sizeof(ngx_http_spdy_connection_t)); ++ if (sc == NULL) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ sc->connection = c; ++ sc->http_connection = hc; ++ ++ sc->send_window = NGX_SPDY_CONNECTION_WINDOW; ++ sc->recv_window = NGX_SPDY_CONNECTION_WINDOW; ++ ++ sc->init_window = NGX_SPDY_INIT_STREAM_WINDOW; ++ ++ sc->handler = hc->proxy_protocol ? ngx_http_spdy_proxy_protocol ++ : ngx_http_spdy_state_head; ++ ++ sc->zstream_in.zalloc = ngx_http_spdy_zalloc; ++ sc->zstream_in.zfree = ngx_http_spdy_zfree; ++ sc->zstream_in.opaque = sc; ++ ++ rc = inflateInit(&sc->zstream_in); ++ if (rc != Z_OK) { ++ ngx_log_error(NGX_LOG_ALERT, c->log, 0, ++ "inflateInit() failed: %d", rc); ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ sc->zstream_out.zalloc = ngx_http_spdy_zalloc; ++ sc->zstream_out.zfree = ngx_http_spdy_zfree; ++ sc->zstream_out.opaque = sc; ++ ++ sscf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_spdy_module); ++ ++ rc = deflateInit2(&sc->zstream_out, (int) sscf->headers_comp, ++ Z_DEFLATED, 11, 4, Z_DEFAULT_STRATEGY); ++ ++ if (rc != Z_OK) { ++ ngx_log_error(NGX_LOG_ALERT, c->log, 0, ++ "deflateInit2() failed: %d", rc); ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ rc = deflateSetDictionary(&sc->zstream_out, ngx_http_spdy_dict, ++ sizeof(ngx_http_spdy_dict)); ++ if (rc != Z_OK) { ++ ngx_log_error(NGX_LOG_ALERT, c->log, 0, ++ "deflateSetDictionary() failed: %d", rc); ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ sc->pool = ngx_create_pool(sscf->pool_size, sc->connection->log); ++ if (sc->pool == NULL) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ cln = ngx_pool_cleanup_add(c->pool, sizeof(ngx_pool_cleanup_file_t)); ++ if (cln == NULL) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ cln->handler = ngx_http_spdy_pool_cleanup; ++ cln->data = sc; ++ ++ sc->streams_index = ngx_pcalloc(sc->pool, ++ ngx_http_spdy_streams_index_size(sscf) ++ * sizeof(ngx_http_spdy_stream_t *)); ++ if (sc->streams_index == NULL) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ if (ngx_http_spdy_send_settings(sc) == NGX_ERROR) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ if (ngx_http_spdy_send_window_update(sc, 0, NGX_SPDY_MAX_WINDOW ++ - sc->recv_window) ++ == NGX_ERROR) ++ { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ sc->recv_window = NGX_SPDY_MAX_WINDOW; ++ ++ ngx_queue_init(&sc->waiting); ++ ngx_queue_init(&sc->posted); ++ ++ c->data = sc; ++ ++ rev->handler = ngx_http_spdy_read_handler; ++ c->write->handler = ngx_http_spdy_write_handler; ++ ++ ngx_http_spdy_read_handler(rev); ++} ++ ++ ++static void ++ngx_http_spdy_read_handler(ngx_event_t *rev) ++{ ++ u_char *p, *end; ++ size_t available; ++ ssize_t n; ++ ngx_connection_t *c; ++ ngx_http_spdy_main_conf_t *smcf; ++ ngx_http_spdy_connection_t *sc; ++ ++ c = rev->data; ++ sc = c->data; ++ ++ if (rev->timedout) { ++ ngx_log_error(NGX_LOG_INFO, c->log, NGX_ETIMEDOUT, "client timed out"); ++ ngx_http_spdy_finalize_connection(sc, NGX_HTTP_REQUEST_TIME_OUT); ++ return; ++ } ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "spdy read handler"); ++ ++ sc->blocked = 1; ++ ++ smcf = ngx_http_get_module_main_conf(sc->http_connection->conf_ctx, ++ ngx_http_spdy_module); ++ ++ available = smcf->recv_buffer_size - 2 * NGX_SPDY_STATE_BUFFER_SIZE; ++ ++ do { ++ p = smcf->recv_buffer; ++ ++ ngx_memcpy(p, sc->buffer, NGX_SPDY_STATE_BUFFER_SIZE); ++ end = p + sc->buffer_used; ++ ++ n = c->recv(c, end, available); ++ ++ if (n == NGX_AGAIN) { ++ break; ++ } ++ ++ if (n == 0 && (sc->incomplete || sc->processing)) { ++ ngx_log_error(NGX_LOG_INFO, c->log, 0, ++ "client prematurely closed connection"); ++ } ++ ++ if (n == 0 || n == NGX_ERROR) { ++ ngx_http_spdy_finalize_connection(sc, ++ NGX_HTTP_CLIENT_CLOSED_REQUEST); ++ return; ++ } ++ ++ end += n; ++ ++ sc->buffer_used = 0; ++ sc->incomplete = 0; ++ ++ do { ++ p = sc->handler(sc, p, end); ++ ++ if (p == NULL) { ++ return; ++ } ++ ++ } while (p != end); ++ ++ } while (rev->ready); ++ ++ if (ngx_handle_read_event(rev, 0) != NGX_OK) { ++ ngx_http_spdy_finalize_connection(sc, NGX_HTTP_INTERNAL_SERVER_ERROR); ++ return; ++ } ++ ++ if (sc->last_out && ngx_http_spdy_send_output_queue(sc) == NGX_ERROR) { ++ ngx_http_spdy_finalize_connection(sc, NGX_HTTP_CLIENT_CLOSED_REQUEST); ++ return; ++ } ++ ++ sc->blocked = 0; ++ ++ if (sc->processing) { ++ if (rev->timer_set) { ++ ngx_del_timer(rev); ++ } ++ return; ++ } ++ ++ ngx_http_spdy_handle_connection(sc); ++} ++ ++ ++static void ++ngx_http_spdy_write_handler(ngx_event_t *wev) ++{ ++ ngx_int_t rc; ++ ngx_queue_t *q; ++ ngx_connection_t *c; ++ ngx_http_spdy_stream_t *stream; ++ ngx_http_spdy_connection_t *sc; ++ ++ c = wev->data; ++ sc = c->data; ++ ++ if (wev->timedout) { ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, ++ "spdy write event timed out"); ++ ngx_http_spdy_finalize_connection(sc, NGX_HTTP_CLIENT_CLOSED_REQUEST); ++ return; ++ } ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "spdy write handler"); ++ ++ sc->blocked = 1; ++ ++ rc = ngx_http_spdy_send_output_queue(sc); ++ ++ if (rc == NGX_ERROR) { ++ ngx_http_spdy_finalize_connection(sc, NGX_HTTP_CLIENT_CLOSED_REQUEST); ++ return; ++ } ++ ++ while (!ngx_queue_empty(&sc->posted)) { ++ q = ngx_queue_head(&sc->posted); ++ ++ ngx_queue_remove(q); ++ ++ stream = ngx_queue_data(q, ngx_http_spdy_stream_t, queue); ++ ++ stream->handled = 0; ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, ++ "run spdy stream %ui", stream->id); ++ ++ wev = stream->request->connection->write; ++ wev->handler(wev); ++ } ++ ++ sc->blocked = 0; ++ ++ if (rc == NGX_AGAIN) { ++ return; ++ } ++ ++ ngx_http_spdy_handle_connection(sc); ++} ++ ++ ++ngx_int_t ++ngx_http_spdy_send_output_queue(ngx_http_spdy_connection_t *sc) ++{ ++ int tcp_nodelay; ++ ngx_chain_t *cl; ++ ngx_event_t *wev; ++ ngx_connection_t *c; ++ ngx_http_core_loc_conf_t *clcf; ++ ngx_http_spdy_out_frame_t *out, *frame, *fn; ++ ++ c = sc->connection; ++ ++ if (c->error) { ++ return NGX_ERROR; ++ } ++ ++ wev = c->write; ++ ++ if (!wev->ready) { ++ return NGX_OK; ++ } ++ ++ cl = NULL; ++ out = NULL; ++ ++ for (frame = sc->last_out; frame; frame = fn) { ++ frame->last->next = cl; ++ cl = frame->first; ++ ++ fn = frame->next; ++ frame->next = out; ++ out = frame; ++ ++ ngx_log_debug5(NGX_LOG_DEBUG_HTTP, c->log, 0, ++ "spdy frame out: %p sid:%ui prio:%ui bl:%d len:%uz", ++ out, out->stream ? out->stream->id : 0, out->priority, ++ out->blocked, out->length); ++ } ++ ++ cl = c->send_chain(c, cl, 0); ++ ++ if (cl == NGX_CHAIN_ERROR) { ++ goto error; ++ } ++ ++ clcf = ngx_http_get_module_loc_conf(sc->http_connection->conf_ctx, ++ ngx_http_core_module); ++ ++ if (ngx_handle_write_event(wev, clcf->send_lowat) != NGX_OK) { ++ goto error; ++ } ++ ++ if (c->tcp_nopush == NGX_TCP_NOPUSH_SET) { ++ if (ngx_tcp_push(c->fd) == -1) { ++ ngx_connection_error(c, ngx_socket_errno, ngx_tcp_push_n " failed"); ++ goto error; ++ } ++ ++ c->tcp_nopush = NGX_TCP_NOPUSH_UNSET; ++ tcp_nodelay = ngx_tcp_nodelay_and_tcp_nopush ? 1 : 0; ++ ++ } else { ++ tcp_nodelay = 1; ++ } ++ ++ if (tcp_nodelay ++ && clcf->tcp_nodelay ++ && c->tcp_nodelay == NGX_TCP_NODELAY_UNSET) ++ { ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "tcp_nodelay"); ++ ++ if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY, ++ (const void *) &tcp_nodelay, sizeof(int)) ++ == -1) ++ { ++#if (NGX_SOLARIS) ++ /* Solaris returns EINVAL if a socket has been shut down */ ++ c->log_error = NGX_ERROR_IGNORE_EINVAL; ++#endif ++ ++ ngx_connection_error(c, ngx_socket_errno, ++ "setsockopt(TCP_NODELAY) failed"); ++ ++ c->log_error = NGX_ERROR_INFO; ++ goto error; ++ } ++ ++ c->tcp_nodelay = NGX_TCP_NODELAY_SET; ++ } ++ ++ if (cl) { ++ ngx_add_timer(wev, clcf->send_timeout); ++ ++ } else { ++ if (wev->timer_set) { ++ ngx_del_timer(wev); ++ } ++ } ++ ++ for ( /* void */ ; out; out = fn) { ++ fn = out->next; ++ ++ if (out->handler(sc, out) != NGX_OK) { ++ out->blocked = 1; ++ out->priority = NGX_SPDY_HIGHEST_PRIORITY; ++ break; ++ } ++ ++ ngx_log_debug4(NGX_LOG_DEBUG_HTTP, c->log, 0, ++ "spdy frame sent: %p sid:%ui bl:%d len:%uz", ++ out, out->stream ? out->stream->id : 0, ++ out->blocked, out->length); ++ } ++ ++ frame = NULL; ++ ++ for ( /* void */ ; out; out = fn) { ++ fn = out->next; ++ out->next = frame; ++ frame = out; ++ } ++ ++ sc->last_out = frame; ++ ++ return NGX_OK; ++ ++error: ++ ++ c->error = 1; ++ ++ if (!sc->blocked) { ++ ngx_post_event(wev, &ngx_posted_events); ++ } ++ ++ return NGX_ERROR; ++} ++ ++ ++static void ++ngx_http_spdy_handle_connection(ngx_http_spdy_connection_t *sc) ++{ ++ ngx_connection_t *c; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ++ if (sc->last_out || sc->processing) { ++ return; ++ } ++ ++ c = sc->connection; ++ ++ if (c->error) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ if (c->buffered) { ++ return; ++ } ++ ++ sscf = ngx_http_get_module_srv_conf(sc->http_connection->conf_ctx, ++ ngx_http_spdy_module); ++ if (sc->incomplete) { ++ ngx_add_timer(c->read, sscf->recv_timeout); ++ return; ++ } ++ ++ if (ngx_terminate || ngx_exiting) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ ngx_destroy_pool(sc->pool); ++ ++ sc->pool = NULL; ++ sc->free_ctl_frames = NULL; ++ sc->free_fake_connections = NULL; ++ ++#if (NGX_HTTP_SSL) ++ if (c->ssl) { ++ ngx_ssl_free_buffer(c); ++ } ++#endif ++ ++ c->destroyed = 1; ++ c->idle = 1; ++ ngx_reusable_connection(c, 1); ++ ++ c->write->handler = ngx_http_empty_handler; ++ c->read->handler = ngx_http_spdy_keepalive_handler; ++ ++ if (c->write->timer_set) { ++ ngx_del_timer(c->write); ++ } ++ ++ ngx_add_timer(c->read, sscf->keepalive_timeout); ++} ++ ++ ++static u_char * ++ngx_http_spdy_proxy_protocol(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ ngx_log_t *log; ++ ++ log = sc->connection->log; ++ log->action = "reading PROXY protocol"; ++ ++ pos = ngx_proxy_protocol_read(sc->connection, pos, end); ++ ++ log->action = "processing SPDY"; ++ ++ if (pos == NULL) { ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_head(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ uint32_t head, flen; ++ ngx_uint_t type; ++ ++ if (end - pos < NGX_SPDY_FRAME_HEADER_SIZE) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_head); ++ } ++ ++ head = ngx_spdy_frame_parse_uint32(pos); ++ ++ pos += sizeof(uint32_t); ++ ++ flen = ngx_spdy_frame_parse_uint32(pos); ++ ++ sc->flags = ngx_spdy_frame_flags(flen); ++ sc->length = ngx_spdy_frame_length(flen); ++ ++ pos += sizeof(uint32_t); ++ ++ ngx_log_debug3(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "process spdy frame head:%08XD f:%Xd l:%uz", ++ head, sc->flags, sc->length); ++ ++ if (ngx_spdy_ctl_frame_check(head)) { ++ type = ngx_spdy_ctl_frame_type(head); ++ ++ switch (type) { ++ ++ case NGX_SPDY_SYN_STREAM: ++ return ngx_http_spdy_state_syn_stream(sc, pos, end); ++ ++ case NGX_SPDY_SYN_REPLY: ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent unexpected SYN_REPLY frame"); ++ return ngx_http_spdy_state_protocol_error(sc); ++ ++ case NGX_SPDY_RST_STREAM: ++ return ngx_http_spdy_state_rst_stream(sc, pos, end); ++ ++ case NGX_SPDY_SETTINGS: ++ return ngx_http_spdy_state_settings(sc, pos, end); ++ ++ case NGX_SPDY_PING: ++ return ngx_http_spdy_state_ping(sc, pos, end); ++ ++ case NGX_SPDY_GOAWAY: ++ return ngx_http_spdy_state_skip(sc, pos, end); /* TODO */ ++ ++ case NGX_SPDY_HEADERS: ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent unexpected HEADERS frame"); ++ return ngx_http_spdy_state_protocol_error(sc); ++ ++ case NGX_SPDY_WINDOW_UPDATE: ++ return ngx_http_spdy_state_window_update(sc, pos, end); ++ ++ default: ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy control frame with unknown type %ui", type); ++ return ngx_http_spdy_state_skip(sc, pos, end); ++ } ++ } ++ ++ if (ngx_spdy_data_frame_check(head)) { ++ sc->stream = ngx_http_spdy_get_stream_by_id(sc, head); ++ return ngx_http_spdy_state_data(sc, pos, end); ++ } ++ ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent invalid frame"); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_syn_stream(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ ngx_uint_t sid, prio; ++ ngx_http_spdy_stream_t *stream; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ++ if (end - pos < NGX_SPDY_SYN_STREAM_SIZE) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_syn_stream); ++ } ++ ++ if (sc->length <= NGX_SPDY_SYN_STREAM_SIZE) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent SYN_STREAM frame with incorrect length %uz", ++ sc->length); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ sc->length -= NGX_SPDY_SYN_STREAM_SIZE; ++ ++ sid = ngx_spdy_frame_parse_sid(pos); ++ prio = pos[8] >> 5; ++ ++ pos += NGX_SPDY_SYN_STREAM_SIZE; ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy SYN_STREAM frame sid:%ui prio:%ui", sid, prio); ++ ++ if (sid % 2 == 0 || sid <= sc->last_sid) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent SYN_STREAM frame " ++ "with invalid Stream-ID %ui", sid); ++ ++ stream = ngx_http_spdy_get_stream_by_id(sc, sid); ++ ++ if (stream) { ++ if (ngx_http_spdy_terminate_stream(sc, stream, ++ NGX_SPDY_PROTOCOL_ERROR) ++ != NGX_OK) ++ { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ } ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ sc->last_sid = sid; ++ ++ sscf = ngx_http_get_module_srv_conf(sc->http_connection->conf_ctx, ++ ngx_http_spdy_module); ++ ++ if (sc->processing >= sscf->concurrent_streams) { ++ ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "concurrent streams exceeded %ui", sc->processing); ++ ++ if (ngx_http_spdy_send_rst_stream(sc, sid, NGX_SPDY_REFUSED_STREAM, ++ prio) ++ != NGX_OK) ++ { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ return ngx_http_spdy_state_headers_skip(sc, pos, end); ++ } ++ ++ stream = ngx_http_spdy_create_stream(sc, sid, prio); ++ if (stream == NULL) { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ stream->in_closed = (sc->flags & NGX_SPDY_FLAG_FIN) ? 1 : 0; ++ ++ stream->request->request_length = NGX_SPDY_FRAME_HEADER_SIZE ++ + NGX_SPDY_SYN_STREAM_SIZE ++ + sc->length; ++ ++ sc->stream = stream; ++ ++ return ngx_http_spdy_state_headers(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_headers(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ int z; ++ size_t size; ++ ngx_buf_t *buf; ++ ngx_int_t rc; ++ ngx_http_request_t *r; ++ ++ size = end - pos; ++ ++ if (size == 0) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_headers); ++ } ++ ++ if (size > sc->length) { ++ size = sc->length; ++ } ++ ++ r = sc->stream->request; ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "process spdy header block %uz of %uz", size, sc->length); ++ ++ buf = r->header_in; ++ ++ sc->zstream_in.next_in = pos; ++ sc->zstream_in.avail_in = size; ++ sc->zstream_in.next_out = buf->last; ++ ++ /* one byte is reserved for null-termination of the last header value */ ++ sc->zstream_in.avail_out = buf->end - buf->last - 1; ++ ++ z = inflate(&sc->zstream_in, Z_NO_FLUSH); ++ ++ if (z == Z_NEED_DICT) { ++ z = inflateSetDictionary(&sc->zstream_in, ngx_http_spdy_dict, ++ sizeof(ngx_http_spdy_dict)); ++ ++ if (z != Z_OK) { ++ if (z == Z_DATA_ERROR) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent SYN_STREAM frame with header " ++ "block encoded using wrong dictionary: %ul", ++ (u_long) sc->zstream_in.adler); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, ++ "inflateSetDictionary() failed: %d", z); ++ ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy inflateSetDictionary(): %d", z); ++ ++ z = sc->zstream_in.avail_in ? inflate(&sc->zstream_in, Z_NO_FLUSH) ++ : Z_OK; ++ } ++ ++ ngx_log_debug5(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy inflate out: ni:%p no:%p ai:%ud ao:%ud rc:%d", ++ sc->zstream_in.next_in, sc->zstream_in.next_out, ++ sc->zstream_in.avail_in, sc->zstream_in.avail_out, ++ z); ++ ++ if (z != Z_OK) { ++ return ngx_http_spdy_state_inflate_error(sc, z); ++ } ++ ++ sc->length -= sc->zstream_in.next_in - pos; ++ pos = sc->zstream_in.next_in; ++ ++ buf->last = sc->zstream_in.next_out; ++ ++ if (r->headers_in.headers.part.elts == NULL) { ++ ++ if (buf->last - buf->pos < NGX_SPDY_NV_NUM_SIZE) { ++ ++ if (sc->length == 0) { ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "premature end of spdy header block"); ++ ++ return ngx_http_spdy_state_headers_error(sc, pos, end); ++ } ++ ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_headers); ++ } ++ ++ sc->entries = ngx_spdy_frame_parse_uint32(buf->pos); ++ ++ buf->pos += NGX_SPDY_NV_NUM_SIZE; ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy header block has %ui entries", ++ sc->entries); ++ ++ if (ngx_list_init(&r->headers_in.headers, r->pool, 20, ++ sizeof(ngx_table_elt_t)) ++ != NGX_OK) ++ { ++ ngx_http_spdy_close_stream(sc->stream, ++ NGX_HTTP_INTERNAL_SERVER_ERROR); ++ return ngx_http_spdy_state_headers_skip(sc, pos, end); ++ } ++ ++ if (ngx_array_init(&r->headers_in.cookies, r->pool, 2, ++ sizeof(ngx_table_elt_t *)) ++ != NGX_OK) ++ { ++ ngx_http_spdy_close_stream(sc->stream, ++ NGX_HTTP_INTERNAL_SERVER_ERROR); ++ return ngx_http_spdy_state_headers_skip(sc, pos, end); ++ } ++ } ++ ++ while (sc->entries) { ++ ++ rc = ngx_http_spdy_parse_header(r); ++ ++ switch (rc) { ++ ++ case NGX_DONE: ++ sc->entries--; ++ ++ case NGX_OK: ++ break; ++ ++ case NGX_AGAIN: ++ ++ if (sc->zstream_in.avail_in) { ++ ++ rc = ngx_http_spdy_alloc_large_header_buffer(r); ++ ++ if (rc == NGX_DECLINED) { ++ ngx_http_finalize_request(r, ++ NGX_HTTP_REQUEST_HEADER_TOO_LARGE); ++ return ngx_http_spdy_state_headers_skip(sc, pos, end); ++ } ++ ++ if (rc != NGX_OK) { ++ ngx_http_spdy_close_stream(sc->stream, ++ NGX_HTTP_INTERNAL_SERVER_ERROR); ++ return ngx_http_spdy_state_headers_skip(sc, pos, end); ++ } ++ ++ /* null-terminate the last processed header name or value */ ++ *buf->pos = '\0'; ++ ++ buf = r->header_in; ++ ++ sc->zstream_in.next_out = buf->last; ++ ++ /* one byte is reserved for null-termination */ ++ sc->zstream_in.avail_out = buf->end - buf->last - 1; ++ ++ z = inflate(&sc->zstream_in, Z_NO_FLUSH); ++ ++ ngx_log_debug5(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy inflate out: ni:%p no:%p ai:%ud ao:%ud rc:%d", ++ sc->zstream_in.next_in, sc->zstream_in.next_out, ++ sc->zstream_in.avail_in, sc->zstream_in.avail_out, ++ z); ++ ++ if (z != Z_OK) { ++ return ngx_http_spdy_state_inflate_error(sc, z); ++ } ++ ++ sc->length -= sc->zstream_in.next_in - pos; ++ pos = sc->zstream_in.next_in; ++ ++ buf->last = sc->zstream_in.next_out; ++ ++ continue; ++ } ++ ++ if (sc->length == 0) { ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "premature end of spdy header block"); ++ ++ return ngx_http_spdy_state_headers_error(sc, pos, end); ++ } ++ ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_headers); ++ ++ case NGX_HTTP_PARSE_INVALID_HEADER: ++ ngx_http_finalize_request(r, NGX_HTTP_BAD_REQUEST); ++ return ngx_http_spdy_state_headers_skip(sc, pos, end); ++ ++ default: /* NGX_ERROR */ ++ return ngx_http_spdy_state_headers_error(sc, pos, end); ++ } ++ ++ /* a header line has been parsed successfully */ ++ ++ rc = ngx_http_spdy_handle_request_header(r); ++ ++ if (rc != NGX_OK) { ++ if (rc == NGX_HTTP_PARSE_INVALID_HEADER) { ++ ngx_http_finalize_request(r, NGX_HTTP_BAD_REQUEST); ++ return ngx_http_spdy_state_headers_skip(sc, pos, end); ++ } ++ ++ if (rc != NGX_ABORT) { ++ ngx_http_spdy_close_stream(sc->stream, ++ NGX_HTTP_INTERNAL_SERVER_ERROR); ++ } ++ ++ return ngx_http_spdy_state_headers_skip(sc, pos, end); ++ } ++ } ++ ++ if (buf->pos != buf->last || sc->zstream_in.avail_in) { ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "incorrect number of spdy header block entries"); ++ ++ return ngx_http_spdy_state_headers_error(sc, pos, end); ++ } ++ ++ if (sc->length) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_headers); ++ } ++ ++ /* null-terminate the last header value */ ++ *buf->pos = '\0'; ++ ++ ngx_http_spdy_run_request(r); ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_headers_skip(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ int n; ++ size_t size; ++ u_char buffer[NGX_SPDY_SKIP_HEADERS_BUFFER_SIZE]; ++ ++ if (sc->length == 0) { ++ return ngx_http_spdy_state_complete(sc, pos, end); ++ } ++ ++ size = end - pos; ++ ++ if (size == 0) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_headers_skip); ++ } ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy header block skip %uz of %uz", size, sc->length); ++ ++ sc->zstream_in.next_in = pos; ++ sc->zstream_in.avail_in = (size < sc->length) ? size : sc->length; ++ ++ while (sc->zstream_in.avail_in) { ++ sc->zstream_in.next_out = buffer; ++ sc->zstream_in.avail_out = NGX_SPDY_SKIP_HEADERS_BUFFER_SIZE; ++ ++ n = inflate(&sc->zstream_in, Z_NO_FLUSH); ++ ++ ngx_log_debug5(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy inflate out: ni:%p no:%p ai:%ud ao:%ud rc:%d", ++ sc->zstream_in.next_in, sc->zstream_in.next_out, ++ sc->zstream_in.avail_in, sc->zstream_in.avail_out, ++ n); ++ ++ if (n != Z_OK) { ++ return ngx_http_spdy_state_inflate_error(sc, n); ++ } ++ } ++ ++ pos = sc->zstream_in.next_in; ++ ++ if (size < sc->length) { ++ sc->length -= size; ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_headers_skip); ++ } ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_headers_error(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ ngx_http_spdy_stream_t *stream; ++ ++ stream = sc->stream; ++ ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent SYN_STREAM frame for stream %ui " ++ "with invalid header block", stream->id); ++ ++ if (ngx_http_spdy_send_rst_stream(sc, stream->id, NGX_SPDY_PROTOCOL_ERROR, ++ stream->priority) ++ != NGX_OK) ++ { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ stream->out_closed = 1; ++ ++ ngx_http_spdy_close_stream(stream, NGX_HTTP_BAD_REQUEST); ++ ++ return ngx_http_spdy_state_headers_skip(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_window_update(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ size_t delta; ++ ngx_uint_t sid; ++ ngx_event_t *wev; ++ ngx_queue_t *q; ++ ngx_http_spdy_stream_t *stream; ++ ++ if (end - pos < NGX_SPDY_WINDOW_UPDATE_SIZE) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_window_update); ++ } ++ ++ if (sc->length != NGX_SPDY_WINDOW_UPDATE_SIZE) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent WINDOW_UPDATE frame " ++ "with incorrect length %uz", sc->length); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ sid = ngx_spdy_frame_parse_sid(pos); ++ ++ pos += NGX_SPDY_SID_SIZE; ++ ++ delta = ngx_spdy_frame_parse_delta(pos); ++ ++ pos += NGX_SPDY_DELTA_SIZE; ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy WINDOW_UPDATE sid:%ui delta:%uz", sid, delta); ++ ++ if (sid) { ++ stream = ngx_http_spdy_get_stream_by_id(sc, sid); ++ ++ if (stream == NULL) { ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "unknown spdy stream"); ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++ } ++ ++ if (stream->send_window > (ssize_t) (NGX_SPDY_MAX_WINDOW - delta)) { ++ ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client violated flow control for stream %ui: " ++ "received WINDOW_UPDATE frame with delta %uz " ++ "not allowed for window %z", ++ sid, delta, stream->send_window); ++ ++ if (ngx_http_spdy_terminate_stream(sc, stream, ++ NGX_SPDY_FLOW_CONTROL_ERROR) ++ == NGX_ERROR) ++ { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++ } ++ ++ stream->send_window += delta; ++ ++ if (stream->exhausted) { ++ stream->exhausted = 0; ++ ++ wev = stream->request->connection->write; ++ ++ if (!wev->timer_set) { ++ wev->delayed = 0; ++ wev->handler(wev); ++ } ++ } ++ ++ } else { ++ sc->send_window += delta; ++ ++ if (sc->send_window > NGX_SPDY_MAX_WINDOW) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client violated connection flow control: " ++ "received WINDOW_UPDATE frame with delta %uz " ++ "not allowed for window %uz", ++ delta, sc->send_window); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ while (!ngx_queue_empty(&sc->waiting)) { ++ q = ngx_queue_head(&sc->waiting); ++ ++ ngx_queue_remove(q); ++ ++ stream = ngx_queue_data(q, ngx_http_spdy_stream_t, queue); ++ ++ stream->handled = 0; ++ ++ wev = stream->request->connection->write; ++ ++ if (!wev->timer_set) { ++ wev->delayed = 0; ++ wev->handler(wev); ++ ++ if (sc->send_window == 0) { ++ break; ++ } ++ } ++ } ++ } ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_data(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ ngx_http_spdy_stream_t *stream; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy DATA frame"); ++ ++ if (sc->length > sc->recv_window) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client violated connection flow control: " ++ "received DATA frame length %uz, available window %uz", ++ sc->length, sc->recv_window); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ sc->recv_window -= sc->length; ++ ++ if (sc->recv_window < NGX_SPDY_MAX_WINDOW / 4) { ++ ++ if (ngx_http_spdy_send_window_update(sc, 0, ++ NGX_SPDY_MAX_WINDOW ++ - sc->recv_window) ++ == NGX_ERROR) ++ { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ sc->recv_window = NGX_SPDY_MAX_WINDOW; ++ } ++ ++ stream = sc->stream; ++ ++ if (stream == NULL) { ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "unknown spdy stream"); ++ ++ return ngx_http_spdy_state_skip(sc, pos, end); ++ } ++ ++ if (sc->length > stream->recv_window) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client violated flow control for stream %ui: " ++ "received DATA frame length %uz, available window %uz", ++ stream->id, sc->length, stream->recv_window); ++ ++ if (ngx_http_spdy_terminate_stream(sc, stream, ++ NGX_SPDY_FLOW_CONTROL_ERROR) ++ == NGX_ERROR) ++ { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ return ngx_http_spdy_state_skip(sc, pos, end); ++ } ++ ++ stream->recv_window -= sc->length; ++ ++ if (stream->recv_window < NGX_SPDY_STREAM_WINDOW / 4) { ++ ++ if (ngx_http_spdy_send_window_update(sc, stream->id, ++ NGX_SPDY_STREAM_WINDOW ++ - stream->recv_window) ++ == NGX_ERROR) ++ { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ stream->recv_window = NGX_SPDY_STREAM_WINDOW; ++ } ++ ++ if (stream->in_closed) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent DATA frame for half-closed stream %ui", ++ stream->id); ++ ++ if (ngx_http_spdy_terminate_stream(sc, stream, ++ NGX_SPDY_STREAM_ALREADY_CLOSED) ++ == NGX_ERROR) ++ { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ return ngx_http_spdy_state_skip(sc, pos, end); ++ } ++ ++ return ngx_http_spdy_state_read_data(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_read_data(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ size_t size; ++ ssize_t n; ++ ngx_buf_t *buf; ++ ngx_int_t rc; ++ ngx_temp_file_t *tf; ++ ngx_http_request_t *r; ++ ngx_http_spdy_stream_t *stream; ++ ngx_http_request_body_t *rb; ++ ngx_http_core_loc_conf_t *clcf; ++ ++ stream = sc->stream; ++ ++ if (stream == NULL) { ++ return ngx_http_spdy_state_skip(sc, pos, end); ++ } ++ ++ if (stream->skip_data) { ++ ++ if (sc->flags & NGX_SPDY_FLAG_FIN) { ++ stream->in_closed = 1; ++ } ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "skipping spdy DATA frame, reason: %d", ++ stream->skip_data); ++ ++ return ngx_http_spdy_state_skip(sc, pos, end); ++ } ++ ++ size = end - pos; ++ ++ if (size > sc->length) { ++ size = sc->length; ++ } ++ ++ r = stream->request; ++ ++ if (r->request_body == NULL ++ && ngx_http_spdy_init_request_body(r) != NGX_OK) ++ { ++ stream->skip_data = NGX_SPDY_DATA_INTERNAL_ERROR; ++ return ngx_http_spdy_state_skip(sc, pos, end); ++ } ++ ++ rb = r->request_body; ++ tf = rb->temp_file; ++ buf = rb->buf; ++ ++ if (size) { ++ rb->rest += size; ++ ++ if (r->headers_in.content_length_n != -1 ++ && r->headers_in.content_length_n < rb->rest) ++ { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client intended to send body data " ++ "larger than declared"); ++ ++ stream->skip_data = NGX_SPDY_DATA_ERROR; ++ goto error; ++ ++ } else { ++ clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); ++ ++ if (clcf->client_max_body_size ++ && clcf->client_max_body_size < rb->rest) ++ { ++ ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, ++ "client intended to send " ++ "too large chunked body: %O bytes", rb->rest); ++ ++ stream->skip_data = NGX_SPDY_DATA_ERROR; ++ goto error; ++ } ++ } ++ ++ sc->length -= size; ++ ++ if (tf) { ++ buf->start = pos; ++ buf->pos = pos; ++ ++ pos += size; ++ ++ buf->end = pos; ++ buf->last = pos; ++ ++ n = ngx_write_chain_to_temp_file(tf, rb->bufs); ++ ++ /* TODO: n == 0 or not complete and level event */ ++ ++ if (n == NGX_ERROR) { ++ stream->skip_data = NGX_SPDY_DATA_INTERNAL_ERROR; ++ goto error; ++ } ++ ++ tf->offset += n; ++ ++ } else { ++ buf->last = ngx_cpymem(buf->last, pos, size); ++ pos += size; ++ } ++ ++ r->request_length += size; ++ } ++ ++ if (sc->length) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_read_data); ++ } ++ ++ if (sc->flags & NGX_SPDY_FLAG_FIN) { ++ ++ stream->in_closed = 1; ++ ++ if (r->headers_in.content_length_n < 0) { ++ r->headers_in.content_length_n = rb->rest; ++ ++ } else if (r->headers_in.content_length_n != rb->rest) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client prematurely closed stream: " ++ "only %O out of %O bytes of request body received", ++ rb->rest, r->headers_in.content_length_n); ++ ++ stream->skip_data = NGX_SPDY_DATA_ERROR; ++ goto error; ++ } ++ ++ if (tf) { ++ ngx_memzero(buf, sizeof(ngx_buf_t)); ++ ++ buf->in_file = 1; ++ buf->file_last = tf->file.offset; ++ buf->file = &tf->file; ++ ++ rb->buf = NULL; ++ } ++ ++ if (rb->post_handler) { ++ r->read_event_handler = ngx_http_block_reading; ++ rb->post_handler(r); ++ } ++ } ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++ ++error: ++ ++ if (rb->post_handler) { ++ ++ if (stream->skip_data == NGX_SPDY_DATA_ERROR) { ++ rc = (r->headers_in.content_length_n == -1) ++ ? NGX_HTTP_REQUEST_ENTITY_TOO_LARGE ++ : NGX_HTTP_BAD_REQUEST; ++ ++ } else { ++ rc = NGX_HTTP_INTERNAL_SERVER_ERROR; ++ } ++ ++ ngx_http_finalize_request(r, rc); ++ } ++ ++ return ngx_http_spdy_state_skip(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_rst_stream(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ ngx_uint_t sid, status; ++ ngx_event_t *ev; ++ ngx_connection_t *fc; ++ ngx_http_spdy_stream_t *stream; ++ ++ if (end - pos < NGX_SPDY_RST_STREAM_SIZE) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_rst_stream); ++ } ++ ++ if (sc->length != NGX_SPDY_RST_STREAM_SIZE) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent RST_STREAM frame with incorrect length %uz", ++ sc->length); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ sid = ngx_spdy_frame_parse_sid(pos); ++ ++ pos += NGX_SPDY_SID_SIZE; ++ ++ status = ngx_spdy_frame_parse_uint32(pos); ++ ++ pos += sizeof(uint32_t); ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy RST_STREAM sid:%ui st:%ui", sid, status); ++ ++ stream = ngx_http_spdy_get_stream_by_id(sc, sid); ++ ++ if (stream == NULL) { ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "unknown spdy stream"); ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++ } ++ ++ stream->in_closed = 1; ++ stream->out_closed = 1; ++ ++ fc = stream->request->connection; ++ fc->error = 1; ++ ++ switch (status) { ++ ++ case NGX_SPDY_CANCEL: ++ ngx_log_error(NGX_LOG_INFO, fc->log, 0, ++ "client canceled stream %ui", sid); ++ break; ++ ++ case NGX_SPDY_INTERNAL_ERROR: ++ ngx_log_error(NGX_LOG_INFO, fc->log, 0, ++ "client terminated stream %ui due to internal error", ++ sid); ++ break; ++ ++ default: ++ ngx_log_error(NGX_LOG_INFO, fc->log, 0, ++ "client terminated stream %ui with status %ui", ++ sid, status); ++ break; ++ } ++ ++ ev = fc->read; ++ ev->handler(ev); ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_ping(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ u_char *p; ++ ngx_buf_t *buf; ++ ngx_http_spdy_out_frame_t *frame; ++ ++ if (end - pos < NGX_SPDY_PING_SIZE) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_ping); ++ } ++ ++ if (sc->length != NGX_SPDY_PING_SIZE) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent PING frame with incorrect length %uz", ++ sc->length); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy PING frame"); ++ ++ frame = ngx_http_spdy_get_ctl_frame(sc, NGX_SPDY_PING_SIZE, ++ NGX_SPDY_HIGHEST_PRIORITY); ++ if (frame == NULL) { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ buf = frame->first->buf; ++ ++ p = buf->pos; ++ ++ p = ngx_spdy_frame_write_head(p, NGX_SPDY_PING); ++ p = ngx_spdy_frame_write_flags_and_len(p, 0, NGX_SPDY_PING_SIZE); ++ ++ p = ngx_cpymem(p, pos, NGX_SPDY_PING_SIZE); ++ ++ buf->last = p; ++ ++ ngx_http_spdy_queue_frame(sc, frame); ++ ++ pos += NGX_SPDY_PING_SIZE; ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_skip(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ size_t size; ++ ++ size = end - pos; ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy frame skip %uz of %uz", size, sc->length); ++ ++ if (size < sc->length) { ++ sc->length -= size; ++ return ngx_http_spdy_state_save(sc, end, end, ++ ngx_http_spdy_state_skip); ++ } ++ ++ return ngx_http_spdy_state_complete(sc, pos + sc->length, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_settings(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ ngx_uint_t fid, val; ++ ++ if (sc->entries == 0) { ++ ++ if (end - pos < NGX_SPDY_SETTINGS_NUM_SIZE) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_settings); ++ } ++ ++ sc->entries = ngx_spdy_frame_parse_uint32(pos); ++ ++ pos += NGX_SPDY_SETTINGS_NUM_SIZE; ++ sc->length -= NGX_SPDY_SETTINGS_NUM_SIZE; ++ ++ if (sc->length < sc->entries * NGX_SPDY_SETTINGS_PAIR_SIZE) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent SETTINGS frame with incorrect " ++ "length %uz or number of entries %ui", ++ sc->length, sc->entries); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy SETTINGS frame has %ui entries", sc->entries); ++ } ++ ++ while (sc->entries) { ++ if (end - pos < NGX_SPDY_SETTINGS_PAIR_SIZE) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_settings); ++ } ++ ++ sc->entries--; ++ sc->length -= NGX_SPDY_SETTINGS_PAIR_SIZE; ++ ++ fid = ngx_spdy_frame_parse_uint32(pos); ++ ++ pos += NGX_SPDY_SETTINGS_FID_SIZE; ++ ++ val = ngx_spdy_frame_parse_uint32(pos); ++ ++ pos += NGX_SPDY_SETTINGS_VAL_SIZE; ++ ++ ngx_log_debug3(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy SETTINGS entry fl:%ui id:%ui val:%ui", ++ ngx_spdy_frame_flags(fid), ngx_spdy_frame_id(fid), val); ++ ++ if (ngx_spdy_frame_flags(fid) == NGX_SPDY_SETTINGS_FLAG_PERSISTED) { ++ continue; ++ } ++ ++ switch (ngx_spdy_frame_id(fid)) { ++ ++ case NGX_SPDY_SETTINGS_INIT_WINDOW: ++ ++ if (val > NGX_SPDY_MAX_WINDOW) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent SETTINGS frame with " ++ "incorrect INIT_WINDOW value: %ui", val); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ if (ngx_http_spdy_adjust_windows(sc, val - sc->init_window) ++ != NGX_OK) ++ { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ sc->init_window = val; ++ ++ continue; ++ } ++ } ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_complete(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy frame complete pos:%p end:%p", pos, end); ++ ++ if (pos > end) { ++ ngx_log_error(NGX_LOG_ALERT, sc->connection->log, 0, ++ "receive buffer overrun"); ++ ++ ngx_debug_point(); ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ sc->handler = ngx_http_spdy_state_head; ++ sc->stream = NULL; ++ ++ return pos; ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_save(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end, ngx_http_spdy_handler_pt handler) ++{ ++ size_t size; ++ ++ ngx_log_debug3(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy frame state save pos:%p end:%p handler:%p", ++ pos, end, handler); ++ ++ size = end - pos; ++ ++ if (size > NGX_SPDY_STATE_BUFFER_SIZE) { ++ ngx_log_error(NGX_LOG_ALERT, sc->connection->log, 0, ++ "state buffer overflow: %uz bytes required", size); ++ ++ ngx_debug_point(); ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ ngx_memcpy(sc->buffer, pos, NGX_SPDY_STATE_BUFFER_SIZE); ++ ++ sc->buffer_used = size; ++ sc->handler = handler; ++ sc->incomplete = 1; ++ ++ return end; ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_inflate_error(ngx_http_spdy_connection_t *sc, int rc) ++{ ++ if (rc == Z_DATA_ERROR || rc == Z_STREAM_END) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent SYN_STREAM frame with " ++ "corrupted header block, inflate() failed: %d", rc); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ ngx_log_error(NGX_LOG_ERR, sc->connection->log, 0, ++ "inflate() failed: %d", rc); ++ ++ return ngx_http_spdy_state_internal_error(sc); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_protocol_error(ngx_http_spdy_connection_t *sc) ++{ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy state protocol error"); ++ ++ if (sc->stream) { ++ sc->stream->out_closed = 1; ++ ngx_http_spdy_close_stream(sc->stream, NGX_HTTP_BAD_REQUEST); ++ } ++ ++ ngx_http_spdy_finalize_connection(sc, NGX_HTTP_CLIENT_CLOSED_REQUEST); ++ ++ return NULL; ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_internal_error(ngx_http_spdy_connection_t *sc) ++{ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy state internal error"); ++ ++ if (sc->stream) { ++ sc->stream->out_closed = 1; ++ ngx_http_spdy_close_stream(sc->stream, NGX_HTTP_INTERNAL_SERVER_ERROR); ++ } ++ ++ ngx_http_spdy_finalize_connection(sc, NGX_HTTP_INTERNAL_SERVER_ERROR); ++ ++ return NULL; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_send_window_update(ngx_http_spdy_connection_t *sc, ngx_uint_t sid, ++ ngx_uint_t delta) ++{ ++ u_char *p; ++ ngx_buf_t *buf; ++ ngx_http_spdy_out_frame_t *frame; ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy send WINDOW_UPDATE sid:%ui delta:%ui", sid, delta); ++ ++ frame = ngx_http_spdy_get_ctl_frame(sc, NGX_SPDY_WINDOW_UPDATE_SIZE, ++ NGX_SPDY_HIGHEST_PRIORITY); ++ if (frame == NULL) { ++ return NGX_ERROR; ++ } ++ ++ buf = frame->first->buf; ++ ++ p = buf->pos; ++ ++ p = ngx_spdy_frame_write_head(p, NGX_SPDY_WINDOW_UPDATE); ++ p = ngx_spdy_frame_write_flags_and_len(p, 0, NGX_SPDY_WINDOW_UPDATE_SIZE); ++ ++ p = ngx_spdy_frame_write_sid(p, sid); ++ p = ngx_spdy_frame_aligned_write_uint32(p, delta); ++ ++ buf->last = p; ++ ++ ngx_http_spdy_queue_frame(sc, frame); ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_send_rst_stream(ngx_http_spdy_connection_t *sc, ngx_uint_t sid, ++ ngx_uint_t status, ngx_uint_t priority) ++{ ++ u_char *p; ++ ngx_buf_t *buf; ++ ngx_http_spdy_out_frame_t *frame; ++ ++ if (sc->connection->error) { ++ return NGX_OK; ++ } ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy send RST_STREAM sid:%ui st:%ui", sid, status); ++ ++ frame = ngx_http_spdy_get_ctl_frame(sc, NGX_SPDY_RST_STREAM_SIZE, ++ priority); ++ if (frame == NULL) { ++ return NGX_ERROR; ++ } ++ ++ buf = frame->first->buf; ++ ++ p = buf->pos; ++ ++ p = ngx_spdy_frame_write_head(p, NGX_SPDY_RST_STREAM); ++ p = ngx_spdy_frame_write_flags_and_len(p, 0, NGX_SPDY_RST_STREAM_SIZE); ++ ++ p = ngx_spdy_frame_write_sid(p, sid); ++ p = ngx_spdy_frame_aligned_write_uint32(p, status); ++ ++ buf->last = p; ++ ++ ngx_http_spdy_queue_frame(sc, frame); ++ ++ return NGX_OK; ++} ++ ++ ++#if 0 ++static ngx_int_t ++ngx_http_spdy_send_goaway(ngx_http_spdy_connection_t *sc) ++{ ++ u_char *p; ++ ngx_buf_t *buf; ++ ngx_http_spdy_out_frame_t *frame; ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy send GOAWAY sid:%ui", sc->last_sid); ++ ++ frame = ngx_http_spdy_get_ctl_frame(sc, NGX_SPDY_GOAWAY_SIZE, ++ NGX_SPDY_HIGHEST_PRIORITY); ++ if (frame == NULL) { ++ return NGX_ERROR; ++ } ++ ++ buf = frame->first->buf; ++ ++ p = buf->pos; ++ ++ p = ngx_spdy_frame_write_head(p, NGX_SPDY_GOAWAY); ++ p = ngx_spdy_frame_write_flags_and_len(p, 0, NGX_SPDY_GOAWAY_SIZE); ++ ++ p = ngx_spdy_frame_write_sid(p, sc->last_sid); ++ ++ buf->last = p; ++ ++ ngx_http_spdy_queue_frame(sc, frame); ++ ++ return NGX_OK; ++} ++#endif ++ ++ ++static ngx_int_t ++ngx_http_spdy_send_settings(ngx_http_spdy_connection_t *sc) ++{ ++ u_char *p; ++ ngx_buf_t *buf; ++ ngx_chain_t *cl; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ngx_http_spdy_out_frame_t *frame; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy send SETTINGS frame"); ++ ++ frame = ngx_palloc(sc->pool, sizeof(ngx_http_spdy_out_frame_t)); ++ if (frame == NULL) { ++ return NGX_ERROR; ++ } ++ ++ cl = ngx_alloc_chain_link(sc->pool); ++ if (cl == NULL) { ++ return NGX_ERROR; ++ } ++ ++ buf = ngx_create_temp_buf(sc->pool, NGX_SPDY_FRAME_HEADER_SIZE ++ + NGX_SPDY_SETTINGS_NUM_SIZE ++ + 2 * NGX_SPDY_SETTINGS_PAIR_SIZE); ++ if (buf == NULL) { ++ return NGX_ERROR; ++ } ++ ++ buf->last_buf = 1; ++ ++ cl->buf = buf; ++ cl->next = NULL; ++ ++ frame->first = cl; ++ frame->last = cl; ++ frame->handler = ngx_http_spdy_settings_frame_handler; ++ frame->stream = NULL; ++#if (NGX_DEBUG) ++ frame->length = NGX_SPDY_SETTINGS_NUM_SIZE ++ + 2 * NGX_SPDY_SETTINGS_PAIR_SIZE; ++#endif ++ frame->priority = NGX_SPDY_HIGHEST_PRIORITY; ++ frame->blocked = 0; ++ ++ p = buf->pos; ++ ++ p = ngx_spdy_frame_write_head(p, NGX_SPDY_SETTINGS); ++ p = ngx_spdy_frame_write_flags_and_len(p, NGX_SPDY_FLAG_CLEAR_SETTINGS, ++ NGX_SPDY_SETTINGS_NUM_SIZE ++ + 2 * NGX_SPDY_SETTINGS_PAIR_SIZE); ++ ++ p = ngx_spdy_frame_aligned_write_uint32(p, 2); ++ ++ sscf = ngx_http_get_module_srv_conf(sc->http_connection->conf_ctx, ++ ngx_http_spdy_module); ++ ++ p = ngx_spdy_frame_write_flags_and_id(p, 0, NGX_SPDY_SETTINGS_MAX_STREAMS); ++ p = ngx_spdy_frame_aligned_write_uint32(p, sscf->concurrent_streams); ++ ++ p = ngx_spdy_frame_write_flags_and_id(p, 0, NGX_SPDY_SETTINGS_INIT_WINDOW); ++ p = ngx_spdy_frame_aligned_write_uint32(p, NGX_SPDY_STREAM_WINDOW); ++ ++ buf->last = p; ++ ++ ngx_http_spdy_queue_frame(sc, frame); ++ ++ return NGX_OK; ++} ++ ++ ++ngx_int_t ++ngx_http_spdy_settings_frame_handler(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_out_frame_t *frame) ++{ ++ ngx_buf_t *buf; ++ ++ buf = frame->first->buf; ++ ++ if (buf->pos != buf->last) { ++ return NGX_AGAIN; ++ } ++ ++ ngx_free_chain(sc->pool, frame->first); ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_http_spdy_out_frame_t * ++ngx_http_spdy_get_ctl_frame(ngx_http_spdy_connection_t *sc, size_t length, ++ ngx_uint_t priority) ++{ ++ ngx_chain_t *cl; ++ ngx_http_spdy_out_frame_t *frame; ++ ++ frame = sc->free_ctl_frames; ++ ++ if (frame) { ++ sc->free_ctl_frames = frame->next; ++ ++ cl = frame->first; ++ cl->buf->pos = cl->buf->start; ++ ++ } else { ++ frame = ngx_palloc(sc->pool, sizeof(ngx_http_spdy_out_frame_t)); ++ if (frame == NULL) { ++ return NULL; ++ } ++ ++ cl = ngx_alloc_chain_link(sc->pool); ++ if (cl == NULL) { ++ return NULL; ++ } ++ ++ cl->buf = ngx_create_temp_buf(sc->pool, ++ NGX_SPDY_CTL_FRAME_BUFFER_SIZE); ++ if (cl->buf == NULL) { ++ return NULL; ++ } ++ ++ cl->buf->last_buf = 1; ++ ++ frame->first = cl; ++ frame->last = cl; ++ frame->handler = ngx_http_spdy_ctl_frame_handler; ++ frame->stream = NULL; ++ } ++ ++#if (NGX_DEBUG) ++ if (length > NGX_SPDY_CTL_FRAME_BUFFER_SIZE - NGX_SPDY_FRAME_HEADER_SIZE) { ++ ngx_log_error(NGX_LOG_ALERT, sc->pool->log, 0, ++ "requested control frame is too large: %uz", length); ++ return NULL; ++ } ++ ++ frame->length = length; ++#endif ++ ++ frame->priority = priority; ++ frame->blocked = 0; ++ ++ return frame; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_ctl_frame_handler(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_out_frame_t *frame) ++{ ++ ngx_buf_t *buf; ++ ++ buf = frame->first->buf; ++ ++ if (buf->pos != buf->last) { ++ return NGX_AGAIN; ++ } ++ ++ frame->next = sc->free_ctl_frames; ++ sc->free_ctl_frames = frame; ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_http_spdy_stream_t * ++ngx_http_spdy_create_stream(ngx_http_spdy_connection_t *sc, ngx_uint_t id, ++ ngx_uint_t priority) ++{ ++ ngx_log_t *log; ++ ngx_uint_t index; ++ ngx_event_t *rev, *wev; ++ ngx_connection_t *fc; ++ ngx_http_log_ctx_t *ctx; ++ ngx_http_request_t *r; ++ ngx_http_spdy_stream_t *stream; ++ ngx_http_core_srv_conf_t *cscf; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ++ fc = sc->free_fake_connections; ++ ++ if (fc) { ++ sc->free_fake_connections = fc->data; ++ ++ rev = fc->read; ++ wev = fc->write; ++ log = fc->log; ++ ctx = log->data; ++ ++ } else { ++ fc = ngx_palloc(sc->pool, sizeof(ngx_connection_t)); ++ if (fc == NULL) { ++ return NULL; ++ } ++ ++ rev = ngx_palloc(sc->pool, sizeof(ngx_event_t)); ++ if (rev == NULL) { ++ return NULL; ++ } ++ ++ wev = ngx_palloc(sc->pool, sizeof(ngx_event_t)); ++ if (wev == NULL) { ++ return NULL; ++ } ++ ++ log = ngx_palloc(sc->pool, sizeof(ngx_log_t)); ++ if (log == NULL) { ++ return NULL; ++ } ++ ++ ctx = ngx_palloc(sc->pool, sizeof(ngx_http_log_ctx_t)); ++ if (ctx == NULL) { ++ return NULL; ++ } ++ ++ ctx->connection = fc; ++ ctx->request = NULL; ++ } ++ ++ ngx_memcpy(log, sc->connection->log, sizeof(ngx_log_t)); ++ ++ log->data = ctx; ++ ++ ngx_memzero(rev, sizeof(ngx_event_t)); ++ ++ rev->data = fc; ++ rev->ready = 1; ++ rev->handler = ngx_http_spdy_close_stream_handler; ++ rev->log = log; ++ ++ ngx_memcpy(wev, rev, sizeof(ngx_event_t)); ++ ++ wev->write = 1; ++ ++ ngx_memcpy(fc, sc->connection, sizeof(ngx_connection_t)); ++ ++ fc->data = sc->http_connection; ++ fc->read = rev; ++ fc->write = wev; ++ fc->sent = 0; ++ fc->log = log; ++ fc->buffered = 0; ++ fc->sndlowat = 1; ++ fc->tcp_nodelay = NGX_TCP_NODELAY_DISABLED; ++ ++ r = ngx_http_create_request(fc); ++ if (r == NULL) { ++ return NULL; ++ } ++ ++ r->valid_location = 1; ++ ++ fc->data = r; ++ sc->connection->requests++; ++ ++ cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module); ++ ++ r->header_in = ngx_create_temp_buf(r->pool, ++ cscf->client_header_buffer_size); ++ if (r->header_in == NULL) { ++ ngx_http_free_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR); ++ return NULL; ++ } ++ ++ r->headers_in.connection_type = NGX_HTTP_CONNECTION_CLOSE; ++ ++ stream = ngx_pcalloc(r->pool, sizeof(ngx_http_spdy_stream_t)); ++ if (stream == NULL) { ++ ngx_http_free_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR); ++ return NULL; ++ } ++ ++ r->spdy_stream = stream; ++ ++ stream->id = id; ++ stream->request = r; ++ stream->connection = sc; ++ ++ stream->send_window = sc->init_window; ++ stream->recv_window = NGX_SPDY_STREAM_WINDOW; ++ ++ stream->priority = priority; ++ ++ sscf = ngx_http_get_module_srv_conf(r, ngx_http_spdy_module); ++ ++ index = ngx_http_spdy_stream_index(sscf, id); ++ ++ stream->index = sc->streams_index[index]; ++ sc->streams_index[index] = stream; ++ ++ sc->processing++; ++ ++ return stream; ++} ++ ++ ++static ngx_http_spdy_stream_t * ++ngx_http_spdy_get_stream_by_id(ngx_http_spdy_connection_t *sc, ++ ngx_uint_t sid) ++{ ++ ngx_http_spdy_stream_t *stream; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ++ sscf = ngx_http_get_module_srv_conf(sc->http_connection->conf_ctx, ++ ngx_http_spdy_module); ++ ++ stream = sc->streams_index[ngx_http_spdy_stream_index(sscf, sid)]; ++ ++ while (stream) { ++ if (stream->id == sid) { ++ return stream; ++ } ++ ++ stream = stream->index; ++ } ++ ++ return NULL; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_parse_header(ngx_http_request_t *r) ++{ ++ u_char *p, *end, ch; ++ ngx_uint_t hash; ++ ngx_http_core_srv_conf_t *cscf; ++ ++ enum { ++ sw_name_len = 0, ++ sw_name, ++ sw_value_len, ++ sw_value ++ } state; ++ ++ state = r->state; ++ ++ p = r->header_in->pos; ++ end = r->header_in->last; ++ ++ switch (state) { ++ ++ case sw_name_len: ++ ++ if (end - p < NGX_SPDY_NV_NLEN_SIZE) { ++ return NGX_AGAIN; ++ } ++ ++ r->lowcase_index = ngx_spdy_frame_parse_uint32(p); ++ ++ if (r->lowcase_index == 0) { ++ return NGX_ERROR; ++ } ++ ++ /* null-terminate the previous header value */ ++ *p = '\0'; ++ ++ p += NGX_SPDY_NV_NLEN_SIZE; ++ ++ r->invalid_header = 0; ++ ++ state = sw_name; ++ ++ /* fall through */ ++ ++ case sw_name: ++ ++ if ((ngx_uint_t) (end - p) < r->lowcase_index) { ++ break; ++ } ++ ++ cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module); ++ ++ r->header_name_start = p; ++ r->header_name_end = p + r->lowcase_index; ++ ++ if (p[0] == ':') { ++ p++; ++ } ++ ++ hash = 0; ++ ++ for ( /* void */ ; p != r->header_name_end; p++) { ++ ++ ch = *p; ++ ++ hash = ngx_hash(hash, ch); ++ ++ if ((ch >= 'a' && ch <= 'z') ++ || (ch == '-') ++ || (ch >= '0' && ch <= '9') ++ || (ch == '_' && cscf->underscores_in_headers)) ++ { ++ continue; ++ } ++ ++ switch (ch) { ++ case '\0': ++ case LF: ++ case CR: ++ case ':': ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent invalid header name: \"%*s\"", ++ r->lowcase_index, r->header_name_start); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ if (ch >= 'A' && ch <= 'Z') { ++ return NGX_ERROR; ++ } ++ ++ r->invalid_header = 1; ++ } ++ ++ r->header_hash = hash; ++ ++ state = sw_value_len; ++ ++ /* fall through */ ++ ++ case sw_value_len: ++ ++ if (end - p < NGX_SPDY_NV_VLEN_SIZE) { ++ break; ++ } ++ ++ r->lowcase_index = ngx_spdy_frame_parse_uint32(p); ++ ++ /* null-terminate header name */ ++ *p = '\0'; ++ ++ p += NGX_SPDY_NV_VLEN_SIZE; ++ ++ state = sw_value; ++ ++ /* fall through */ ++ ++ case sw_value: ++ ++ if ((ngx_uint_t) (end - p) < r->lowcase_index) { ++ break; ++ } ++ ++ r->header_start = p; ++ ++ while (r->lowcase_index--) { ++ ch = *p; ++ ++ if (ch == '\0') { ++ ++ if (p == r->header_start) { ++ return NGX_ERROR; ++ } ++ ++ r->header_end = p; ++ r->header_in->pos = p + 1; ++ ++ r->state = sw_value; ++ ++ return NGX_OK; ++ } ++ ++ if (ch == CR || ch == LF) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent header \"%*s\" with " ++ "invalid value: \"%*s\\%c...\"", ++ r->header_name_end - r->header_name_start, ++ r->header_name_start, ++ p - r->header_start, ++ r->header_start, ++ ch == CR ? 'r' : 'n'); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ p++; ++ } ++ ++ r->header_end = p; ++ r->header_in->pos = p; ++ ++ r->state = 0; ++ ++ return NGX_DONE; ++ } ++ ++ r->header_in->pos = p; ++ r->state = state; ++ ++ return NGX_AGAIN; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_alloc_large_header_buffer(ngx_http_request_t *r) ++{ ++ u_char *old, *new, *p; ++ size_t rest; ++ ngx_buf_t *buf; ++ ngx_http_spdy_stream_t *stream; ++ ngx_http_core_srv_conf_t *cscf; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy alloc large header buffer"); ++ ++ stream = r->spdy_stream; ++ ++ cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module); ++ ++ if (stream->header_buffers ++ == (ngx_uint_t) cscf->large_client_header_buffers.num) ++ { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent too large request"); ++ ++ return NGX_DECLINED; ++ } ++ ++ rest = r->header_in->last - r->header_in->pos; ++ ++ /* ++ * One more byte is needed for null-termination ++ * and another one for further progress. ++ */ ++ if (rest > cscf->large_client_header_buffers.size - 2) { ++ p = r->header_in->pos; ++ ++ if (rest > NGX_MAX_ERROR_STR - 300) { ++ rest = NGX_MAX_ERROR_STR - 300; ++ } ++ ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent too long header name or value: \"%*s...\"", ++ rest, p); ++ ++ return NGX_DECLINED; ++ } ++ ++ buf = ngx_create_temp_buf(r->pool, cscf->large_client_header_buffers.size); ++ if (buf == NULL) { ++ return NGX_ERROR; ++ } ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy large header alloc: %p %uz", ++ buf->pos, buf->end - buf->last); ++ ++ old = r->header_in->pos; ++ new = buf->pos; ++ ++ if (rest) { ++ buf->last = ngx_cpymem(new, old, rest); ++ } ++ ++ r->header_in = buf; ++ ++ stream->header_buffers++; ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_handle_request_header(ngx_http_request_t *r) ++{ ++ ngx_uint_t i; ++ ngx_table_elt_t *h; ++ ngx_http_core_srv_conf_t *cscf; ++ ngx_http_spdy_request_header_t *sh; ++ ++ if (r->invalid_header) { ++ cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module); ++ ++ if (cscf->ignore_invalid_headers) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent invalid header: \"%*s\"", ++ r->header_end - r->header_name_start, ++ r->header_name_start); ++ return NGX_OK; ++ } ++ ++ } ++ ++ if (r->header_name_start[0] == ':') { ++ r->header_name_start++; ++ ++ for (i = 0; i < NGX_SPDY_REQUEST_HEADERS; i++) { ++ sh = &ngx_http_spdy_request_headers[i]; ++ ++ if (sh->hash != r->header_hash ++ || sh->len != r->header_name_end - r->header_name_start ++ || ngx_strncmp(sh->header, r->header_name_start, sh->len) != 0) ++ { ++ continue; ++ } ++ ++ return sh->handler(r); ++ } ++ ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent invalid header name: \":%*s\"", ++ r->header_end - r->header_name_start, ++ r->header_name_start); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ h = ngx_list_push(&r->headers_in.headers); ++ if (h == NULL) { ++ return NGX_ERROR; ++ } ++ ++ h->hash = r->header_hash; ++ ++ h->key.len = r->header_name_end - r->header_name_start; ++ h->key.data = r->header_name_start; ++ ++ h->value.len = r->header_end - r->header_start; ++ h->value.data = r->header_start; ++ ++ h->lowcase_key = h->key.data; ++ ++ return NGX_OK; ++} ++ ++ ++void ++ngx_http_spdy_request_headers_init(void) ++{ ++ ngx_uint_t i; ++ ngx_http_spdy_request_header_t *h; ++ ++ for (i = 0; i < NGX_SPDY_REQUEST_HEADERS; i++) { ++ h = &ngx_http_spdy_request_headers[i]; ++ h->hash = ngx_hash_key(h->header, h->len); ++ } ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_parse_method(ngx_http_request_t *r) ++{ ++ size_t k, len; ++ ngx_uint_t n; ++ const u_char *p, *m; ++ ++ /* ++ * This array takes less than 256 sequential bytes, ++ * and if typical CPU cache line size is 64 bytes, ++ * it is prefetched for 4 load operations. ++ */ ++ static const struct { ++ u_char len; ++ const u_char method[11]; ++ uint32_t value; ++ } tests[] = { ++ { 3, "GET", NGX_HTTP_GET }, ++ { 4, "POST", NGX_HTTP_POST }, ++ { 4, "HEAD", NGX_HTTP_HEAD }, ++ { 7, "OPTIONS", NGX_HTTP_OPTIONS }, ++ { 8, "PROPFIND", NGX_HTTP_PROPFIND }, ++ { 3, "PUT", NGX_HTTP_PUT }, ++ { 5, "MKCOL", NGX_HTTP_MKCOL }, ++ { 6, "DELETE", NGX_HTTP_DELETE }, ++ { 4, "COPY", NGX_HTTP_COPY }, ++ { 4, "MOVE", NGX_HTTP_MOVE }, ++ { 9, "PROPPATCH", NGX_HTTP_PROPPATCH }, ++ { 4, "LOCK", NGX_HTTP_LOCK }, ++ { 6, "UNLOCK", NGX_HTTP_UNLOCK }, ++ { 5, "PATCH", NGX_HTTP_PATCH }, ++ { 5, "TRACE", NGX_HTTP_TRACE } ++ }, *test; ++ ++ if (r->method_name.len) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent duplicate :method header"); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ len = r->header_end - r->header_start; ++ ++ r->method_name.len = len; ++ r->method_name.data = r->header_start; ++ ++ test = tests; ++ n = sizeof(tests) / sizeof(tests[0]); ++ ++ do { ++ if (len == test->len) { ++ p = r->method_name.data; ++ m = test->method; ++ k = len; ++ ++ do { ++ if (*p++ != *m++) { ++ goto next; ++ } ++ } while (--k); ++ ++ r->method = test->value; ++ return NGX_OK; ++ } ++ ++ next: ++ test++; ++ ++ } while (--n); ++ ++ p = r->method_name.data; ++ ++ do { ++ if ((*p < 'A' || *p > 'Z') && *p != '_') { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent invalid method: \"%V\"", ++ &r->method_name); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ p++; ++ ++ } while (--len); ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_parse_scheme(ngx_http_request_t *r) ++{ ++ if (r->schema_start) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent duplicate :schema header"); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ r->schema_start = r->header_start; ++ r->schema_end = r->header_end; ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_parse_host(ngx_http_request_t *r) ++{ ++ ngx_table_elt_t *h; ++ ++ if (r->headers_in.host) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent duplicate :host header"); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ h = ngx_list_push(&r->headers_in.headers); ++ if (h == NULL) { ++ return NGX_ERROR; ++ } ++ ++ r->headers_in.host = h; ++ ++ h->hash = r->header_hash; ++ ++ h->key.len = r->header_name_end - r->header_name_start; ++ h->key.data = r->header_name_start; ++ ++ h->value.len = r->header_end - r->header_start; ++ h->value.data = r->header_start; ++ ++ h->lowcase_key = h->key.data; ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_parse_path(ngx_http_request_t *r) ++{ ++ if (r->unparsed_uri.len) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent duplicate :path header"); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ r->uri_start = r->header_start; ++ r->uri_end = r->header_end; ++ ++ if (ngx_http_parse_uri(r) != NGX_OK) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent invalid URI: \"%*s\"", ++ r->uri_end - r->uri_start, r->uri_start); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ if (ngx_http_process_request_uri(r) != NGX_OK) { ++ /* ++ * request has been finalized already ++ * in ngx_http_process_request_uri() ++ */ ++ return NGX_ABORT; ++ } ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_parse_version(ngx_http_request_t *r) ++{ ++ u_char *p, ch; ++ ++ if (r->http_protocol.len) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent duplicate :version header"); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ p = r->header_start; ++ ++ if (r->header_end - p < 8 || !(ngx_str5cmp(p, 'H', 'T', 'T', 'P', '/'))) { ++ goto invalid; ++ } ++ ++ ch = *(p + 5); ++ ++ if (ch < '1' || ch > '9') { ++ goto invalid; ++ } ++ ++ r->http_major = ch - '0'; ++ ++ for (p += 6; p != r->header_end - 2; p++) { ++ ++ ch = *p; ++ ++ if (ch == '.') { ++ break; ++ } ++ ++ if (ch < '0' || ch > '9') { ++ goto invalid; ++ } ++ ++ r->http_major = r->http_major * 10 + ch - '0'; ++ } ++ ++ if (*p != '.') { ++ goto invalid; ++ } ++ ++ ch = *(p + 1); ++ ++ if (ch < '0' || ch > '9') { ++ goto invalid; ++ } ++ ++ r->http_minor = ch - '0'; ++ ++ for (p += 2; p != r->header_end; p++) { ++ ++ ch = *p; ++ ++ if (ch < '0' || ch > '9') { ++ goto invalid; ++ } ++ ++ r->http_minor = r->http_minor * 10 + ch - '0'; ++ } ++ ++ r->http_protocol.len = r->header_end - r->header_start; ++ r->http_protocol.data = r->header_start; ++ r->http_version = r->http_major * 1000 + r->http_minor; ++ ++ return NGX_OK; ++ ++invalid: ++ ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent invalid http version: \"%*s\"", ++ r->header_end - r->header_start, r->header_start); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_construct_request_line(ngx_http_request_t *r) ++{ ++ u_char *p; ++ ++ if (r->method_name.len == 0 ++ || r->unparsed_uri.len == 0 ++ || r->http_protocol.len == 0) ++ { ++ ngx_http_finalize_request(r, NGX_HTTP_BAD_REQUEST); ++ return NGX_ERROR; ++ } ++ ++ r->request_line.len = r->method_name.len + 1 ++ + r->unparsed_uri.len + 1 ++ + r->http_protocol.len; ++ ++ p = ngx_pnalloc(r->pool, r->request_line.len + 1); ++ if (p == NULL) { ++ ngx_http_spdy_close_stream(r->spdy_stream, ++ NGX_HTTP_INTERNAL_SERVER_ERROR); ++ return NGX_ERROR; ++ } ++ ++ r->request_line.data = p; ++ ++ p = ngx_cpymem(p, r->method_name.data, r->method_name.len); ++ ++ *p++ = ' '; ++ ++ p = ngx_cpymem(p, r->unparsed_uri.data, r->unparsed_uri.len); ++ ++ *p++ = ' '; ++ ++ ngx_memcpy(p, r->http_protocol.data, r->http_protocol.len + 1); ++ ++ /* some modules expect the space character after method name */ ++ r->method_name.data = r->request_line.data; ++ ++ return NGX_OK; ++} ++ ++ ++static void ++ngx_http_spdy_run_request(ngx_http_request_t *r) ++{ ++ ngx_uint_t i; ++ ngx_list_part_t *part; ++ ngx_table_elt_t *h; ++ ngx_http_header_t *hh; ++ ngx_http_core_main_conf_t *cmcf; ++ ++ if (ngx_http_spdy_construct_request_line(r) != NGX_OK) { ++ return; ++ } ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy http request line: \"%V\"", &r->request_line); ++ ++ cmcf = ngx_http_get_module_main_conf(r, ngx_http_core_module); ++ ++ part = &r->headers_in.headers.part; ++ h = part->elts; ++ ++ for (i = 0 ;; i++) { ++ ++ if (i >= part->nelts) { ++ if (part->next == NULL) { ++ break; ++ } ++ ++ part = part->next; ++ h = part->elts; ++ i = 0; ++ } ++ ++ hh = ngx_hash_find(&cmcf->headers_in_hash, h[i].hash, ++ h[i].lowcase_key, h[i].key.len); ++ ++ if (hh && hh->handler(r, &h[i], hh->offset) != NGX_OK) { ++ return; ++ } ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy http header: \"%V: %V\"", &h[i].key, &h[i].value); ++ } ++ ++ r->http_state = NGX_HTTP_PROCESS_REQUEST_STATE; ++ ++ if (ngx_http_process_request_header(r) != NGX_OK) { ++ return; ++ } ++ ++ if (r->headers_in.content_length_n > 0 && r->spdy_stream->in_closed) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client prematurely closed stream"); ++ ++ r->spdy_stream->skip_data = NGX_SPDY_DATA_ERROR; ++ ++ ngx_http_finalize_request(r, NGX_HTTP_BAD_REQUEST); ++ return; ++ } ++ ++ ngx_http_process_request(r); ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_init_request_body(ngx_http_request_t *r) ++{ ++ ngx_buf_t *buf; ++ ngx_temp_file_t *tf; ++ ngx_http_request_body_t *rb; ++ ngx_http_core_loc_conf_t *clcf; ++ ++ rb = ngx_pcalloc(r->pool, sizeof(ngx_http_request_body_t)); ++ if (rb == NULL) { ++ return NGX_ERROR; ++ } ++ ++ r->request_body = rb; ++ ++ if (r->spdy_stream->in_closed) { ++ return NGX_OK; ++ } ++ ++ rb->rest = r->headers_in.content_length_n; ++ ++ clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); ++ ++ if (r->request_body_in_file_only ++ || rb->rest > (off_t) clcf->client_body_buffer_size ++ || rb->rest < 0) ++ { ++ tf = ngx_pcalloc(r->pool, sizeof(ngx_temp_file_t)); ++ if (tf == NULL) { ++ return NGX_ERROR; ++ } ++ ++ tf->file.fd = NGX_INVALID_FILE; ++ tf->file.log = r->connection->log; ++ tf->path = clcf->client_body_temp_path; ++ tf->pool = r->pool; ++ tf->warn = "a client request body is buffered to a temporary file"; ++ tf->log_level = r->request_body_file_log_level; ++ tf->persistent = r->request_body_in_persistent_file; ++ tf->clean = r->request_body_in_clean_file; ++ ++ if (r->request_body_file_group_access) { ++ tf->access = 0660; ++ } ++ ++ rb->temp_file = tf; ++ ++ if (r->spdy_stream->in_closed ++ && ngx_create_temp_file(&tf->file, tf->path, tf->pool, ++ tf->persistent, tf->clean, tf->access) ++ != NGX_OK) ++ { ++ return NGX_ERROR; ++ } ++ ++ buf = ngx_calloc_buf(r->pool); ++ if (buf == NULL) { ++ return NGX_ERROR; ++ } ++ ++ } else { ++ ++ if (rb->rest == 0) { ++ return NGX_OK; ++ } ++ ++ buf = ngx_create_temp_buf(r->pool, (size_t) rb->rest); ++ if (buf == NULL) { ++ return NGX_ERROR; ++ } ++ } ++ ++ rb->buf = buf; ++ ++ rb->bufs = ngx_alloc_chain_link(r->pool); ++ if (rb->bufs == NULL) { ++ return NGX_ERROR; ++ } ++ ++ rb->bufs->buf = buf; ++ rb->bufs->next = NULL; ++ ++ rb->rest = 0; ++ ++ return NGX_OK; ++} ++ ++ ++ngx_int_t ++ngx_http_spdy_read_request_body(ngx_http_request_t *r, ++ ngx_http_client_body_handler_pt post_handler) ++{ ++ ngx_http_spdy_stream_t *stream; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy read request body"); ++ ++ stream = r->spdy_stream; ++ ++ switch (stream->skip_data) { ++ ++ case NGX_SPDY_DATA_DISCARD: ++ post_handler(r); ++ return NGX_OK; ++ ++ case NGX_SPDY_DATA_ERROR: ++ if (r->headers_in.content_length_n == -1) { ++ return NGX_HTTP_REQUEST_ENTITY_TOO_LARGE; ++ } else { ++ return NGX_HTTP_BAD_REQUEST; ++ } ++ ++ case NGX_SPDY_DATA_INTERNAL_ERROR: ++ return NGX_HTTP_INTERNAL_SERVER_ERROR; ++ } ++ ++ if (!r->request_body && ngx_http_spdy_init_request_body(r) != NGX_OK) { ++ stream->skip_data = NGX_SPDY_DATA_INTERNAL_ERROR; ++ return NGX_HTTP_INTERNAL_SERVER_ERROR; ++ } ++ ++ if (stream->in_closed) { ++ post_handler(r); ++ return NGX_OK; ++ } ++ ++ r->request_body->post_handler = post_handler; ++ ++ r->read_event_handler = ngx_http_test_reading; ++ r->write_event_handler = ngx_http_request_empty_handler; ++ ++ return NGX_AGAIN; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_terminate_stream(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_stream_t *stream, ngx_uint_t status) ++{ ++ ngx_event_t *rev; ++ ngx_connection_t *fc; ++ ++ if (ngx_http_spdy_send_rst_stream(sc, stream->id, status, ++ NGX_SPDY_HIGHEST_PRIORITY) ++ == NGX_ERROR) ++ { ++ return NGX_ERROR; ++ } ++ ++ stream->out_closed = 1; ++ ++ fc = stream->request->connection; ++ fc->error = 1; ++ ++ rev = fc->read; ++ rev->handler(rev); ++ ++ return NGX_OK; ++} ++ ++ ++static void ++ngx_http_spdy_close_stream_handler(ngx_event_t *ev) ++{ ++ ngx_connection_t *fc; ++ ngx_http_request_t *r; ++ ++ fc = ev->data; ++ r = fc->data; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy close stream handler"); ++ ++ ngx_http_spdy_close_stream(r->spdy_stream, 0); ++} ++ ++ ++void ++ngx_http_spdy_close_stream(ngx_http_spdy_stream_t *stream, ngx_int_t rc) ++{ ++ ngx_event_t *ev; ++ ngx_connection_t *fc; ++ ngx_http_spdy_stream_t **index, *s; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ngx_http_spdy_connection_t *sc; ++ ++ sc = stream->connection; ++ ++ ngx_log_debug3(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy close stream %ui, queued %ui, processing %ui", ++ stream->id, stream->queued, sc->processing); ++ ++ fc = stream->request->connection; ++ ++ if (stream->queued) { ++ fc->write->handler = ngx_http_spdy_close_stream_handler; ++ return; ++ } ++ ++ if (!stream->out_closed) { ++ if (ngx_http_spdy_send_rst_stream(sc, stream->id, ++ NGX_SPDY_INTERNAL_ERROR, ++ stream->priority) ++ != NGX_OK) ++ { ++ sc->connection->error = 1; ++ } ++ } ++ ++ if (sc->stream == stream) { ++ sc->stream = NULL; ++ } ++ ++ sscf = ngx_http_get_module_srv_conf(sc->http_connection->conf_ctx, ++ ngx_http_spdy_module); ++ ++ index = sc->streams_index + ngx_http_spdy_stream_index(sscf, stream->id); ++ ++ for ( ;; ) { ++ s = *index; ++ ++ if (s == NULL) { ++ break; ++ } ++ ++ if (s == stream) { ++ *index = s->index; ++ break; ++ } ++ ++ index = &s->index; ++ } ++ ++ ngx_http_free_request(stream->request, rc); ++ ++ ev = fc->read; ++ ++ if (ev->active || ev->disabled) { ++ ngx_log_error(NGX_LOG_ALERT, sc->connection->log, 0, ++ "fake read event was activated"); ++ } ++ ++ if (ev->timer_set) { ++ ngx_del_timer(ev); ++ } ++ ++ if (ev->posted) { ++ ngx_delete_posted_event(ev); ++ } ++ ++ ev = fc->write; ++ ++ if (ev->active || ev->disabled) { ++ ngx_log_error(NGX_LOG_ALERT, sc->connection->log, 0, ++ "fake write event was activated"); ++ } ++ ++ if (ev->timer_set) { ++ ngx_del_timer(ev); ++ } ++ ++ if (ev->posted) { ++ ngx_delete_posted_event(ev); ++ } ++ ++ fc->data = sc->free_fake_connections; ++ sc->free_fake_connections = fc; ++ ++ sc->processing--; ++ ++ if (sc->processing || sc->blocked) { ++ return; ++ } ++ ++ ev = sc->connection->read; ++ ++ ev->handler = ngx_http_spdy_handle_connection_handler; ++ ngx_post_event(ev, &ngx_posted_events); ++} ++ ++ ++static void ++ngx_http_spdy_handle_connection_handler(ngx_event_t *rev) ++{ ++ ngx_connection_t *c; ++ ++ rev->handler = ngx_http_spdy_read_handler; ++ ++ if (rev->ready) { ++ ngx_http_spdy_read_handler(rev); ++ return; ++ } ++ ++ c = rev->data; ++ ++ ngx_http_spdy_handle_connection(c->data); ++} ++ ++ ++static void ++ngx_http_spdy_keepalive_handler(ngx_event_t *rev) ++{ ++ ngx_connection_t *c; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ngx_http_spdy_connection_t *sc; ++ ++ c = rev->data; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "spdy keepalive handler"); ++ ++ if (rev->timedout || c->close) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++#if (NGX_HAVE_KQUEUE) ++ ++ if (ngx_event_flags & NGX_USE_KQUEUE_EVENT) { ++ if (rev->pending_eof) { ++ c->log->handler = NULL; ++ ngx_log_error(NGX_LOG_INFO, c->log, rev->kq_errno, ++ "kevent() reported that client %V closed " ++ "keepalive connection", &c->addr_text); ++#if (NGX_HTTP_SSL) ++ if (c->ssl) { ++ c->ssl->no_send_shutdown = 1; ++ } ++#endif ++ ngx_http_close_connection(c); ++ return; ++ } ++ } ++ ++#endif ++ ++ c->destroyed = 0; ++ c->idle = 0; ++ ngx_reusable_connection(c, 0); ++ ++ sc = c->data; ++ ++ sscf = ngx_http_get_module_srv_conf(sc->http_connection->conf_ctx, ++ ngx_http_spdy_module); ++ ++ sc->pool = ngx_create_pool(sscf->pool_size, sc->connection->log); ++ if (sc->pool == NULL) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ sc->streams_index = ngx_pcalloc(sc->pool, ++ ngx_http_spdy_streams_index_size(sscf) ++ * sizeof(ngx_http_spdy_stream_t *)); ++ if (sc->streams_index == NULL) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ c->write->handler = ngx_http_spdy_write_handler; ++ ++ rev->handler = ngx_http_spdy_read_handler; ++ ngx_http_spdy_read_handler(rev); ++} ++ ++ ++static void ++ngx_http_spdy_finalize_connection(ngx_http_spdy_connection_t *sc, ++ ngx_int_t rc) ++{ ++ ngx_uint_t i, size; ++ ngx_event_t *ev; ++ ngx_connection_t *c, *fc; ++ ngx_http_request_t *r; ++ ngx_http_spdy_stream_t *stream; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ++ c = sc->connection; ++ ++ if (!sc->processing) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ c->error = 1; ++ c->read->handler = ngx_http_empty_handler; ++ c->write->handler = ngx_http_empty_handler; ++ ++ sc->last_out = NULL; ++ ++ sc->blocked = 1; ++ ++ sscf = ngx_http_get_module_srv_conf(sc->http_connection->conf_ctx, ++ ngx_http_spdy_module); ++ ++ size = ngx_http_spdy_streams_index_size(sscf); ++ ++ for (i = 0; i < size; i++) { ++ stream = sc->streams_index[i]; ++ ++ while (stream) { ++ stream->handled = 0; ++ ++ r = stream->request; ++ fc = r->connection; ++ ++ fc->error = 1; ++ ++ if (stream->queued) { ++ stream->queued = 0; ++ ++ ev = fc->write; ++ ev->delayed = 0; ++ ++ } else { ++ ev = fc->read; ++ } ++ ++ stream = stream->index; ++ ++ ev->eof = 1; ++ ev->handler(ev); ++ } ++ } ++ ++ sc->blocked = 0; ++ ++ if (sc->processing) { ++ return; ++ } ++ ++ ngx_http_close_connection(c); ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_adjust_windows(ngx_http_spdy_connection_t *sc, ssize_t delta) ++{ ++ ngx_uint_t i, size; ++ ngx_event_t *wev; ++ ngx_http_spdy_stream_t *stream, *sn; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ++ sscf = ngx_http_get_module_srv_conf(sc->http_connection->conf_ctx, ++ ngx_http_spdy_module); ++ ++ size = ngx_http_spdy_streams_index_size(sscf); ++ ++ for (i = 0; i < size; i++) { ++ ++ for (stream = sc->streams_index[i]; stream; stream = sn) { ++ sn = stream->index; ++ ++ if (delta > 0 ++ && stream->send_window ++ > (ssize_t) (NGX_SPDY_MAX_WINDOW - delta)) ++ { ++ if (ngx_http_spdy_terminate_stream(sc, stream, ++ NGX_SPDY_FLOW_CONTROL_ERROR) ++ == NGX_ERROR) ++ { ++ return NGX_ERROR; ++ } ++ ++ continue; ++ } ++ ++ stream->send_window += delta; ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy:%ui adjust window:%z", ++ stream->id, stream->send_window); ++ ++ if (stream->send_window > 0 && stream->exhausted) { ++ stream->exhausted = 0; ++ ++ wev = stream->request->connection->write; ++ ++ if (!wev->timer_set) { ++ wev->delayed = 0; ++ wev->handler(wev); ++ } ++ } ++ } ++ } ++ ++ return NGX_OK; ++} ++ ++ ++static void ++ngx_http_spdy_pool_cleanup(void *data) ++{ ++ ngx_http_spdy_connection_t *sc = data; ++ ++ if (sc->pool) { ++ ngx_destroy_pool(sc->pool); ++ } ++} ++ ++ ++static void * ++ngx_http_spdy_zalloc(void *opaque, u_int items, u_int size) ++{ ++ ngx_http_spdy_connection_t *sc = opaque; ++ ++ return ngx_palloc(sc->connection->pool, items * size); ++} ++ ++ ++static void ++ngx_http_spdy_zfree(void *opaque, void *address) ++{ ++#if 0 ++ ngx_http_spdy_connection_t *sc = opaque; ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy zfree: %p", address); ++#endif ++} +diff --git a/src/http/ngx_http_spdy.h b/src/http/ngx_http_spdy.h +new file mode 100644 +index 0000000..df24495 +--- /dev/null ++++ b/src/http/ngx_http_spdy.h +@@ -0,0 +1,261 @@ ++/* ++ * Copyright (C) Nginx, Inc. ++ * Copyright (C) Valentin V. Bartenev ++ */ ++ ++ ++#ifndef _NGX_HTTP_SPDY_H_INCLUDED_ ++#define _NGX_HTTP_SPDY_H_INCLUDED_ ++ ++ ++#include ++#include ++#include ++ ++#include ++ ++ ++#define NGX_SPDY_VERSION 3 ++ ++#define NGX_SPDY_NPN_ADVERTISE "\x08spdy/3.1" ++#define NGX_SPDY_NPN_NEGOTIATED "spdy/3.1" ++ ++#define NGX_SPDY_STATE_BUFFER_SIZE 16 ++ ++#define NGX_SPDY_CTL_BIT 1 ++ ++#define NGX_SPDY_SYN_STREAM 1 ++#define NGX_SPDY_SYN_REPLY 2 ++#define NGX_SPDY_RST_STREAM 3 ++#define NGX_SPDY_SETTINGS 4 ++#define NGX_SPDY_PING 6 ++#define NGX_SPDY_GOAWAY 7 ++#define NGX_SPDY_HEADERS 8 ++#define NGX_SPDY_WINDOW_UPDATE 9 ++ ++#define NGX_SPDY_FRAME_HEADER_SIZE 8 ++ ++#define NGX_SPDY_SID_SIZE 4 ++#define NGX_SPDY_DELTA_SIZE 4 ++ ++#define NGX_SPDY_SYN_STREAM_SIZE 10 ++#define NGX_SPDY_SYN_REPLY_SIZE 4 ++#define NGX_SPDY_RST_STREAM_SIZE 8 ++#define NGX_SPDY_PING_SIZE 4 ++#define NGX_SPDY_GOAWAY_SIZE 8 ++#define NGX_SPDY_WINDOW_UPDATE_SIZE 8 ++#define NGX_SPDY_NV_NUM_SIZE 4 ++#define NGX_SPDY_NV_NLEN_SIZE 4 ++#define NGX_SPDY_NV_VLEN_SIZE 4 ++#define NGX_SPDY_SETTINGS_NUM_SIZE 4 ++#define NGX_SPDY_SETTINGS_FID_SIZE 4 ++#define NGX_SPDY_SETTINGS_VAL_SIZE 4 ++ ++#define NGX_SPDY_SETTINGS_PAIR_SIZE \ ++ (NGX_SPDY_SETTINGS_FID_SIZE + NGX_SPDY_SETTINGS_VAL_SIZE) ++ ++#define NGX_SPDY_HIGHEST_PRIORITY 0 ++#define NGX_SPDY_LOWEST_PRIORITY 7 ++ ++#define NGX_SPDY_FLAG_FIN 0x01 ++#define NGX_SPDY_FLAG_UNIDIRECTIONAL 0x02 ++#define NGX_SPDY_FLAG_CLEAR_SETTINGS 0x01 ++ ++#define NGX_SPDY_MAX_FRAME_SIZE ((1 << 24) - 1) ++ ++#define NGX_SPDY_DATA_DISCARD 1 ++#define NGX_SPDY_DATA_ERROR 2 ++#define NGX_SPDY_DATA_INTERNAL_ERROR 3 ++ ++ ++typedef struct ngx_http_spdy_connection_s ngx_http_spdy_connection_t; ++typedef struct ngx_http_spdy_out_frame_s ngx_http_spdy_out_frame_t; ++ ++ ++typedef u_char *(*ngx_http_spdy_handler_pt) (ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++ ++struct ngx_http_spdy_connection_s { ++ ngx_connection_t *connection; ++ ngx_http_connection_t *http_connection; ++ ++ ngx_uint_t processing; ++ ++ size_t send_window; ++ size_t recv_window; ++ size_t init_window; ++ ++ ngx_queue_t waiting; ++ ++ u_char buffer[NGX_SPDY_STATE_BUFFER_SIZE]; ++ size_t buffer_used; ++ ngx_http_spdy_handler_pt handler; ++ ++ z_stream zstream_in; ++ z_stream zstream_out; ++ ++ ngx_pool_t *pool; ++ ++ ngx_http_spdy_out_frame_t *free_ctl_frames; ++ ngx_connection_t *free_fake_connections; ++ ++ ngx_http_spdy_stream_t **streams_index; ++ ++ ngx_http_spdy_out_frame_t *last_out; ++ ++ ngx_queue_t posted; ++ ++ ngx_http_spdy_stream_t *stream; ++ ++ ngx_uint_t entries; ++ size_t length; ++ u_char flags; ++ ++ ngx_uint_t last_sid; ++ ++ unsigned blocked:1; ++ unsigned incomplete:1; ++}; ++ ++ ++struct ngx_http_spdy_stream_s { ++ ngx_uint_t id; ++ ngx_http_request_t *request; ++ ngx_http_spdy_connection_t *connection; ++ ngx_http_spdy_stream_t *index; ++ ++ ngx_uint_t header_buffers; ++ ngx_uint_t queued; ++ ++ /* ++ * A change to SETTINGS_INITIAL_WINDOW_SIZE could cause the ++ * send_window to become negative, hence it's signed. ++ */ ++ ssize_t send_window; ++ size_t recv_window; ++ ++ ngx_http_spdy_out_frame_t *free_frames; ++ ngx_chain_t *free_data_headers; ++ ngx_chain_t *free_bufs; ++ ++ ngx_queue_t queue; ++ ++ unsigned priority:3; ++ unsigned handled:1; ++ unsigned blocked:1; ++ unsigned exhausted:1; ++ unsigned in_closed:1; ++ unsigned out_closed:1; ++ unsigned skip_data:2; ++}; ++ ++ ++struct ngx_http_spdy_out_frame_s { ++ ngx_http_spdy_out_frame_t *next; ++ ngx_chain_t *first; ++ ngx_chain_t *last; ++ ngx_int_t (*handler)(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_out_frame_t *frame); ++ ++ ngx_http_spdy_stream_t *stream; ++ size_t length; ++ ++ ngx_uint_t priority; ++ unsigned blocked:1; ++ unsigned fin:1; ++}; ++ ++ ++static ngx_inline void ++ngx_http_spdy_queue_frame(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_out_frame_t *frame) ++{ ++ ngx_http_spdy_out_frame_t **out; ++ ++ for (out = &sc->last_out; *out; out = &(*out)->next) ++ { ++ /* ++ * NB: higher values represent lower priorities. ++ */ ++ if (frame->priority >= (*out)->priority) { ++ break; ++ } ++ } ++ ++ frame->next = *out; ++ *out = frame; ++} ++ ++ ++static ngx_inline void ++ngx_http_spdy_queue_blocked_frame(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_out_frame_t *frame) ++{ ++ ngx_http_spdy_out_frame_t **out; ++ ++ for (out = &sc->last_out; *out; out = &(*out)->next) ++ { ++ if ((*out)->blocked) { ++ break; ++ } ++ } ++ ++ frame->next = *out; ++ *out = frame; ++} ++ ++ ++void ngx_http_spdy_init(ngx_event_t *rev); ++void ngx_http_spdy_request_headers_init(void); ++ ++ngx_int_t ngx_http_spdy_read_request_body(ngx_http_request_t *r, ++ ngx_http_client_body_handler_pt post_handler); ++ ++void ngx_http_spdy_close_stream(ngx_http_spdy_stream_t *stream, ngx_int_t rc); ++ ++ngx_int_t ngx_http_spdy_send_output_queue(ngx_http_spdy_connection_t *sc); ++ ++ ++#define ngx_spdy_frame_aligned_write_uint16(p, s) \ ++ (*(uint16_t *) (p) = htons((uint16_t) (s)), (p) + sizeof(uint16_t)) ++ ++#define ngx_spdy_frame_aligned_write_uint32(p, s) \ ++ (*(uint32_t *) (p) = htonl((uint32_t) (s)), (p) + sizeof(uint32_t)) ++ ++#if (NGX_HAVE_NONALIGNED) ++ ++#define ngx_spdy_frame_write_uint16 ngx_spdy_frame_aligned_write_uint16 ++#define ngx_spdy_frame_write_uint32 ngx_spdy_frame_aligned_write_uint32 ++ ++#else ++ ++#define ngx_spdy_frame_write_uint16(p, s) \ ++ ((p)[0] = (u_char) ((s) >> 8), \ ++ (p)[1] = (u_char) (s), \ ++ (p) + sizeof(uint16_t)) ++ ++#define ngx_spdy_frame_write_uint32(p, s) \ ++ ((p)[0] = (u_char) ((s) >> 24), \ ++ (p)[1] = (u_char) ((s) >> 16), \ ++ (p)[2] = (u_char) ((s) >> 8), \ ++ (p)[3] = (u_char) (s), \ ++ (p) + sizeof(uint32_t)) ++ ++#endif ++ ++ ++#define ngx_spdy_ctl_frame_head(t) \ ++ ((uint32_t) NGX_SPDY_CTL_BIT << 31 | NGX_SPDY_VERSION << 16 | (t)) ++ ++#define ngx_spdy_frame_write_head(p, t) \ ++ ngx_spdy_frame_aligned_write_uint32(p, ngx_spdy_ctl_frame_head(t)) ++ ++#define ngx_spdy_frame_write_flags_and_len(p, f, l) \ ++ ngx_spdy_frame_aligned_write_uint32(p, (f) << 24 | (l)) ++#define ngx_spdy_frame_write_flags_and_id(p, f, i) \ ++ ngx_spdy_frame_aligned_write_uint32(p, (f) << 24 | (i)) ++ ++#define ngx_spdy_frame_write_sid ngx_spdy_frame_aligned_write_uint32 ++#define ngx_spdy_frame_write_window ngx_spdy_frame_aligned_write_uint32 ++ ++#endif /* _NGX_HTTP_SPDY_H_INCLUDED_ */ +diff --git a/src/http/ngx_http_spdy_filter_module.c b/src/http/ngx_http_spdy_filter_module.c +new file mode 100644 +index 0000000..377e935 +--- /dev/null ++++ b/src/http/ngx_http_spdy_filter_module.c +@@ -0,0 +1,1222 @@ ++ ++/* ++ * Copyright (C) Nginx, Inc. ++ * Copyright (C) Valentin V. Bartenev ++ */ ++ ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++ ++#define ngx_http_spdy_nv_nsize(h) (NGX_SPDY_NV_NLEN_SIZE + sizeof(h) - 1) ++#define ngx_http_spdy_nv_vsize(h) (NGX_SPDY_NV_VLEN_SIZE + sizeof(h) - 1) ++ ++#define ngx_http_spdy_nv_write_num ngx_spdy_frame_write_uint32 ++#define ngx_http_spdy_nv_write_nlen ngx_spdy_frame_write_uint32 ++#define ngx_http_spdy_nv_write_vlen ngx_spdy_frame_write_uint32 ++ ++#define ngx_http_spdy_nv_write_name(p, h) \ ++ ngx_cpymem(ngx_http_spdy_nv_write_nlen(p, sizeof(h) - 1), h, sizeof(h) - 1) ++ ++#define ngx_http_spdy_nv_write_val(p, h) \ ++ ngx_cpymem(ngx_http_spdy_nv_write_vlen(p, sizeof(h) - 1), h, sizeof(h) - 1) ++ ++ ++static ngx_chain_t *ngx_http_spdy_send_chain(ngx_connection_t *fc, ++ ngx_chain_t *in, off_t limit); ++ ++static ngx_inline ngx_int_t ngx_http_spdy_filter_send( ++ ngx_connection_t *fc, ngx_http_spdy_stream_t *stream); ++static ngx_inline ngx_int_t ngx_http_spdy_flow_control( ++ ngx_http_spdy_connection_t *sc, ngx_http_spdy_stream_t *stream); ++static void ngx_http_spdy_waiting_queue(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_stream_t *stream); ++ ++static ngx_chain_t *ngx_http_spdy_filter_get_shadow( ++ ngx_http_spdy_stream_t *stream, ngx_buf_t *buf, off_t offset, off_t size); ++static ngx_http_spdy_out_frame_t *ngx_http_spdy_filter_get_data_frame( ++ ngx_http_spdy_stream_t *stream, size_t len, ngx_chain_t *first, ++ ngx_chain_t *last); ++ ++static ngx_int_t ngx_http_spdy_syn_frame_handler( ++ ngx_http_spdy_connection_t *sc, ngx_http_spdy_out_frame_t *frame); ++static ngx_int_t ngx_http_spdy_data_frame_handler( ++ ngx_http_spdy_connection_t *sc, ngx_http_spdy_out_frame_t *frame); ++static ngx_inline void ngx_http_spdy_handle_frame( ++ ngx_http_spdy_stream_t *stream, ngx_http_spdy_out_frame_t *frame); ++static ngx_inline void ngx_http_spdy_handle_stream( ++ ngx_http_spdy_connection_t *sc, ngx_http_spdy_stream_t *stream); ++ ++static void ngx_http_spdy_filter_cleanup(void *data); ++ ++static ngx_int_t ngx_http_spdy_filter_init(ngx_conf_t *cf); ++ ++ ++static ngx_http_module_t ngx_http_spdy_filter_module_ctx = { ++ NULL, /* preconfiguration */ ++ ngx_http_spdy_filter_init, /* postconfiguration */ ++ ++ NULL, /* create main configuration */ ++ NULL, /* init main configuration */ ++ ++ NULL, /* create server configuration */ ++ NULL, /* merge server configuration */ ++ ++ NULL, /* create location configuration */ ++ NULL /* merge location configuration */ ++}; ++ ++ ++ngx_module_t ngx_http_spdy_filter_module = { ++ NGX_MODULE_V1, ++ &ngx_http_spdy_filter_module_ctx, /* module context */ ++ NULL, /* module directives */ ++ NGX_HTTP_MODULE, /* module type */ ++ NULL, /* init master */ ++ NULL, /* init module */ ++ NULL, /* init process */ ++ NULL, /* init thread */ ++ NULL, /* exit thread */ ++ NULL, /* exit process */ ++ NULL, /* exit master */ ++ NGX_MODULE_V1_PADDING ++}; ++ ++ ++static ngx_http_output_header_filter_pt ngx_http_next_header_filter; ++ ++ ++static ngx_int_t ++ngx_http_spdy_header_filter(ngx_http_request_t *r) ++{ ++ int rc; ++ size_t len; ++ u_char *p, *buf, *last; ++ ngx_buf_t *b; ++ ngx_str_t host; ++ ngx_uint_t i, j, count, port; ++ ngx_chain_t *cl; ++ ngx_list_part_t *part, *pt; ++ ngx_table_elt_t *header, *h; ++ ngx_connection_t *c; ++ ngx_http_cleanup_t *cln; ++ ngx_http_core_loc_conf_t *clcf; ++ ngx_http_core_srv_conf_t *cscf; ++ ngx_http_spdy_stream_t *stream; ++ ngx_http_spdy_out_frame_t *frame; ++ ngx_http_spdy_connection_t *sc; ++ struct sockaddr_in *sin; ++#if (NGX_HAVE_INET6) ++ struct sockaddr_in6 *sin6; ++#endif ++ u_char addr[NGX_SOCKADDR_STRLEN]; ++ ++ if (!r->spdy_stream) { ++ return ngx_http_next_header_filter(r); ++ } ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy header filter"); ++ ++ if (r->header_sent) { ++ return NGX_OK; ++ } ++ ++ r->header_sent = 1; ++ ++ if (r != r->main) { ++ return NGX_OK; ++ } ++ ++ c = r->connection; ++ ++ if (r->method == NGX_HTTP_HEAD) { ++ r->header_only = 1; ++ } ++ ++ switch (r->headers_out.status) { ++ ++ case NGX_HTTP_OK: ++ case NGX_HTTP_PARTIAL_CONTENT: ++ break; ++ ++ case NGX_HTTP_NOT_MODIFIED: ++ r->header_only = 1; ++ break; ++ ++ case NGX_HTTP_NO_CONTENT: ++ r->header_only = 1; ++ ++ ngx_str_null(&r->headers_out.content_type); ++ ++ r->headers_out.content_length = NULL; ++ r->headers_out.content_length_n = -1; ++ ++ /* fall through */ ++ ++ default: ++ r->headers_out.last_modified_time = -1; ++ r->headers_out.last_modified = NULL; ++ } ++ ++ len = NGX_SPDY_NV_NUM_SIZE ++ + ngx_http_spdy_nv_nsize(":version") ++ + ngx_http_spdy_nv_vsize("HTTP/1.1") ++ + ngx_http_spdy_nv_nsize(":status") ++ + (r->headers_out.status_line.len ++ ? NGX_SPDY_NV_VLEN_SIZE + r->headers_out.status_line.len ++ : ngx_http_spdy_nv_vsize("418")); ++ ++ clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); ++ ++ if (r->headers_out.server == NULL) { ++ len += ngx_http_spdy_nv_nsize("server"); ++ len += clcf->server_tokens ? ngx_http_spdy_nv_vsize(NGINX_VER) ++ : ngx_http_spdy_nv_vsize("nginx"); ++ } ++ ++ if (r->headers_out.date == NULL) { ++ len += ngx_http_spdy_nv_nsize("date") ++ + ngx_http_spdy_nv_vsize("Wed, 31 Dec 1986 10:00:00 GMT"); ++ } ++ ++ if (r->headers_out.content_type.len) { ++ len += ngx_http_spdy_nv_nsize("content-type") ++ + NGX_SPDY_NV_VLEN_SIZE + r->headers_out.content_type.len; ++ ++ if (r->headers_out.content_type_len == r->headers_out.content_type.len ++ && r->headers_out.charset.len) ++ { ++ len += sizeof("; charset=") - 1 + r->headers_out.charset.len; ++ } ++ } ++ ++ if (r->headers_out.content_length == NULL ++ && r->headers_out.content_length_n >= 0) ++ { ++ len += ngx_http_spdy_nv_nsize("content-length") ++ + NGX_SPDY_NV_VLEN_SIZE + NGX_OFF_T_LEN; ++ } ++ ++ if (r->headers_out.last_modified == NULL ++ && r->headers_out.last_modified_time != -1) ++ { ++ len += ngx_http_spdy_nv_nsize("last-modified") ++ + ngx_http_spdy_nv_vsize("Wed, 31 Dec 1986 10:00:00 GMT"); ++ } ++ ++ if (r->headers_out.location ++ && r->headers_out.location->value.len ++ && r->headers_out.location->value.data[0] == '/') ++ { ++ r->headers_out.location->hash = 0; ++ ++ if (clcf->server_name_in_redirect) { ++ cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module); ++ host = cscf->server_name; ++ ++ } else if (r->headers_in.server.len) { ++ host = r->headers_in.server; ++ ++ } else { ++ host.len = NGX_SOCKADDR_STRLEN; ++ host.data = addr; ++ ++ if (ngx_connection_local_sockaddr(c, &host, 0) != NGX_OK) { ++ return NGX_ERROR; ++ } ++ } ++ ++ switch (c->local_sockaddr->sa_family) { ++ ++#if (NGX_HAVE_INET6) ++ case AF_INET6: ++ sin6 = (struct sockaddr_in6 *) c->local_sockaddr; ++ port = ntohs(sin6->sin6_port); ++ break; ++#endif ++#if (NGX_HAVE_UNIX_DOMAIN) ++ case AF_UNIX: ++ port = 0; ++ break; ++#endif ++ default: /* AF_INET */ ++ sin = (struct sockaddr_in *) c->local_sockaddr; ++ port = ntohs(sin->sin_port); ++ break; ++ } ++ ++ len += ngx_http_spdy_nv_nsize("location") ++ + ngx_http_spdy_nv_vsize("https://") ++ + host.len ++ + r->headers_out.location->value.len; ++ ++ if (clcf->port_in_redirect) { ++ ++#if (NGX_HTTP_SSL) ++ if (c->ssl) ++ port = (port == 443) ? 0 : port; ++ else ++#endif ++ port = (port == 80) ? 0 : port; ++ ++ } else { ++ port = 0; ++ } ++ ++ if (port) { ++ len += sizeof(":65535") - 1; ++ } ++ ++ } else { ++ ngx_str_null(&host); ++ port = 0; ++ } ++ ++ part = &r->headers_out.headers.part; ++ header = part->elts; ++ ++ for (i = 0; /* void */; i++) { ++ ++ if (i >= part->nelts) { ++ if (part->next == NULL) { ++ break; ++ } ++ ++ part = part->next; ++ header = part->elts; ++ i = 0; ++ } ++ ++ if (header[i].hash == 0) { ++ continue; ++ } ++ ++ len += NGX_SPDY_NV_NLEN_SIZE + header[i].key.len ++ + NGX_SPDY_NV_VLEN_SIZE + header[i].value.len; ++ } ++ ++ buf = ngx_alloc(len, r->pool->log); ++ if (buf == NULL) { ++ return NGX_ERROR; ++ } ++ ++ last = buf + NGX_SPDY_NV_NUM_SIZE; ++ ++ last = ngx_http_spdy_nv_write_name(last, ":version"); ++ last = ngx_http_spdy_nv_write_val(last, "HTTP/1.1"); ++ ++ last = ngx_http_spdy_nv_write_name(last, ":status"); ++ ++ if (r->headers_out.status_line.len) { ++ last = ngx_http_spdy_nv_write_vlen(last, ++ r->headers_out.status_line.len); ++ last = ngx_cpymem(last, r->headers_out.status_line.data, ++ r->headers_out.status_line.len); ++ } else { ++ last = ngx_http_spdy_nv_write_vlen(last, 3); ++ last = ngx_sprintf(last, "%03ui", r->headers_out.status); ++ } ++ ++ count = 2; ++ ++ if (r->headers_out.server == NULL) { ++ last = ngx_http_spdy_nv_write_name(last, "server"); ++ last = clcf->server_tokens ++ ? ngx_http_spdy_nv_write_val(last, NGINX_VER) ++ : ngx_http_spdy_nv_write_val(last, "nginx"); ++ ++ count++; ++ } ++ ++ if (r->headers_out.date == NULL) { ++ last = ngx_http_spdy_nv_write_name(last, "date"); ++ ++ last = ngx_http_spdy_nv_write_vlen(last, ngx_cached_http_time.len); ++ ++ last = ngx_cpymem(last, ngx_cached_http_time.data, ++ ngx_cached_http_time.len); ++ ++ count++; ++ } ++ ++ if (r->headers_out.content_type.len) { ++ ++ last = ngx_http_spdy_nv_write_name(last, "content-type"); ++ ++ p = last + NGX_SPDY_NV_VLEN_SIZE; ++ ++ last = ngx_cpymem(p, r->headers_out.content_type.data, ++ r->headers_out.content_type.len); ++ ++ if (r->headers_out.content_type_len == r->headers_out.content_type.len ++ && r->headers_out.charset.len) ++ { ++ last = ngx_cpymem(last, "; charset=", sizeof("; charset=") - 1); ++ ++ last = ngx_cpymem(last, r->headers_out.charset.data, ++ r->headers_out.charset.len); ++ ++ /* update r->headers_out.content_type for possible logging */ ++ ++ r->headers_out.content_type.len = last - p; ++ r->headers_out.content_type.data = p; ++ } ++ ++ (void) ngx_http_spdy_nv_write_vlen(p - NGX_SPDY_NV_VLEN_SIZE, ++ r->headers_out.content_type.len); ++ ++ count++; ++ } ++ ++ if (r->headers_out.content_length == NULL ++ && r->headers_out.content_length_n >= 0) ++ { ++ last = ngx_http_spdy_nv_write_name(last, "content-length"); ++ ++ p = last + NGX_SPDY_NV_VLEN_SIZE; ++ ++ last = ngx_sprintf(p, "%O", r->headers_out.content_length_n); ++ ++ (void) ngx_http_spdy_nv_write_vlen(p - NGX_SPDY_NV_VLEN_SIZE, ++ last - p); ++ ++ count++; ++ } ++ ++ if (r->headers_out.last_modified == NULL ++ && r->headers_out.last_modified_time != -1) ++ { ++ last = ngx_http_spdy_nv_write_name(last, "last-modified"); ++ ++ p = last + NGX_SPDY_NV_VLEN_SIZE; ++ ++ last = ngx_http_time(p, r->headers_out.last_modified_time); ++ ++ (void) ngx_http_spdy_nv_write_vlen(p - NGX_SPDY_NV_VLEN_SIZE, ++ last - p); ++ ++ count++; ++ } ++ ++ if (host.data) { ++ ++ last = ngx_http_spdy_nv_write_name(last, "location"); ++ ++ p = last + NGX_SPDY_NV_VLEN_SIZE; ++ ++ last = ngx_cpymem(p, "http", sizeof("http") - 1); ++ ++#if (NGX_HTTP_SSL) ++ if (c->ssl) { ++ *last++ ='s'; ++ } ++#endif ++ ++ *last++ = ':'; *last++ = '/'; *last++ = '/'; ++ ++ last = ngx_cpymem(last, host.data, host.len); ++ ++ if (port) { ++ last = ngx_sprintf(last, ":%ui", port); ++ } ++ ++ last = ngx_cpymem(last, r->headers_out.location->value.data, ++ r->headers_out.location->value.len); ++ ++ /* update r->headers_out.location->value for possible logging */ ++ ++ r->headers_out.location->value.len = last - p; ++ r->headers_out.location->value.data = p; ++ ngx_str_set(&r->headers_out.location->key, "location"); ++ ++ (void) ngx_http_spdy_nv_write_vlen(p - NGX_SPDY_NV_VLEN_SIZE, ++ r->headers_out.location->value.len); ++ ++ count++; ++ } ++ ++ part = &r->headers_out.headers.part; ++ header = part->elts; ++ ++ for (i = 0; /* void */; i++) { ++ ++ if (i >= part->nelts) { ++ if (part->next == NULL) { ++ break; ++ } ++ ++ part = part->next; ++ header = part->elts; ++ i = 0; ++ } ++ ++ if (header[i].hash == 0 || header[i].hash == 2) { ++ continue; ++ } ++ ++ last = ngx_http_spdy_nv_write_nlen(last, header[i].key.len); ++ ++ ngx_strlow(last, header[i].key.data, header[i].key.len); ++ last += header[i].key.len; ++ ++ p = last + NGX_SPDY_NV_VLEN_SIZE; ++ ++ last = ngx_cpymem(p, header[i].value.data, header[i].value.len); ++ ++ pt = part; ++ h = header; ++ ++ for (j = i + 1; /* void */; j++) { ++ ++ if (j >= pt->nelts) { ++ if (pt->next == NULL) { ++ break; ++ } ++ ++ pt = pt->next; ++ h = pt->elts; ++ j = 0; ++ } ++ ++ if (h[j].hash == 0 || h[j].hash == 2 ++ || h[j].key.len != header[i].key.len ++ || ngx_strncasecmp(header[i].key.data, h[j].key.data, ++ header[i].key.len)) ++ { ++ continue; ++ } ++ ++ if (h[j].value.len) { ++ if (last != p) { ++ *last++ = '\0'; ++ } ++ ++ last = ngx_cpymem(last, h[j].value.data, h[j].value.len); ++ } ++ ++ h[j].hash = 2; ++ } ++ ++ (void) ngx_http_spdy_nv_write_vlen(p - NGX_SPDY_NV_VLEN_SIZE, ++ last - p); ++ ++ count++; ++ } ++ ++ (void) ngx_http_spdy_nv_write_num(buf, count); ++ ++ stream = r->spdy_stream; ++ sc = stream->connection; ++ ++ len = last - buf; ++ ++ b = ngx_create_temp_buf(r->pool, NGX_SPDY_FRAME_HEADER_SIZE ++ + NGX_SPDY_SYN_REPLY_SIZE ++ + deflateBound(&sc->zstream_out, len)); ++ if (b == NULL) { ++ ngx_free(buf); ++ return NGX_ERROR; ++ } ++ ++ b->last += NGX_SPDY_FRAME_HEADER_SIZE + NGX_SPDY_SYN_REPLY_SIZE; ++ ++ sc->zstream_out.next_in = buf; ++ sc->zstream_out.avail_in = len; ++ sc->zstream_out.next_out = b->last; ++ sc->zstream_out.avail_out = b->end - b->last; ++ ++ rc = deflate(&sc->zstream_out, Z_SYNC_FLUSH); ++ ++ ngx_free(buf); ++ ++ if (rc != Z_OK) { ++ ngx_log_error(NGX_LOG_ALERT, c->log, 0, "deflate() failed: %d", rc); ++ return NGX_ERROR; ++ } ++ ++ ngx_log_debug5(NGX_LOG_DEBUG_HTTP, c->log, 0, ++ "spdy deflate out: ni:%p no:%p ai:%ud ao:%ud rc:%d", ++ sc->zstream_out.next_in, sc->zstream_out.next_out, ++ sc->zstream_out.avail_in, sc->zstream_out.avail_out, ++ rc); ++ ++ b->last = sc->zstream_out.next_out; ++ ++ p = b->pos; ++ p = ngx_spdy_frame_write_head(p, NGX_SPDY_SYN_REPLY); ++ ++ len = b->last - b->pos; ++ ++ r->header_size = len; ++ ++ len -= NGX_SPDY_FRAME_HEADER_SIZE; ++ ++ if (r->header_only) { ++ b->last_buf = 1; ++ p = ngx_spdy_frame_write_flags_and_len(p, NGX_SPDY_FLAG_FIN, len); ++ ++ } else { ++ p = ngx_spdy_frame_write_flags_and_len(p, 0, len); ++ } ++ ++ (void) ngx_spdy_frame_write_sid(p, stream->id); ++ ++ cl = ngx_alloc_chain_link(r->pool); ++ if (cl == NULL) { ++ return NGX_ERROR; ++ } ++ ++ cl->buf = b; ++ cl->next = NULL; ++ ++ frame = ngx_palloc(r->pool, sizeof(ngx_http_spdy_out_frame_t)); ++ if (frame == NULL) { ++ return NGX_ERROR; ++ } ++ ++ frame->first = cl; ++ frame->last = cl; ++ frame->handler = ngx_http_spdy_syn_frame_handler; ++ frame->stream = stream; ++ frame->length = len; ++ frame->priority = stream->priority; ++ frame->blocked = 1; ++ frame->fin = r->header_only; ++ ++ ngx_log_debug3(NGX_LOG_DEBUG_HTTP, stream->request->connection->log, 0, ++ "spdy:%ui create SYN_REPLY frame %p: len:%uz", ++ stream->id, frame, frame->length); ++ ++ ngx_http_spdy_queue_blocked_frame(sc, frame); ++ ++ cln = ngx_http_cleanup_add(r, 0); ++ if (cln == NULL) { ++ return NGX_ERROR; ++ } ++ ++ cln->handler = ngx_http_spdy_filter_cleanup; ++ cln->data = stream; ++ ++ stream->queued = 1; ++ ++ c->send_chain = ngx_http_spdy_send_chain; ++ c->need_last_buf = 1; ++ ++ return ngx_http_spdy_filter_send(c, stream); ++} ++ ++ ++static ngx_chain_t * ++ngx_http_spdy_send_chain(ngx_connection_t *fc, ngx_chain_t *in, off_t limit) ++{ ++ off_t size, offset; ++ size_t rest, frame_size; ++ ngx_chain_t *cl, *out, **ln; ++ ngx_http_request_t *r; ++ ngx_http_spdy_stream_t *stream; ++ ngx_http_spdy_loc_conf_t *slcf; ++ ngx_http_spdy_out_frame_t *frame; ++ ngx_http_spdy_connection_t *sc; ++ ++ r = fc->data; ++ stream = r->spdy_stream; ++ ++#if (NGX_SUPPRESS_WARN) ++ size = 0; ++#endif ++ ++ while (in) { ++ size = ngx_buf_size(in->buf); ++ ++ if (size || in->buf->last_buf) { ++ break; ++ } ++ ++ in = in->next; ++ } ++ ++ if (in == NULL) { ++ ++ if (stream->queued) { ++ fc->write->delayed = 1; ++ } else { ++ fc->buffered &= ~NGX_SPDY_BUFFERED; ++ } ++ ++ return NULL; ++ } ++ ++ sc = stream->connection; ++ ++ if (size && ngx_http_spdy_flow_control(sc, stream) == NGX_DECLINED) { ++ fc->write->delayed = 1; ++ return in; ++ } ++ ++ if (limit == 0 || limit > (off_t) sc->send_window) { ++ limit = sc->send_window; ++ } ++ ++ if (limit > stream->send_window) { ++ limit = (stream->send_window > 0) ? stream->send_window : 0; ++ } ++ ++ if (in->buf->tag == (ngx_buf_tag_t) &ngx_http_spdy_filter_get_shadow) { ++ cl = ngx_alloc_chain_link(r->pool); ++ if (cl == NULL) { ++ return NGX_CHAIN_ERROR; ++ } ++ ++ cl->buf = in->buf; ++ in->buf = cl->buf->shadow; ++ ++ offset = ngx_buf_in_memory(in->buf) ++ ? (cl->buf->pos - in->buf->pos) ++ : (cl->buf->file_pos - in->buf->file_pos); ++ ++ cl->next = stream->free_bufs; ++ stream->free_bufs = cl; ++ ++ } else { ++ offset = 0; ++ } ++ ++#if (NGX_SUPPRESS_WARN) ++ cl = NULL; ++#endif ++ ++ slcf = ngx_http_get_module_loc_conf(r, ngx_http_spdy_module); ++ ++ frame_size = (limit <= (off_t) slcf->chunk_size) ? (size_t) limit ++ : slcf->chunk_size; ++ ++ for ( ;; ) { ++ ln = &out; ++ rest = frame_size; ++ ++ while ((off_t) rest >= size) { ++ ++ if (offset) { ++ cl = ngx_http_spdy_filter_get_shadow(stream, in->buf, ++ offset, size); ++ if (cl == NULL) { ++ return NGX_CHAIN_ERROR; ++ } ++ ++ offset = 0; ++ ++ } else { ++ cl = ngx_alloc_chain_link(r->pool); ++ if (cl == NULL) { ++ return NGX_CHAIN_ERROR; ++ } ++ ++ cl->buf = in->buf; ++ } ++ ++ *ln = cl; ++ ln = &cl->next; ++ ++ rest -= (size_t) size; ++ in = in->next; ++ ++ if (in == NULL) { ++ frame_size -= rest; ++ rest = 0; ++ break; ++ } ++ ++ size = ngx_buf_size(in->buf); ++ } ++ ++ if (rest) { ++ cl = ngx_http_spdy_filter_get_shadow(stream, in->buf, ++ offset, rest); ++ if (cl == NULL) { ++ return NGX_CHAIN_ERROR; ++ } ++ ++ cl->buf->flush = 0; ++ cl->buf->last_buf = 0; ++ ++ *ln = cl; ++ ++ offset += rest; ++ size -= rest; ++ } ++ ++ frame = ngx_http_spdy_filter_get_data_frame(stream, frame_size, ++ out, cl); ++ if (frame == NULL) { ++ return NGX_CHAIN_ERROR; ++ } ++ ++ ngx_http_spdy_queue_frame(sc, frame); ++ ++ sc->send_window -= frame_size; ++ ++ stream->send_window -= frame_size; ++ stream->queued++; ++ ++ if (in == NULL) { ++ break; ++ } ++ ++ limit -= frame_size; ++ ++ if (limit == 0) { ++ break; ++ } ++ ++ if (limit < (off_t) slcf->chunk_size) { ++ frame_size = (size_t) limit; ++ } ++ } ++ ++ if (offset) { ++ cl = ngx_http_spdy_filter_get_shadow(stream, in->buf, offset, size); ++ if (cl == NULL) { ++ return NGX_CHAIN_ERROR; ++ } ++ ++ in->buf = cl->buf; ++ ngx_free_chain(r->pool, cl); ++ } ++ ++ if (ngx_http_spdy_filter_send(fc, stream) == NGX_ERROR) { ++ return NGX_CHAIN_ERROR; ++ } ++ ++ if (in && ngx_http_spdy_flow_control(sc, stream) == NGX_DECLINED) { ++ fc->write->delayed = 1; ++ } ++ ++ return in; ++} ++ ++ ++static ngx_chain_t * ++ngx_http_spdy_filter_get_shadow(ngx_http_spdy_stream_t *stream, ngx_buf_t *buf, ++ off_t offset, off_t size) ++{ ++ ngx_buf_t *chunk; ++ ngx_chain_t *cl; ++ ++ cl = ngx_chain_get_free_buf(stream->request->pool, &stream->free_bufs); ++ if (cl == NULL) { ++ return NULL; ++ } ++ ++ chunk = cl->buf; ++ ++ ngx_memcpy(chunk, buf, sizeof(ngx_buf_t)); ++ ++ chunk->tag = (ngx_buf_tag_t) &ngx_http_spdy_filter_get_shadow; ++ chunk->shadow = buf; ++ ++ if (ngx_buf_in_memory(chunk)) { ++ chunk->pos += offset; ++ chunk->last = chunk->pos + size; ++ } ++ ++ if (chunk->in_file) { ++ chunk->file_pos += offset; ++ chunk->file_last = chunk->file_pos + size; ++ } ++ ++ return cl; ++} ++ ++ ++static ngx_http_spdy_out_frame_t * ++ngx_http_spdy_filter_get_data_frame(ngx_http_spdy_stream_t *stream, ++ size_t len, ngx_chain_t *first, ngx_chain_t *last) ++{ ++ u_char *p; ++ ngx_buf_t *buf; ++ ngx_uint_t flags; ++ ngx_chain_t *cl; ++ ngx_http_spdy_out_frame_t *frame; ++ ++ ++ frame = stream->free_frames; ++ ++ if (frame) { ++ stream->free_frames = frame->next; ++ ++ } else { ++ frame = ngx_palloc(stream->request->pool, ++ sizeof(ngx_http_spdy_out_frame_t)); ++ if (frame == NULL) { ++ return NULL; ++ } ++ } ++ ++ flags = last->buf->last_buf ? NGX_SPDY_FLAG_FIN : 0; ++ ++ ngx_log_debug4(NGX_LOG_DEBUG_HTTP, stream->request->connection->log, 0, ++ "spdy:%ui create DATA frame %p: len:%uz flags:%ui", ++ stream->id, frame, len, flags); ++ ++ cl = ngx_chain_get_free_buf(stream->request->pool, ++ &stream->free_data_headers); ++ if (cl == NULL) { ++ return NULL; ++ } ++ ++ buf = cl->buf; ++ ++ if (buf->start) { ++ p = buf->start; ++ buf->pos = p; ++ ++ p += NGX_SPDY_SID_SIZE; ++ ++ (void) ngx_spdy_frame_write_flags_and_len(p, flags, len); ++ ++ } else { ++ p = ngx_palloc(stream->request->pool, NGX_SPDY_FRAME_HEADER_SIZE); ++ if (p == NULL) { ++ return NULL; ++ } ++ ++ buf->pos = p; ++ buf->start = p; ++ ++ p = ngx_spdy_frame_write_sid(p, stream->id); ++ p = ngx_spdy_frame_write_flags_and_len(p, flags, len); ++ ++ buf->last = p; ++ buf->end = p; ++ ++ buf->tag = (ngx_buf_tag_t) &ngx_http_spdy_filter_get_data_frame; ++ buf->memory = 1; ++ } ++ ++ cl->next = first; ++ first = cl; ++ ++ last->buf->flush = 1; ++ ++ frame->first = first; ++ frame->last = last; ++ frame->handler = ngx_http_spdy_data_frame_handler; ++ frame->stream = stream; ++ frame->length = len; ++ frame->priority = stream->priority; ++ frame->blocked = 0; ++ frame->fin = last->buf->last_buf; ++ ++ return frame; ++} ++ ++ ++static ngx_inline ngx_int_t ++ngx_http_spdy_filter_send(ngx_connection_t *fc, ngx_http_spdy_stream_t *stream) ++{ ++ stream->blocked = 1; ++ ++ if (ngx_http_spdy_send_output_queue(stream->connection) == NGX_ERROR) { ++ fc->error = 1; ++ return NGX_ERROR; ++ } ++ ++ stream->blocked = 0; ++ ++ if (stream->queued) { ++ fc->buffered |= NGX_SPDY_BUFFERED; ++ fc->write->delayed = 1; ++ return NGX_AGAIN; ++ } ++ ++ fc->buffered &= ~NGX_SPDY_BUFFERED; ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_inline ngx_int_t ++ngx_http_spdy_flow_control(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_stream_t *stream) ++{ ++ if (stream->send_window <= 0) { ++ stream->exhausted = 1; ++ return NGX_DECLINED; ++ } ++ ++ if (sc->send_window == 0) { ++ ngx_http_spdy_waiting_queue(sc, stream); ++ return NGX_DECLINED; ++ } ++ ++ return NGX_OK; ++} ++ ++ ++static void ++ngx_http_spdy_waiting_queue(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_stream_t *stream) ++{ ++ ngx_queue_t *q; ++ ngx_http_spdy_stream_t *s; ++ ++ if (stream->handled) { ++ return; ++ } ++ ++ stream->handled = 1; ++ ++ for (q = ngx_queue_last(&sc->waiting); ++ q != ngx_queue_sentinel(&sc->waiting); ++ q = ngx_queue_prev(q)) ++ { ++ s = ngx_queue_data(q, ngx_http_spdy_stream_t, queue); ++ ++ /* ++ * NB: higher values represent lower priorities. ++ */ ++ if (stream->priority >= s->priority) { ++ break; ++ } ++ } ++ ++ ngx_queue_insert_after(q, &stream->queue); ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_syn_frame_handler(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_out_frame_t *frame) ++{ ++ ngx_buf_t *buf; ++ ngx_http_spdy_stream_t *stream; ++ ++ buf = frame->first->buf; ++ ++ if (buf->pos != buf->last) { ++ return NGX_AGAIN; ++ } ++ ++ stream = frame->stream; ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy:%ui SYN_REPLY frame %p was sent", stream->id, frame); ++ ++ ngx_free_chain(stream->request->pool, frame->first); ++ ++ ngx_http_spdy_handle_frame(stream, frame); ++ ++ ngx_http_spdy_handle_stream(sc, stream); ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_data_frame_handler(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_out_frame_t *frame) ++{ ++ ngx_buf_t *buf; ++ ngx_chain_t *cl, *ln; ++ ngx_http_spdy_stream_t *stream; ++ ++ stream = frame->stream; ++ ++ cl = frame->first; ++ ++ if (cl->buf->tag == (ngx_buf_tag_t) &ngx_http_spdy_filter_get_data_frame) { ++ ++ if (cl->buf->pos != cl->buf->last) { ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy:%ui DATA frame %p was sent partially", ++ stream->id, frame); ++ ++ return NGX_AGAIN; ++ } ++ ++ ln = cl->next; ++ ++ cl->next = stream->free_data_headers; ++ stream->free_data_headers = cl; ++ ++ if (cl == frame->last) { ++ goto done; ++ } ++ ++ cl = ln; ++ } ++ ++ for ( ;; ) { ++ if (cl->buf->tag == (ngx_buf_tag_t) &ngx_http_spdy_filter_get_shadow) { ++ buf = cl->buf->shadow; ++ ++ if (ngx_buf_in_memory(buf)) { ++ buf->pos = cl->buf->pos; ++ } ++ ++ if (buf->in_file) { ++ buf->file_pos = cl->buf->file_pos; ++ } ++ } ++ ++ if (ngx_buf_size(cl->buf) != 0) { ++ ++ if (cl != frame->first) { ++ frame->first = cl; ++ ngx_http_spdy_handle_stream(sc, stream); ++ } ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy:%ui DATA frame %p was sent partially", ++ stream->id, frame); ++ ++ return NGX_AGAIN; ++ } ++ ++ ln = cl->next; ++ ++ if (cl->buf->tag == (ngx_buf_tag_t) &ngx_http_spdy_filter_get_shadow) { ++ cl->next = stream->free_bufs; ++ stream->free_bufs = cl; ++ ++ } else { ++ ngx_free_chain(stream->request->pool, cl); ++ } ++ ++ if (cl == frame->last) { ++ goto done; ++ } ++ ++ cl = ln; ++ } ++ ++done: ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy:%ui DATA frame %p was sent", stream->id, frame); ++ ++ stream->request->header_size += NGX_SPDY_FRAME_HEADER_SIZE; ++ ++ ngx_http_spdy_handle_frame(stream, frame); ++ ++ ngx_http_spdy_handle_stream(sc, stream); ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_inline void ++ngx_http_spdy_handle_frame(ngx_http_spdy_stream_t *stream, ++ ngx_http_spdy_out_frame_t *frame) ++{ ++ ngx_http_request_t *r; ++ ++ r = stream->request; ++ ++ r->connection->sent += NGX_SPDY_FRAME_HEADER_SIZE + frame->length; ++ ++ if (frame->fin) { ++ stream->out_closed = 1; ++ } ++ ++ frame->next = stream->free_frames; ++ stream->free_frames = frame; ++ ++ stream->queued--; ++} ++ ++ ++static ngx_inline void ++ngx_http_spdy_handle_stream(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_stream_t *stream) ++{ ++ ngx_event_t *wev; ++ ++ if (stream->handled || stream->blocked || stream->exhausted) { ++ return; ++ } ++ ++ wev = stream->request->connection->write; ++ ++ /* ++ * This timer can only be set if the stream was delayed because of rate ++ * limit. In that case the event should be triggered by the timer. ++ */ ++ ++ if (!wev->timer_set) { ++ wev->delayed = 0; ++ ++ stream->handled = 1; ++ ngx_queue_insert_tail(&sc->posted, &stream->queue); ++ } ++} ++ ++ ++static void ++ngx_http_spdy_filter_cleanup(void *data) ++{ ++ ngx_http_spdy_stream_t *stream = data; ++ ++ size_t delta; ++ ngx_http_spdy_out_frame_t *frame, **fn; ++ ngx_http_spdy_connection_t *sc; ++ ++ if (stream->handled) { ++ stream->handled = 0; ++ ngx_queue_remove(&stream->queue); ++ } ++ ++ if (stream->queued == 0) { ++ return; ++ } ++ ++ delta = 0; ++ sc = stream->connection; ++ fn = &sc->last_out; ++ ++ for ( ;; ) { ++ frame = *fn; ++ ++ if (frame == NULL) { ++ break; ++ } ++ ++ if (frame->stream == stream && !frame->blocked) { ++ *fn = frame->next; ++ ++ delta += frame->length; ++ ++ if (--stream->queued == 0) { ++ break; ++ } ++ ++ continue; ++ } ++ ++ fn = &frame->next; ++ } ++ ++ if (sc->send_window == 0 && delta && !ngx_queue_empty(&sc->waiting)) { ++ ngx_queue_add(&sc->posted, &sc->waiting); ++ ngx_queue_init(&sc->waiting); ++ } ++ ++ sc->send_window += delta; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_filter_init(ngx_conf_t *cf) ++{ ++ ngx_http_next_header_filter = ngx_http_top_header_filter; ++ ngx_http_top_header_filter = ngx_http_spdy_header_filter; ++ ++ return NGX_OK; ++} +diff --git a/src/http/ngx_http_spdy_module.c b/src/http/ngx_http_spdy_module.c +new file mode 100644 +index 0000000..5178a36 +--- /dev/null ++++ b/src/http/ngx_http_spdy_module.c +@@ -0,0 +1,408 @@ ++ ++/* ++ * Copyright (C) Nginx, Inc. ++ * Copyright (C) Valentin V. Bartenev ++ */ ++ ++ ++#include ++#include ++#include ++#include ++ ++ ++static ngx_int_t ngx_http_spdy_add_variables(ngx_conf_t *cf); ++ ++static ngx_int_t ngx_http_spdy_variable(ngx_http_request_t *r, ++ ngx_http_variable_value_t *v, uintptr_t data); ++static ngx_int_t ngx_http_spdy_request_priority_variable(ngx_http_request_t *r, ++ ngx_http_variable_value_t *v, uintptr_t data); ++ ++static ngx_int_t ngx_http_spdy_module_init(ngx_cycle_t *cycle); ++ ++static void *ngx_http_spdy_create_main_conf(ngx_conf_t *cf); ++static char *ngx_http_spdy_init_main_conf(ngx_conf_t *cf, void *conf); ++static void *ngx_http_spdy_create_srv_conf(ngx_conf_t *cf); ++static char *ngx_http_spdy_merge_srv_conf(ngx_conf_t *cf, void *parent, ++ void *child); ++static void *ngx_http_spdy_create_loc_conf(ngx_conf_t *cf); ++static char *ngx_http_spdy_merge_loc_conf(ngx_conf_t *cf, void *parent, ++ void *child); ++ ++static char *ngx_http_spdy_recv_buffer_size(ngx_conf_t *cf, void *post, ++ void *data); ++static char *ngx_http_spdy_pool_size(ngx_conf_t *cf, void *post, void *data); ++static char *ngx_http_spdy_streams_index_mask(ngx_conf_t *cf, void *post, ++ void *data); ++static char *ngx_http_spdy_chunk_size(ngx_conf_t *cf, void *post, void *data); ++ ++ ++static ngx_conf_num_bounds_t ngx_http_spdy_headers_comp_bounds = { ++ ngx_conf_check_num_bounds, 0, 9 ++}; ++ ++static ngx_conf_post_t ngx_http_spdy_recv_buffer_size_post = ++ { ngx_http_spdy_recv_buffer_size }; ++static ngx_conf_post_t ngx_http_spdy_pool_size_post = ++ { ngx_http_spdy_pool_size }; ++static ngx_conf_post_t ngx_http_spdy_streams_index_mask_post = ++ { ngx_http_spdy_streams_index_mask }; ++static ngx_conf_post_t ngx_http_spdy_chunk_size_post = ++ { ngx_http_spdy_chunk_size }; ++ ++ ++static ngx_command_t ngx_http_spdy_commands[] = { ++ ++ { ngx_string("spdy_recv_buffer_size"), ++ NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1, ++ ngx_conf_set_size_slot, ++ NGX_HTTP_MAIN_CONF_OFFSET, ++ offsetof(ngx_http_spdy_main_conf_t, recv_buffer_size), ++ &ngx_http_spdy_recv_buffer_size_post }, ++ ++ { ngx_string("spdy_pool_size"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, ++ ngx_conf_set_size_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_spdy_srv_conf_t, pool_size), ++ &ngx_http_spdy_pool_size_post }, ++ ++ { ngx_string("spdy_max_concurrent_streams"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, ++ ngx_conf_set_num_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_spdy_srv_conf_t, concurrent_streams), ++ NULL }, ++ ++ { ngx_string("spdy_streams_index_size"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, ++ ngx_conf_set_num_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_spdy_srv_conf_t, streams_index_mask), ++ &ngx_http_spdy_streams_index_mask_post }, ++ ++ { ngx_string("spdy_recv_timeout"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, ++ ngx_conf_set_msec_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_spdy_srv_conf_t, recv_timeout), ++ NULL }, ++ ++ { ngx_string("spdy_keepalive_timeout"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, ++ ngx_conf_set_msec_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_spdy_srv_conf_t, keepalive_timeout), ++ NULL }, ++ ++ { ngx_string("spdy_headers_comp"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, ++ ngx_conf_set_num_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_spdy_srv_conf_t, headers_comp), ++ &ngx_http_spdy_headers_comp_bounds }, ++ ++ { ngx_string("spdy_chunk_size"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, ++ ngx_conf_set_size_slot, ++ NGX_HTTP_LOC_CONF_OFFSET, ++ offsetof(ngx_http_spdy_loc_conf_t, chunk_size), ++ &ngx_http_spdy_chunk_size_post }, ++ ++ ngx_null_command ++}; ++ ++ ++static ngx_http_module_t ngx_http_spdy_module_ctx = { ++ ngx_http_spdy_add_variables, /* preconfiguration */ ++ NULL, /* postconfiguration */ ++ ++ ngx_http_spdy_create_main_conf, /* create main configuration */ ++ ngx_http_spdy_init_main_conf, /* init main configuration */ ++ ++ ngx_http_spdy_create_srv_conf, /* create server configuration */ ++ ngx_http_spdy_merge_srv_conf, /* merge server configuration */ ++ ++ ngx_http_spdy_create_loc_conf, /* create location configuration */ ++ ngx_http_spdy_merge_loc_conf /* merge location configuration */ ++}; ++ ++ ++ngx_module_t ngx_http_spdy_module = { ++ NGX_MODULE_V1, ++ &ngx_http_spdy_module_ctx, /* module context */ ++ ngx_http_spdy_commands, /* module directives */ ++ NGX_HTTP_MODULE, /* module type */ ++ NULL, /* init master */ ++ ngx_http_spdy_module_init, /* init module */ ++ NULL, /* init process */ ++ NULL, /* init thread */ ++ NULL, /* exit thread */ ++ NULL, /* exit process */ ++ NULL, /* exit master */ ++ NGX_MODULE_V1_PADDING ++}; ++ ++ ++static ngx_http_variable_t ngx_http_spdy_vars[] = { ++ ++ { ngx_string("spdy"), NULL, ++ ngx_http_spdy_variable, 0, 0, 0 }, ++ ++ { ngx_string("spdy_request_priority"), NULL, ++ ngx_http_spdy_request_priority_variable, 0, 0, 0 }, ++ ++ { ngx_null_string, NULL, NULL, 0, 0, 0 } ++}; ++ ++ ++static ngx_int_t ++ngx_http_spdy_add_variables(ngx_conf_t *cf) ++{ ++ ngx_http_variable_t *var, *v; ++ ++ for (v = ngx_http_spdy_vars; v->name.len; v++) { ++ var = ngx_http_add_variable(cf, &v->name, v->flags); ++ if (var == NULL) { ++ return NGX_ERROR; ++ } ++ ++ var->get_handler = v->get_handler; ++ var->data = v->data; ++ } ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_variable(ngx_http_request_t *r, ++ ngx_http_variable_value_t *v, uintptr_t data) ++{ ++ if (r->spdy_stream) { ++ v->len = sizeof("3.1") - 1; ++ v->valid = 1; ++ v->no_cacheable = 0; ++ v->not_found = 0; ++ v->data = (u_char *) "3.1"; ++ ++ return NGX_OK; ++ } ++ ++ *v = ngx_http_variable_null_value; ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_request_priority_variable(ngx_http_request_t *r, ++ ngx_http_variable_value_t *v, uintptr_t data) ++{ ++ if (r->spdy_stream) { ++ v->len = 1; ++ v->valid = 1; ++ v->no_cacheable = 0; ++ v->not_found = 0; ++ ++ v->data = ngx_pnalloc(r->pool, 1); ++ if (v->data == NULL) { ++ return NGX_ERROR; ++ } ++ ++ v->data[0] = '0' + (u_char) r->spdy_stream->priority; ++ ++ return NGX_OK; ++ } ++ ++ *v = ngx_http_variable_null_value; ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_module_init(ngx_cycle_t *cycle) ++{ ++ ngx_http_spdy_request_headers_init(); ++ ++ return NGX_OK; ++} ++ ++ ++static void * ++ngx_http_spdy_create_main_conf(ngx_conf_t *cf) ++{ ++ ngx_http_spdy_main_conf_t *smcf; ++ ++ smcf = ngx_pcalloc(cf->pool, sizeof(ngx_http_spdy_main_conf_t)); ++ if (smcf == NULL) { ++ return NULL; ++ } ++ ++ smcf->recv_buffer_size = NGX_CONF_UNSET_SIZE; ++ ++ return smcf; ++} ++ ++ ++static char * ++ngx_http_spdy_init_main_conf(ngx_conf_t *cf, void *conf) ++{ ++ ngx_http_spdy_main_conf_t *smcf = conf; ++ ++ ngx_conf_init_size_value(smcf->recv_buffer_size, 256 * 1024); ++ ++ return NGX_CONF_OK; ++} ++ ++ ++static void * ++ngx_http_spdy_create_srv_conf(ngx_conf_t *cf) ++{ ++ ngx_http_spdy_srv_conf_t *sscf; ++ ++ sscf = ngx_pcalloc(cf->pool, sizeof(ngx_http_spdy_srv_conf_t)); ++ if (sscf == NULL) { ++ return NULL; ++ } ++ ++ sscf->pool_size = NGX_CONF_UNSET_SIZE; ++ ++ sscf->concurrent_streams = NGX_CONF_UNSET_UINT; ++ sscf->streams_index_mask = NGX_CONF_UNSET_UINT; ++ ++ sscf->recv_timeout = NGX_CONF_UNSET_MSEC; ++ sscf->keepalive_timeout = NGX_CONF_UNSET_MSEC; ++ ++ sscf->headers_comp = NGX_CONF_UNSET; ++ ++ return sscf; ++} ++ ++ ++static char * ++ngx_http_spdy_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) ++{ ++ ngx_http_spdy_srv_conf_t *prev = parent; ++ ngx_http_spdy_srv_conf_t *conf = child; ++ ++ ngx_conf_merge_size_value(conf->pool_size, prev->pool_size, 4096); ++ ++ ngx_conf_merge_uint_value(conf->concurrent_streams, ++ prev->concurrent_streams, 100); ++ ++ ngx_conf_merge_uint_value(conf->streams_index_mask, ++ prev->streams_index_mask, 32 - 1); ++ ++ ngx_conf_merge_msec_value(conf->recv_timeout, ++ prev->recv_timeout, 30000); ++ ngx_conf_merge_msec_value(conf->keepalive_timeout, ++ prev->keepalive_timeout, 180000); ++ ++ ngx_conf_merge_value(conf->headers_comp, prev->headers_comp, 0); ++ ++ return NGX_CONF_OK; ++} ++ ++ ++static void * ++ngx_http_spdy_create_loc_conf(ngx_conf_t *cf) ++{ ++ ngx_http_spdy_loc_conf_t *slcf; ++ ++ slcf = ngx_pcalloc(cf->pool, sizeof(ngx_http_spdy_loc_conf_t)); ++ if (slcf == NULL) { ++ return NULL; ++ } ++ ++ slcf->chunk_size = NGX_CONF_UNSET_SIZE; ++ ++ return slcf; ++} ++ ++ ++static char * ++ngx_http_spdy_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) ++{ ++ ngx_http_spdy_loc_conf_t *prev = parent; ++ ngx_http_spdy_loc_conf_t *conf = child; ++ ++ ngx_conf_merge_size_value(conf->chunk_size, prev->chunk_size, 8 * 1024); ++ ++ return NGX_CONF_OK; ++} ++ ++ ++static char * ++ngx_http_spdy_recv_buffer_size(ngx_conf_t *cf, void *post, void *data) ++{ ++ size_t *sp = data; ++ ++ if (*sp <= 2 * NGX_SPDY_STATE_BUFFER_SIZE) { ++ return "value is too small"; ++ } ++ ++ return NGX_CONF_OK; ++} ++ ++ ++static char * ++ngx_http_spdy_pool_size(ngx_conf_t *cf, void *post, void *data) ++{ ++ size_t *sp = data; ++ ++ if (*sp < NGX_MIN_POOL_SIZE) { ++ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, ++ "the pool size must be no less than %uz", ++ NGX_MIN_POOL_SIZE); ++ return NGX_CONF_ERROR; ++ } ++ ++ if (*sp % NGX_POOL_ALIGNMENT) { ++ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, ++ "the pool size must be a multiple of %uz", ++ NGX_POOL_ALIGNMENT); ++ return NGX_CONF_ERROR; ++ } ++ ++ return NGX_CONF_OK; ++} ++ ++ ++static char * ++ngx_http_spdy_streams_index_mask(ngx_conf_t *cf, void *post, void *data) ++{ ++ ngx_uint_t *np = data; ++ ++ ngx_uint_t mask; ++ ++ mask = *np - 1; ++ ++ if (*np == 0 || (*np & mask)) { ++ return "must be a power of two"; ++ } ++ ++ *np = mask; ++ ++ return NGX_CONF_OK; ++} ++ ++ ++static char * ++ngx_http_spdy_chunk_size(ngx_conf_t *cf, void *post, void *data) ++{ ++ size_t *sp = data; ++ ++ if (*sp == 0) { ++ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, ++ "the spdy chunk size cannot be zero"); ++ return NGX_CONF_ERROR; ++ } ++ ++ if (*sp > NGX_SPDY_MAX_FRAME_SIZE) { ++ *sp = NGX_SPDY_MAX_FRAME_SIZE; ++ } ++ ++ return NGX_CONF_OK; ++} +diff --git a/src/http/ngx_http_spdy_module.h b/src/http/ngx_http_spdy_module.h +new file mode 100644 +index 0000000..5242322 +--- /dev/null ++++ b/src/http/ngx_http_spdy_module.h +@@ -0,0 +1,41 @@ ++ ++/* ++ * Copyright (C) Nginx, Inc. ++ * Copyright (C) Valentin V. Bartenev ++ */ ++ ++ ++#ifndef _NGX_HTTP_SPDY_MODULE_H_INCLUDED_ ++#define _NGX_HTTP_SPDY_MODULE_H_INCLUDED_ ++ ++ ++#include ++#include ++#include ++ ++ ++typedef struct { ++ size_t recv_buffer_size; ++ u_char *recv_buffer; ++} ngx_http_spdy_main_conf_t; ++ ++ ++typedef struct { ++ size_t pool_size; ++ ngx_uint_t concurrent_streams; ++ ngx_uint_t streams_index_mask; ++ ngx_msec_t recv_timeout; ++ ngx_msec_t keepalive_timeout; ++ ngx_int_t headers_comp; ++} ngx_http_spdy_srv_conf_t; ++ ++ ++typedef struct { ++ size_t chunk_size; ++} ngx_http_spdy_loc_conf_t; ++ ++ ++extern ngx_module_t ngx_http_spdy_module; ++ ++ ++#endif /* _NGX_HTTP_SPDY_MODULE_H_INCLUDED_ */ +diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c +index 74fffd5..aa75284 100644 +--- a/src/http/ngx_http_upstream.c ++++ b/src/http/ngx_http_upstream.c +@@ -481,6 +481,12 @@ ngx_http_upstream_init(ngx_http_request_t *r) + return; + } + #endif ++#if (NGX_HTTP_SPDY) ++ if (r->spdy_stream) { ++ ngx_http_upstream_init_request(r); ++ return; ++ } ++#endif + + if (c->read->timer_set) { + ngx_del_timer(c->read); +@@ -1156,6 +1162,11 @@ ngx_http_upstream_check_broken_connection(ngx_http_request_t *r, + return; + } + #endif ++#if (NGX_HTTP_SPDY) ++ if (r->spdy_stream) { ++ return; ++ } ++#endif + + #if (NGX_HAVE_KQUEUE) + +-- +2.7.4 (Apple Git-66) + From 554f0ce5d4a0a5e08d2b61b7eaa01fa6f7c3a228 Mon Sep 17 00:00:00 2001 From: John Graham-Cumming Date: Fri, 10 Jun 2016 11:53:07 +0100 Subject: [PATCH 02/16] Open source TLS dynamic record sizing --- patches/nginx__dynamic_tls_records.patch | 271 +++++++++++++++++++++++ 1 file changed, 271 insertions(+) create mode 100644 patches/nginx__dynamic_tls_records.patch diff --git a/patches/nginx__dynamic_tls_records.patch b/patches/nginx__dynamic_tls_records.patch new file mode 100644 index 0000000..7bb24cd --- /dev/null +++ b/patches/nginx__dynamic_tls_records.patch @@ -0,0 +1,271 @@ +From a424fefb0a638eb6d32756b5a0c471efc63e5384 Mon Sep 17 00:00:00 2001 +From: Vlad Krasnov +Date: Sat, 9 Jan 2016 06:53:14 -0800 +Subject: [PATCH] - Add TLS Dynamic Record Resizing + +What we do now: +We use a static record size of 4K. This gives a good balance of latency and +throughput. + +Optimize latency: +By initialy sending small (1 TCP segment) sized records, we are able to avoid +HoL blocking of the first byte. This means TTFB is sometime lower by a whole +RTT. + +Optimizing throughput: +By sending increasingly larger records later in the connection, when HoL is not +a problem, we reduce the overhead of TLS record (29 bytes per record with +GCM/CHACHA-POLY). + +Logic: +Start each connection with small records (1369 byte default, change with +ssl_dyn_rec_size_lo). After a given number of records (40, change with +ssl_dyn_rec_threshold) start sending larger records (4229, ssl_dyn_rec_size_lo). +Eventually after the same number of records, start sending the largest records +(ssl_buffer_size). +In case the connection idles for a given amount of time (1s, +ssl_dyn_rec_timeout), the process repeats itself (i.e. begin sending small +records again). + +--- + src/event/ngx_event_openssl.c | 39 +++++++++++++++++ + src/event/ngx_event_openssl.h | 15 ++++++- + src/http/modules/ngx_http_ssl_module.c | 76 ++++++++++++++++++++++++++++++++++ + src/http/modules/ngx_http_ssl_module.h | 6 +++ + 4 files changed, 135 insertions(+), 1 deletion(-) + +diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c +index 57dfc6c..4a0d41a 100644 +--- a/src/event/ngx_event_openssl.c ++++ b/src/event/ngx_event_openssl.c +@@ -1037,6 +1037,7 @@ ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c, ngx_uint_t flags) + + sc->buffer = ((flags & NGX_SSL_BUFFER) != 0); + sc->buffer_size = ssl->buffer_size; ++ sc->dyn_rec = ssl->dyn_rec; + + sc->session_ctx = ssl->ctx; + +@@ -1575,6 +1576,41 @@ ngx_ssl_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) + + for ( ;; ) { + ++ /* Dynamic record resizing: ++ We want the initial records to fit into one TCP segment ++ so we don't get TCP HoL blocking due to TCP Slow Start. ++ A connection always starts with small records, but after ++ a given amount of records sent, we make the records larger ++ to reduce header overhead. ++ After a connection has idled for a given timeout, begin ++ the process from the start. The actual parameters are ++ configurable. If dyn_rec_timeout is 0, we assume dyn_rec is off. */ ++ ++ if (c->ssl->dyn_rec.timeout > 0 ) { ++ ++ if (ngx_current_msec - c->ssl->dyn_rec_last_write > ++ c->ssl->dyn_rec.timeout) ++ { ++ buf->end = buf->start + c->ssl->dyn_rec.size_lo; ++ c->ssl->dyn_rec_records_sent = 0; ++ ++ } else { ++ if (c->ssl->dyn_rec_records_sent > ++ c->ssl->dyn_rec.threshold * 2) ++ { ++ buf->end = buf->start + c->ssl->buffer_size; ++ ++ } else if (c->ssl->dyn_rec_records_sent > ++ c->ssl->dyn_rec.threshold) ++ { ++ buf->end = buf->start + c->ssl->dyn_rec.size_hi; ++ ++ } else { ++ buf->end = buf->start + c->ssl->dyn_rec.size_lo; ++ } ++ } ++ } ++ + while (in && buf->last < buf->end && send < limit) { + if (in->buf->last_buf || in->buf->flush) { + flush = 1; +@@ -1676,6 +1712,9 @@ ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size) + + if (n > 0) { + ++ c->ssl->dyn_rec_records_sent++; ++ c->ssl->dyn_rec_last_write = ngx_current_msec; ++ + if (c->ssl->saved_read_handler) { + + c->read->handler = c->ssl->saved_read_handler; +diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h +index c86be2a..4a45934 100644 +--- a/src/event/ngx_event_openssl.h ++++ b/src/event/ngx_event_openssl.h +@@ -38,9 +38,18 @@ + + + typedef struct { ++ ngx_msec_t timeout; ++ ngx_uint_t threshold; ++ size_t size_lo; ++ size_t size_hi; ++} ngx_ssl_dyn_rec_t; ++ ++ ++typedef struct { + SSL_CTX *ctx; + ngx_log_t *log; + size_t buffer_size; ++ ngx_ssl_dyn_rec_t dyn_rec; + } ngx_ssl_t; + + +@@ -63,6 +72,10 @@ typedef struct { + unsigned no_wait_shutdown:1; + unsigned no_send_shutdown:1; + unsigned handshake_buffer_set:1; ++ ++ ngx_ssl_dyn_rec_t dyn_rec; ++ ngx_msec_t dyn_rec_last_write; ++ ngx_uint_t dyn_rec_records_sent; + } ngx_ssl_connection_t; + + +@@ -72,7 +85,7 @@ typedef struct { + #define NGX_SSL_DFLT_BUILTIN_SCACHE -5 + + +-#define NGX_SSL_MAX_SESSION_SIZE 4096 ++#define NGX_SSL_MAX_SESSION_SIZE 16384 + + typedef struct ngx_ssl_sess_id_s ngx_ssl_sess_id_t; + +diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c +index 7b051ea..e2941af 100644 +--- a/src/http/modules/ngx_http_ssl_module.c ++++ b/src/http/modules/ngx_http_ssl_module.c +@@ -233,6 +233,41 @@ static ngx_command_t ngx_http_ssl_commands[] = { + offsetof(ngx_http_ssl_srv_conf_t, stapling_verify), + NULL }, + ++ { ngx_string("ssl_dyn_rec_enable"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, ++ ngx_conf_set_flag_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_enable), ++ NULL }, ++ ++ { ngx_string("ssl_dyn_rec_timeout"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, ++ ngx_conf_set_msec_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_timeout), ++ NULL }, ++ ++ { ngx_string("ssl_dyn_rec_size_lo"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, ++ ngx_conf_set_size_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_lo), ++ NULL }, ++ ++ { ngx_string("ssl_dyn_rec_size_hi"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, ++ ngx_conf_set_size_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_hi), ++ NULL }, ++ ++ { ngx_string("ssl_dyn_rec_threshold"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, ++ ngx_conf_set_num_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_threshold), ++ NULL }, ++ + ngx_null_command + }; + +@@ -532,6 +567,11 @@ ngx_http_ssl_create_srv_conf(ngx_conf_t *cf) + sscf->session_ticket_keys = NGX_CONF_UNSET_PTR; + sscf->stapling = NGX_CONF_UNSET; + sscf->stapling_verify = NGX_CONF_UNSET; ++ sscf->dyn_rec_enable = NGX_CONF_UNSET; ++ sscf->dyn_rec_timeout = NGX_CONF_UNSET_MSEC; ++ sscf->dyn_rec_size_lo = NGX_CONF_UNSET_SIZE; ++ sscf->dyn_rec_size_hi = NGX_CONF_UNSET_SIZE; ++ sscf->dyn_rec_threshold = NGX_CONF_UNSET_UINT; + + return sscf; + } +@@ -596,6 +636,20 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) + ngx_conf_merge_str_value(conf->stapling_responder, + prev->stapling_responder, ""); + ++ ngx_conf_merge_value(conf->dyn_rec_enable, prev->dyn_rec_enable, 0); ++ ngx_conf_merge_msec_value(conf->dyn_rec_timeout, prev->dyn_rec_timeout, ++ 1000); ++ /* Default sizes for the dynamic record sizes are defined to fit maximal ++ TLS + IPv6 overhead in a single TCP segment for lo and 3 segments for hi: ++ 1369 = 1500 - 40 (IP) - 20 (TCP) - 10 (Time) - 61 (Max TLS overhead) */ ++ ngx_conf_merge_size_value(conf->dyn_rec_size_lo, prev->dyn_rec_size_lo, ++ 1369); ++ /* 4229 = (1500 - 40 - 20 - 10) * 3 - 61 */ ++ ngx_conf_merge_size_value(conf->dyn_rec_size_hi, prev->dyn_rec_size_hi, ++ 4229); ++ ngx_conf_merge_uint_value(conf->dyn_rec_threshold, prev->dyn_rec_threshold, ++ 40); ++ + conf->ssl.log = cf->log; + + if (conf->enable) { +@@ -773,6 +827,28 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) + + } + ++ if (conf->dyn_rec_enable) { ++ conf->ssl.dyn_rec.timeout = conf->dyn_rec_timeout; ++ conf->ssl.dyn_rec.threshold = conf->dyn_rec_threshold; ++ ++ if (conf->buffer_size > conf->dyn_rec_size_lo) { ++ conf->ssl.dyn_rec.size_lo = conf->dyn_rec_size_lo; ++ ++ } else { ++ conf->ssl.dyn_rec.size_lo = conf->buffer_size; ++ } ++ ++ if (conf->buffer_size > conf->dyn_rec_size_hi) { ++ conf->ssl.dyn_rec.size_hi = conf->dyn_rec_size_hi; ++ ++ } else { ++ conf->ssl.dyn_rec.size_hi = conf->buffer_size; ++ } ++ ++ } else { ++ conf->ssl.dyn_rec.timeout = 0; ++ } ++ + return NGX_CONF_OK; + } + +diff --git a/src/http/modules/ngx_http_ssl_module.h b/src/http/modules/ngx_http_ssl_module.h +index 8e69e9e..05967d4 100644 +--- a/src/http/modules/ngx_http_ssl_module.h ++++ b/src/http/modules/ngx_http_ssl_module.h +@@ -56,6 +56,12 @@ typedef struct { + + u_char *file; + ngx_uint_t line; ++ ++ ngx_flag_t dyn_rec_enable; ++ ngx_msec_t dyn_rec_timeout; ++ size_t dyn_rec_size_lo; ++ size_t dyn_rec_size_hi; ++ ngx_uint_t dyn_rec_threshold; + } ngx_http_ssl_srv_conf_t; + + +-- +2.7.4 (Apple Git-66) + From 23b5651129e3f61685f928da6d3f88a8fdc06e7e Mon Sep 17 00:00:00 2001 From: 0x7E Date: Sat, 11 Jun 2016 11:07:02 +0800 Subject: [PATCH 03/16] change Logic: (4229,ssl_dyn_rec_size_lo) to (4229,ssl_dyn_rec_size_hi) --- patches/nginx__dynamic_tls_records.patch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/patches/nginx__dynamic_tls_records.patch b/patches/nginx__dynamic_tls_records.patch index 7bb24cd..843d7ff 100644 --- a/patches/nginx__dynamic_tls_records.patch +++ b/patches/nginx__dynamic_tls_records.patch @@ -20,7 +20,7 @@ GCM/CHACHA-POLY). Logic: Start each connection with small records (1369 byte default, change with ssl_dyn_rec_size_lo). After a given number of records (40, change with -ssl_dyn_rec_threshold) start sending larger records (4229, ssl_dyn_rec_size_lo). +ssl_dyn_rec_threshold) start sending larger records (4229, ssl_dyn_rec_size_hi). Eventually after the same number of records, start sending the largest records (ssl_buffer_size). In case the connection idles for a given amount of time (1s, From 3e45b99b47caec3bbbb6e7d7fc4dbd52bd6de5f4 Mon Sep 17 00:00:00 2001 From: Leon Klingele Date: Wed, 12 Oct 2016 18:41:22 +0200 Subject: [PATCH 04/16] Refresh 'nginx__dynamic_tls_records.patch' for nginx 1.11.5 under the name 'nginx__1.11.5_dynamic_tls_records.patch' --- .../nginx__1.11.5_dynamic_tls_records.patch | 249 ++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 patches/nginx__1.11.5_dynamic_tls_records.patch diff --git a/patches/nginx__1.11.5_dynamic_tls_records.patch b/patches/nginx__1.11.5_dynamic_tls_records.patch new file mode 100644 index 0000000..01968be --- /dev/null +++ b/patches/nginx__1.11.5_dynamic_tls_records.patch @@ -0,0 +1,249 @@ +What we do now: +We use a static record size of 4K. This gives a good balance of latency and +throughput. + +Optimize latency: +By initialy sending small (1 TCP segment) sized records, we are able to avoid +HoL blocking of the first byte. This means TTFB is sometime lower by a whole +RTT. + +Optimizing throughput: +By sending increasingly larger records later in the connection, when HoL is not +a problem, we reduce the overhead of TLS record (29 bytes per record with +GCM/CHACHA-POLY). + +Logic: +Start each connection with small records (1369 byte default, change with +ssl_dyn_rec_size_lo). After a given number of records (40, change with +ssl_dyn_rec_threshold) start sending larger records (4229, ssl_dyn_rec_size_hi). +Eventually after the same number of records, start sending the largest records +(ssl_buffer_size). +In case the connection idles for a given amount of time (1s, +ssl_dyn_rec_timeout), the process repeats itself (i.e. begin sending small +records again). + +--- a/src/event/ngx_event_openssl.c ++++ b/src/event/ngx_event_openssl.c +@@ -1131,6 +1131,7 @@ + + sc->buffer = ((flags & NGX_SSL_BUFFER) != 0); + sc->buffer_size = ssl->buffer_size; ++ sc->dyn_rec = ssl->dyn_rec; + + sc->session_ctx = ssl->ctx; + +@@ -1669,6 +1670,41 @@ + + for ( ;; ) { + ++ /* Dynamic record resizing: ++ We want the initial records to fit into one TCP segment ++ so we don't get TCP HoL blocking due to TCP Slow Start. ++ A connection always starts with small records, but after ++ a given amount of records sent, we make the records larger ++ to reduce header overhead. ++ After a connection has idled for a given timeout, begin ++ the process from the start. The actual parameters are ++ configurable. If dyn_rec_timeout is 0, we assume dyn_rec is off. */ ++ ++ if (c->ssl->dyn_rec.timeout > 0 ) { ++ ++ if (ngx_current_msec - c->ssl->dyn_rec_last_write > ++ c->ssl->dyn_rec.timeout) ++ { ++ buf->end = buf->start + c->ssl->dyn_rec.size_lo; ++ c->ssl->dyn_rec_records_sent = 0; ++ ++ } else { ++ if (c->ssl->dyn_rec_records_sent > ++ c->ssl->dyn_rec.threshold * 2) ++ { ++ buf->end = buf->start + c->ssl->buffer_size; ++ ++ } else if (c->ssl->dyn_rec_records_sent > ++ c->ssl->dyn_rec.threshold) ++ { ++ buf->end = buf->start + c->ssl->dyn_rec.size_hi; ++ ++ } else { ++ buf->end = buf->start + c->ssl->dyn_rec.size_lo; ++ } ++ } ++ } ++ + while (in && buf->last < buf->end && send < limit) { + if (in->buf->last_buf || in->buf->flush) { + flush = 1; +@@ -1770,6 +1806,9 @@ + + if (n > 0) { + ++ c->ssl->dyn_rec_records_sent++; ++ c->ssl->dyn_rec_last_write = ngx_current_msec; ++ + if (c->ssl->saved_read_handler) { + + c->read->handler = c->ssl->saved_read_handler; +--- a/src/event/ngx_event_openssl.h ++++ b/src/event/ngx_event_openssl.h +@@ -54,10 +54,19 @@ + #define ngx_ssl_conn_t SSL + + ++typedef struct { ++ ngx_msec_t timeout; ++ ngx_uint_t threshold; ++ size_t size_lo; ++ size_t size_hi; ++} ngx_ssl_dyn_rec_t; ++ ++ + struct ngx_ssl_s { + SSL_CTX *ctx; + ngx_log_t *log; + size_t buffer_size; ++ ngx_ssl_dyn_rec_t dyn_rec; + }; + + +@@ -80,6 +89,10 @@ + unsigned no_wait_shutdown:1; + unsigned no_send_shutdown:1; + unsigned handshake_buffer_set:1; ++ ++ ngx_ssl_dyn_rec_t dyn_rec; ++ ngx_msec_t dyn_rec_last_write; ++ ngx_uint_t dyn_rec_records_sent; + }; + + +@@ -89,7 +102,7 @@ + #define NGX_SSL_DFLT_BUILTIN_SCACHE -5 + + +-#define NGX_SSL_MAX_SESSION_SIZE 4096 ++#define NGX_SSL_MAX_SESSION_SIZE 16384 + + typedef struct ngx_ssl_sess_id_s ngx_ssl_sess_id_t; + +--- a/src/http/modules/ngx_http_ssl_module.c ++++ b/src/http/modules/ngx_http_ssl_module.c +@@ -233,6 +233,41 @@ + offsetof(ngx_http_ssl_srv_conf_t, stapling_verify), + NULL }, + ++ { ngx_string("ssl_dyn_rec_enable"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, ++ ngx_conf_set_flag_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_enable), ++ NULL }, ++ ++ { ngx_string("ssl_dyn_rec_timeout"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, ++ ngx_conf_set_msec_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_timeout), ++ NULL }, ++ ++ { ngx_string("ssl_dyn_rec_size_lo"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, ++ ngx_conf_set_size_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_lo), ++ NULL }, ++ ++ { ngx_string("ssl_dyn_rec_size_hi"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, ++ ngx_conf_set_size_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_size_hi), ++ NULL }, ++ ++ { ngx_string("ssl_dyn_rec_threshold"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, ++ ngx_conf_set_num_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_ssl_srv_conf_t, dyn_rec_threshold), ++ NULL }, ++ + ngx_null_command + }; + +@@ -533,6 +568,11 @@ + sscf->session_ticket_keys = NGX_CONF_UNSET_PTR; + sscf->stapling = NGX_CONF_UNSET; + sscf->stapling_verify = NGX_CONF_UNSET; ++ sscf->dyn_rec_enable = NGX_CONF_UNSET; ++ sscf->dyn_rec_timeout = NGX_CONF_UNSET_MSEC; ++ sscf->dyn_rec_size_lo = NGX_CONF_UNSET_SIZE; ++ sscf->dyn_rec_size_hi = NGX_CONF_UNSET_SIZE; ++ sscf->dyn_rec_threshold = NGX_CONF_UNSET_UINT; + + return sscf; + } +@@ -598,6 +638,20 @@ + ngx_conf_merge_str_value(conf->stapling_responder, + prev->stapling_responder, ""); + ++ ngx_conf_merge_value(conf->dyn_rec_enable, prev->dyn_rec_enable, 0); ++ ngx_conf_merge_msec_value(conf->dyn_rec_timeout, prev->dyn_rec_timeout, ++ 1000); ++ /* Default sizes for the dynamic record sizes are defined to fit maximal ++ TLS + IPv6 overhead in a single TCP segment for lo and 3 segments for hi: ++ 1369 = 1500 - 40 (IP) - 20 (TCP) - 10 (Time) - 61 (Max TLS overhead) */ ++ ngx_conf_merge_size_value(conf->dyn_rec_size_lo, prev->dyn_rec_size_lo, ++ 1369); ++ /* 4229 = (1500 - 40 - 20 - 10) * 3 - 61 */ ++ ngx_conf_merge_size_value(conf->dyn_rec_size_hi, prev->dyn_rec_size_hi, ++ 4229); ++ ngx_conf_merge_uint_value(conf->dyn_rec_threshold, prev->dyn_rec_threshold, ++ 40); ++ + conf->ssl.log = cf->log; + + if (conf->enable) { +@@ -778,6 +832,28 @@ + + } + ++ if (conf->dyn_rec_enable) { ++ conf->ssl.dyn_rec.timeout = conf->dyn_rec_timeout; ++ conf->ssl.dyn_rec.threshold = conf->dyn_rec_threshold; ++ ++ if (conf->buffer_size > conf->dyn_rec_size_lo) { ++ conf->ssl.dyn_rec.size_lo = conf->dyn_rec_size_lo; ++ ++ } else { ++ conf->ssl.dyn_rec.size_lo = conf->buffer_size; ++ } ++ ++ if (conf->buffer_size > conf->dyn_rec_size_hi) { ++ conf->ssl.dyn_rec.size_hi = conf->dyn_rec_size_hi; ++ ++ } else { ++ conf->ssl.dyn_rec.size_hi = conf->buffer_size; ++ } ++ ++ } else { ++ conf->ssl.dyn_rec.timeout = 0; ++ } ++ + return NGX_CONF_OK; + } + +--- a/src/http/modules/ngx_http_ssl_module.h ++++ b/src/http/modules/ngx_http_ssl_module.h +@@ -57,6 +57,12 @@ + + u_char *file; + ngx_uint_t line; ++ ++ ngx_flag_t dyn_rec_enable; ++ ngx_msec_t dyn_rec_timeout; ++ size_t dyn_rec_size_lo; ++ size_t dyn_rec_size_hi; ++ ngx_uint_t dyn_rec_threshold; + } ngx_http_ssl_srv_conf_t; + + From 2c72c7fb235f66d01b7f0a1c3613f01935a4dd20 Mon Sep 17 00:00:00 2001 From: vkrasnov Date: Thu, 27 Oct 2016 10:39:00 -0700 Subject: [PATCH 05/16] New patch for OpenSSL 1.0.2j Remove older patch versions, and add a new OpenSSL 1.0.2j patch. * Significantly improved performance for the RFC version, similar to the implementation in Go 1.8. * Builds on MacOS. * Improved cipher negotiation: a common ChaCha cipher will always be found, even when first priority is not ChaCha. --- patches/openssl__chacha20_poly1305_cf.patch | 4462 ---------------- ...ssl__chacha20_poly1305_cf_latest_git.patch | 4368 --------------- ...ha20_poly1305_draft_and_rfc_ossl102j.patch | 4718 +++++++++++++++++ 3 files changed, 4718 insertions(+), 8830 deletions(-) delete mode 100644 patches/openssl__chacha20_poly1305_cf.patch delete mode 100644 patches/openssl__chacha20_poly1305_cf_latest_git.patch create mode 100644 patches/openssl__chacha20_poly1305_draft_and_rfc_ossl102j.patch diff --git a/patches/openssl__chacha20_poly1305_cf.patch b/patches/openssl__chacha20_poly1305_cf.patch deleted file mode 100644 index 4949c40..0000000 --- a/patches/openssl__chacha20_poly1305_cf.patch +++ /dev/null @@ -1,4462 +0,0 @@ -From 68bc18b82f7437373f9c301dce8fa811490e9567 Mon Sep 17 00:00:00 2001 -From: Vlad Krasnov -Date: Thu, 17 Sep 2015 17:36:53 -0700 -Subject: [PATCH] CHACHA20-POLY1305 Draft 1 - -Rebase of the original patch on top of OpenSSL 1.0.2 stable as of September 19, -2015 ---- - Configure | 50 +- - Makefile.org | 4 +- - apps/speed.c | 34 +- - crypto/chacha20poly1305/Makefile | 92 +++ - crypto/chacha20poly1305/asm/chacha20_avx.pl | 388 +++++++++++ - crypto/chacha20poly1305/asm/chacha20_avx2.pl | 424 +++++++++++++ - crypto/chacha20poly1305/asm/poly1305_avx.pl | 717 +++++++++++++++++++++ - crypto/chacha20poly1305/asm/poly1305_avx2.pl | 918 +++++++++++++++++++++++++++ - crypto/chacha20poly1305/chacha20.c | 157 +++++ - crypto/chacha20poly1305/chacha20poly1305.h | 63 ++ - crypto/chacha20poly1305/chapolytest.c | 287 +++++++++ - crypto/chacha20poly1305/poly1305.c | 285 +++++++++ - crypto/cryptlib.c | 14 +- - crypto/crypto.h | 2 +- - crypto/evp/Makefile | 7 +- - crypto/evp/e_chacha20poly1305.c | 323 ++++++++++ - crypto/evp/evp.h | 3 + - crypto/objects/obj_dat.h | 10 +- - crypto/objects/obj_mac.h | 4 + - ssl/s3_lib.c | 62 +- - ssl/ssl.h | 1 + - ssl/ssl_algs.c | 4 + - ssl/ssl_ciph.c | 15 +- - ssl/ssl_locl.h | 1 + - ssl/tls1.h | 10 + - test/Makefile | 17 +- - 26 files changed, 3845 insertions(+), 47 deletions(-) - create mode 100644 crypto/chacha20poly1305/Makefile - create mode 100644 crypto/chacha20poly1305/asm/chacha20_avx.pl - create mode 100644 crypto/chacha20poly1305/asm/chacha20_avx2.pl - create mode 100644 crypto/chacha20poly1305/asm/poly1305_avx.pl - create mode 100644 crypto/chacha20poly1305/asm/poly1305_avx2.pl - create mode 100644 crypto/chacha20poly1305/chacha20.c - create mode 100644 crypto/chacha20poly1305/chacha20poly1305.h - create mode 100644 crypto/chacha20poly1305/chapolytest.c - create mode 100644 crypto/chacha20poly1305/poly1305.c - create mode 100644 crypto/evp/e_chacha20poly1305.c - -diff --git a/Configure b/Configure -index d99eed7..1a7f5f3 100755 ---- a/Configure -+++ b/Configure -@@ -143,25 +143,25 @@ my $tlib="-lnsl -lsocket"; - my $bits1="THIRTY_TWO_BIT "; - my $bits2="SIXTY_FOUR_BIT "; - --my $x86_asm="x86cpuid.o:bn-586.o co-586.o x86-mont.o x86-gf2m.o::des-586.o crypt586.o:aes-586.o vpaes-x86.o aesni-x86.o:bf-586.o:md5-586.o:sha1-586.o sha256-586.o sha512-586.o:cast-586.o:rc4-586.o:rmd-586.o:rc5-586.o:wp_block.o wp-mmx.o:cmll-x86.o:ghash-x86.o:"; -+my $x86_asm="x86cpuid.o:bn-586.o co-586.o x86-mont.o x86-gf2m.o::des-586.o crypt586.o:aes-586.o vpaes-x86.o aesni-x86.o:bf-586.o:md5-586.o:sha1-586.o sha256-586.o sha512-586.o:cast-586.o:rc4-586.o:rmd-586.o:rc5-586.o:wp_block.o wp-mmx.o:cmll-x86.o:ghash-x86.o::"; - - my $x86_elf_asm="$x86_asm:elf"; - --my $x86_64_asm="x86_64cpuid.o:x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o rsaz_exp.o rsaz-x86_64.o rsaz-avx2.o:ecp_nistz256.o ecp_nistz256-x86_64.o::aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o aesni-sha256-x86_64.o aesni-mb-x86_64.o::md5-x86_64.o:sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o sha1-mb-x86_64.o sha256-mb-x86_64.o::rc4-x86_64.o rc4-md5-x86_64.o:::wp-x86_64.o:cmll-x86_64.o cmll_misc.o:ghash-x86_64.o aesni-gcm-x86_64.o:"; --my $ia64_asm="ia64cpuid.o:bn-ia64.o ia64-mont.o:::aes_core.o aes_cbc.o aes-ia64.o::md5-ia64.o:sha1-ia64.o sha256-ia64.o sha512-ia64.o::rc4-ia64.o rc4_skey.o:::::ghash-ia64.o::void"; --my $sparcv9_asm="sparcv9cap.o sparccpuid.o:bn-sparcv9.o sparcv9-mont.o sparcv9a-mont.o vis3-mont.o sparct4-mont.o sparcv9-gf2m.o::des_enc-sparc.o fcrypt_b.o dest4-sparcv9.o:aes_core.o aes_cbc.o aes-sparcv9.o aest4-sparcv9.o::md5-sparcv9.o:sha1-sparcv9.o sha256-sparcv9.o sha512-sparcv9.o::::::camellia.o cmll_misc.o cmll_cbc.o cmllt4-sparcv9.o:ghash-sparcv9.o::void"; --my $sparcv8_asm=":sparcv8.o::des_enc-sparc.o fcrypt_b.o:::::::::::::void"; --my $alpha_asm="alphacpuid.o:bn_asm.o alpha-mont.o::::::sha1-alpha.o:::::::ghash-alpha.o::void"; --my $mips64_asm=":bn-mips.o mips-mont.o:::aes_cbc.o aes-mips.o:::sha1-mips.o sha256-mips.o sha512-mips.o::::::::"; -+my $x86_64_asm="x86_64cpuid.o:x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o rsaz_exp.o rsaz-x86_64.o rsaz-avx2.o:ecp_nistz256.o ecp_nistz256-x86_64.o::aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o aesni-sha256-x86_64.o aesni-mb-x86_64.o::md5-x86_64.o:sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o sha1-mb-x86_64.o sha256-mb-x86_64.o::rc4-x86_64.o rc4-md5-x86_64.o:::wp-x86_64.o:cmll-x86_64.o cmll_misc.o:ghash-x86_64.o aesni-gcm-x86_64.o::chacha20_avx.o poly1305_avx.o chacha20_avx2.o poly1305_avx2.o"; -+my $ia64_asm="ia64cpuid.o:bn-ia64.o ia64-mont.o:::aes_core.o aes_cbc.o aes-ia64.o::md5-ia64.o:sha1-ia64.o sha256-ia64.o sha512-ia64.o::rc4-ia64.o rc4_skey.o:::::ghash-ia64.o:::void"; -+my $sparcv9_asm="sparcv9cap.o sparccpuid.o:bn-sparcv9.o sparcv9-mont.o sparcv9a-mont.o vis3-mont.o sparct4-mont.o sparcv9-gf2m.o::des_enc-sparc.o fcrypt_b.o dest4-sparcv9.o:aes_core.o aes_cbc.o aes-sparcv9.o aest4-sparcv9.o::md5-sparcv9.o:sha1-sparcv9.o sha256-sparcv9.o sha512-sparcv9.o::::::camellia.o cmll_misc.o cmll_cbc.o cmllt4-sparcv9.o:ghash-sparcv9.o:::void"; -+my $sparcv8_asm=":sparcv8.o::des_enc-sparc.o fcrypt_b.o::::::::::::::void"; -+my $alpha_asm="alphacpuid.o:bn_asm.o alpha-mont.o::::::sha1-alpha.o:::::::ghash-alpha.o:::void"; -+my $mips64_asm=":bn-mips.o mips-mont.o:::aes_cbc.o aes-mips.o:::sha1-mips.o sha256-mips.o sha512-mips.o:::::::::"; - my $mips32_asm=$mips64_asm; $mips32_asm =~ s/\s*sha512\-mips\.o//; --my $s390x_asm="s390xcap.o s390xcpuid.o:bn-s390x.o s390x-mont.o s390x-gf2m.o:::aes-s390x.o aes-ctr.o aes-xts.o:::sha1-s390x.o sha256-s390x.o sha512-s390x.o::rc4-s390x.o:::::ghash-s390x.o:"; --my $armv4_asm="armcap.o armv4cpuid.o:bn_asm.o armv4-mont.o armv4-gf2m.o:::aes_cbc.o aes-armv4.o bsaes-armv7.o aesv8-armx.o:::sha1-armv4-large.o sha256-armv4.o sha512-armv4.o:::::::ghash-armv4.o ghashv8-armx.o::void"; --my $aarch64_asm="armcap.o arm64cpuid.o mem_clr.o::::aes_core.o aes_cbc.o aesv8-armx.o:::sha1-armv8.o sha256-armv8.o sha512-armv8.o:::::::ghashv8-armx.o:"; --my $parisc11_asm="pariscid.o:bn_asm.o parisc-mont.o:::aes_core.o aes_cbc.o aes-parisc.o:::sha1-parisc.o sha256-parisc.o sha512-parisc.o::rc4-parisc.o:::::ghash-parisc.o::32"; --my $parisc20_asm="pariscid.o:pa-risc2W.o parisc-mont.o:::aes_core.o aes_cbc.o aes-parisc.o:::sha1-parisc.o sha256-parisc.o sha512-parisc.o::rc4-parisc.o:::::ghash-parisc.o::64"; --my $ppc64_asm="ppccpuid.o ppccap.o:bn-ppc.o ppc-mont.o ppc64-mont.o:::aes_core.o aes_cbc.o aes-ppc.o vpaes-ppc.o aesp8-ppc.o:::sha1-ppc.o sha256-ppc.o sha512-ppc.o sha256p8-ppc.o sha512p8-ppc.o:::::::ghashp8-ppc.o:"; -+my $s390x_asm="s390xcap.o s390xcpuid.o:bn-s390x.o s390x-mont.o s390x-gf2m.o:::aes-s390x.o aes-ctr.o aes-xts.o:::sha1-s390x.o sha256-s390x.o sha512-s390x.o::rc4-s390x.o:::::ghash-s390x.o::"; -+my $armv4_asm="armcap.o armv4cpuid.o:bn_asm.o armv4-mont.o armv4-gf2m.o:::aes_cbc.o aes-armv4.o bsaes-armv7.o aesv8-armx.o:::sha1-armv4-large.o sha256-armv4.o sha512-armv4.o:::::::ghash-armv4.o ghashv8-armx.o:::void"; -+my $aarch64_asm="armcap.o arm64cpuid.o mem_clr.o::::aes_core.o aes_cbc.o aesv8-armx.o:::sha1-armv8.o sha256-armv8.o sha512-armv8.o:::::::ghashv8-armx.o::"; -+my $parisc11_asm="pariscid.o:bn_asm.o parisc-mont.o:::aes_core.o aes_cbc.o aes-parisc.o:::sha1-parisc.o sha256-parisc.o sha512-parisc.o::rc4-parisc.o:::::ghash-parisc.o:::32"; -+my $parisc20_asm="pariscid.o:pa-risc2W.o parisc-mont.o:::aes_core.o aes_cbc.o aes-parisc.o:::sha1-parisc.o sha256-parisc.o sha512-parisc.o::rc4-parisc.o:::::ghash-parisc.o:::64"; -+my $ppc64_asm="ppccpuid.o ppccap.o:bn-ppc.o ppc-mont.o ppc64-mont.o:::aes_core.o aes_cbc.o aes-ppc.o vpaes-ppc.o aesp8-ppc.o:::sha1-ppc.o sha256-ppc.o sha512-ppc.o sha256p8-ppc.o sha512p8-ppc.o:::::::ghashp8-ppc.o::"; - my $ppc32_asm=$ppc64_asm; --my $no_asm="::::::::::::::::void"; -+my $no_asm=":::::::::::::::::void"; - - # As for $BSDthreads. Idea is to maintain "collective" set of flags, - # which would cover all BSD flavors. -pthread applies to them all, -@@ -706,6 +706,7 @@ my $idx_wp_obj = $idx++; - my $idx_cmll_obj = $idx++; - my $idx_modes_obj = $idx++; - my $idx_engines_obj = $idx++; -+my $idx_chapoly_obj = $idx++; - my $idx_perlasm_scheme = $idx++; - my $idx_dso_scheme = $idx++; - my $idx_shared_target = $idx++; -@@ -748,6 +749,7 @@ my $bf ="crypto/bf/bf_locl.h"; - my $bn_asm ="bn_asm.o"; - my $des_enc="des_enc.o fcrypt_b.o"; - my $aes_enc="aes_core.o aes_cbc.o"; -+my $chapoly_enc=""; - my $bf_enc ="bf_enc.o"; - my $cast_enc="c_enc.o"; - my $rc4_enc="rc4_enc.o rc4_skey.o"; -@@ -1206,7 +1208,7 @@ $openssldir=$prefix . "/" . $openssldir if $openssldir !~ /(^\/|^[a-zA-Z]:[\\\/] - - print "IsMK1MF=$IsMK1MF\n"; - --my @fields = split(/\s*:\s*/,$table{$target} . ":" x 30 , -1); -+my @fields = split(/\s*:\s*/,$table{$target} . ":" x 31 , -1); - my $cc = $fields[$idx_cc]; - # Allow environment CC to override compiler... - if($ENV{CC}) { -@@ -1235,6 +1237,7 @@ my $wp_obj = $fields[$idx_wp_obj]; - my $cmll_obj = $fields[$idx_cmll_obj]; - my $modes_obj = $fields[$idx_modes_obj]; - my $engines_obj = $fields[$idx_engines_obj]; -+my $chapoly_obj = $fields[$idx_chapoly_obj]; - my $perlasm_scheme = $fields[$idx_perlasm_scheme]; - my $dso_scheme = $fields[$idx_dso_scheme]; - my $shared_target = $fields[$idx_shared_target]; -@@ -1401,7 +1404,7 @@ if ($no_asm) - { - $cpuid_obj=$bn_obj=$ec_obj= - $des_obj=$aes_obj=$bf_obj=$cast_obj=$rc4_obj=$rc5_obj=$cmll_obj= -- $modes_obj=$sha1_obj=$md5_obj=$rmd160_obj=$wp_obj=$engines_obj=""; -+ $modes_obj=$sha1_obj=$md5_obj=$rmd160_obj=$wp_obj=$engines_obj=$chapoly_obj=""; - } - - if (!$no_shared) -@@ -1554,6 +1557,14 @@ $bf_obj=$bf_enc unless ($bf_obj =~ /\.o$/); - $cast_obj=$cast_enc unless ($cast_obj =~ /\.o$/); - $rc4_obj=$rc4_enc unless ($rc4_obj =~ /\.o$/); - $rc5_obj=$rc5_enc unless ($rc5_obj =~ /\.o$/); -+if ($chapoly_obj =~ /\.o$/) -+ { -+ $cflags.=" -DCHAPOLY_x86_64_ASM"; -+ } -+else -+ { -+ $chapoly_obj=$chapoly_enc; -+ } - if ($sha1_obj =~ /\.o$/) - { - # $sha1_obj=$sha1_enc; -@@ -1733,7 +1744,8 @@ while () - s/^RMD160_ASM_OBJ=.*$/RMD160_ASM_OBJ= $rmd160_obj/; - s/^WP_ASM_OBJ=.*$/WP_ASM_OBJ= $wp_obj/; - s/^CMLL_ENC=.*$/CMLL_ENC= $cmll_obj/; -- s/^MODES_ASM_OBJ.=*$/MODES_ASM_OBJ= $modes_obj/; -+ s/^MODES_ASM_OBJ.=*$/MODES_ASM_OBJ= $modes_obj/; -+ s/^CHAPOLY_ENC=.*$/CHAPOLY_ENC= $chapoly_obj/; - s/^ENGINES_ASM_OBJ.=*$/ENGINES_ASM_OBJ= $engines_obj/; - s/^PERLASM_SCHEME=.*$/PERLASM_SCHEME= $perlasm_scheme/; - s/^PROCESSOR=.*/PROCESSOR= $processor/; -@@ -1796,6 +1808,7 @@ print "RMD160_OBJ_ASM=$rmd160_obj\n"; - print "CMLL_ENC =$cmll_obj\n"; - print "MODES_OBJ =$modes_obj\n"; - print "ENGINES_OBJ =$engines_obj\n"; -+print "CHAPOLY_ENC =$chapoly_obj\n"; - print "PROCESSOR =$processor\n"; - print "RANLIB =$ranlib\n"; - print "ARFLAGS =$arflags\n"; -@@ -2194,7 +2207,7 @@ sub print_table_entry - my ($cc, $cflags, $unistd, $thread_cflag, $sys_id, $lflags, - $bn_ops, $cpuid_obj, $bn_obj, $ec_obj, $des_obj, $aes_obj, $bf_obj, - $md5_obj, $sha1_obj, $cast_obj, $rc4_obj, $rmd160_obj, -- $rc5_obj, $wp_obj, $cmll_obj, $modes_obj, $engines_obj, -+ $rc5_obj, $wp_obj, $cmll_obj, $modes_obj, $engines_obj, $chapoly_obj, - $perlasm_scheme, $dso_scheme, $shared_target, $shared_cflag, - $shared_ldflag, $shared_extension, $ranlib, $arflags, $multilib)= - split(/\s*:\s*/,$table{$target} . ":" x 30 , -1); -@@ -2225,6 +2238,7 @@ sub print_table_entry - \$cmll_obj = $cmll_obj - \$modes_obj = $modes_obj - \$engines_obj = $engines_obj -+\$chapoly_obj = $chapoly_obj - \$perlasm_scheme = $perlasm_scheme - \$dso_scheme = $dso_scheme - \$shared_target= $shared_target -diff --git a/Makefile.org b/Makefile.org -index 48469c5..c2c5107 100644 ---- a/Makefile.org -+++ b/Makefile.org -@@ -91,6 +91,7 @@ BN_ASM= bn_asm.o - EC_ASM= - DES_ENC= des_enc.o fcrypt_b.o - AES_ENC= aes_core.o aes_cbc.o -+CHAPOLY_ENC= - BF_ENC= bf_enc.o - CAST_ENC= c_enc.o - RC4_ENC= rc4_enc.o -@@ -148,7 +149,7 @@ SDIRS= \ - bn ec rsa dsa ecdsa dh ecdh dso engine \ - buffer bio stack lhash rand err \ - evp asn1 pem x509 x509v3 conf txt_db pkcs7 pkcs12 comp ocsp ui krb5 \ -- cms pqueue ts jpake srp store cmac -+ cms pqueue ts jpake srp store cmac chacha20poly1305 - # keep in mind that the above list is adjusted by ./Configure - # according to no-xxx arguments... - -@@ -233,6 +234,7 @@ BUILDENV= PLATFORM='$(PLATFORM)' PROCESSOR='$(PROCESSOR)' \ - WP_ASM_OBJ='$(WP_ASM_OBJ)' \ - MODES_ASM_OBJ='$(MODES_ASM_OBJ)' \ - ENGINES_ASM_OBJ='$(ENGINES_ASM_OBJ)' \ -+ CHAPOLY_ENC='$(CHAPOLY_ENC)' \ - PERLASM_SCHEME='$(PERLASM_SCHEME)' \ - FIPSLIBDIR='${FIPSLIBDIR}' \ - FIPSDIR='${FIPSDIR}' \ -diff --git a/apps/speed.c b/apps/speed.c -index 3697b71..ecf7817 100644 ---- a/apps/speed.c -+++ b/apps/speed.c -@@ -226,7 +226,7 @@ - # endif - - # undef BUFSIZE --# define BUFSIZE ((long)1024*8+1) -+# define BUFSIZE ((long)1024*8+16) - static volatile int run = 0; - - static int mr = 0; -@@ -241,7 +241,7 @@ static void print_result(int alg, int run_no, int count, double time_used); - static int do_multi(int multi); - # endif - --# define ALGOR_NUM 30 -+# define ALGOR_NUM 31 - # define SIZE_NUM 5 - # define RSA_NUM 4 - # define DSA_NUM 3 -@@ -256,7 +256,7 @@ static const char *names[ALGOR_NUM] = { - "aes-128 cbc", "aes-192 cbc", "aes-256 cbc", - "camellia-128 cbc", "camellia-192 cbc", "camellia-256 cbc", - "evp", "sha256", "sha512", "whirlpool", -- "aes-128 ige", "aes-192 ige", "aes-256 ige", "ghash" -+ "aes-128 ige", "aes-192 ige", "aes-256 ige", "ghash", "chacha20-poly1305" - }; - - static double results[ALGOR_NUM][SIZE_NUM]; -@@ -516,6 +516,7 @@ int MAIN(int argc, char **argv) - # define D_IGE_192_AES 27 - # define D_IGE_256_AES 28 - # define D_GHASH 29 -+# define D_CHAPOLY 30 - double d = 0.0; - long c[ALGOR_NUM][SIZE_NUM]; - # define R_DSA_512 0 -@@ -972,6 +973,11 @@ int MAIN(int argc, char **argv) - doit[D_CBC_256_CML] = 1; - } else - # endif -+# ifndef OPENSSL_NO_CHACHA_POLY -+ if (strcmp(*argv,"chacha20-poly1305") == 0) { -+ doit[D_CHAPOLY] = 1; -+ } else -+# endif - # ifndef OPENSSL_NO_RSA - if (strcmp(*argv, "rsa") == 0) { - rsa_doit[R_RSA_512] = 1; -@@ -1139,7 +1145,9 @@ int MAIN(int argc, char **argv) - BIO_printf(bio_err, "rc4"); - # endif - BIO_printf(bio_err, "\n"); -- -+# ifndef OPENSSL_NO_CHACHA_POLY -+ BIO_printf(bio_err,"chacha20-poly1305\n"); -+# endif - # ifndef OPENSSL_NO_RSA - BIO_printf(bio_err, "rsa512 rsa1024 rsa2048 rsa4096\n"); - # endif -@@ -1370,6 +1378,7 @@ int MAIN(int argc, char **argv) - c[D_IGE_192_AES][0] = count; - c[D_IGE_256_AES][0] = count; - c[D_GHASH][0] = count; -+ c[D_CHAPOLY][0] = count; - - for (i = 1; i < SIZE_NUM; i++) { - c[D_MD2][i] = c[D_MD2][0] * 4 * lengths[0] / lengths[i]; -@@ -1862,6 +1871,23 @@ int MAIN(int argc, char **argv) - } - } - # endif -+# ifndef OPENSSL_NO_CHACHA_POLY -+ if (doit[D_CHAPOLY]) { -+ EVP_CIPHER_CTX ctx; -+ EVP_CIPHER_CTX_init(&ctx); -+ EVP_CipherInit_ex(&ctx,EVP_chacha20_poly1305(),NULL,key32,NULL,1); -+ for (j=0; j $@ -+poly1305_avx.s:asm/poly1305_avx.pl -+ $(PERL) asm/poly1305_avx.pl $(PERLASM_SCHEME) > $@ -+chacha20_avx2.s:asm/chacha20_avx2.pl -+ $(PERL) asm/chacha20_avx2.pl $(PERLASM_SCHEME) > $@ -+poly1305_avx2.s:asm/poly1305_avx2.pl -+ $(PERL) asm/poly1305_avx2.pl $(PERLASM_SCHEME) > $@ -+ -+files: -+ $(PERL) $(TOP)/util/files.pl Makefile >> $(TOP)/MINFO -+ -+links: -+ @$(PERL) $(TOP)/util/mklink.pl ../../include/openssl $(EXHEADER) -+ @$(PERL) $(TOP)/util/mklink.pl ../../test $(TEST) -+ @$(PERL) $(TOP)/util/mklink.pl ../../apps $(APPS) -+ -+install: -+ @[ -n "$(INSTALLTOP)" ] # should be set by top Makefile... -+ @headerlist="$(EXHEADER)"; for i in $$headerlist ; \ -+ do \ -+ (cp $$i $(INSTALL_PREFIX)$(INSTALLTOP)/include/openssl/$$i; \ -+ chmod 644 $(INSTALL_PREFIX)$(INSTALLTOP)/include/openssl/$$i ); \ -+ done; -+ -+tags: -+ ctags $(SRC) -+ -+tests: -+ -+lint: -+ lint -DLINT $(INCLUDES) $(SRC)>fluff -+ -+depend: -+ @[ -n "$(MAKEDEPEND)" ] # should be set by upper Makefile... -+ $(MAKEDEPEND) -- $(CFLAG) $(INCLUDES) $(DEPFLAG) -- $(PROGS) $(LIBSRC) -+ -+dclean: -+ $(PERL) -pe 'if (/^# DO NOT DELETE THIS LINE/) {print; exit(0);}' $(MAKEFILE) >Makefile.new -+ mv -f Makefile.new $(MAKEFILE) -+ -+clean: -+ rm -f *.s *.o *.obj lib tags core .pure .nfs* *.old *.bak fluff -+ -+# DO NOT DELETE THIS LINE -- make depend depends on it. -+ -+chacha20.o: ../../include/openssl/chacha20poly1305.h chacha20.c -+poly1305.o: ../../include/openssl/chacha20poly1305.h poly1305.c -diff --git a/crypto/chacha20poly1305/asm/chacha20_avx.pl b/crypto/chacha20poly1305/asm/chacha20_avx.pl -new file mode 100644 -index 0000000..7b5b763 ---- /dev/null -+++ b/crypto/chacha20poly1305/asm/chacha20_avx.pl -@@ -0,0 +1,388 @@ -+#!/usr/bin/env perl -+ -+############################################################################## -+# # -+# Copyright 2014 Intel Corporation # -+# # -+# Licensed under the Apache License, Version 2.0 (the "License"); # -+# you may not use this file except in compliance with the License. # -+# You may obtain a copy of the License at # -+# # -+# http://www.apache.org/licenses/LICENSE-2.0 # -+# # -+# Unless required by applicable law or agreed to in writing, software # -+# distributed under the License is distributed on an "AS IS" BASIS, # -+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -+# See the License for the specific language governing permissions and # -+# limitations under the License. # -+# # -+############################################################################## -+# # -+# Developers and authors: # -+# Shay Gueron (1, 2), and Vlad Krasnov (1) # -+# (1) Intel Corporation, Israel Development Center # -+# (2) University of Haifa # -+# # -+# Related work: # -+# M. Goll, S. Gueron, "Vectorization on ChaCha Stream Cipher", IEEE # -+# Proceedings of 11th International Conference on Information # -+# Technology: New Generations (ITNG 2014), 612-615 (2014). # -+# M. Goll, S. Gueron, "Vectorization on Poly1305 Message Authentication Code"# -+# to be published. # -+# A. Langley, chacha20poly1305 for the AEAD head # -+# https://git.openssl.org/gitweb/?p=openssl.git;a=commit;h=9a8646510b3d0a48e950748f7a2aaa12ed40d5e0 # -+############################################################################## -+ -+ -+ -+$flavour = shift; -+$output = shift; -+if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } -+ -+$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); -+ -+$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; -+( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or -+( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or -+die "can't locate x86_64-xlate.pl"; -+ -+open OUT,"| \"$^X\" $xlate $flavour $output"; -+*STDOUT=*OUT; -+ -+if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` -+ =~ /GNU assembler version ([2-9]\.[0-9]+)/) { -+ $avx = ($1>=2.19) + ($1>=2.22); -+} -+ -+if ($win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && -+ `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { -+ $avx = ($1>=2.09) + ($1>=2.10); -+} -+ -+if ($win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && -+ `ml64 2>&1` =~ /Version ([0-9]+)\./) { -+ $avx = ($1>=10) + ($1>=11); -+} -+ -+if (`$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9])\.([0-9]+)/) { -+ my $ver = $2 + $3/100.0; # 3.1->3.01, 3.10->3.10 -+ $avx = ($ver>=3.0) + ($ver>=3.01); -+} -+ -+if ($avx>=1) {{ -+ -+sub chacha_qr { -+my ($a,$b,$c,$d,$tmp)=@_; -+$code.=<<___; -+ -+ vpaddd $b, $a, $a # a += b -+ vpxor $a, $d, $d # d ^= a -+ vpshufb .rol16(%rip), $d, $d # d <<<= 16 -+ -+ vpaddd $d, $c, $c # c += d -+ vpxor $c, $b, $b # b ^= c -+ vpslld \$12, $b, $tmp -+ vpsrld \$20, $b, $b -+ vpxor $tmp, $b, $b # b <<<= 12 -+ -+ vpaddd $b, $a, $a # a += b -+ vpxor $a, $d, $d # d ^= a -+ vpshufb .rol8(%rip), $d, $d # d <<<= 8 -+ -+ vpaddd $d, $c, $c # c += d -+ vpxor $c, $b, $b # b ^= c -+ -+ vpslld \$7, $b, $tmp -+ vpsrld \$25, $b, $b -+ vpxor $tmp, $b, $b # b <<<= 7 -+___ -+} -+ -+ -+$code.=<<___; -+.text -+.align 16 -+chacha20_consts: -+.byte 'e','x','p','a','n','d',' ','3','2','-','b','y','t','e',' ','k' -+.rol8: -+.byte 3,0,1,2, 7,4,5,6, 11,8,9,10, 15,12,13,14 -+.rol16: -+.byte 2,3,0,1, 6,7,4,5, 10,11,8,9, 14,15,12,13 -+.avxInc: -+.quad 1,0 -+___ -+ -+{ -+my ($state_4567, $state_89ab, $state_cdef, $tmp, -+ $v0, $v1, $v2, $v3, $v4, $v5, $v6, $v7, -+ $v8, $v9, $v10, $v11)=map("%xmm$_",(0..15)); -+ -+my ($out, $in, $in_len, $key_ptr, $nonce_ptr, $counter, $nr) -+ =("%rdi", "%rsi", "%rdx", "%rcx", "%r8", "%r9", "%rax"); -+ -+$code.=<<___; -+.globl chacha_20_core_avx -+.type chacha_20_core_avx ,\@function,2 -+.align 64 -+chacha_20_core_avx: -+ vzeroupper -+ -+ # Init state -+ vmovdqu 16*0($key_ptr), $state_4567 -+ vmovdqu 16*1($key_ptr), $state_89ab -+ vmovq $counter, $state_cdef -+ vpinsrq \$1, ($nonce_ptr), $state_cdef, $state_cdef -+2: -+ cmp \$3*64, $in_len -+ jb 2f -+ -+ vmovdqa chacha20_consts(%rip), $v0 -+ vmovdqa chacha20_consts(%rip), $v4 -+ vmovdqa chacha20_consts(%rip), $v8 -+ -+ vmovdqa $state_4567, $v1 -+ vmovdqa $state_4567, $v5 -+ vmovdqa $state_4567, $v9 -+ -+ vmovdqa $state_89ab, $v2 -+ vmovdqa $state_89ab, $v6 -+ vmovdqa $state_89ab, $v10 -+ -+ vmovdqa $state_cdef, $v3 -+ vpaddq .avxInc(%rip), $v3, $v7 -+ vpaddq .avxInc(%rip), $v7, $v11 -+ -+ mov \$10, $nr -+ -+ 1: -+___ -+ &chacha_qr($v0,$v1,$v2,$v3,$tmp); -+ &chacha_qr($v4,$v5,$v6,$v7,$tmp); -+ &chacha_qr($v8,$v9,$v10,$v11,$tmp); -+$code.=<<___; -+ vpalignr \$4, $v1, $v1, $v1 -+ vpalignr \$8, $v2, $v2, $v2 -+ vpalignr \$12, $v3, $v3, $v3 -+ vpalignr \$4, $v5, $v5, $v5 -+ vpalignr \$8, $v6, $v6, $v6 -+ vpalignr \$12, $v7, $v7, $v7 -+ vpalignr \$4, $v9, $v9, $v9 -+ vpalignr \$8, $v10, $v10, $v10 -+ vpalignr \$12, $v11, $v11, $v11 -+___ -+ &chacha_qr($v0,$v1,$v2,$v3,$tmp); -+ &chacha_qr($v4,$v5,$v6,$v7,$tmp); -+ &chacha_qr($v8,$v9,$v10,$v11,$tmp); -+$code.=<<___; -+ vpalignr \$12, $v1, $v1, $v1 -+ vpalignr \$8, $v2, $v2, $v2 -+ vpalignr \$4, $v3, $v3, $v3 -+ vpalignr \$12, $v5, $v5, $v5 -+ vpalignr \$8, $v6, $v6, $v6 -+ vpalignr \$4, $v7, $v7, $v7 -+ vpalignr \$12, $v9, $v9, $v9 -+ vpalignr \$8, $v10, $v10, $v10 -+ vpalignr \$4, $v11, $v11, $v11 -+ -+ dec $nr -+ -+ jnz 1b -+ -+ vpaddd chacha20_consts(%rip), $v0, $v0 -+ vpaddd chacha20_consts(%rip), $v4, $v4 -+ vpaddd chacha20_consts(%rip), $v8, $v8 -+ -+ vpaddd $state_4567, $v1, $v1 -+ vpaddd $state_4567, $v5, $v5 -+ vpaddd $state_4567, $v9, $v9 -+ -+ vpaddd $state_89ab, $v2, $v2 -+ vpaddd $state_89ab, $v6, $v6 -+ vpaddd $state_89ab, $v10, $v10 -+ -+ vpaddd $state_cdef, $v3, $v3 -+ vpaddq .avxInc(%rip), $state_cdef, $state_cdef -+ vpaddd $state_cdef, $v7, $v7 -+ vpaddq .avxInc(%rip), $state_cdef, $state_cdef -+ vpaddd $state_cdef, $v11, $v11 -+ vpaddq .avxInc(%rip), $state_cdef, $state_cdef -+ -+ vpxor 16*0($in), $v0, $v0 -+ vpxor 16*1($in), $v1, $v1 -+ vpxor 16*2($in), $v2, $v2 -+ vpxor 16*3($in), $v3, $v3 -+ -+ vmovdqu $v0, 16*0($out) -+ vmovdqu $v1, 16*1($out) -+ vmovdqu $v2, 16*2($out) -+ vmovdqu $v3, 16*3($out) -+ -+ vpxor 16*4($in), $v4, $v4 -+ vpxor 16*5($in), $v5, $v5 -+ vpxor 16*6($in), $v6, $v6 -+ vpxor 16*7($in), $v7, $v7 -+ -+ vmovdqu $v4, 16*4($out) -+ vmovdqu $v5, 16*5($out) -+ vmovdqu $v6, 16*6($out) -+ vmovdqu $v7, 16*7($out) -+ -+ vpxor 16*8($in), $v8, $v8 -+ vpxor 16*9($in), $v9, $v9 -+ vpxor 16*10($in), $v10, $v10 -+ vpxor 16*11($in), $v11, $v11 -+ -+ vmovdqu $v8, 16*8($out) -+ vmovdqu $v9, 16*9($out) -+ vmovdqu $v10, 16*10($out) -+ vmovdqu $v11, 16*11($out) -+ -+ lea 16*12($in), $in -+ lea 16*12($out), $out -+ sub \$16*12, $in_len -+ -+ jmp 2b -+ -+2: -+ cmp \$2*64, $in_len -+ jb 2f -+ -+ vmovdqa chacha20_consts(%rip), $v0 -+ vmovdqa chacha20_consts(%rip), $v4 -+ vmovdqa $state_4567, $v1 -+ vmovdqa $state_4567, $v5 -+ vmovdqa $state_89ab, $v2 -+ vmovdqa $state_89ab, $v6 -+ vmovdqa $state_89ab, $v10 -+ vmovdqa $state_cdef, $v3 -+ vpaddq .avxInc(%rip), $v3, $v7 -+ -+ mov \$10, $nr -+ -+ 1: -+___ -+ &chacha_qr($v0,$v1,$v2,$v3,$tmp); -+ &chacha_qr($v4,$v5,$v6,$v7,$tmp); -+$code.=<<___; -+ vpalignr \$4, $v1, $v1, $v1 -+ vpalignr \$8, $v2, $v2, $v2 -+ vpalignr \$12, $v3, $v3, $v3 -+ vpalignr \$4, $v5, $v5, $v5 -+ vpalignr \$8, $v6, $v6, $v6 -+ vpalignr \$12, $v7, $v7, $v7 -+___ -+ &chacha_qr($v0,$v1,$v2,$v3,$tmp); -+ &chacha_qr($v4,$v5,$v6,$v7,$tmp); -+$code.=<<___; -+ vpalignr \$12, $v1, $v1, $v1 -+ vpalignr \$8, $v2, $v2, $v2 -+ vpalignr \$4, $v3, $v3, $v3 -+ vpalignr \$12, $v5, $v5, $v5 -+ vpalignr \$8, $v6, $v6, $v6 -+ vpalignr \$4, $v7, $v7, $v7 -+ -+ dec $nr -+ -+ jnz 1b -+ -+ vpaddd chacha20_consts(%rip), $v0, $v0 -+ vpaddd chacha20_consts(%rip), $v4, $v4 -+ -+ vpaddd $state_4567, $v1, $v1 -+ vpaddd $state_4567, $v5, $v5 -+ -+ vpaddd $state_89ab, $v2, $v2 -+ vpaddd $state_89ab, $v6, $v6 -+ -+ vpaddd $state_cdef, $v3, $v3 -+ vpaddq .avxInc(%rip), $state_cdef, $state_cdef -+ vpaddd $state_cdef, $v7, $v7 -+ vpaddq .avxInc(%rip), $state_cdef, $state_cdef -+ -+ vpxor 16*0($in), $v0, $v0 -+ vpxor 16*1($in), $v1, $v1 -+ vpxor 16*2($in), $v2, $v2 -+ vpxor 16*3($in), $v3, $v3 -+ -+ vmovdqu $v0, 16*0($out) -+ vmovdqu $v1, 16*1($out) -+ vmovdqu $v2, 16*2($out) -+ vmovdqu $v3, 16*3($out) -+ -+ vpxor 16*4($in), $v4, $v4 -+ vpxor 16*5($in), $v5, $v5 -+ vpxor 16*6($in), $v6, $v6 -+ vpxor 16*7($in), $v7, $v7 -+ -+ vmovdqu $v4, 16*4($out) -+ vmovdqu $v5, 16*5($out) -+ vmovdqu $v6, 16*6($out) -+ vmovdqu $v7, 16*7($out) -+ -+ lea 16*8($in), $in -+ lea 16*8($out), $out -+ sub \$16*8, $in_len -+ -+ jmp 2b -+2: -+ cmp \$64, $in_len -+ jb 2f -+ -+ vmovdqa chacha20_consts(%rip), $v0 -+ vmovdqa $state_4567, $v1 -+ vmovdqa $state_89ab, $v2 -+ vmovdqa $state_cdef, $v3 -+ -+ mov \$10, $nr -+ -+ 1: -+___ -+ &chacha_qr($v0,$v1,$v2,$v3,$tmp); -+$code.=<<___; -+ vpalignr \$4, $v1, $v1, $v1 -+ vpalignr \$8, $v2, $v2, $v2 -+ vpalignr \$12, $v3, $v3, $v3 -+___ -+ &chacha_qr($v0,$v1,$v2,$v3,$tmp); -+$code.=<<___; -+ vpalignr \$12, $v1, $v1, $v1 -+ vpalignr \$8, $v2, $v2, $v2 -+ vpalignr \$4, $v3, $v3, $v3 -+ -+ dec $nr -+ jnz 1b -+ -+ vpaddd chacha20_consts(%rip), $v0, $v0 -+ vpaddd $state_4567, $v1, $v1 -+ vpaddd $state_89ab, $v2, $v2 -+ vpaddd $state_cdef, $v3, $v3 -+ vpaddq .avxInc(%rip), $state_cdef, $state_cdef -+ -+ vpxor 16*0($in), $v0, $v0 -+ vpxor 16*1($in), $v1, $v1 -+ vpxor 16*2($in), $v2, $v2 -+ vpxor 16*3($in), $v3, $v3 -+ -+ vmovdqu $v0, 16*0($out) -+ vmovdqu $v1, 16*1($out) -+ vmovdqu $v2, 16*2($out) -+ vmovdqu $v3, 16*3($out) -+ -+ lea 16*4($in), $in -+ lea 16*4($out), $out -+ sub \$16*4, $in_len -+ jmp 2b -+2: -+ vzeroupper -+ ret -+.size chacha_20_core_avx,.-chacha_20_core_avx -+___ -+} -+}} -+ -+ -+$code =~ s/\`([^\`]*)\`/eval($1)/gem; -+ -+print $code; -+ -+close STDOUT; -diff --git a/crypto/chacha20poly1305/asm/chacha20_avx2.pl b/crypto/chacha20poly1305/asm/chacha20_avx2.pl -new file mode 100644 -index 0000000..31ae721 ---- /dev/null -+++ b/crypto/chacha20poly1305/asm/chacha20_avx2.pl -@@ -0,0 +1,424 @@ -+#!/usr/bin/env perl -+ -+############################################################################## -+# # -+# Copyright 2014 Intel Corporation # -+# # -+# Licensed under the Apache License, Version 2.0 (the "License"); # -+# you may not use this file except in compliance with the License. # -+# You may obtain a copy of the License at # -+# # -+# http://www.apache.org/licenses/LICENSE-2.0 # -+# # -+# Unless required by applicable law or agreed to in writing, software # -+# distributed under the License is distributed on an "AS IS" BASIS, # -+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -+# See the License for the specific language governing permissions and # -+# limitations under the License. # -+# # -+############################################################################## -+# # -+# Developers and authors: # -+# Shay Gueron (1, 2), and Vlad Krasnov (1) # -+# (1) Intel Corporation, Israel Development Center # -+# (2) University of Haifa # -+# # -+# Related work: # -+# M. Goll, S. Gueron, "Vectorization on ChaCha Stream Cipher", IEEE # -+# Proceedings of 11th International Conference on Information # -+# Technology: New Generations (ITNG 2014), 612-615 (2014). # -+# M. Goll, S. Gueron, "Vectorization on Poly1305 Message Authentication Code"# -+# to be published. # -+# A. Langley, chacha20poly1305 for the AEAD head # -+# https://git.openssl.org/gitweb/?p=openssl.git;a=commit;h=9a8646510b3d0a48e950748f7a2aaa12ed40d5e0 # -+############################################################################## -+ -+$flavour = shift; -+$output = shift; -+if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } -+ -+$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); -+ -+$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; -+( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or -+( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or -+die "can't locate x86_64-xlate.pl"; -+ -+open OUT,"| \"$^X\" $xlate $flavour $output"; -+*STDOUT=*OUT; -+ -+if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` -+ =~ /GNU assembler version ([2-9]\.[0-9]+)/) { -+ $avx = ($1>=2.19) + ($1>=2.22); -+} -+ -+if ($win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && -+ `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { -+ $avx = ($1>=2.09) + ($1>=2.10); -+} -+ -+if ($win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && -+ `ml64 2>&1` =~ /Version ([0-9]+)\./) { -+ $avx = ($1>=10) + ($1>=11); -+} -+ -+if (`$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9])\.([0-9]+)/) { -+ my $ver = $2 + $3/100.0; # 3.1->3.01, 3.10->3.10 -+ $avx = ($ver>=3.0) + ($ver>=3.01); -+} -+ -+if ($avx>=2) {{ -+ -+sub chacha_qr { -+my ($a,$b,$c,$d,$tmp)=@_; -+$code.=<<___; -+ -+ vpaddd $b, $a, $a # a += b -+ vpxor $a, $d, $d # d ^= a -+ vpshufb .rol16(%rip), $d, $d # d <<<= 16 -+ -+ vpaddd $d, $c, $c # c += d -+ vpxor $c, $b, $b # b ^= c -+ vpslld \$12, $b, $tmp -+ vpsrld \$20, $b, $b -+ vpxor $tmp, $b, $b # b <<<= 12 -+ -+ vpaddd $b, $a, $a # a += b -+ vpxor $a, $d, $d # d ^= a -+ vpshufb .rol8(%rip), $d, $d # d <<<= 8 -+ -+ vpaddd $d, $c, $c # c += d -+ vpxor $c, $b, $b # b ^= c -+ -+ vpslld \$7, $b, $tmp -+ vpsrld \$25, $b, $b -+ vpxor $tmp, $b, $b # b <<<= 7 -+___ -+} -+ -+ -+$code.=<<___; -+.text -+.align 32 -+chacha20_consts: -+.byte 'e','x','p','a','n','d',' ','3','2','-','b','y','t','e',' ','k' -+.byte 'e','x','p','a','n','d',' ','3','2','-','b','y','t','e',' ','k' -+.rol8: -+.byte 3,0,1,2, 7,4,5,6, 11,8,9,10, 15,12,13,14 -+.byte 3,0,1,2, 7,4,5,6, 11,8,9,10, 15,12,13,14 -+.rol16: -+.byte 2,3,0,1, 6,7,4,5, 10,11,8,9, 14,15,12,13 -+.byte 2,3,0,1, 6,7,4,5, 10,11,8,9, 14,15,12,13 -+.avx2Init: -+.quad 0,0,1,0 -+.avx2Inc: -+.quad 2,0,2,0 -+___ -+ -+{ -+my ($state_4567, $state_89ab, $state_cdef, $tmp, -+ $v0, $v1, $v2, $v3, $v4, $v5, $v6, $v7, -+ $v8, $v9, $v10, $v11)=map("%ymm$_",(0..15)); -+ -+my $state_cdef_xmm="%xmm2"; -+ -+my ($out, $in, $in_len, $key_ptr, $nonce_ptr, $counter, $nr) -+ =("%rdi", "%rsi", "%rdx", "%rcx", "%r8", "%r9", "%rax"); -+ -+$code.=<<___; -+.globl chacha_20_core_avx2 -+.type chacha_20_core_avx2 ,\@function,2 -+.align 64 -+chacha_20_core_avx2: -+ vzeroupper -+ -+ # Init state -+ vbroadcasti128 16*0($key_ptr), $state_4567 -+ vbroadcasti128 16*1($key_ptr), $state_89ab -+ vmovq $counter, $state_cdef_xmm -+ vpinsrq \$1, ($nonce_ptr), $state_cdef_xmm, $state_cdef_xmm -+ vperm2i128 \$0x00, $state_cdef, $state_cdef, $state_cdef -+ vpaddq .avx2Init(%rip), $state_cdef, $state_cdef -+ -+2: -+ cmp \$6*64, $in_len -+ jb 2f -+ -+ vmovdqa chacha20_consts(%rip), $v0 -+ vmovdqa chacha20_consts(%rip), $v4 -+ vmovdqa chacha20_consts(%rip), $v8 -+ -+ vmovdqa $state_4567, $v1 -+ vmovdqa $state_4567, $v5 -+ vmovdqa $state_4567, $v9 -+ -+ vmovdqa $state_89ab, $v2 -+ vmovdqa $state_89ab, $v6 -+ vmovdqa $state_89ab, $v10 -+ -+ vmovdqa $state_cdef, $v3 -+ vpaddq .avx2Inc(%rip), $v3, $v7 -+ vpaddq .avx2Inc(%rip), $v7, $v11 -+ -+ mov \$10, $nr -+ -+ 1: -+___ -+ &chacha_qr($v0,$v1,$v2,$v3,$tmp); -+ &chacha_qr($v4,$v5,$v6,$v7,$tmp); -+ &chacha_qr($v8,$v9,$v10,$v11,$tmp); -+$code.=<<___; -+ vpalignr \$4, $v1, $v1, $v1 -+ vpalignr \$8, $v2, $v2, $v2 -+ vpalignr \$12, $v3, $v3, $v3 -+ vpalignr \$4, $v5, $v5, $v5 -+ vpalignr \$8, $v6, $v6, $v6 -+ vpalignr \$12, $v7, $v7, $v7 -+ vpalignr \$4, $v9, $v9, $v9 -+ vpalignr \$8, $v10, $v10, $v10 -+ vpalignr \$12, $v11, $v11, $v11 -+___ -+ &chacha_qr($v0,$v1,$v2,$v3,$tmp); -+ &chacha_qr($v4,$v5,$v6,$v7,$tmp); -+ &chacha_qr($v8,$v9,$v10,$v11,$tmp); -+$code.=<<___; -+ vpalignr \$12, $v1, $v1, $v1 -+ vpalignr \$8, $v2, $v2, $v2 -+ vpalignr \$4, $v3, $v3, $v3 -+ vpalignr \$12, $v5, $v5, $v5 -+ vpalignr \$8, $v6, $v6, $v6 -+ vpalignr \$4, $v7, $v7, $v7 -+ vpalignr \$12, $v9, $v9, $v9 -+ vpalignr \$8, $v10, $v10, $v10 -+ vpalignr \$4, $v11, $v11, $v11 -+ -+ dec $nr -+ -+ jnz 1b -+ -+ vpaddd chacha20_consts(%rip), $v0, $v0 -+ vpaddd chacha20_consts(%rip), $v4, $v4 -+ vpaddd chacha20_consts(%rip), $v8, $v8 -+ -+ vpaddd $state_4567, $v1, $v1 -+ vpaddd $state_4567, $v5, $v5 -+ vpaddd $state_4567, $v9, $v9 -+ -+ vpaddd $state_89ab, $v2, $v2 -+ vpaddd $state_89ab, $v6, $v6 -+ vpaddd $state_89ab, $v10, $v10 -+ -+ vpaddd $state_cdef, $v3, $v3 -+ vpaddq .avx2Inc(%rip), $state_cdef, $state_cdef -+ vpaddd $state_cdef, $v7, $v7 -+ vpaddq .avx2Inc(%rip), $state_cdef, $state_cdef -+ vpaddd $state_cdef, $v11, $v11 -+ vpaddq .avx2Inc(%rip), $state_cdef, $state_cdef -+ -+ vperm2i128 \$0x02, $v0, $v1, $tmp -+ vpxor 32*0($in), $tmp, $tmp -+ vmovdqu $tmp, 32*0($out) -+ vperm2i128 \$0x02, $v2, $v3, $tmp -+ vpxor 32*1($in), $tmp, $tmp -+ vmovdqu $tmp, 32*1($out) -+ vperm2i128 \$0x13, $v0, $v1, $tmp -+ vpxor 32*2($in), $tmp, $tmp -+ vmovdqu $tmp, 32*2($out) -+ vperm2i128 \$0x13, $v2, $v3, $tmp -+ vpxor 32*3($in), $tmp, $tmp -+ vmovdqu $tmp, 32*3($out) -+ -+ vperm2i128 \$0x02, $v4, $v5, $v0 -+ vperm2i128 \$0x02, $v6, $v7, $v1 -+ vperm2i128 \$0x13, $v4, $v5, $v2 -+ vperm2i128 \$0x13, $v6, $v7, $v3 -+ -+ vpxor 32*4($in), $v0, $v0 -+ vpxor 32*5($in), $v1, $v1 -+ vpxor 32*6($in), $v2, $v2 -+ vpxor 32*7($in), $v3, $v3 -+ -+ vmovdqu $v0, 32*4($out) -+ vmovdqu $v1, 32*5($out) -+ vmovdqu $v2, 32*6($out) -+ vmovdqu $v3, 32*7($out) -+ -+ vperm2i128 \$0x02, $v8, $v9, $v0 -+ vperm2i128 \$0x02, $v10, $v11, $v1 -+ vperm2i128 \$0x13, $v8, $v9, $v2 -+ vperm2i128 \$0x13, $v10, $v11, $v3 -+ -+ vpxor 32*8($in), $v0, $v0 -+ vpxor 32*9($in), $v1, $v1 -+ vpxor 32*10($in), $v2, $v2 -+ vpxor 32*11($in), $v3, $v3 -+ -+ vmovdqu $v0, 32*8($out) -+ vmovdqu $v1, 32*9($out) -+ vmovdqu $v2, 32*10($out) -+ vmovdqu $v3, 32*11($out) -+ -+ lea 64*6($in), $in -+ lea 64*6($out), $out -+ sub \$64*6, $in_len -+ -+ jmp 2b -+ -+2: -+ cmp \$4*64, $in_len -+ jb 2f -+ -+ vmovdqa chacha20_consts(%rip), $v0 -+ vmovdqa chacha20_consts(%rip), $v4 -+ vmovdqa $state_4567, $v1 -+ vmovdqa $state_4567, $v5 -+ vmovdqa $state_89ab, $v2 -+ vmovdqa $state_89ab, $v6 -+ vmovdqa $state_89ab, $v10 -+ vmovdqa $state_cdef, $v3 -+ vpaddq .avx2Inc(%rip), $v3, $v7 -+ -+ mov \$10, $nr -+ -+ 1: -+___ -+ &chacha_qr($v0,$v1,$v2,$v3,$tmp); -+ &chacha_qr($v4,$v5,$v6,$v7,$tmp); -+$code.=<<___; -+ vpalignr \$4, $v1, $v1, $v1 -+ vpalignr \$8, $v2, $v2, $v2 -+ vpalignr \$12, $v3, $v3, $v3 -+ vpalignr \$4, $v5, $v5, $v5 -+ vpalignr \$8, $v6, $v6, $v6 -+ vpalignr \$12, $v7, $v7, $v7 -+___ -+ &chacha_qr($v0,$v1,$v2,$v3,$tmp); -+ &chacha_qr($v4,$v5,$v6,$v7,$tmp); -+$code.=<<___; -+ vpalignr \$12, $v1, $v1, $v1 -+ vpalignr \$8, $v2, $v2, $v2 -+ vpalignr \$4, $v3, $v3, $v3 -+ vpalignr \$12, $v5, $v5, $v5 -+ vpalignr \$8, $v6, $v6, $v6 -+ vpalignr \$4, $v7, $v7, $v7 -+ -+ dec $nr -+ -+ jnz 1b -+ -+ vpaddd chacha20_consts(%rip), $v0, $v0 -+ vpaddd chacha20_consts(%rip), $v4, $v4 -+ -+ vpaddd $state_4567, $v1, $v1 -+ vpaddd $state_4567, $v5, $v5 -+ -+ vpaddd $state_89ab, $v2, $v2 -+ vpaddd $state_89ab, $v6, $v6 -+ -+ vpaddd $state_cdef, $v3, $v3 -+ vpaddq .avx2Inc(%rip), $state_cdef, $state_cdef -+ vpaddd $state_cdef, $v7, $v7 -+ vpaddq .avx2Inc(%rip), $state_cdef, $state_cdef -+ -+ vperm2i128 \$0x02, $v0, $v1, $v8 -+ vperm2i128 \$0x02, $v2, $v3, $v9 -+ vperm2i128 \$0x13, $v0, $v1, $v10 -+ vperm2i128 \$0x13, $v2, $v3, $v11 -+ -+ vpxor 32*0($in), $v8, $v8 -+ vpxor 32*1($in), $v9, $v9 -+ vpxor 32*2($in), $v10, $v10 -+ vpxor 32*3($in), $v11, $v11 -+ -+ vmovdqu $v8, 32*0($out) -+ vmovdqu $v9, 32*1($out) -+ vmovdqu $v10, 32*2($out) -+ vmovdqu $v11, 32*3($out) -+ -+ vperm2i128 \$0x02, $v4, $v5, $v0 -+ vperm2i128 \$0x02, $v6, $v7, $v1 -+ vperm2i128 \$0x13, $v4, $v5, $v2 -+ vperm2i128 \$0x13, $v6, $v7, $v3 -+ -+ vpxor 32*4($in), $v0, $v0 -+ vpxor 32*5($in), $v1, $v1 -+ vpxor 32*6($in), $v2, $v2 -+ vpxor 32*7($in), $v3, $v3 -+ -+ vmovdqu $v0, 32*4($out) -+ vmovdqu $v1, 32*5($out) -+ vmovdqu $v2, 32*6($out) -+ vmovdqu $v3, 32*7($out) -+ -+ lea 64*4($in), $in -+ lea 64*4($out), $out -+ sub \$64*4, $in_len -+ -+ jmp 2b -+2: -+ cmp \$128, $in_len -+ jb 2f -+ -+ vmovdqa chacha20_consts(%rip), $v0 -+ vmovdqa $state_4567, $v1 -+ vmovdqa $state_89ab, $v2 -+ vmovdqa $state_cdef, $v3 -+ -+ mov \$10, $nr -+ -+ 1: -+___ -+ &chacha_qr($v0,$v1,$v2,$v3,$tmp); -+$code.=<<___; -+ vpalignr \$4, $v1, $v1, $v1 -+ vpalignr \$8, $v2, $v2, $v2 -+ vpalignr \$12, $v3, $v3, $v3 -+___ -+ &chacha_qr($v0,$v1,$v2,$v3,$tmp); -+$code.=<<___; -+ vpalignr \$12, $v1, $v1, $v1 -+ vpalignr \$8, $v2, $v2, $v2 -+ vpalignr \$4, $v3, $v3, $v3 -+ -+ dec $nr -+ jnz 1b -+ -+ vpaddd chacha20_consts(%rip), $v0, $v0 -+ vpaddd $state_4567, $v1, $v1 -+ vpaddd $state_89ab, $v2, $v2 -+ vpaddd $state_cdef, $v3, $v3 -+ vpaddq .avx2Inc(%rip), $state_cdef, $state_cdef -+ -+ vperm2i128 \$0x02, $v0, $v1, $v8 -+ vperm2i128 \$0x02, $v2, $v3, $v9 -+ vperm2i128 \$0x13, $v0, $v1, $v10 -+ vperm2i128 \$0x13, $v2, $v3, $v11 -+ -+ vpxor 32*0($in), $v8, $v8 -+ vpxor 32*1($in), $v9, $v9 -+ vpxor 32*2($in), $v10, $v10 -+ vpxor 32*3($in), $v11, $v11 -+ -+ vmovdqu $v8, 32*0($out) -+ vmovdqu $v9, 32*1($out) -+ vmovdqu $v10, 32*2($out) -+ vmovdqu $v11, 32*3($out) -+ -+ lea 64*2($in), $in -+ lea 64*2($out), $out -+ sub \$64*2, $in_len -+ jmp 2b -+2: -+ vzeroupper -+ ret -+.size chacha_20_core_avx2,.-chacha_20_core_avx2 -+___ -+} -+}} -+ -+ -+$code =~ s/\`([^\`]*)\`/eval($1)/gem; -+ -+print $code; -+ -+close STDOUT; -diff --git a/crypto/chacha20poly1305/asm/poly1305_avx.pl b/crypto/chacha20poly1305/asm/poly1305_avx.pl -new file mode 100644 -index 0000000..2d06e41 ---- /dev/null -+++ b/crypto/chacha20poly1305/asm/poly1305_avx.pl -@@ -0,0 +1,717 @@ -+############################################################################## -+# # -+# Copyright 2014 Intel Corporation # -+# # -+# Licensed under the Apache License, Version 2.0 (the "License"); # -+# you may not use this file except in compliance with the License. # -+# You may obtain a copy of the License at # -+# # -+# http://www.apache.org/licenses/LICENSE-2.0 # -+# # -+# Unless required by applicable law or agreed to in writing, software # -+# distributed under the License is distributed on an "AS IS" BASIS, # -+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -+# See the License for the specific language governing permissions and # -+# limitations under the License. # -+# # -+############################################################################## -+# # -+# Developers and authors: # -+# Shay Gueron (1, 2), and Vlad Krasnov (1) # -+# (1) Intel Corporation, Israel Development Center # -+# (2) University of Haifa # -+# # -+############################################################################## -+# state: -+# 0: r[0] || r^2[0] -+# 16: r[1] || r^2[1] -+# 32: r[2] || r^2[2] -+# 48: r[3] || r^2[3] -+# 64: r[4] || r^2[4] -+# 80: r[1]*5 || r^2[1]*5 -+# 96: r[2]*5 || r^2[2]*5 -+#112: r[3]*5 || r^2[3]*5 -+#128: r[4]*5 || r^2[4]*5 -+#144: k -+#160: A0 -+#164: A1 -+#168: A2 -+#172: A3 -+#176: A4 -+#180: END -+ -+$flavour = shift; -+$output = shift; -+if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } -+ -+$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); -+ -+$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; -+( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or -+( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or -+die "can't locate x86_64-xlate.pl"; -+ -+open OUT,"| \"$^X\" $xlate $flavour $output"; -+*STDOUT=*OUT; -+ -+if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` -+ =~ /GNU assembler version ([2-9]\.[0-9]+)/) { -+ $avx = ($1>=2.19) + ($1>=2.22); -+} -+ -+if ($win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && -+ `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { -+ $avx = ($1>=2.09) + ($1>=2.10); -+} -+ -+if ($win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && -+ `ml64 2>&1` =~ /Version ([0-9]+)\./) { -+ $avx = ($1>=10) + ($1>=11); -+} -+ -+if (`$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9])\.([0-9]+)/) { -+ my $ver = $2 + $3/100.0; # 3.1->3.01, 3.10->3.10 -+ $avx = ($ver>=3.0) + ($ver>=3.01); -+} -+ -+if ($avx>=1) {{ -+ -+my ($_r0_, $_r1_, $_r2_, $_r3_, $_r4_, $_r1_x5, $_r2_x5, $_r3_x5, $_r4_x5, $_k_, $_A0_, $_A1_, $_A2_, $_A3_, $_A4_) -+= (0,16,32,48,64,80,96,112,128,144,160,164,168,172,176); -+ -+$code.=<<___; -+.text -+.align 32 -+.LandMask: -+.quad 0x3FFFFFF, 0x3FFFFFF -+.LsetBit: -+.quad 0x1000000, 0x1000000 -+.LrSet: -+.quad 0xFFFFFFC0FFFFFFF, 0xFFFFFFC0FFFFFFF -+.quad 0xFFFFFFC0FFFFFFC, 0xFFFFFFC0FFFFFFC -+.Lone: -+.quad 1,0 -+___ -+ -+ -+{ -+my ($A0, $A1, $A2, $A3, $A4, -+ $r0, $r1, $r2, $r3, $r4, -+ $T0, $T1, $A5, $A6, $A7, $A8)=map("%xmm$_",(0..15)); -+my ($state, $key) -+ =("%rdi", "%rsi"); -+ -+$code.=<<___; -+################################################################################ -+# void poly1305_init_avx(void *state, uint8_t key[32]) -+ -+.globl poly1305_init_avx -+.type poly1305_init_avx, \@function, 2 -+.align 64 -+poly1305_init_avx: -+ vzeroupper -+ # load and convert r -+ vmovq 8*0($key), $r0 -+ vmovq 8*1($key), $T0 -+ vpand .LrSet(%rip), $r0, $r0 -+ vpand .LrSet+16(%rip), $T0, $T0 -+ -+ vpsrlq \$26, $r0, $r1 -+ vpand .LandMask(%rip), $r0, $r0 -+ vpsrlq \$26, $r1, $r2 -+ vpand .LandMask(%rip), $r1, $r1 -+ vpsllq \$12, $T0, $T1 -+ vpxor $T1, $r2, $r2 -+ vpsrlq \$26, $r2, $r3 -+ vpsrlq \$40, $T0, $r4 -+ vpand .LandMask(%rip), $r2, $r2 -+ vpand .LandMask(%rip), $r3, $r3 -+ -+ # SQR R -+ vpmuludq $r0, $r0, $A0 -+ vpmuludq $r1, $r0, $A1 -+ vpmuludq $r2, $r0, $A2 -+ vpmuludq $r3, $r0, $A3 -+ vpmuludq $r4, $r0, $A4 -+ -+ vpsllq \$1, $A1, $A1 -+ vpsllq \$1, $A2, $A2 -+ vpmuludq $r1, $r1, $T0 -+ vpaddq $T0, $A2, $A2 -+ vpmuludq $r2, $r1, $T0 -+ vpaddq $T0, $A3, $A3 -+ vpmuludq $r3, $r1, $T0 -+ vpaddq $T0, $A4, $A4 -+ vpmuludq $r4, $r1, $A5 -+ -+ vpsllq \$1, $A3, $A3 -+ vpsllq \$1, $A4, $A4 -+ vpmuludq $r2, $r2, $T0 -+ vpaddq $T0, $A4, $A4 -+ vpmuludq $r3, $r2, $T0 -+ vpaddq $T0, $A5, $A5 -+ vpmuludq $r4, $r2, $A6 -+ -+ vpsllq \$1, $A5, $A5 -+ vpsllq \$1, $A6, $A6 -+ vpmuludq $r3, $r3, $T0 -+ vpaddq $T0, $A6, $A6 -+ vpmuludq $r4, $r3, $A7 -+ -+ vpsllq \$1, $A7, $A7 -+ vpmuludq $r4, $r4, $A8 -+ -+ # Reduce -+ vpsrlq \$26, $A4, $T0 -+ vpand .LandMask(%rip), $A4, $A4 -+ vpaddq $T0, $A5, $A5 -+ -+ vpsllq \$2, $A5, $T0 -+ vpaddq $T0, $A5, $A5 -+ vpsllq \$2, $A6, $T0 -+ vpaddq $T0, $A6, $A6 -+ vpsllq \$2, $A7, $T0 -+ vpaddq $T0, $A7, $A7 -+ vpsllq \$2, $A8, $T0 -+ vpaddq $T0, $A8, $A8 -+ -+ vpaddq $A5, $A0, $A0 -+ vpaddq $A6, $A1, $A1 -+ vpaddq $A7, $A2, $A2 -+ vpaddq $A8, $A3, $A3 -+ -+ vpsrlq \$26, $A0, $T0 -+ vpand .LandMask(%rip), $A0, $A0 -+ vpaddq $T0, $A1, $A1 -+ vpsrlq \$26, $A1, $T0 -+ vpand .LandMask(%rip), $A1, $A1 -+ vpaddq $T0, $A2, $A2 -+ vpsrlq \$26, $A2, $T0 -+ vpand .LandMask(%rip), $A2, $A2 -+ vpaddq $T0, $A3, $A3 -+ vpsrlq \$26, $A3, $T0 -+ vpand .LandMask(%rip), $A3, $A3 -+ vpaddq $T0, $A4, $A4 -+ -+ vpunpcklqdq $r0, $A0, $r0 -+ vpunpcklqdq $r1, $A1, $r1 -+ vpunpcklqdq $r2, $A2, $r2 -+ vpunpcklqdq $r3, $A3, $r3 -+ vpunpcklqdq $r4, $A4, $r4 -+ -+ vmovdqu $r0, $_r0_($state) -+ vmovdqu $r1, $_r1_($state) -+ vmovdqu $r2, $_r2_($state) -+ vmovdqu $r3, $_r3_($state) -+ vmovdqu $r4, $_r4_($state) -+ -+ vpsllq \$2, $r1, $A1 -+ vpsllq \$2, $r2, $A2 -+ vpsllq \$2, $r3, $A3 -+ vpsllq \$2, $r4, $A4 -+ -+ vpaddq $A1, $r1, $A1 -+ vpaddq $A2, $r2, $A2 -+ vpaddq $A3, $r3, $A3 -+ vpaddq $A4, $r4, $A4 -+ -+ vmovdqu $A1, $_r1_x5($state) -+ vmovdqu $A2, $_r2_x5($state) -+ vmovdqu $A3, $_r3_x5($state) -+ vmovdqu $A4, $_r4_x5($state) -+ # Store k -+ vmovdqu 16*1($key), $T0 -+ vmovdqu $T0, $_k_($state) -+ # Init the MAC value -+ vpxor $T0, $T0, $T0 -+ vmovdqu $T0, $_A0_($state) -+ vmovd $T0, $_A4_($state) -+ vzeroupper -+ ret -+.size poly1305_init_avx,.-poly1305_init_avx -+___ -+} -+ -+{ -+ -+my ($A0, $A1, $A2, $A3, $A4, -+ $T0, $T1, $R0, $R1, $R2, -+ $R3, $R4, $AND_MASK)=map("%xmm$_",(0..12)); -+ -+my ($state, $in, $in_len)=("%rdi", "%rsi", "%rdx"); -+ -+$code.=<<___; -+ -+############################################################################### -+# void* poly1305_update_avx(void* $state, void* in, uint64_t in_len) -+.globl poly1305_update_avx -+.type poly1305_update_avx, \@function, 2 -+.align 64 -+poly1305_update_avx: -+ -+ vzeroupper -+ vmovd $_A0_($state), $A0 -+ vmovd $_A1_($state), $A1 -+ vmovd $_A2_($state), $A2 -+ vmovd $_A3_($state), $A3 -+ vmovd $_A4_($state), $A4 -+ vmovdqa .LandMask(%rip), $AND_MASK -+ # Skip to single block case -+ cmp \$32, $in_len -+ jb 3f -+1: -+ cmp \$16*4, $in_len -+ jb 1f -+ sub \$16*2, $in_len -+ # load the next two blocks -+ vmovdqu 16*0($in), $R2 -+ vmovdqu 16*1($in), $R3 -+ add \$16*2, $in -+ -+ vpunpcklqdq $R3, $R2, $R0 -+ vpunpckhqdq $R3, $R2, $R1 -+ -+ vpsrlq \$26, $R0, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A0, $A0 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpand $AND_MASK, $R2, $R2 -+ vpaddq $R2, $A1, $A1 -+ -+ vpsllq \$12, $R1, $R2 -+ vpxor $R2, $R0, $R0 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A2, $A2 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpsrlq \$40, $R1, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpxor .LsetBit(%rip), $R2, $R2 -+ vpaddq $R0, $A3, $A3 -+ vpaddq $R2, $A4, $A4 -+ -+ # Multiply input by R[0] -+ vbroadcastss $_r0_($state), $T0 -+ vpmuludq $T0, $A0, $R0 -+ vpmuludq $T0, $A1, $R1 -+ vpmuludq $T0, $A2, $R2 -+ vpmuludq $T0, $A3, $R3 -+ vpmuludq $T0, $A4, $R4 -+ # Multiply input by R[1] (and R[1]*5) -+ vbroadcastss $_r1_x5($state), $T0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R0, $R0 -+ vbroadcastss $_r1_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R4, $R4 -+ # Etc -+ vbroadcastss $_r2_x5($state), $T0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R1, $R1 -+ vbroadcastss $_r2_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R4, $R4 -+ -+ vbroadcastss $_r3_x5($state), $T0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R2, $R2 -+ vbroadcastss $_r3_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R4, $R4 -+ -+ vbroadcastss $_r4_x5($state), $T0 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R3, $R3 -+ vbroadcastss $_r4_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R4, $R4 -+ # Reduce -+ vpsrlq \$26, $R3, $T0 -+ vpaddq $T0, $R4, $R4 -+ vpand $AND_MASK, $R3, $R3 -+ -+ vpsrlq \$26, $R4, $T0 -+ vpsllq \$2, $T0, $T1 -+ vpaddq $T1, $T0, $T0 -+ vpaddq $T0, $R0, $R0 -+ vpand $AND_MASK, $R4, $R4 -+ -+ vpsrlq \$26, $R0, $T0 -+ vpand $AND_MASK, $R0, $A0 -+ vpaddq $T0, $R1, $R1 -+ vpsrlq \$26, $R1, $T0 -+ vpand $AND_MASK, $R1, $A1 -+ vpaddq $T0, $R2, $R2 -+ vpsrlq \$26, $R2, $T0 -+ vpand $AND_MASK, $R2, $A2 -+ vpaddq $T0, $R3, $R3 -+ vpsrlq \$26, $R3, $T0 -+ vpand $AND_MASK, $R3, $A3 -+ vpaddq $T0, $R4, $A4 -+ jmp 1b -+1: -+ cmp \$16*2, $in_len -+ jb 1f -+ sub \$16*2, $in_len -+ # load the next two blocks -+ vmovdqu 16*0($in), $R2 -+ vmovdqu 16*1($in), $R3 -+ add \$16*2, $in -+ -+ vpunpcklqdq $R3, $R2, $R0 -+ vpunpckhqdq $R3, $R2, $R1 -+ -+ vpsrlq \$26, $R0, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A0, $A0 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpand $AND_MASK, $R2, $R2 -+ vpaddq $R2, $A1, $A1 -+ -+ vpsllq \$12, $R1, $R2 -+ vpxor $R2, $R0, $R0 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A2, $A2 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpsrlq \$40, $R1, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpxor .LsetBit(%rip), $R2, $R2 -+ vpaddq $R0, $A3, $A3 -+ vpaddq $R2, $A4, $A4 -+ -+ # Multiply input by R[0] -+ vmovdqu $_r0_($state), $T0 -+ vpmuludq $T0, $A0, $R0 -+ vpmuludq $T0, $A1, $R1 -+ vpmuludq $T0, $A2, $R2 -+ vpmuludq $T0, $A3, $R3 -+ vpmuludq $T0, $A4, $R4 -+ # Multiply input by R[1] (and R[1]*5) -+ vmovdqu $_r1_x5($state), $T0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R0, $R0 -+ vmovdqu $_r1_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R4, $R4 -+ # Etc -+ vmovdqu $_r2_x5($state), $T0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R1, $R1 -+ vmovdqu $_r2_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R4, $R4 -+ -+ vmovdqu $_r3_x5($state), $T0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R2, $R2 -+ vmovdqu $_r3_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R4, $R4 -+ -+ vmovdqu $_r4_x5($state), $T0 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R3, $R3 -+ vmovdqu $_r4_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R4, $R4 -+1: -+ vpsrldq \$8, $R0, $A0 -+ vpsrldq \$8, $R1, $A1 -+ vpsrldq \$8, $R2, $A2 -+ vpsrldq \$8, $R3, $A3 -+ vpsrldq \$8, $R4, $A4 -+ -+ vpaddq $R0, $A0, $A0 -+ vpaddq $R1, $A1, $A1 -+ vpaddq $R2, $A2, $A2 -+ vpaddq $R3, $A3, $A3 -+ vpaddq $R4, $A4, $A4 -+ # Reduce -+ vpsrlq \$26, $A3, $T0 -+ vpaddq $T0, $A4, $A4 -+ vpand $AND_MASK, $A3, $A3 -+ vpsrlq \$26, $A4, $T0 -+ vpsllq \$2, $T0, $T1 -+ vpaddq $T1, $T0, $T0 -+ vpaddq $T0, $A0, $A0 -+ vpand $AND_MASK, $A4, $A4 -+ vpsrlq \$26, $A0, $T0 -+ vpand $AND_MASK, $A0, $A0 -+ vpaddq $T0, $A1, $A1 -+ vpsrlq \$26, $A1, $T0 -+ vpand $AND_MASK, $A1, $A1 -+ vpaddq $T0, $A2, $A2 -+ vpsrlq \$26, $A2, $T0 -+ vpand $AND_MASK, $A2, $A2 -+ vpaddq $T0, $A3, $A3 -+ vpsrlq \$26, $A3, $T0 -+ vpand $AND_MASK, $A3, $A3 -+ vpaddq $T0, $A4, $A4 -+3: -+ cmp \$16, $in_len -+ jb 1f -+ -+ # load the next block -+ vmovq 8*0($in), $R0 -+ vmovq 8*1($in), $R1 -+ add \$16, $in -+ sub \$16, $in_len -+ -+ vpsrlq \$26, $R0, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A0, $A0 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpand $AND_MASK, $R2, $R2 -+ vpaddq $R2, $A1, $A1 -+ -+ vpsllq \$12, $R1, $R2 -+ vpxor $R2, $R0, $R0 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A2, $A2 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpsrlq \$40, $R1, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpxor .LsetBit(%rip), $R2, $R2 -+ vpaddq $R0, $A3, $A3 -+ vpaddq $R2, $A4, $A4 -+2: -+ # Multiply input by R[0] -+ vmovq $_r0_+8($state), $T0 -+ vpmuludq $T0, $A0, $R0 -+ vpmuludq $T0, $A1, $R1 -+ vpmuludq $T0, $A2, $R2 -+ vpmuludq $T0, $A3, $R3 -+ vpmuludq $T0, $A4, $R4 -+ # Multiply input by R[1] (and R[1]*5) -+ vmovq $_r1_x5+8($state), $T0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R0, $R0 -+ vmovq $_r1_+8($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R4, $R4 -+ # Etc -+ vmovq $_r2_x5+8($state), $T0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R1, $R1 -+ vmovq $_r2_+8($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R4, $R4 -+ -+ vmovq $_r3_x5+8($state), $T0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R2, $R2 -+ vmovq $_r3_+8($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R4, $R4 -+ -+ vmovq $_r4_x5+8($state), $T0 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R3, $R3 -+ vmovq $_r4_+8($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R4, $R4 -+ -+ # Reduce -+ vpsrlq \$26, $R3, $T0 -+ vpaddq $T0, $R4, $R4 -+ vpand $AND_MASK, $R3, $R3 -+ vpsrlq \$26, $R4, $T0 -+ vpsllq \$2, $T0, $T1 -+ vpaddq $T1, $T0, $T0 -+ vpaddq $T0, $R0, $R0 -+ vpand $AND_MASK, $R4, $R4 -+ vpsrlq \$26, $R0, $T0 -+ vpand $AND_MASK, $R0, $A0 -+ vpaddq $T0, $R1, $R1 -+ vpsrlq \$26, $R1, $T0 -+ vpand $AND_MASK, $R1, $A1 -+ vpaddq $T0, $R2, $R2 -+ vpsrlq \$26, $R2, $T0 -+ vpand $AND_MASK, $R2, $A2 -+ vpaddq $T0, $R3, $R3 -+ vpsrlq \$26, $R3, $T0 -+ vpand $AND_MASK, $R3, $A3 -+ vpaddq $T0, $R4, $A4 -+ -+1: -+ test $in_len, $in_len -+ jz 1f -+ -+ vmovdqa .Lone(%rip), $R0 -+3: -+ dec $in_len -+ vpslldq \$1, $R0, $R0 -+ vpinsrb \$0, ($in, $in_len), $R0, $R0 -+ test $in_len, $in_len -+ jnz 3b -+ -+ vpsrldq \$8, $R0, $R1 -+ vpsrlq \$26, $R0, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A0, $A0 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpand $AND_MASK, $R2, $R2 -+ vpaddq $R2, $A1, $A1 -+ -+ vpsllq \$12, $R1, $R2 -+ vpxor $R2, $R0, $R0 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A2, $A2 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpsrlq \$40, $R1, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A3, $A3 -+ vpaddq $R2, $A4, $A4 -+ xor $in_len, $in_len -+ jmp 2b -+1: -+ vmovd $A0, $_A0_($state) -+ vmovd $A1, $_A1_($state) -+ vmovd $A2, $_A2_($state) -+ vmovd $A3, $_A3_($state) -+ vmovd $A4, $_A4_($state) -+ -+ -+ mov $in, %rax -+ vzeroupper -+ ret -+.size poly1305_update_avx,.-poly1305_update_avx -+############################################################################### -+# void poly1305_finish_avx(void* $state, uint64_t mac[2]); -+.type poly1305_finish_avx,\@function, 2 -+.globl poly1305_finish_avx -+poly1305_finish_avx: -+___ -+my $mac="%rsi"; -+$code.=<<___; -+ vzeroupper -+ vmovd $_A0_($state), $A0 -+ vmovd $_A1_($state), $A1 -+ vmovd $_A2_($state), $A2 -+ vmovd $_A3_($state), $A3 -+ vmovd $_A4_($state), $A4 -+ # Reduce one last time in case there was a carry from 130 bit -+ vpsrlq \$26, $A4, $T0 -+ vpsllq \$2, $T0, $T1 -+ vpaddq $T1, $T0, $T0 -+ vpaddq $T0, $A0, $A0 -+ vpand .LandMask(%rip), $A4, $A4 -+ -+ vpsrlq \$26, $A0, $T0 -+ vpand .LandMask(%rip), $A0, $A0 -+ vpaddq $T0, $A1, $A1 -+ vpsrlq \$26, $A1, $T0 -+ vpand .LandMask(%rip), $A1, $A1 -+ vpaddq $T0, $A2, $A2 -+ vpsrlq \$26, $A2, $T0 -+ vpand .LandMask(%rip), $A2, $A2 -+ vpaddq $T0, $A3, $A3 -+ vpsrlq \$26, $A3, $T0 -+ vpand .LandMask(%rip), $A3, $A3 -+ vpaddq $T0, $A4, $A4 -+ # Convert to normal -+ vpsllq \$26, $A1, $T0 -+ vpxor $T0, $A0, $A0 -+ vpsllq \$52, $A2, $T0 -+ vpxor $T0, $A0, $A0 -+ vpsrlq \$12, $A2, $A1 -+ vpsllq \$14, $A3, $T0 -+ vpxor $T0, $A1, $A1 -+ vpsllq \$40, $A4, $T0 -+ vpxor $T0, $A1, $A1 -+ vmovq $A0, %rax -+ vmovq $A1, %rdx -+ -+ add $_k_($state), %rax -+ adc $_k_+8($state), %rdx -+ mov %rax, ($mac) -+ mov %rdx, 8($mac) -+ vzeroupper -+ ret -+.size poly1305_finish_avx,.-poly1305_finish_avx -+___ -+} -+}} -+ -+$code =~ s/\`([^\`]*)\`/eval($1)/gem; -+print $code; -+close STDOUT; -diff --git a/crypto/chacha20poly1305/asm/poly1305_avx2.pl b/crypto/chacha20poly1305/asm/poly1305_avx2.pl -new file mode 100644 -index 0000000..8134542 ---- /dev/null -+++ b/crypto/chacha20poly1305/asm/poly1305_avx2.pl -@@ -0,0 +1,918 @@ -+############################################################################## -+# # -+# Copyright 2014 Intel Corporation # -+# # -+# Licensed under the Apache License, Version 2.0 (the "License"); # -+# you may not use this file except in compliance with the License. # -+# You may obtain a copy of the License at # -+# # -+# http://www.apache.org/licenses/LICENSE-2.0 # -+# # -+# Unless required by applicable law or agreed to in writing, software # -+# distributed under the License is distributed on an "AS IS" BASIS, # -+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -+# See the License for the specific language governing permissions and # -+# limitations under the License. # -+# # -+############################################################################## -+# # -+# Developers and authors: # -+# Shay Gueron (1, 2), and Vlad Krasnov (1) # -+# (1) Intel Corporation, Israel Development Center # -+# (2) University of Haifa # -+# # -+############################################################################## -+# state: -+# 0: r[0] || r^2[0] -+# 16: r[1] || r^2[1] -+# 32: r[2] || r^2[2] -+# 48: r[3] || r^2[3] -+# 64: r[4] || r^2[4] -+# 80: r[1]*5 || r^2[1]*5 -+# 96: r[2]*5 || r^2[2]*5 -+#112: r[3]*5 || r^2[3]*5 -+#128: r[4]*5 || r^2[4]*5 -+#144: k -+#160: A0 -+#164: A1 -+#168: A2 -+#172: A3 -+#176: A4 -+#180: END -+ -+$flavour = shift; -+$output = shift; -+if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } -+ -+$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); -+ -+$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; -+( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or -+( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or -+die "can't locate x86_64-xlate.pl"; -+ -+open OUT,"| \"$^X\" $xlate $flavour $output"; -+*STDOUT=*OUT; -+ -+if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` -+ =~ /GNU assembler version ([2-9]\.[0-9]+)/) { -+ $avx = ($1>=2.19) + ($1>=2.22); -+} -+ -+if ($win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && -+ `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { -+ $avx = ($1>=2.09) + ($1>=2.10); -+} -+ -+if ($win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && -+ `ml64 2>&1` =~ /Version ([0-9]+)\./) { -+ $avx = ($1>=10) + ($1>=11); -+} -+ -+if (`$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9])\.([0-9]+)/) { -+ my $ver = $2 + $3/100.0; # 3.1->3.01, 3.10->3.10 -+ $avx = ($ver>=3.0) + ($ver>=3.01); -+} -+ -+if ($avx>=1) {{ -+ -+my ($_r0_, $_r1_, $_r2_, $_r3_, $_r4_, $_r1_x5, $_r2_x5, $_r3_x5, $_r4_x5, $_k_, $_A0_, $_A1_, $_A2_, $_A3_, $_A4_) -+= (0,32,64,96,128,160,192,224,256,288,304,308,312,316,320); -+ -+$code.=<<___; -+.text -+.align 32 -+.LandMask: -+.quad 0x3FFFFFF, 0x3FFFFFF, 0x3FFFFFF, 0x3FFFFFF -+.LsetBit: -+.quad 0x1000000, 0x1000000, 0x1000000, 0x1000000 -+.LrSet: -+.quad 0xFFFFFFC0FFFFFFF, 0xFFFFFFC0FFFFFFF, 0xFFFFFFC0FFFFFFF, 0xFFFFFFC0FFFFFFF -+.quad 0xFFFFFFC0FFFFFFC, 0xFFFFFFC0FFFFFFC, 0xFFFFFFC0FFFFFFC, 0xFFFFFFC0FFFFFFC -+ -+.LpermFix: -+.long 6,7,6,7,6,7,6,7 -+.long 4,5,6,7,6,7,6,7 -+.long 2,3,6,7,4,5,6,7 -+.long 0,1,4,5,2,3,6,7 -+___ -+ -+ -+{ -+my ($A0, $A1, $A2, $A3, $A4, -+ $r0, $r1, $r2, $r3, $r4, -+ $T0, $T1, $A5, $A6, $A7, $A8)=map("%xmm$_",(0..15)); -+my ($A0_y, $A1_y, $A2_y, $A3_y, $A4_y, -+ $r0_y, $r1_y, $r2_y, $r3_y, $r4_y)=map("%ymm$_",(0..9)); -+my ($state, $key) -+ =("%rdi", "%rsi"); -+ -+$code.=<<___; -+################################################################################ -+# void poly1305_init_avx2(void *state, uint8_t key[32]) -+ -+.globl poly1305_init_avx2 -+.type poly1305_init_avx2, \@function, 2 -+.align 64 -+poly1305_init_avx2: -+ vzeroupper -+ -+ # Store k -+ vmovdqu 16*1($key), $T0 -+ vmovdqu $T0, $_k_($state) -+ # Init the MAC value -+ vpxor $T0, $T0, $T0 -+ vmovdqu $T0, $_A0_($state) -+ vmovd $T0, $_A4_($state) -+ # load and convert r -+ vmovq 8*0($key), $r0 -+ vmovq 8*1($key), $T0 -+ vpand .LrSet(%rip), $r0, $r0 -+ vpand .LrSet+32(%rip), $T0, $T0 -+ -+ vpsrlq \$26, $r0, $r1 -+ vpand .LandMask(%rip), $r0, $r0 -+ vpsrlq \$26, $r1, $r2 -+ vpand .LandMask(%rip), $r1, $r1 -+ vpsllq \$12, $T0, $T1 -+ vpxor $T1, $r2, $r2 -+ vpsrlq \$26, $r2, $r3 -+ vpsrlq \$40, $T0, $r4 -+ vpand .LandMask(%rip), $r2, $r2 -+ vpand .LandMask(%rip), $r3, $r3 -+ # SQR R -+ vpmuludq $r0, $r0, $A0 -+ vpmuludq $r1, $r0, $A1 -+ vpmuludq $r2, $r0, $A2 -+ vpmuludq $r3, $r0, $A3 -+ vpmuludq $r4, $r0, $A4 -+ -+ vpsllq \$1, $A1, $A1 -+ vpsllq \$1, $A2, $A2 -+ vpmuludq $r1, $r1, $T0 -+ vpaddq $T0, $A2, $A2 -+ vpmuludq $r2, $r1, $T0 -+ vpaddq $T0, $A3, $A3 -+ vpmuludq $r3, $r1, $T0 -+ vpaddq $T0, $A4, $A4 -+ vpmuludq $r4, $r1, $A5 -+ -+ vpsllq \$1, $A3, $A3 -+ vpsllq \$1, $A4, $A4 -+ vpmuludq $r2, $r2, $T0 -+ vpaddq $T0, $A4, $A4 -+ vpmuludq $r3, $r2, $T0 -+ vpaddq $T0, $A5, $A5 -+ vpmuludq $r4, $r2, $A6 -+ -+ vpsllq \$1, $A5, $A5 -+ vpsllq \$1, $A6, $A6 -+ vpmuludq $r3, $r3, $T0 -+ vpaddq $T0, $A6, $A6 -+ vpmuludq $r4, $r3, $A7 -+ -+ vpsllq \$1, $A7, $A7 -+ vpmuludq $r4, $r4, $A8 -+ -+ # Reduce -+ vpsrlq \$26, $A4, $T0 -+ vpand .LandMask(%rip), $A4, $A4 -+ vpaddq $T0, $A5, $A5 -+ -+ vpsllq \$2, $A5, $T0 -+ vpaddq $T0, $A5, $A5 -+ vpsllq \$2, $A6, $T0 -+ vpaddq $T0, $A6, $A6 -+ vpsllq \$2, $A7, $T0 -+ vpaddq $T0, $A7, $A7 -+ vpsllq \$2, $A8, $T0 -+ vpaddq $T0, $A8, $A8 -+ -+ vpaddq $A5, $A0, $A0 -+ vpaddq $A6, $A1, $A1 -+ vpaddq $A7, $A2, $A2 -+ vpaddq $A8, $A3, $A3 -+ -+ vpsrlq \$26, $A0, $T0 -+ vpand .LandMask(%rip), $A0, $A0 -+ vpaddq $T0, $A1, $A1 -+ vpsrlq \$26, $A1, $T0 -+ vpand .LandMask(%rip), $A1, $A1 -+ vpaddq $T0, $A2, $A2 -+ vpsrlq \$26, $A2, $T0 -+ vpand .LandMask(%rip), $A2, $A2 -+ vpaddq $T0, $A3, $A3 -+ vpsrlq \$26, $A3, $T0 -+ vpand .LandMask(%rip), $A3, $A3 -+ vpaddq $T0, $A4, $A4 -+ -+ vpunpcklqdq $r0, $A0, $r0 -+ vpunpcklqdq $r1, $A1, $r1 -+ vpunpcklqdq $r2, $A2, $r2 -+ vpunpcklqdq $r3, $A3, $r3 -+ vpunpcklqdq $r4, $A4, $r4 -+ -+ vmovdqu $r0, $_r0_+16($state) -+ vmovdqu $r1, $_r1_+16($state) -+ vmovdqu $r2, $_r2_+16($state) -+ vmovdqu $r3, $_r3_+16($state) -+ vmovdqu $r4, $_r4_+16($state) -+ -+ vpsllq \$2, $r1, $A1 -+ vpsllq \$2, $r2, $A2 -+ vpsllq \$2, $r3, $A3 -+ vpsllq \$2, $r4, $A4 -+ -+ vpaddq $A1, $r1, $A1 -+ vpaddq $A2, $r2, $A2 -+ vpaddq $A3, $r3, $A3 -+ vpaddq $A4, $r4, $A4 -+ -+ vmovdqu $A1, $_r1_x5+16($state) -+ vmovdqu $A2, $_r2_x5+16($state) -+ vmovdqu $A3, $_r3_x5+16($state) -+ vmovdqu $A4, $_r4_x5+16($state) -+ -+ # Compute r^3 and r^4 -+ vpshufd \$0x44, $r0, $A0 -+ vpshufd \$0x44, $r1, $A1 -+ vpshufd \$0x44, $r2, $A2 -+ vpshufd \$0x44, $r3, $A3 -+ vpshufd \$0x44, $r4, $A4 -+ -+ # Multiply input by R[0] -+ vmovdqu $_r0_+16($state), $T0 -+ vpmuludq $T0, $A0, $r0 -+ vpmuludq $T0, $A1, $r1 -+ vpmuludq $T0, $A2, $r2 -+ vpmuludq $T0, $A3, $r3 -+ vpmuludq $T0, $A4, $r4 -+ # Multiply input by R[1] (and R[1]*5) -+ vmovdqu $_r1_x5+16($state), $T0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $r0, $r0 -+ vmovdqu $_r1_+16($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $r1, $r1 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $r2, $r2 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $r3, $r3 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $r4, $r4 -+ # Etc -+ vmovdqu $_r2_x5+16($state), $T0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $r0, $r0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $r1, $r1 -+ vmovdqu $_r2_+16($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $r2, $r2 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $r3, $r3 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $r4, $r4 -+ -+ vmovdqu $_r3_x5+16($state), $T0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $r0, $r0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $r1, $r1 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $r2, $r2 -+ vmovdqu $_r3_+16($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $r3, $r3 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $r4, $r4 -+ -+ vmovdqu $_r4_x5+16($state), $T0 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $r0, $r0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $r1, $r1 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $r2, $r2 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $r3, $r3 -+ vmovdqu $_r4_+16($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $r4, $r4 -+ # Reduce -+ vpsrlq \$26, $r3, $T0 -+ vpaddq $T0, $r4, $r4 -+ vpand .LandMask(%rip), $r3, $r3 -+ vpsrlq \$26, $r4, $T0 -+ vpsllq \$2, $T0, $T1 -+ vpaddq $T1, $T0, $T0 -+ vpaddq $T0, $r0, $r0 -+ vpand .LandMask(%rip), $r4, $r4 -+ vpsrlq \$26, $r0, $T0 -+ vpand .LandMask(%rip), $r0, $r0 -+ vpaddq $T0, $r1, $r1 -+ vpsrlq \$26, $r1, $T0 -+ vpand .LandMask(%rip), $r1, $r1 -+ vpaddq $T0, $r2, $r2 -+ vpsrlq \$26, $r2, $T0 -+ vpand .LandMask(%rip), $r2, $r2 -+ vpaddq $T0, $r3, $r3 -+ vpsrlq \$26, $r3, $T0 -+ vpand .LandMask(%rip), $r3, $r3 -+ vpaddq $T0, $r4, $r4 -+ -+ vmovdqu $r0, $_r0_($state) -+ vmovdqu $r1, $_r1_($state) -+ vmovdqu $r2, $_r2_($state) -+ vmovdqu $r3, $_r3_($state) -+ vmovdqu $r4, $_r4_($state) -+ -+ vpsllq \$2, $r1, $A1 -+ vpsllq \$2, $r2, $A2 -+ vpsllq \$2, $r3, $A3 -+ vpsllq \$2, $r4, $A4 -+ -+ vpaddq $A1, $r1, $A1 -+ vpaddq $A2, $r2, $A2 -+ vpaddq $A3, $r3, $A3 -+ vpaddq $A4, $r4, $A4 -+ -+ vmovdqu $A1, $_r1_x5($state) -+ vmovdqu $A2, $_r2_x5($state) -+ vmovdqu $A3, $_r3_x5($state) -+ vmovdqu $A4, $_r4_x5($state) -+ -+ ret -+.size poly1305_init_avx2,.-poly1305_init_avx2 -+___ -+} -+ -+{ -+ -+my ($A0, $A1, $A2, $A3, $A4, -+ $T0, $T1, $R0, $R1, $R2, -+ $R3, $R4, $AND_MASK, $PERM_MASK, $SET_MASK)=map("%ymm$_",(0..14)); -+ -+my ($A0_x, $A1_x, $A2_x, $A3_x, $A4_x, -+ $T0_x, $T1_x, $R0_x, $R1_x, $R2_x, -+ $R3_x, $R4_x, $AND_MASK_x, $PERM_MASK_x, $SET_MASK_x)=map("%xmm$_",(0..14)); -+ -+my ($state, $in, $in_len, $hlp, $rsp_save)=("%rdi", "%rsi", "%rdx", "%rcx", "%rax"); -+ -+$code.=<<___; -+ -+############################################################################### -+# void poly1305_update_avx2(void* $state, void* in, uint64_t in_len2) -+.globl poly1305_update_avx2 -+.type poly1305_update_avx2, \@function, 2 -+.align 64 -+poly1305_update_avx2: -+ -+ vmovd $_A0_($state), $A0_x -+ vmovd $_A1_($state), $A1_x -+ vmovd $_A2_($state), $A2_x -+ vmovd $_A3_($state), $A3_x -+ vmovd $_A4_($state), $A4_x -+ -+ vmovdqa .LandMask(%rip), $AND_MASK -+1: -+ cmp \$32*4, $in_len -+ jb 1f -+ sub \$32*2, $in_len -+ -+ # load the next four blocks -+ vmovdqu 32*0($in), $R2 -+ vmovdqu 32*1($in), $R3 -+ add \$32*2, $in -+ -+ vpunpcklqdq $R3, $R2, $R0 -+ vpunpckhqdq $R3, $R2, $R1 -+ -+ vpermq \$0xD8, $R0, $R0 # it is possible to rearrange the precomputations, and save this shuffle -+ vpermq \$0xD8, $R1, $R1 -+ -+ vpsrlq \$26, $R0, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A0, $A0 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpand $AND_MASK, $R2, $R2 -+ vpaddq $R2, $A1, $A1 -+ -+ vpsllq \$12, $R1, $R2 -+ vpxor $R2, $R0, $R0 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A2, $A2 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpsrlq \$40, $R1, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpxor .LsetBit(%rip), $R2, $R2 -+ vpaddq $R0, $A3, $A3 -+ vpaddq $R2, $A4, $A4 -+ -+ # Multiply input by R[0] -+ vpbroadcastq $_r0_($state), $T0 -+ vpmuludq $T0, $A0, $R0 -+ vpmuludq $T0, $A1, $R1 -+ vpmuludq $T0, $A2, $R2 -+ vpmuludq $T0, $A3, $R3 -+ vpmuludq $T0, $A4, $R4 -+ # Multiply input by R[1] (and R[1]*5) -+ vpbroadcastq $_r1_x5($state), $T0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpbroadcastq $_r1_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R4, $R4 -+ # Etc -+ vpbroadcastq $_r2_x5($state), $T0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpbroadcastq $_r2_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R4, $R4 -+ -+ vpbroadcastq $_r3_x5($state), $T0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpbroadcastq $_r3_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R4, $R4 -+ -+ vpbroadcastq $_r4_x5($state), $T0 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpbroadcastq $_r4_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R4, $R4 -+ # Reduce -+ vpsrlq \$26, $R3, $T0 -+ vpaddq $T0, $R4, $R4 -+ vpand $AND_MASK, $R3, $R3 -+ -+ vpsrlq \$26, $R4, $T0 -+ vpsllq \$2, $T0, $T1 -+ vpaddq $T1, $T0, $T0 -+ vpaddq $T0, $R0, $R0 -+ vpand $AND_MASK, $R4, $R4 -+ -+ vpsrlq \$26, $R0, $T0 -+ vpand $AND_MASK, $R0, $A0 -+ vpaddq $T0, $R1, $R1 -+ vpsrlq \$26, $R1, $T0 -+ vpand $AND_MASK, $R1, $A1 -+ vpaddq $T0, $R2, $R2 -+ vpsrlq \$26, $R2, $T0 -+ vpand $AND_MASK, $R2, $A2 -+ vpaddq $T0, $R3, $R3 -+ vpsrlq \$26, $R3, $T0 -+ vpand $AND_MASK, $R3, $A3 -+ vpaddq $T0, $R4, $A4 -+ jmp 1b -+1: -+ -+ cmp \$32*2, $in_len -+ jb 1f -+ sub \$32*2, $in_len -+ # load the next four blocks -+ vmovdqu 32*0($in), $R2 -+ vmovdqu 32*1($in), $R3 -+ add \$32*2, $in -+ -+ vpunpcklqdq $R3, $R2, $R0 -+ vpunpckhqdq $R3, $R2, $R1 -+ -+ vpermq \$0xD8, $R0, $R0 -+ vpermq \$0xD8, $R1, $R1 -+ -+ vpsrlq \$26, $R0, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A0, $A0 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpand $AND_MASK, $R2, $R2 -+ vpaddq $R2, $A1, $A1 -+ -+ vpsllq \$12, $R1, $R2 -+ vpxor $R2, $R0, $R0 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A2, $A2 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpsrlq \$40, $R1, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpxor .LsetBit(%rip), $R2, $R2 -+ vpaddq $R0, $A3, $A3 -+ vpaddq $R2, $A4, $A4 -+ -+ # Multiply input by R[0] -+ vmovdqu $_r0_($state), $T0 -+ vpmuludq $T0, $A0, $R0 -+ vpmuludq $T0, $A1, $R1 -+ vpmuludq $T0, $A2, $R2 -+ vpmuludq $T0, $A3, $R3 -+ vpmuludq $T0, $A4, $R4 -+ # Multiply input by R[1] (and R[1]*5) -+ vmovdqu $_r1_x5($state), $T0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R0, $R0 -+ vmovdqu $_r1_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R4, $R4 -+ # Etc -+ vmovdqu $_r2_x5($state), $T0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R1, $R1 -+ vmovdqu $_r2_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R4, $R4 -+ -+ vmovdqu $_r3_x5($state), $T0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R2, $R2 -+ vmovdqu $_r3_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R4, $R4 -+ -+ vmovdqu $_r4_x5($state), $T0 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R3, $R3 -+ vmovdqu $_r4_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R4, $R4 -+ # Reduce -+ vpsrlq \$26, $R3, $T0 -+ vpaddq $T0, $R4, $R4 -+ vpand $AND_MASK, $R3, $R3 -+ vpsrlq \$26, $R4, $T0 -+ vpsllq \$2, $T0, $T1 -+ vpaddq $T1, $T0, $T0 -+ vpaddq $T0, $R0, $R0 -+ vpand $AND_MASK, $R4, $R4 -+ vpsrlq \$26, $R0, $T0 -+ vpand $AND_MASK, $R0, $A0 -+ vpaddq $T0, $R1, $R1 -+ vpsrlq \$26, $R1, $T0 -+ vpand $AND_MASK, $R1, $A1 -+ vpaddq $T0, $R2, $R2 -+ vpsrlq \$26, $R2, $T0 -+ vpand $AND_MASK, $R2, $A2 -+ vpaddq $T0, $R3, $R3 -+ vpsrlq \$26, $R3, $T0 -+ vpand $AND_MASK, $R3, $A3 -+ vpaddq $T0, $R4, $A4 -+ -+ vpsrldq \$8, $A0, $R0 -+ vpsrldq \$8, $A1, $R1 -+ vpsrldq \$8, $A2, $R2 -+ vpsrldq \$8, $A3, $R3 -+ vpsrldq \$8, $A4, $R4 -+ -+ vpaddq $R0, $A0, $A0 -+ vpaddq $R1, $A1, $A1 -+ vpaddq $R2, $A2, $A2 -+ vpaddq $R3, $A3, $A3 -+ vpaddq $R4, $A4, $A4 -+ -+ vpermq \$0xAA, $A0, $R0 -+ vpermq \$0xAA, $A1, $R1 -+ vpermq \$0xAA, $A2, $R2 -+ vpermq \$0xAA, $A3, $R3 -+ vpermq \$0xAA, $A4, $R4 -+ -+ vpaddq $R0, $A0, $A0 -+ vpaddq $R1, $A1, $A1 -+ vpaddq $R2, $A2, $A2 -+ vpaddq $R3, $A3, $A3 -+ vpaddq $R4, $A4, $A4 -+1: -+ test $in_len, $in_len -+ jz 5f -+ # In case 1,2 or 3 blocks remain, we want to multiply them correctly -+ vmovq $A0_x, $A0_x -+ vmovq $A1_x, $A1_x -+ vmovq $A2_x, $A2_x -+ vmovq $A3_x, $A3_x -+ vmovq $A4_x, $A4_x -+ -+ mov .LsetBit(%rip), $hlp -+ mov %rsp, $rsp_save -+ test \$15, $in_len -+ jz 1f -+ xor $hlp, $hlp -+ sub \$64, %rsp -+ vpxor $R0, $R0, $R0 -+ vmovdqu $R0, (%rsp) -+ vmovdqu $R0, 32(%rsp) -+3: -+ movb ($in, $hlp), %r8b -+ movb %r8b, (%rsp, $hlp) -+ inc $hlp -+ cmp $hlp, $in_len -+ jne 3b -+ -+ movb \$1, (%rsp, $hlp) -+ xor $hlp, $hlp -+ mov %rsp, $in -+ -+1: -+ -+ cmp \$16, $in_len -+ ja 2f -+ vmovq 8*0($in), $R0_x -+ vmovq 8*1($in), $R1_x -+ vmovq $hlp, $SET_MASK_x -+ vmovdqa .LpermFix(%rip), $PERM_MASK -+ jmp 1f -+2: -+ cmp \$32, $in_len -+ ja 2f -+ vmovdqu 16*0($in), $R2_x -+ vmovdqu 16*1($in), $R3_x -+ vmovq .LsetBit(%rip), $SET_MASK_x -+ vpinsrq \$1, $hlp, $SET_MASK_x, $SET_MASK_x -+ vmovdqa .LpermFix+32(%rip), $PERM_MASK -+ -+ vpunpcklqdq $R3, $R2, $R0 -+ vpunpckhqdq $R3, $R2, $R1 -+ jmp 1f -+2: -+ cmp \$48, $in_len -+ ja 2f -+ vmovdqu 32*0($in), $R2 -+ vmovdqu 32*1($in), $R3_x -+ vmovq .LsetBit(%rip), $SET_MASK_x -+ vpinsrq \$1, $hlp, $SET_MASK_x, $SET_MASK_x -+ vpermq \$0xc4, $SET_MASK, $SET_MASK -+ vmovdqa .LpermFix+64(%rip), $PERM_MASK -+ -+ vpunpcklqdq $R3, $R2, $R0 -+ vpunpckhqdq $R3, $R2, $R1 -+ jmp 1f -+2: -+ vmovdqu 32*0($in), $R2 -+ vmovdqu 32*1($in), $R3 -+ vmovq .LsetBit(%rip), $SET_MASK_x -+ vpinsrq \$1, $hlp, $SET_MASK_x, $SET_MASK_x -+ vpermq \$0x40, $SET_MASK, $SET_MASK -+ vmovdqa .LpermFix+96(%rip), $PERM_MASK -+ -+ vpunpcklqdq $R3, $R2, $R0 -+ vpunpckhqdq $R3, $R2, $R1 -+ -+1: -+ mov $rsp_save, %rsp -+ -+ vpsrlq \$26, $R0, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A0, $A0 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpand $AND_MASK, $R2, $R2 -+ vpaddq $R2, $A1, $A1 -+ -+ vpsllq \$12, $R1, $R2 -+ vpxor $R2, $R0, $R0 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A2, $A2 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpsrlq \$40, $R1, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpxor $SET_MASK, $R2, $R2 -+ vpaddq $R0, $A3, $A3 -+ vpaddq $R2, $A4, $A4 -+ -+ # Multiply input by R[0] -+ vmovdqu $_r0_($state), $T0 -+ vpermd $T0, $PERM_MASK, $T0 -+ vpmuludq $T0, $A0, $R0 -+ vpmuludq $T0, $A1, $R1 -+ vpmuludq $T0, $A2, $R2 -+ vpmuludq $T0, $A3, $R3 -+ vpmuludq $T0, $A4, $R4 -+ # Multiply input by R[1] (and R[1]*5) -+ vmovdqu $_r1_x5($state), $T0 -+ vpermd $T0, $PERM_MASK, $T0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R0, $R0 -+ vmovdqu $_r1_($state), $T0 -+ vpermd $T0, $PERM_MASK, $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R4, $R4 -+ # Etc -+ vmovdqu $_r2_x5($state), $T0 -+ vpermd $T0, $PERM_MASK, $T0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R1, $R1 -+ vmovdqu $_r2_($state), $T0 -+ vpermd $T0, $PERM_MASK, $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R4, $R4 -+ -+ vmovdqu $_r3_x5($state), $T0 -+ vpermd $T0, $PERM_MASK, $T0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R2, $R2 -+ vmovdqu $_r3_($state), $T0 -+ vpermd $T0, $PERM_MASK, $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R4, $R4 -+ -+ vmovdqu $_r4_x5($state), $T0 -+ vpermd $T0, $PERM_MASK, $T0 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R3, $R3 -+ vmovdqu $_r4_($state), $T0 -+ vpermd $T0, $PERM_MASK, $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R4, $R4 -+ # Reduce -+ vpsrlq \$26, $R3, $T0 -+ vpaddq $T0, $R4, $R4 -+ vpand $AND_MASK, $R3, $R3 -+ vpsrlq \$26, $R4, $T0 -+ vpsllq \$2, $T0, $T1 -+ vpaddq $T1, $T0, $T0 -+ vpaddq $T0, $R0, $R0 -+ vpand $AND_MASK, $R4, $R4 -+ vpsrlq \$26, $R0, $T0 -+ vpand $AND_MASK, $R0, $A0 -+ vpaddq $T0, $R1, $R1 -+ vpsrlq \$26, $R1, $T0 -+ vpand $AND_MASK, $R1, $A1 -+ vpaddq $T0, $R2, $R2 -+ vpsrlq \$26, $R2, $T0 -+ vpand $AND_MASK, $R2, $A2 -+ vpaddq $T0, $R3, $R3 -+ vpsrlq \$26, $R3, $T0 -+ vpand $AND_MASK, $R3, $A3 -+ vpaddq $T0, $R4, $A4 -+ -+ vpsrldq \$8, $A0, $R0 -+ vpsrldq \$8, $A1, $R1 -+ vpsrldq \$8, $A2, $R2 -+ vpsrldq \$8, $A3, $R3 -+ vpsrldq \$8, $A4, $R4 -+ -+ vpaddq $R0, $A0, $A0 -+ vpaddq $R1, $A1, $A1 -+ vpaddq $R2, $A2, $A2 -+ vpaddq $R3, $A3, $A3 -+ vpaddq $R4, $A4, $A4 -+ -+ vpermq \$0xAA, $A0, $R0 -+ vpermq \$0xAA, $A1, $R1 -+ vpermq \$0xAA, $A2, $R2 -+ vpermq \$0xAA, $A3, $R3 -+ vpermq \$0xAA, $A4, $R4 -+ -+ vpaddq $R0, $A0, $A0 -+ vpaddq $R1, $A1, $A1 -+ vpaddq $R2, $A2, $A2 -+ vpaddq $R3, $A3, $A3 -+ vpaddq $R4, $A4, $A4 -+ -+5: -+ vmovd $A0_x, $_A0_($state) -+ vmovd $A1_x, $_A1_($state) -+ vmovd $A2_x, $_A2_($state) -+ vmovd $A3_x, $_A3_($state) -+ vmovd $A4_x, $_A4_($state) -+ -+ ret -+.size poly1305_update_avx2,.-poly1305_update_avx2 -+############################################################################### -+# void poly1305_finish_avx2(void* $state, uint8_t mac[16]); -+.type poly1305_finish_avx2,\@function,2 -+.globl poly1305_finish_avx2 -+poly1305_finish_avx2: -+___ -+my $mac="%rsi"; -+my ($A0, $A1, $A2, $A3, $A4, $T0, $T1)=map("%xmm$_",(0..6)); -+ -+$code.=<<___; -+ vmovd $_A0_($state), $A0 -+ vmovd $_A1_($state), $A1 -+ vmovd $_A2_($state), $A2 -+ vmovd $_A3_($state), $A3 -+ vmovd $_A4_($state), $A4 -+ # Reduce one last time in case there was a carry from 130 bit -+ vpsrlq \$26, $A4, $T0 -+ vpsllq \$2, $T0, $T1 -+ vpaddq $T1, $T0, $T0 -+ vpaddq $T0, $A0, $A0 -+ vpand .LandMask(%rip), $A4, $A4 -+ -+ vpsrlq \$26, $A0, $T0 -+ vpand .LandMask(%rip), $A0, $A0 -+ vpaddq $T0, $A1, $A1 -+ vpsrlq \$26, $A1, $T0 -+ vpand .LandMask(%rip), $A1, $A1 -+ vpaddq $T0, $A2, $A2 -+ vpsrlq \$26, $A2, $T0 -+ vpand .LandMask(%rip), $A2, $A2 -+ vpaddq $T0, $A3, $A3 -+ vpsrlq \$26, $A3, $T0 -+ vpand .LandMask(%rip), $A3, $A3 -+ vpaddq $T0, $A4, $A4 -+ # Convert to normal -+ vpsllq \$26, $A1, $T0 -+ vpxor $T0, $A0, $A0 -+ vpsllq \$52, $A2, $T0 -+ vpxor $T0, $A0, $A0 -+ vpsrlq \$12, $A2, $A1 -+ vpsllq \$14, $A3, $T0 -+ vpxor $T0, $A1, $A1 -+ vpsllq \$40, $A4, $T0 -+ vpxor $T0, $A1, $A1 -+ vmovq $A0, %rax -+ vmovq $A1, %rdx -+ -+ add $_k_($state), %rax -+ adc $_k_+8($state), %rdx -+ mov %rax, ($mac) -+ mov %rdx, 8($mac) -+ -+ ret -+.size poly1305_finish_avx2,.-poly1305_finish_avx2 -+___ -+} -+}} -+ -+$code =~ s/\`([^\`]*)\`/eval(\$1)/gem; -+print $code; -+close STDOUT; -diff --git a/crypto/chacha20poly1305/chacha20.c b/crypto/chacha20poly1305/chacha20.c -new file mode 100644 -index 0000000..72ab173 ---- /dev/null -+++ b/crypto/chacha20poly1305/chacha20.c -@@ -0,0 +1,157 @@ -+/* Copyright (c) 2014, Google Inc. -+ * -+ * Permission to use, copy, modify, and/or distribute this software for any -+ * purpose with or without fee is hereby granted, provided that the above -+ * copyright notice and this permission notice appear in all copies. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY -+ * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION -+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ -+ -+/* Adapted from the public domain, estream code by D. Bernstein. */ -+ -+#include -+ -+/* sigma contains the ChaCha constants, which happen to be an ASCII string. */ -+static const char sigma[16] = "expand 32-byte k"; -+ -+#define ROTATE(v, n) (((v) << (n)) | ((v) >> (32 - (n)))) -+#define XOR(v, w) ((v) ^ (w)) -+#define PLUS(x, y) ((x) + (y)) -+#define PLUSONE(v) (PLUS((v), 1)) -+ -+#define U32TO8_LITTLE(p, v) \ -+ { \ -+ (p)[0] = (v >> 0) & 0xff; \ -+ (p)[1] = (v >> 8) & 0xff; \ -+ (p)[2] = (v >> 16) & 0xff; \ -+ (p)[3] = (v >> 24) & 0xff; \ -+ } -+ -+#define U8TO32_LITTLE(p) \ -+ (((uint32_t)((p)[0])) | ((uint32_t)((p)[1]) << 8) | \ -+ ((uint32_t)((p)[2]) << 16) | ((uint32_t)((p)[3]) << 24)) -+ -+/* QUARTERROUND updates a, b, c, d with a ChaCha "quarter" round. */ -+#define QUARTERROUND(a,b,c,d) \ -+ x[a] = PLUS(x[a],x[b]); x[d] = ROTATE(XOR(x[d],x[a]),16); \ -+ x[c] = PLUS(x[c],x[d]); x[b] = ROTATE(XOR(x[b],x[c]),12); \ -+ x[a] = PLUS(x[a],x[b]); x[d] = ROTATE(XOR(x[d],x[a]), 8); \ -+ x[c] = PLUS(x[c],x[d]); x[b] = ROTATE(XOR(x[b],x[c]), 7); -+ -+/* chacha_core performs |num_rounds| rounds of ChaCha20 on the input words in -+ * |input| and writes the 64 output bytes to |output|. */ -+static void chacha_core(uint8_t output[64], const uint32_t input[16]) { -+ uint32_t x[16]; -+ int i; -+ -+ memcpy(x, input, sizeof(uint32_t) * 16); -+ for (i = 20; i > 0; i -= 2) { -+ QUARTERROUND(0, 4, 8, 12) -+ QUARTERROUND(1, 5, 9, 13) -+ QUARTERROUND(2, 6, 10, 14) -+ QUARTERROUND(3, 7, 11, 15) -+ QUARTERROUND(0, 5, 10, 15) -+ QUARTERROUND(1, 6, 11, 12) -+ QUARTERROUND(2, 7, 8, 13) -+ QUARTERROUND(3, 4, 9, 14) -+ } -+ -+ for (i = 0; i < 16; ++i) { -+ x[i] = PLUS(x[i], input[i]); -+ } -+ for (i = 0; i < 16; ++i) { -+ U32TO8_LITTLE(output + 4 * i, x[i]); -+ } -+} -+ -+void CRYPTO_chacha_20(uint8_t *out, const uint8_t *in, size_t in_len, -+ const uint8_t key[32], const uint8_t nonce[8], -+ size_t counter) { -+#ifdef CHAPOLY_x86_64_ASM -+ uint8_t buf[256]; -+ size_t buf_size, ctr_msk; -+ void (*core_func)(uint8_t *out, const uint8_t *in, size_t in_len, -+ const uint8_t key[32], const uint8_t nonce[8], -+ size_t counter) = NULL; -+#else -+ uint8_t buf[64]; -+#endif -+ uint32_t input[16]; -+ size_t todo, i; -+ -+#ifdef CHAPOLY_x86_64_ASM -+ -+ if ((OPENSSL_ia32cap_loc()[2] >> 5) & 1) -+ { -+ buf_size = 128; -+ core_func = chacha_20_core_avx2; -+ ctr_msk = -2; -+ } -+ else if ((OPENSSL_ia32cap_loc()[1] >> 28) & 1) -+ { -+ buf_size = 64; -+ core_func = chacha_20_core_avx; -+ ctr_msk = -1; -+ } -+ else goto do_legacy; -+ -+ core_func(out, in, in_len, key, nonce, counter); -+ todo = in_len & (~(-buf_size)); -+ if(todo) -+ { -+ out += in_len&(-buf_size); -+ in += in_len&(-buf_size); -+ counter += (in_len/64) & ctr_msk; -+ memcpy(buf, in, todo); -+ core_func(buf, buf, buf_size, key, nonce, counter); -+ memcpy(out, buf, todo); -+ memset(buf, 0, buf_size); -+ } -+ return; -+ -+do_legacy: -+#endif -+ -+ input[0] = U8TO32_LITTLE(sigma + 0); -+ input[1] = U8TO32_LITTLE(sigma + 4); -+ input[2] = U8TO32_LITTLE(sigma + 8); -+ input[3] = U8TO32_LITTLE(sigma + 12); -+ -+ input[4] = U8TO32_LITTLE(key + 0); -+ input[5] = U8TO32_LITTLE(key + 4); -+ input[6] = U8TO32_LITTLE(key + 8); -+ input[7] = U8TO32_LITTLE(key + 12); -+ -+ input[8] = U8TO32_LITTLE(key + 16); -+ input[9] = U8TO32_LITTLE(key + 20); -+ input[10] = U8TO32_LITTLE(key + 24); -+ input[11] = U8TO32_LITTLE(key + 28); -+ -+ input[12] = counter; -+ input[13] = (uint64_t)counter >> 32; -+ input[14] = U8TO32_LITTLE(nonce + 0); -+ input[15] = U8TO32_LITTLE(nonce + 4); -+ -+ while (in_len > 0) { -+ todo = 64; -+ if (in_len < todo) { -+ todo = in_len; -+ } -+ -+ chacha_core(buf, input); -+ for (i = 0; i < todo; i++) { -+ out[i] = in[i] ^ buf[i]; -+ } -+ -+ out += todo; -+ in += todo; -+ in_len -= todo; -+ -+ ((uint64_t*)input)[6]++; -+ } -+} -diff --git a/crypto/chacha20poly1305/chacha20poly1305.h b/crypto/chacha20poly1305/chacha20poly1305.h -new file mode 100644 -index 0000000..bcabbb4 ---- /dev/null -+++ b/crypto/chacha20poly1305/chacha20poly1305.h -@@ -0,0 +1,63 @@ -+#ifndef OPENSSL_HEADER_POLY1305_H -+#define OPENSSL_HEADER_POLY1305_H -+ -+#include -+#include -+#include -+#include "crypto.h" -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+#define POLY1305_MAC_LEN (16) -+ -+typedef unsigned char poly1305_state[512]; -+ -+ -+/* CRYPTO_poly1305_init sets up |state| so that it can be used to calculate an -+ * authentication tag with the one-time key |key|. Note that |key| is a -+ * one-time key and therefore there is no `reset' method because that would -+ * enable several messages to be authenticated with the same key. */ -+void CRYPTO_poly1305_init(poly1305_state* state, const uint8_t key[32]); -+ -+/* CRYPTO_poly1305_update processes |in_len| bytes from |in|. It can be called -+ * zero or more times after poly1305_init. */ -+void CRYPTO_poly1305_update(poly1305_state* state, const uint8_t* in, -+ size_t in_len); -+ -+/* CRYPTO_poly1305_finish completes the poly1305 calculation and writes a 16 -+ * byte authentication tag to |mac|. */ -+void CRYPTO_poly1305_finish(poly1305_state* state, uint8_t mac[16]); -+ -+/* CRYPTO_chacha_20 encrypts |in_len| bytes from |in| with the given key and -+ * nonce and writes the result to |out|, which may be equal to |in|. The -+ * initial block counter is specified by |counter|. */ -+void CRYPTO_chacha_20(uint8_t *out, const uint8_t *in, size_t in_len, -+ const uint8_t key[32], const uint8_t nonce[8], -+ size_t counter); -+ -+#ifdef CHAPOLY_x86_64_ASM -+void poly1305_init_avx(poly1305_state* state, const uint8_t key[32]); -+void poly1305_update_avx(poly1305_state* state, const uint8_t *in, size_t in_len); -+void poly1305_finish_avx(poly1305_state* state, uint8_t mac[16]); -+ -+void poly1305_init_avx2(poly1305_state* state, const uint8_t key[32]); -+void poly1305_update_avx2(poly1305_state* state, const uint8_t *in, size_t in_len); -+void poly1305_finish_avx2(poly1305_state* state, uint8_t mac[16]); -+ -+void chacha_20_core_avx(uint8_t *out, const uint8_t *in, size_t in_len, -+ const uint8_t key[32], const uint8_t nonce[8], -+ size_t counter); -+ -+void chacha_20_core_avx2(uint8_t *out, const uint8_t *in, size_t in_len, -+ const uint8_t key[32], const uint8_t nonce[8], -+ size_t counter); -+#endif -+ -+ -+#if defined(__cplusplus) -+} /* extern C */ -+#endif -+ -+#endif /* OPENSSL_HEADER_POLY1305_H */ -diff --git a/crypto/chacha20poly1305/chapolytest.c b/crypto/chacha20poly1305/chapolytest.c -new file mode 100644 -index 0000000..56e713e ---- /dev/null -+++ b/crypto/chacha20poly1305/chapolytest.c -@@ -0,0 +1,287 @@ -+/* ==================================================================== -+ * Copyright (c) 2011-2013 The OpenSSL Project. All rights reserved. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions -+ * are met: -+ * -+ * 1. Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * 2. Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in -+ * the documentation and/or other materials provided with the -+ * distribution. -+ * -+ * 3. All advertising materials mentioning features or use of this -+ * software must display the following acknowledgment: -+ * "This product includes software developed by the OpenSSL Project -+ * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)" -+ * -+ * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to -+ * endorse or promote products derived from this software without -+ * prior written permission. For written permission, please contact -+ * licensing@OpenSSL.org. -+ * -+ * 5. Products derived from this software may not be called "OpenSSL" -+ * nor may "OpenSSL" appear in their names without prior written -+ * permission of the OpenSSL Project. -+ * -+ * 6. Redistributions of any form whatsoever must retain the following -+ * acknowledgment: -+ * "This product includes software developed by the OpenSSL Project -+ * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)" -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY -+ * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR -+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED -+ * OF THE POSSIBILITY OF SUCH DAMAGE. -+ * ==================================================================== -+ */ -+ -+ -+#include -+#include -+#include -+#include -+ -+#include -+ -+struct chacha_test { -+ const char *keyhex; -+ const char *noncehex; -+ const char *outhex; -+}; -+ -+struct poly1305_test -+ { -+ const char *inputhex; -+ const char *keyhex; -+ const char *outhex; -+ }; -+ -+static const struct chacha_test chacha_tests[] = { -+ { -+ "0000000000000000000000000000000000000000000000000000000000000000", -+ "0000000000000000", -+ "76b8e0ada0f13d90405d6ae55386bd28bdd219b8a08ded1aa836efcc8b770dc7da41597c5157488d7724e03fb8d84a376a43b8f41518a11cc387b669b2ee6586", -+ }, -+ { -+ "0000000000000000000000000000000000000000000000000000000000000001", -+ "0000000000000000", -+ "4540f05a9f1fb296d7736e7b208e3c96eb4fe1834688d2604f450952ed432d41bbe2a0b6ea7566d2a5d1e7e20d42af2c53d792b1c43fea817e9ad275ae546963", -+ }, -+ { -+ "0000000000000000000000000000000000000000000000000000000000000000", -+ "0000000000000001", -+ "de9cba7bf3d69ef5e786dc63973f653a0b49e015adbff7134fcb7df137821031e85a050278a7084527214f73efc7fa5b5277062eb7a0433e445f41e31afab757", -+ }, -+ { -+ "0000000000000000000000000000000000000000000000000000000000000000", -+ "0100000000000000", -+ "ef3fdfd6c61578fbf5cf35bd3dd33b8009631634d21e42ac33960bd138e50d32111e4caf237ee53ca8ad6426194a88545ddc497a0b466e7d6bbdb0041b2f586b", -+ }, -+ { -+ "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", -+ "0001020304050607", -+ "f798a189f195e66982105ffb640bb7757f579da31602fc93ec01ac56f85ac3c134a4547b733b46413042c9440049176905d3be59ea1c53f15916155c2be8241a38008b9a26bc35941e2444177c8ade6689de95264986d95889fb60e84629c9bd9a5acb1cc118be563eb9b3a4a472f82e09a7e778492b562ef7130e88dfe031c79db9d4f7c7a899151b9a475032b63fc385245fe054e3dd5a97a5f576fe064025d3ce042c566ab2c507b138db853e3d6959660996546cc9c4a6eafdc777c040d70eaf46f76dad3979e5c5360c3317166a1c894c94a371876a94df7628fe4eaaf2ccb27d5aaae0ad7ad0f9d4b6ad3b54098746d4524d38407a6deb", -+ }, -+}; -+ -+static const struct poly1305_test poly1305_tests[] = { -+ { -+ "", -+ "c8afaac331ee372cd6082de134943b174710130e9f6fea8d72293850a667d86c", -+ "4710130e9f6fea8d72293850a667d86c", -+ }, -+ { -+ "48656c6c6f20776f726c6421", -+ "746869732069732033322d62797465206b657920666f7220506f6c7931333035", -+ "a6f745008f81c916a20dcc74eef2b2f0", -+ }, -+ { -+ "0000000000000000000000000000000000000000000000000000000000000000", -+ "746869732069732033322d62797465206b657920666f7220506f6c7931333035", -+ "49ec78090e481ec6c26b33b91ccc0307", -+ }, -+}; -+ -+static unsigned char hex_digit(char h) -+ { -+ if (h >= '0' && h <= '9') -+ return h - '0'; -+ else if (h >= 'a' && h <= 'f') -+ return h - 'a' + 10; -+ else if (h >= 'A' && h <= 'F') -+ return h - 'A' + 10; -+ else -+ abort(); -+ } -+ -+static void hex_decode(unsigned char *out, const char* hex) -+ { -+ size_t j = 0; -+ -+ while (*hex != 0) -+ { -+ unsigned char v = hex_digit(*hex++); -+ v <<= 4; -+ v |= hex_digit(*hex++); -+ out[j++] = v; -+ } -+ } -+ -+static void hexdump(unsigned char *a, size_t len) -+ { -+ size_t i; -+ -+ for (i = 0; i < len; i++) -+ printf("%02x", a[i]); -+ } -+ -+/* misalign returns a pointer that points 0 to 15 bytes into |in| such that the -+ * returned pointer has alignment 1 mod 16. */ -+static void* misalign(void* in) -+ { -+ intptr_t x = (intptr_t) in; -+ x += (17 - (x % 16)) % 16; -+ return (void*) x; -+ } -+ -+int main() -+ { -+ unsigned num_tests = -+ sizeof(chacha_tests) / sizeof(struct chacha_test); -+ unsigned i; -+ unsigned char key_bytes[32 + 16]; -+ unsigned char nonce_bytes[8 + 16] = {0}; -+ -+ -+ for (i = 0; i < num_tests; i++) -+ { -+ unsigned char *key = misalign(key_bytes); -+ unsigned char *nonce = misalign(nonce_bytes); -+ -+ printf("ChaCha20 test #%d\n", i); -+ const struct chacha_test *test = &chacha_tests[i]; -+ unsigned char *expected, *out_bytes, *zero_bytes, *out, *zeros; -+ size_t len = strlen(test->outhex); -+ -+ if (strlen(test->keyhex) != 32*2 || -+ strlen(test->noncehex) != 8*2 || -+ (len & 1) == 1) -+ return 1; -+ -+ len /= 2; -+ -+ hex_decode(key, test->keyhex); -+ hex_decode(nonce, test->noncehex); -+ -+ expected = malloc(len); -+ out_bytes = malloc(len+16); -+ zero_bytes = malloc(len+16); -+ /* Attempt to test unaligned inputs. */ -+ out = misalign(out_bytes); -+ zeros = misalign(zero_bytes); -+ memset(zeros, 0, len); -+ -+ hex_decode(expected, test->outhex); -+ CRYPTO_chacha_20(out, zeros, len, key, nonce, 0); -+ -+ if (memcmp(out, expected, len) != 0) -+ { -+ printf("ChaCha20 test #%d failed.\n", i); -+ printf("got: "); -+ hexdump(out, len); -+ printf("\nexpected: "); -+ hexdump(expected, len); -+ printf("\n"); -+ return 1; -+ } -+ -+ /* The last test has a large output. We test whether the -+ * counter works as expected by skipping the first 64 bytes of -+ * it. */ -+ if (i == num_tests - 1) -+ { -+ CRYPTO_chacha_20(out, zeros, len - 64, key, nonce, 1); -+ if (memcmp(out, expected + 64, len - 64) != 0) -+ { -+ printf("ChaCha20 skip test failed.\n"); -+ return 1; -+ } -+ } -+ -+ free(expected); -+ free(zero_bytes); -+ free(out_bytes); -+ } -+ num_tests = -+ sizeof(poly1305_tests) / sizeof(struct poly1305_test); -+ unsigned char key[32], out[16], expected[16]; -+ poly1305_state poly1305; -+ -+ for (i = 0; i < num_tests; i++) -+ { -+ printf("Poly1305 test #%d\n", i); -+ const struct poly1305_test *test = &poly1305_tests[i]; -+ unsigned char *in; -+ size_t inlen = strlen(test->inputhex); -+ -+ if (strlen(test->keyhex) != sizeof(key)*2 || -+ strlen(test->outhex) != sizeof(out)*2 || -+ (inlen & 1) == 1) -+ return 1; -+ -+ inlen /= 2; -+ -+ hex_decode(key, test->keyhex); -+ hex_decode(expected, test->outhex); -+ -+ in = malloc(inlen); -+ -+ hex_decode(in, test->inputhex); -+ -+#ifdef CHAPOLY_x86_64_ASM -+ if((OPENSSL_ia32cap_loc()[1] >> 5) & 1) { -+ poly1305_init_avx2(&poly1305, key); -+ poly1305_update_avx2(&poly1305, in, inlen); -+ poly1305_finish_avx2(&poly1305, out); -+ } -+ else if ((OPENSSL_ia32cap_loc()[0] >> 60) & 1) { -+ poly1305_init_avx(&poly1305, key); -+ poly1305_update_avx(&poly1305, in, inlen); -+ poly1305_finish_avx(&poly1305, out); -+ } -+ else -+#endif -+ { -+ CRYPTO_poly1305_init(&poly1305, key); -+ CRYPTO_poly1305_update(&poly1305, in, inlen); -+ CRYPTO_poly1305_finish(&poly1305, out); -+ } -+ if (memcmp(out, expected, sizeof(expected)) != 0) -+ { -+ printf("Poly1305 test #%d failed.\n", i); -+ printf("got: "); -+ hexdump(out, sizeof(out)); -+ printf("\nexpected: "); -+ hexdump(expected, sizeof(expected)); -+ printf("\n"); -+ return 1; -+ } -+ -+ free(in); -+ } -+ -+ printf("PASS\n"); -+ return 0; -+ } -diff --git a/crypto/chacha20poly1305/poly1305.c b/crypto/chacha20poly1305/poly1305.c -new file mode 100644 -index 0000000..8b065cd ---- /dev/null -+++ b/crypto/chacha20poly1305/poly1305.c -@@ -0,0 +1,285 @@ -+/* Copyright (c) 2014, Google Inc. -+ * -+ * Permission to use, copy, modify, and/or distribute this software for any -+ * purpose with or without fee is hereby granted, provided that the above -+ * copyright notice and this permission notice appear in all copies. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY -+ * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION -+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ -+ -+/* This implementation of poly1305 is by Andrew Moon -+ * (https://github.com/floodyberry/poly1305-donna) and released as public -+ * domain. */ -+ -+#include -+#include -+ -+#if !defined(B_ENDIAN) -+/* We can assume little-endian. */ -+static uint32_t U8TO32_LE(const uint8_t *m) { -+ uint32_t r; -+ memcpy(&r, m, sizeof(r)); -+ return r; -+} -+ -+static void U32TO8_LE(uint8_t *m, uint32_t v) { memcpy(m, &v, sizeof(v)); } -+#else -+static uint32_t U8TO32_LE(const uint8_t *m) { -+ return (uint32_t)m[0] | (uint32_t)m[1] << 8 | (uint32_t)m[2] << 16 | -+ (uint32_t)m[3] << 24; -+} -+ -+static void U32TO8_LE(uint8_t *m, uint32_t v) { -+ m[0] = v; -+ m[1] = v >> 8; -+ m[2] = v >> 16; -+ m[3] = v >> 24; -+} -+#endif -+ -+static uint64_t mul32x32_64(uint32_t a, uint32_t b) { return (uint64_t)a * b; } -+ -+struct poly1305_state_st { -+ uint32_t r0, r1, r2, r3, r4; -+ uint32_t s1, s2, s3, s4; -+ uint32_t h0, h1, h2, h3, h4; -+ uint8_t buf[16]; -+ unsigned int buf_used; -+ uint8_t key[16]; -+}; -+ -+/* poly1305_blocks updates |state| given some amount of input data. This -+ * function may only be called with a |len| that is not a multiple of 16 at the -+ * end of the data. Otherwise the input must be buffered into 16 byte blocks. */ -+static void poly1305_update(struct poly1305_state_st *state, const uint8_t *in, -+ size_t len) { -+ uint32_t t0, t1, t2, t3; -+ uint64_t t[5]; -+ uint32_t b; -+ uint64_t c; -+ size_t j; -+ uint8_t mp[16]; -+ -+ if (len < 16) { -+ goto poly1305_donna_atmost15bytes; -+ } -+ -+poly1305_donna_16bytes: -+ t0 = U8TO32_LE(in); -+ t1 = U8TO32_LE(in + 4); -+ t2 = U8TO32_LE(in + 8); -+ t3 = U8TO32_LE(in + 12); -+ -+ in += 16; -+ len -= 16; -+ -+ state->h0 += t0 & 0x3ffffff; -+ state->h1 += ((((uint64_t)t1 << 32) | t0) >> 26) & 0x3ffffff; -+ state->h2 += ((((uint64_t)t2 << 32) | t1) >> 20) & 0x3ffffff; -+ state->h3 += ((((uint64_t)t3 << 32) | t2) >> 14) & 0x3ffffff; -+ state->h4 += (t3 >> 8) | (1 << 24); -+ -+poly1305_donna_mul: -+ t[0] = mul32x32_64(state->h0, state->r0) + mul32x32_64(state->h1, state->s4) + -+ mul32x32_64(state->h2, state->s3) + mul32x32_64(state->h3, state->s2) + -+ mul32x32_64(state->h4, state->s1); -+ t[1] = mul32x32_64(state->h0, state->r1) + mul32x32_64(state->h1, state->r0) + -+ mul32x32_64(state->h2, state->s4) + mul32x32_64(state->h3, state->s3) + -+ mul32x32_64(state->h4, state->s2); -+ t[2] = mul32x32_64(state->h0, state->r2) + mul32x32_64(state->h1, state->r1) + -+ mul32x32_64(state->h2, state->r0) + mul32x32_64(state->h3, state->s4) + -+ mul32x32_64(state->h4, state->s3); -+ t[3] = mul32x32_64(state->h0, state->r3) + mul32x32_64(state->h1, state->r2) + -+ mul32x32_64(state->h2, state->r1) + mul32x32_64(state->h3, state->r0) + -+ mul32x32_64(state->h4, state->s4); -+ t[4] = mul32x32_64(state->h0, state->r4) + mul32x32_64(state->h1, state->r3) + -+ mul32x32_64(state->h2, state->r2) + mul32x32_64(state->h3, state->r1) + -+ mul32x32_64(state->h4, state->r0); -+ -+ state->h0 = (uint32_t)t[0] & 0x3ffffff; -+ c = (t[0] >> 26); -+ t[1] += c; -+ state->h1 = (uint32_t)t[1] & 0x3ffffff; -+ b = (uint32_t)(t[1] >> 26); -+ t[2] += b; -+ state->h2 = (uint32_t)t[2] & 0x3ffffff; -+ b = (uint32_t)(t[2] >> 26); -+ t[3] += b; -+ state->h3 = (uint32_t)t[3] & 0x3ffffff; -+ b = (uint32_t)(t[3] >> 26); -+ t[4] += b; -+ state->h4 = (uint32_t)t[4] & 0x3ffffff; -+ b = (uint32_t)(t[4] >> 26); -+ state->h0 += b * 5; -+ -+ if (len >= 16) -+ goto poly1305_donna_16bytes; -+ -+/* final bytes */ -+poly1305_donna_atmost15bytes: -+ if (!len) -+ return; -+ -+ for (j = 0; j < len; j++) -+ mp[j] = in[j]; -+ mp[j++] = 1; -+ for (; j < 16; j++) -+ mp[j] = 0; -+ len = 0; -+ -+ t0 = U8TO32_LE(mp + 0); -+ t1 = U8TO32_LE(mp + 4); -+ t2 = U8TO32_LE(mp + 8); -+ t3 = U8TO32_LE(mp + 12); -+ -+ state->h0 += t0 & 0x3ffffff; -+ state->h1 += ((((uint64_t)t1 << 32) | t0) >> 26) & 0x3ffffff; -+ state->h2 += ((((uint64_t)t2 << 32) | t1) >> 20) & 0x3ffffff; -+ state->h3 += ((((uint64_t)t3 << 32) | t2) >> 14) & 0x3ffffff; -+ state->h4 += (t3 >> 8); -+ -+ goto poly1305_donna_mul; -+} -+ -+void CRYPTO_poly1305_init(poly1305_state *statep, const uint8_t key[32]) { -+ struct poly1305_state_st *state = (struct poly1305_state_st *)statep; -+ uint32_t t0, t1, t2, t3; -+ -+ t0 = U8TO32_LE(key + 0); -+ t1 = U8TO32_LE(key + 4); -+ t2 = U8TO32_LE(key + 8); -+ t3 = U8TO32_LE(key + 12); -+ -+ /* precompute multipliers */ -+ state->r0 = t0 & 0x3ffffff; -+ t0 >>= 26; -+ t0 |= t1 << 6; -+ state->r1 = t0 & 0x3ffff03; -+ t1 >>= 20; -+ t1 |= t2 << 12; -+ state->r2 = t1 & 0x3ffc0ff; -+ t2 >>= 14; -+ t2 |= t3 << 18; -+ state->r3 = t2 & 0x3f03fff; -+ t3 >>= 8; -+ state->r4 = t3 & 0x00fffff; -+ -+ state->s1 = state->r1 * 5; -+ state->s2 = state->r2 * 5; -+ state->s3 = state->r3 * 5; -+ state->s4 = state->r4 * 5; -+ -+ /* init state */ -+ state->h0 = 0; -+ state->h1 = 0; -+ state->h2 = 0; -+ state->h3 = 0; -+ state->h4 = 0; -+ -+ state->buf_used = 0; -+ memcpy(state->key, key + 16, sizeof(state->key)); -+} -+ -+void CRYPTO_poly1305_update(poly1305_state *statep, const uint8_t *in, -+ size_t in_len) { -+ unsigned int i; -+ struct poly1305_state_st *state = (struct poly1305_state_st *)statep; -+ -+ if (state->buf_used) { -+ unsigned int todo = 16 - state->buf_used; -+ if (todo > in_len) -+ todo = in_len; -+ for (i = 0; i < todo; i++) -+ state->buf[state->buf_used + i] = in[i]; -+ state->buf_used += todo; -+ in_len -= todo; -+ in += todo; -+ -+ if (state->buf_used == 16) { -+ poly1305_update(state, state->buf, 16); -+ state->buf_used = 0; -+ } -+ } -+ -+ if (in_len >= 16) { -+ size_t todo = in_len & ~0xf; -+ poly1305_update(state, in, todo); -+ in += todo; -+ in_len &= 0xf; -+ } -+ -+ if (in_len) { -+ for (i = 0; i < in_len; i++) -+ state->buf[i] = in[i]; -+ state->buf_used = in_len; -+ } -+} -+ -+void CRYPTO_poly1305_finish(poly1305_state *statep, uint8_t mac[16]) { -+ struct poly1305_state_st *state = (struct poly1305_state_st *)statep; -+ uint64_t f0, f1, f2, f3; -+ uint32_t g0, g1, g2, g3, g4; -+ uint32_t b, nb; -+ -+ if (state->buf_used) -+ poly1305_update(state, state->buf, state->buf_used); -+ -+ b = state->h0 >> 26; -+ state->h0 = state->h0 & 0x3ffffff; -+ state->h1 += b; -+ b = state->h1 >> 26; -+ state->h1 = state->h1 & 0x3ffffff; -+ state->h2 += b; -+ b = state->h2 >> 26; -+ state->h2 = state->h2 & 0x3ffffff; -+ state->h3 += b; -+ b = state->h3 >> 26; -+ state->h3 = state->h3 & 0x3ffffff; -+ state->h4 += b; -+ b = state->h4 >> 26; -+ state->h4 = state->h4 & 0x3ffffff; -+ state->h0 += b * 5; -+ -+ g0 = state->h0 + 5; -+ b = g0 >> 26; -+ g0 &= 0x3ffffff; -+ g1 = state->h1 + b; -+ b = g1 >> 26; -+ g1 &= 0x3ffffff; -+ g2 = state->h2 + b; -+ b = g2 >> 26; -+ g2 &= 0x3ffffff; -+ g3 = state->h3 + b; -+ b = g3 >> 26; -+ g3 &= 0x3ffffff; -+ g4 = state->h4 + b - (1 << 26); -+ -+ b = (g4 >> 31) - 1; -+ nb = ~b; -+ state->h0 = (state->h0 & nb) | (g0 & b); -+ state->h1 = (state->h1 & nb) | (g1 & b); -+ state->h2 = (state->h2 & nb) | (g2 & b); -+ state->h3 = (state->h3 & nb) | (g3 & b); -+ state->h4 = (state->h4 & nb) | (g4 & b); -+ -+ f0 = ((state->h0) | (state->h1 << 26)) + (uint64_t)U8TO32_LE(&state->key[0]); -+ f1 = ((state->h1 >> 6) | (state->h2 << 20)) + -+ (uint64_t)U8TO32_LE(&state->key[4]); -+ f2 = ((state->h2 >> 12) | (state->h3 << 14)) + -+ (uint64_t)U8TO32_LE(&state->key[8]); -+ f3 = ((state->h3 >> 18) | (state->h4 << 8)) + -+ (uint64_t)U8TO32_LE(&state->key[12]); -+ -+ U32TO8_LE(&mac[0], f0); -+ f1 += (f0 >> 32); -+ U32TO8_LE(&mac[4], f1); -+ f2 += (f1 >> 32); -+ U32TO8_LE(&mac[8], f2); -+ f3 += (f2 >> 32); -+ U32TO8_LE(&mac[12], f3); -+} -diff --git a/crypto/cryptlib.c b/crypto/cryptlib.c -index ca0e3cc..244c17e 100644 ---- a/crypto/cryptlib.c -+++ b/crypto/cryptlib.c -@@ -654,19 +654,9 @@ const char *CRYPTO_get_lock_name(int type) - defined(_M_AMD64) || defined(_M_X64) - - extern unsigned int OPENSSL_ia32cap_P[4]; --unsigned long *OPENSSL_ia32cap_loc(void) -+unsigned int *OPENSSL_ia32cap_loc(void) - { -- if (sizeof(long) == 4) -- /* -- * If 32-bit application pulls address of OPENSSL_ia32cap_P[0] -- * clear second element to maintain the illusion that vector -- * is 32-bit. -- */ -- OPENSSL_ia32cap_P[1] = 0; -- -- OPENSSL_ia32cap_P[2] = 0; -- -- return (unsigned long *)OPENSSL_ia32cap_P; -+ return OPENSSL_ia32cap_P; - } - - # if defined(OPENSSL_CPUID_OBJ) && !defined(OPENSSL_NO_ASM) && !defined(I386_ONLY) -diff --git a/crypto/crypto.h b/crypto/crypto.h -index c450d7a..aeacb00 100644 ---- a/crypto/crypto.h -+++ b/crypto/crypto.h -@@ -590,7 +590,7 @@ void CRYPTO_mem_leaks_cb(CRYPTO_MEM_LEAK_CB *cb); - void OpenSSLDie(const char *file, int line, const char *assertion); - # define OPENSSL_assert(e) (void)((e) ? 0 : (OpenSSLDie(__FILE__, __LINE__, #e),1)) - --unsigned long *OPENSSL_ia32cap_loc(void); -+unsigned int *OPENSSL_ia32cap_loc(void); - # define OPENSSL_ia32cap (*(OPENSSL_ia32cap_loc())) - int OPENSSL_isservice(void); - -diff --git a/crypto/evp/Makefile b/crypto/evp/Makefile -index aaaad98..e30b588 100644 ---- a/crypto/evp/Makefile -+++ b/crypto/evp/Makefile -@@ -29,7 +29,8 @@ LIBSRC= encode.c digest.c evp_enc.c evp_key.c evp_acnf.c evp_cnf.c \ - c_all.c c_allc.c c_alld.c evp_lib.c bio_ok.c \ - evp_pkey.c evp_pbe.c p5_crpt.c p5_crpt2.c \ - e_old.c pmeth_lib.c pmeth_fn.c pmeth_gn.c m_sigver.c \ -- e_aes_cbc_hmac_sha1.c e_aes_cbc_hmac_sha256.c e_rc4_hmac_md5.c -+ e_aes_cbc_hmac_sha1.c e_aes_cbc_hmac_sha256.c e_rc4_hmac_md5.c \ -+ e_chacha20poly1305.c - - LIBOBJ= encode.o digest.o evp_enc.o evp_key.o evp_acnf.o evp_cnf.o \ - e_des.o e_bf.o e_idea.o e_des3.o e_camellia.o\ -@@ -42,7 +43,8 @@ LIBOBJ= encode.o digest.o evp_enc.o evp_key.o evp_acnf.o evp_cnf.o \ - c_all.o c_allc.o c_alld.o evp_lib.o bio_ok.o \ - evp_pkey.o evp_pbe.o p5_crpt.o p5_crpt2.o \ - e_old.o pmeth_lib.o pmeth_fn.o pmeth_gn.o m_sigver.o \ -- e_aes_cbc_hmac_sha1.o e_aes_cbc_hmac_sha256.o e_rc4_hmac_md5.o -+ e_aes_cbc_hmac_sha1.o e_aes_cbc_hmac_sha256.o e_rc4_hmac_md5.o \ -+ e_chacha20poly1305.o - - SRC= $(LIBSRC) - -@@ -263,6 +265,7 @@ e_cast.o: ../../include/openssl/objects.h ../../include/openssl/opensslconf.h - e_cast.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h - e_cast.o: ../../include/openssl/safestack.h ../../include/openssl/stack.h - e_cast.o: ../../include/openssl/symhacks.h ../cryptlib.h e_cast.c evp_locl.h -+e_chacha20poly1305.o: ../../include/openssl/chacha20poly1305.h e_chacha20poly1305.c - e_des.o: ../../e_os.h ../../include/openssl/asn1.h ../../include/openssl/bio.h - e_des.o: ../../include/openssl/buffer.h ../../include/openssl/crypto.h - e_des.o: ../../include/openssl/des.h ../../include/openssl/des_old.h -diff --git a/crypto/evp/e_chacha20poly1305.c b/crypto/evp/e_chacha20poly1305.c -new file mode 100644 -index 0000000..0cb2af7 ---- /dev/null -+++ b/crypto/evp/e_chacha20poly1305.c -@@ -0,0 +1,323 @@ -+/* ==================================================================== -+ * Copyright (c) 2001-2014 The OpenSSL Project. All rights reserved. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions -+ * are met: -+ * -+ * 1. Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * 2. Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in -+ * the documentation and/or other materials provided with the -+ * distribution. -+ * -+ * 3. All advertising materials mentioning features or use of this -+ * software must display the following acknowledgment: -+ * "This product includes software developed by the OpenSSL Project -+ * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" -+ * -+ * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to -+ * endorse or promote products derived from this software without -+ * prior written permission. For written permission, please contact -+ * openssl-core@openssl.org. -+ * -+ * 5. Products derived from this software may not be called "OpenSSL" -+ * nor may "OpenSSL" appear in their names without prior written -+ * permission of the OpenSSL Project. -+ * -+ * 6. Redistributions of any form whatsoever must retain the following -+ * acknowledgment: -+ * "This product includes software developed by the OpenSSL Project -+ * for use in the OpenSSL Toolkit (http://www.openssl.org/)" -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY -+ * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR -+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED -+ * OF THE POSSIBILITY OF SUCH DAMAGE. -+ * ==================================================================== -+ * -+ */ -+ -+#include -+#ifndef OPENSSL_NO_CHACHA_POLY -+#include -+#include -+#include -+#include "evp_locl.h" -+#include -+ -+typedef struct -+ { -+ uint8_t key[32]; -+ /* uint8_t salt[4] */; -+ uint8_t nonce[8]; -+ poly1305_state poly_state; -+ size_t aad_l; -+ size_t ct_l; -+ int valid; -+#ifdef CHAPOLY_x86_64_ASM -+ void (*poly1305_init_ptr)(poly1305_state *, const uint8_t *); -+ void (*poly1305_update_ptr)(poly1305_state *, const uint8_t *, size_t); -+ void (*poly1305_finish_ptr)(poly1305_state *, uint8_t *); -+ #define poly_init aead_ctx->poly1305_init_ptr -+ #define poly_update poly1305_update_wrapper -+ #define poly_finish poly1305_finish_wrapper -+ #define FILL_BUFFER ((size_t)128) -+ uint8_t poly_buffer[FILL_BUFFER]; -+ uint8_t chacha_buffer[FILL_BUFFER]; -+ uint8_t poly_buffer_used; -+ uint8_t chacha_used; -+#else -+ #define poly_init CRYPTO_poly1305_init -+ #define poly_update(c,i,l) CRYPTO_poly1305_update(&c->poly_state,i,l) -+ #define poly_finish(c,m) CRYPTO_poly1305_finish(&c->poly_state,m) -+#endif -+ } EVP_CHACHA20_POLY1305_CTX; -+ -+#ifdef CHAPOLY_x86_64_ASM -+static void poly1305_update_wrapper(EVP_CHACHA20_POLY1305_CTX *ctx, const uint8_t *in, size_t in_len) -+ { -+ int todo; -+ /* Attempt to fill as many bytes as possible before calling the update function */ -+ if(in_len < FILL_BUFFER || ctx->poly_buffer_used) -+ { -+ todo = FILL_BUFFER - ctx->poly_buffer_used; -+ todo = in_len < todo? in_len : todo; -+ memcpy(ctx->poly_buffer + ctx->poly_buffer_used, in, todo); -+ ctx->poly_buffer_used += todo; -+ in += todo; -+ in_len -= todo; -+ if(ctx->poly_buffer_used == FILL_BUFFER) -+ { -+ ctx->poly1305_update_ptr(&ctx->poly_state, ctx->poly_buffer, FILL_BUFFER); -+ ctx->poly_buffer_used = 0; -+ } -+ } -+ if(in_len >= FILL_BUFFER) -+ { -+ ctx->poly1305_update_ptr(&ctx->poly_state, in, in_len&(-FILL_BUFFER)); -+ in += in_len&(-FILL_BUFFER); -+ in_len &= (FILL_BUFFER-1); -+ } -+ if(in_len) -+ { -+ memcpy(ctx->poly_buffer, in, in_len); -+ ctx->poly_buffer_used = in_len; -+ } -+ } -+ -+static void poly1305_finish_wrapper(EVP_CHACHA20_POLY1305_CTX *ctx, uint8_t mac[16]) -+ { -+ if(ctx->poly_buffer_used) -+ { -+ if(ctx->poly_buffer_used % 16) -+ { -+ memset(ctx->poly_buffer + ctx->poly_buffer_used, 0, 16 - (ctx->poly_buffer_used%16)); -+ } -+ ctx->poly1305_update_ptr(&ctx->poly_state, ctx->poly_buffer, ctx->poly_buffer_used); -+ } -+ ctx->poly1305_finish_ptr(&ctx->poly_state, mac); -+ memset(ctx->poly_buffer, 0 ,FILL_BUFFER); -+ } -+#endif -+ -+static int EVP_chacha20_poly1305_init(EVP_CIPHER_CTX *ctx, const unsigned char *key, const unsigned char *iv, int enc) -+ { -+ EVP_CHACHA20_POLY1305_CTX *aead_ctx = ctx->cipher_data; -+ /* simply copy the chacha key and iv*/ -+ memcpy(aead_ctx->key, key, 32); -+ /* memcpy(aead_ctx->salt, iv, 4); */ -+ aead_ctx->valid = 0; -+ return 1; -+ } -+ -+static int EVP_chacha20_poly1305_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl) -+ { -+ EVP_CHACHA20_POLY1305_CTX *aead_ctx = ctx->cipher_data; -+ uint8_t poly_block[16]; -+ uint64_t cl; -+ if(!aead_ctx->valid) -+ return 0; -+ if (inl < 16) -+ return -1; -+ /* Fix for MAC */ -+ inl -= 16; -+ /* Encryption */ -+ if(ctx->encrypt) -+ { -+#ifdef FILL_BUFFER -+ /* we can use the buffer we already accumulated during the parallel computation in init */ -+ if(inl<=FILL_BUFFER-64) -+ { -+ int i; -+ for(i=0; ichacha_buffer[i+64]; -+ } -+ else -+#endif -+ CRYPTO_chacha_20(out, in, inl, aead_ctx->key, aead_ctx->nonce, 1); -+ poly_update(aead_ctx, out, inl); -+ aead_ctx->ct_l += inl; -+ cl = aead_ctx->ct_l; -+ poly_update(aead_ctx, (uint8_t*)&cl, sizeof(cl)); -+ poly_finish(aead_ctx, &out[inl]); -+ aead_ctx->valid = 0; -+ return inl+16; -+ } -+ /* Decryption */ -+ else -+ { -+ /* Fix to accommodate for the MAC */ -+ poly_update(aead_ctx, in, inl); -+#ifdef FILL_BUFFER -+ /* we can use the buffer we already accumulated during the parallel computation in init */ -+ if(inl<=FILL_BUFFER-64) -+ { -+ int i; -+ for(i=0; ichacha_buffer[i+64]; -+ } -+ else -+#endif -+ CRYPTO_chacha_20(out, in, inl, aead_ctx->key, aead_ctx->nonce, 1); -+ aead_ctx->ct_l += inl; -+ cl = aead_ctx->ct_l; -+ poly_update(aead_ctx, (uint8_t*)&cl, sizeof(cl)); -+ poly_finish(aead_ctx, poly_block); -+ -+ uint64_t cmp = ((uint64_t*)poly_block)[0] ^ ((uint64_t*)(in + inl))[0]; -+ cmp |= ((uint64_t*)poly_block)[1] ^ ((uint64_t*)(in + inl))[1]; -+ -+ /*if (memcmp(poly_block, in + inl, POLY1305_MAC_LEN)) */ -+ if (cmp) -+ { -+ OPENSSL_cleanse(out, inl); -+ aead_ctx->valid = 0; -+ return -1; -+ } -+ aead_ctx->valid = 0; -+ return inl; -+ } -+ return 0; -+ } -+ -+static int EVP_chacha20_poly1305_cleanup(EVP_CIPHER_CTX *ctx) -+ { -+ return 1; -+ } -+ -+static int EVP_chacha20_poly1305_ctrl(EVP_CIPHER_CTX *ctx, int type, int arg, void *ptr) -+ { -+ EVP_CHACHA20_POLY1305_CTX *aead_ctx = ctx->cipher_data; -+#ifndef FILL_BUFFER -+ uint8_t poly1305_key[32]; -+#endif -+ uint8_t aad[13 + 8]; -+ uint64_t thirteen = 13; -+ -+ switch(type) -+ { -+ case EVP_CTRL_AEAD_TLS1_AAD: -+ if(arg!=13) -+ return 0; -+ /* Initialize poly keys */ -+#ifndef FILL_BUFFER -+ memset(poly1305_key, 0, sizeof(poly1305_key)); -+#else -+ memset(aead_ctx->chacha_buffer, 0, FILL_BUFFER); -+#endif -+ /* Salt is the IV (not in draft) */ -+ /* memcpy(aead_ctx->nonce, aead_ctx->salt, 4); */ -+ /* Take sequence number from AAD */ -+ /* memcpy(&aead_ctx->nonce[4], ptr, 8); */ -+ memcpy(aead_ctx->nonce, ptr, 8); -+ -+#ifdef CHAPOLY_x86_64_ASM -+ aead_ctx->poly_buffer_used = 0; -+ if((OPENSSL_ia32cap_loc()[2] >> 5) & 1) /* AVX2 */ -+ { -+ aead_ctx->poly1305_init_ptr = poly1305_init_avx2; -+ aead_ctx->poly1305_update_ptr = poly1305_update_avx2; -+ aead_ctx->poly1305_finish_ptr = poly1305_finish_avx2; -+ } -+ else if ((OPENSSL_ia32cap_loc()[1] >> 28) & 1) /* AVX */ -+ { -+ aead_ctx->poly1305_init_ptr = poly1305_init_avx; -+ aead_ctx->poly1305_update_ptr = poly1305_update_avx; -+ aead_ctx->poly1305_finish_ptr = poly1305_finish_avx; -+ } -+ else /*C*/ -+ { -+ aead_ctx->poly1305_init_ptr = CRYPTO_poly1305_init; -+ aead_ctx->poly1305_update_ptr = CRYPTO_poly1305_update; -+ aead_ctx->poly1305_finish_ptr = CRYPTO_poly1305_finish; -+ } -+ -+#endif -+#ifndef FILL_BUFFER -+ CRYPTO_chacha_20(poly1305_key, poly1305_key, sizeof(poly1305_key), aead_ctx->key, aead_ctx->nonce, 0); -+ poly_init(&aead_ctx->poly_state, poly1305_key); -+#else -+ CRYPTO_chacha_20(aead_ctx->chacha_buffer, aead_ctx->chacha_buffer, FILL_BUFFER, aead_ctx->key, aead_ctx->nonce, 0); -+ poly_init(&aead_ctx->poly_state, aead_ctx->chacha_buffer); -+ aead_ctx->chacha_used = 64; /* We keep 64 byte for future use, to accelerate for very short messages */ -+#endif -+ aead_ctx->aad_l = 0; -+ aead_ctx->ct_l = 0; -+ /* Absorb AAD */ -+ memcpy(aad, ptr, arg); -+ memcpy(&aad[arg], &thirteen, sizeof(thirteen)); -+ /* If decrypting fix length for tag */ -+ if (!ctx->encrypt) -+ { -+ unsigned int len=aad[arg-2]<<8|aad[arg-1]; -+ len -= POLY1305_MAC_LEN; -+ aad[arg-2] = len>>8; -+ aad[arg-1] = len & 0xff; -+ } -+ poly_update(aead_ctx, aad, arg + sizeof(thirteen)); -+ /* aead_ctx->aad_l += arg; */ -+ aead_ctx->valid = 1; -+ return POLY1305_MAC_LEN; -+ break; -+ default: -+ return 0; -+ break; -+ } -+ return 0; -+ } -+ -+#define CUSTOM_FLAGS (\ -+ EVP_CIPH_CUSTOM_IV | EVP_CIPH_FLAG_CUSTOM_CIPHER \ -+ | EVP_CIPH_ALWAYS_CALL_INIT \ -+ | EVP_CIPH_CUSTOM_COPY) -+ -+static const EVP_CIPHER chacha20_poly1305 = { -+ NID_chacha20_poly1305, /* nid */ -+ 1, /* block size, sorta */ -+ 32, /* key len */ -+ 0, /* iv len */ -+ CUSTOM_FLAGS|EVP_CIPH_FLAG_AEAD_CIPHER, /* flags */ -+ EVP_chacha20_poly1305_init, -+ EVP_chacha20_poly1305_cipher, -+ EVP_chacha20_poly1305_cleanup, -+ sizeof(EVP_CHACHA20_POLY1305_CTX), /* ctx size */ -+ NULL, NULL, -+ EVP_chacha20_poly1305_ctrl, -+ NULL -+ }; -+ -+const EVP_CIPHER *EVP_chacha20_poly1305(void) -+{ return &chacha20_poly1305; } -+ -+#endif -diff --git a/crypto/evp/evp.h b/crypto/evp/evp.h -index 39ab793..5f2260a 100644 ---- a/crypto/evp/evp.h -+++ b/crypto/evp/evp.h -@@ -893,6 +893,9 @@ const EVP_CIPHER *EVP_camellia_256_cfb128(void); - # define EVP_camellia_256_cfb EVP_camellia_256_cfb128 - const EVP_CIPHER *EVP_camellia_256_ofb(void); - # endif -+# ifndef OPENSSL_NO_CHACHA_POLY -+const EVP_CIPHER *EVP_chacha20_poly1305(void); -+# endif - - # ifndef OPENSSL_NO_SEED - const EVP_CIPHER *EVP_seed_ecb(void); -diff --git a/crypto/objects/obj_dat.h b/crypto/objects/obj_dat.h -index b7e3cf2..4059875 100644 ---- a/crypto/objects/obj_dat.h -+++ b/crypto/objects/obj_dat.h -@@ -62,9 +62,9 @@ - * [including the GNU Public Licence.] - */ - --#define NUM_NID 958 --#define NUM_SN 951 --#define NUM_LN 951 -+#define NUM_NID 959 -+#define NUM_SN 952 -+#define NUM_LN 952 - #define NUM_OBJ 890 - - static const unsigned char lvalues[6255]={ -@@ -2514,6 +2514,8 @@ static const ASN1_OBJECT nid_objs[NUM_NID]={ - NID_jurisdictionStateOrProvinceName,11,&(lvalues[6232]),0}, - {"jurisdictionC","jurisdictionCountryName", - NID_jurisdictionCountryName,11,&(lvalues[6243]),0}, -+{"id-chacha20-poly1305","chacha20-poly1305",NID_chacha20_poly1305,0, -+ NULL,0}, - }; - - static const unsigned int sn_objs[NUM_SN]={ -@@ -2954,6 +2956,7 @@ static const unsigned int sn_objs[NUM_SN]={ - 362, /* "id-cct-PKIResponse" */ - 360, /* "id-cct-crs" */ - 81, /* "id-ce" */ -+958, /* "id-chacha20-poly1305" */ - 680, /* "id-characteristic-two-basis" */ - 263, /* "id-cmc" */ - 334, /* "id-cmc-addExtensions" */ -@@ -3728,6 +3731,7 @@ static const unsigned int ln_objs[NUM_LN]={ - 677, /* "certicom-arc" */ - 517, /* "certificate extensions" */ - 883, /* "certificateRevocationList" */ -+958, /* "chacha20-poly1305" */ - 54, /* "challengePassword" */ - 407, /* "characteristic-two-field" */ - 395, /* "clearance" */ -diff --git a/crypto/objects/obj_mac.h b/crypto/objects/obj_mac.h -index 779c309..2a34635 100644 ---- a/crypto/objects/obj_mac.h -+++ b/crypto/objects/obj_mac.h -@@ -4192,3 +4192,7 @@ - #define LN_jurisdictionCountryName "jurisdictionCountryName" - #define NID_jurisdictionCountryName 957 - #define OBJ_jurisdictionCountryName 1L,3L,6L,1L,4L,1L,311L,60L,2L,1L,3L -+ -+#define SN_chacha20_poly1305 "id-chacha20-poly1305" -+#define LN_chacha20_poly1305 "chacha20-poly1305" -+#define NID_chacha20_poly1305 958 -diff --git a/ssl/s3_lib.c b/ssl/s3_lib.c -index ad9eeb6..a654176 100644 ---- a/ssl/s3_lib.c -+++ b/ssl/s3_lib.c -@@ -2891,6 +2891,53 @@ OPENSSL_GLOBAL SSL_CIPHER ssl3_ciphers[] = { - 256}, - #endif - -+ /* Chacha20-Poly1305 draft cipher suites */ -+#if !defined(OPENSSL_NO_CHACHA_POLY) -+ { -+ 1, -+ TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305, -+ TLS1_CK_ECDHE_RSA_CHACHA20_POLY1305, -+ SSL_kEECDH, -+ SSL_aRSA, -+ SSL_CHACHA20POLY1305, -+ SSL_AEAD, -+ SSL_TLSV1_2, -+ SSL_NOT_EXP|SSL_HIGH, -+ SSL_HANDSHAKE_MAC_SHA256|TLS1_PRF_SHA256, -+ 256, -+ 0, -+ }, -+ -+ { -+ 1, -+ TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, -+ TLS1_CK_ECDHE_ECDSA_CHACHA20_POLY1305, -+ SSL_kEECDH, -+ SSL_aECDSA, -+ SSL_CHACHA20POLY1305, -+ SSL_AEAD, -+ SSL_TLSV1_2, -+ SSL_NOT_EXP|SSL_HIGH, -+ SSL_HANDSHAKE_MAC_SHA256|TLS1_PRF_SHA256, -+ 256, -+ 0, -+ }, -+ -+ { -+ 1, -+ TLS1_TXT_DHE_RSA_WITH_CHACHA20_POLY1305, -+ TLS1_CK_DHE_RSA_CHACHA20_POLY1305, -+ SSL_kEDH, -+ SSL_aRSA, -+ SSL_CHACHA20POLY1305, -+ SSL_AEAD, -+ SSL_TLSV1_2, -+ SSL_NOT_EXP|SSL_HIGH, -+ SSL_HANDSHAKE_MAC_SHA256|TLS1_PRF_SHA256, -+ 256, -+ 0, -+ }, -+#endif - /* end of list */ - }; - -@@ -4047,6 +4094,7 @@ SSL_CIPHER *ssl3_choose_cipher(SSL *s, STACK_OF(SSL_CIPHER) *clnt, - int i, ii, ok; - CERT *cert; - unsigned long alg_k, alg_a, mask_k, mask_a, emask_k, emask_a; -+ int use_chacha = 0; - - /* Let's see which ciphers we can support */ - cert = s->cert; -@@ -4080,9 +4128,16 @@ SSL_CIPHER *ssl3_choose_cipher(SSL *s, STACK_OF(SSL_CIPHER) *clnt, - if (s->options & SSL_OP_CIPHER_SERVER_PREFERENCE || tls1_suiteb(s)) { - prio = srvr; - allow = clnt; -+ /* Use ChaCha20+Poly1305 iff it's client's most preferred cipher suite */ -+ if (sk_SSL_CIPHER_num(clnt) > 0) { -+ c = sk_SSL_CIPHER_value(clnt, 0); -+ if (c->algorithm_enc == SSL_CHACHA20POLY1305) -+ use_chacha = 1; -+ } - } else { - prio = clnt; -- allow = srvr; -+ allow = srvr; -+ use_chacha = 1; - } - - tls1_set_cert_validity(s); -@@ -4094,6 +4149,11 @@ SSL_CIPHER *ssl3_choose_cipher(SSL *s, STACK_OF(SSL_CIPHER) *clnt, - if ((c->algorithm_ssl & SSL_TLSV1_2) && !SSL_USE_TLS1_2_CIPHERS(s)) - continue; - -+ /* Skip ChaCha unless top client priority */ -+ if ((c->algorithm_enc == SSL_CHACHA20POLY1305) && -+ !use_chacha) -+ continue; -+ - ssl_set_cert_masks(cert, c); - mask_k = cert->mask_k; - mask_a = cert->mask_a; -diff --git a/ssl/ssl.h b/ssl/ssl.h -index c6c5bce..6367a52 100644 ---- a/ssl/ssl.h -+++ b/ssl/ssl.h -@@ -297,6 +297,7 @@ extern "C" { - # define SSL_TXT_CAMELLIA128 "CAMELLIA128" - # define SSL_TXT_CAMELLIA256 "CAMELLIA256" - # define SSL_TXT_CAMELLIA "CAMELLIA" -+# define SSL_TXT_CHACHA20 "CHACHA20" - - # define SSL_TXT_MD5 "MD5" - # define SSL_TXT_SHA1 "SHA1" -diff --git a/ssl/ssl_algs.c b/ssl/ssl_algs.c -index e6f515f..4eff5ea 100644 ---- a/ssl/ssl_algs.c -+++ b/ssl/ssl_algs.c -@@ -105,6 +105,10 @@ int SSL_library_init(void) - EVP_add_cipher(EVP_camellia_128_cbc()); - EVP_add_cipher(EVP_camellia_256_cbc()); - #endif -+ -+#ifndef OPENSSL_NO_CHACHA_POLY -+ EVP_add_cipher(EVP_chacha20_poly1305()); -+#endif - - #ifndef OPENSSL_NO_SEED - EVP_add_cipher(EVP_seed_cbc()); -diff --git a/ssl/ssl_ciph.c b/ssl/ssl_ciph.c -index a53f25b..e25db6d 100644 ---- a/ssl/ssl_ciph.c -+++ b/ssl/ssl_ciph.c -@@ -164,7 +164,8 @@ - #define SSL_ENC_SEED_IDX 11 - #define SSL_ENC_AES128GCM_IDX 12 - #define SSL_ENC_AES256GCM_IDX 13 --#define SSL_ENC_NUM_IDX 14 -+#define SSL_ENC_CHACHA20POLY1305_IDX 14 -+#define SSL_ENC_NUM_IDX 15 - - static const EVP_CIPHER *ssl_cipher_methods[SSL_ENC_NUM_IDX] = { - NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, -@@ -316,6 +317,7 @@ static const SSL_CIPHER cipher_aliases[] = { - {0, SSL_TXT_CAMELLIA256, 0, 0, 0, SSL_CAMELLIA256, 0, 0, 0, 0, 0, 0}, - {0, SSL_TXT_CAMELLIA, 0, 0, 0, SSL_CAMELLIA128 | SSL_CAMELLIA256, 0, 0, 0, - 0, 0, 0}, -+ {0, SSL_TXT_CHACHA20, 0, 0, 0, SSL_CHACHA20POLY1305, 0, 0, 0, 0, 0, 0}, - - /* MAC aliases */ - {0, SSL_TXT_MD5, 0, 0, 0, 0, SSL_MD5, 0, 0, 0, 0, 0}, -@@ -429,6 +431,9 @@ void ssl_load_ciphers(void) - ssl_cipher_methods[SSL_ENC_AES256GCM_IDX] = - EVP_get_cipherbyname(SN_aes_256_gcm); - -+ ssl_cipher_methods[SSL_ENC_CHACHA20POLY1305_IDX] = -+ EVP_get_cipherbyname(SN_chacha20_poly1305); -+ - ssl_digest_methods[SSL_MD_MD5_IDX] = EVP_get_digestbyname(SN_md5); - ssl_mac_secret_size[SSL_MD_MD5_IDX] = - EVP_MD_size(ssl_digest_methods[SSL_MD_MD5_IDX]); -@@ -579,6 +584,9 @@ int ssl_cipher_get_evp(const SSL_SESSION *s, const EVP_CIPHER **enc, - case SSL_AES256GCM: - i = SSL_ENC_AES256GCM_IDX; - break; -+ case SSL_CHACHA20POLY1305: -+ i = SSL_ENC_CHACHA20POLY1305_IDX; -+ break; - default: - i = -1; - break; -@@ -803,6 +811,8 @@ static void ssl_cipher_get_disabled(unsigned long *mkey, unsigned long *auth, - (ssl_cipher_methods[SSL_ENC_GOST89_IDX] == - NULL) ? SSL_eGOST2814789CNT : 0; - *enc |= (ssl_cipher_methods[SSL_ENC_SEED_IDX] == NULL) ? SSL_SEED : 0; -+ *enc |= (ssl_cipher_methods[SSL_ENC_CHACHA20POLY1305_IDX] == -+ NULL) ? SSL_CHACHA20POLY1305 : 0; - - *mac |= (ssl_digest_methods[SSL_MD_MD5_IDX] == NULL) ? SSL_MD5 : 0; - *mac |= (ssl_digest_methods[SSL_MD_SHA1_IDX] == NULL) ? SSL_SHA1 : 0; -@@ -1821,6 +1831,9 @@ char *SSL_CIPHER_description(const SSL_CIPHER *cipher, char *buf, int len) - case SSL_eGOST2814789CNT: - enc = "GOST89(256)"; - break; -+ case SSL_CHACHA20POLY1305: -+ enc = "CHACHA20-POLY1305(256)"; -+ break; - default: - enc = "unknown"; - break; -diff --git a/ssl/ssl_locl.h b/ssl/ssl_locl.h -index 6c2c551..9e1cce3 100644 ---- a/ssl/ssl_locl.h -+++ b/ssl/ssl_locl.h -@@ -354,6 +354,7 @@ - # define SSL_SEED 0x00000800L - # define SSL_AES128GCM 0x00001000L - # define SSL_AES256GCM 0x00002000L -+# define SSL_CHACHA20POLY1305 0x00004000L - - # define SSL_AES (SSL_AES128|SSL_AES256|SSL_AES128GCM|SSL_AES256GCM) - # define SSL_CAMELLIA (SSL_CAMELLIA128|SSL_CAMELLIA256) -diff --git a/ssl/tls1.h b/ssl/tls1.h -index 5929607..3a1ff70 100644 ---- a/ssl/tls1.h -+++ b/ssl/tls1.h -@@ -566,6 +566,11 @@ SSL_CTX_callback_ctrl(ssl,SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB,(void (*)(void))cb) - # define TLS1_CK_ECDH_RSA_WITH_AES_128_GCM_SHA256 0x0300C031 - # define TLS1_CK_ECDH_RSA_WITH_AES_256_GCM_SHA384 0x0300C032 - -+/* ChaCha20-Poly1305 ciphersuites draft-agl-tls-chacha20poly1305-01 */ -+# define TLS1_CK_ECDHE_RSA_CHACHA20_POLY1305 0x0300CC13 -+# define TLS1_CK_ECDHE_ECDSA_CHACHA20_POLY1305 0x0300CC14 -+# define TLS1_CK_DHE_RSA_CHACHA20_POLY1305 0x0300CC15 -+ - /* - * XXX * Backward compatibility alert: + * Older versions of OpenSSL gave - * some DHE ciphers names with "EDH" + * instead of "DHE". Going forward, we -@@ -716,6 +721,11 @@ SSL_CTX_callback_ctrl(ssl,SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB,(void (*)(void))cb) - # define TLS1_TXT_ECDH_RSA_WITH_AES_128_GCM_SHA256 "ECDH-RSA-AES128-GCM-SHA256" - # define TLS1_TXT_ECDH_RSA_WITH_AES_256_GCM_SHA384 "ECDH-RSA-AES256-GCM-SHA384" - -+/* ChaCha20-Poly1305 ciphersuites draft-agl-tls-chacha20poly1305-01 */ -+#define TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305 "ECDHE-RSA-CHACHA20-POLY1305" -+#define TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 "ECDHE-ECDSA-CHACHA20-POLY1305" -+#define TLS1_TXT_DHE_RSA_WITH_CHACHA20_POLY1305 "DHE-RSA-CHACHA20-POLY1305" -+ - # define TLS_CT_RSA_SIGN 1 - # define TLS_CT_DSS_SIGN 2 - # define TLS_CT_RSA_FIXED_DH 3 -diff --git a/test/Makefile b/test/Makefile -index 80aeccf..bce42c5 100644 ---- a/test/Makefile -+++ b/test/Makefile -@@ -70,6 +70,7 @@ HEARTBEATTEST= heartbeat_test - CONSTTIMETEST= constant_time_test - VERIFYEXTRATEST= verify_extra_test - CLIENTHELLOTEST= clienthellotest -+CHAPOLYTEST= chapolytest - - TESTS= alltests - -@@ -83,7 +84,7 @@ EXE= $(BNTEST)$(EXE_EXT) $(ECTEST)$(EXE_EXT) $(ECDSATEST)$(EXE_EXT) $(ECDHTEST) - $(EVPTEST)$(EXE_EXT) $(EVPEXTRATEST)$(EXE_EXT) $(IGETEST)$(EXE_EXT) $(JPAKETEST)$(EXE_EXT) $(SRPTEST)$(EXE_EXT) \ - $(ASN1TEST)$(EXE_EXT) $(V3NAMETEST)$(EXE_EXT) $(HEARTBEATTEST)$(EXE_EXT) \ - $(CONSTTIMETEST)$(EXE_EXT) $(VERIFYEXTRATEST)$(EXE_EXT) \ -- $(CLIENTHELLOTEST)$(EXE_EXT) -+ $(CLIENTHELLOTEST)$(EXE_EXT) $(CHAPOLYTEST)$(EXE_EXT) - - # $(METHTEST)$(EXE_EXT) - -@@ -97,7 +98,7 @@ OBJ= $(BNTEST).o $(ECTEST).o $(ECDSATEST).o $(ECDHTEST).o $(IDEATEST).o \ - $(BFTEST).o $(SSLTEST).o $(DSATEST).o $(EXPTEST).o $(RSATEST).o \ - $(EVPTEST).o $(EVPEXTRATEST).o $(IGETEST).o $(JPAKETEST).o $(ASN1TEST).o $(V3NAMETEST).o \ - $(HEARTBEATTEST).o $(CONSTTIMETEST).o $(VERIFYEXTRATEST).o \ -- $(CLIENTHELLOTEST).o -+ $(CLIENTHELLOTEST).o $(CHAPOLYTEST).o - - SRC= $(BNTEST).c $(ECTEST).c $(ECDSATEST).c $(ECDHTEST).c $(IDEATEST).c \ - $(MD2TEST).c $(MD4TEST).c $(MD5TEST).c \ -@@ -108,7 +109,7 @@ SRC= $(BNTEST).c $(ECTEST).c $(ECDSATEST).c $(ECDHTEST).c $(IDEATEST).c \ - $(BFTEST).c $(SSLTEST).c $(DSATEST).c $(EXPTEST).c $(RSATEST).c \ - $(EVPTEST).c $(EVPEXTRATEST).c $(IGETEST).c $(JPAKETEST).c $(SRPTEST).c $(ASN1TEST).c \ - $(V3NAMETEST).c $(HEARTBEATTEST).c $(CONSTTIMETEST).c $(VERIFYEXTRATEST).c \ -- $(CLIENTHELLOTEST).c -+ $(CLIENTHELLOTEST).c $(CHAPOLYTEST).c - - EXHEADER= - HEADER= testutil.h $(EXHEADER) -@@ -144,7 +145,7 @@ apps: - @(cd ..; $(MAKE) DIRS=apps all) - - alltests: \ -- test_des test_idea test_sha test_md4 test_md5 test_hmac \ -+ test_des test_idea test_sha test_md4 test_md5 test_hmac test_chapoly \ - test_md2 test_mdc2 test_wp \ - test_rmd test_rc2 test_rc4 test_rc5 test_bf test_cast test_aes \ - test_rand test_bn test_ec test_ecdsa test_ecdh \ -@@ -361,6 +362,10 @@ test_clienthello: $(CLIENTHELLOTEST)$(EXE_EXT) - @echo $(START) $@ - ../util/shlib_wrap.sh ./$(CLIENTHELLOTEST) - -+test_chapoly: $(CHAPOLYTEST)$(EXE_EXT) -+ @echo "Test ChaCha20 and Poly1305" -+ ../util/shlib_wrap.sh ./$(CHAPOLYTEST) -+ - lint: - lint -DLINT $(INCLUDES) $(SRC)>fluff - -@@ -538,6 +543,9 @@ $(VERIFYEXTRATEST)$(EXE_EXT): $(VERIFYEXTRATEST).o - $(CLIENTHELLOTEST)$(EXE_EXT): $(CLIENTHELLOTEST).o - @target=$(CLIENTHELLOTEST) $(BUILD_CMD) - -+$(CHAPOLYTEST)$(EXE_EXT): $(CHAPOLYTEST).o -+ @target=$(CHAPOLYTEST); $(BUILD_CMD) -+ - #$(AESTEST).o: $(AESTEST).c - # $(CC) -c $(CFLAGS) -DINTERMEDIATE_VALUE_KAT -DTRACE_KAT_MCT $(AESTEST).c - -@@ -605,6 +613,7 @@ clienthellotest.o: clienthellotest.c - constant_time_test.o: ../crypto/constant_time_locl.h ../e_os.h - constant_time_test.o: ../include/openssl/e_os2.h - constant_time_test.o: ../include/openssl/opensslconf.h constant_time_test.c -+chapolytest.o: ../include/openssl/chacha20poly1305.h chapolytest.c - destest.o: ../include/openssl/des.h ../include/openssl/des_old.h - destest.o: ../include/openssl/e_os2.h ../include/openssl/opensslconf.h - destest.o: ../include/openssl/ossl_typ.h ../include/openssl/safestack.h --- -1.9.1 - diff --git a/patches/openssl__chacha20_poly1305_cf_latest_git.patch b/patches/openssl__chacha20_poly1305_cf_latest_git.patch deleted file mode 100644 index ae5f95e..0000000 --- a/patches/openssl__chacha20_poly1305_cf_latest_git.patch +++ /dev/null @@ -1,4368 +0,0 @@ -From a168b10c5a9a97ecba3d91ff98ed0a9c0c604c7d Mon Sep 17 00:00:00 2001 -From: Vlad Krasnov -Date: Thu, 17 Sep 2015 12:54:11 -0700 -Subject: [PATCH] CHACHA20-POLY1305 Draft 1 - -Rebase of the original patch on top of OpenSSL git as of September 19, 2015 ---- - Configure | 22 +- - Makefile.org | 4 +- - apps/speed.c | 24 +- - crypto/chacha20poly1305/Makefile | 92 +++ - crypto/chacha20poly1305/asm/chacha20_avx.pl | 388 +++++++++++ - crypto/chacha20poly1305/asm/chacha20_avx2.pl | 424 +++++++++++++ - crypto/chacha20poly1305/asm/poly1305_avx.pl | 717 +++++++++++++++++++++ - crypto/chacha20poly1305/asm/poly1305_avx2.pl | 918 +++++++++++++++++++++++++++ - crypto/chacha20poly1305/chacha20.c | 157 +++++ - crypto/chacha20poly1305/poly1305.c | 285 +++++++++ - crypto/evp/Makefile | 7 +- - crypto/evp/e_chacha20poly1305.c | 323 ++++++++++ - crypto/objects/obj_dat.h | 10 +- - include/openssl/chacha20poly1305.h | 63 ++ - include/openssl/evp.h | 1 + - include/openssl/obj_mac.h | 4 + - include/openssl/ssl.h | 1 + - include/openssl/tls1.h | 10 + - ssl/s3_lib.c | 61 ++ - ssl/ssl_algs.c | 3 + - ssl/ssl_ciph.c | 46 +- - ssl/ssl_locl.h | 1 + - test/Makefile | 12 +- - test/chapolytest.c | 287 +++++++++ - test/recipes/05-test_chacha20poly1305.t | 5 + - 25 files changed, 3830 insertions(+), 35 deletions(-) - create mode 100644 crypto/chacha20poly1305/Makefile - create mode 100644 crypto/chacha20poly1305/asm/chacha20_avx.pl - create mode 100644 crypto/chacha20poly1305/asm/chacha20_avx2.pl - create mode 100644 crypto/chacha20poly1305/asm/poly1305_avx.pl - create mode 100644 crypto/chacha20poly1305/asm/poly1305_avx2.pl - create mode 100644 crypto/chacha20poly1305/chacha20.c - create mode 100644 crypto/chacha20poly1305/poly1305.c - create mode 100644 crypto/evp/e_chacha20poly1305.c - create mode 100644 include/openssl/chacha20poly1305.h - create mode 100644 test/chapolytest.c - create mode 100644 test/recipes/05-test_chacha20poly1305.t - -diff --git a/Configure b/Configure -index d7b8190..1a8841f 100755 ---- a/Configure -+++ b/Configure -@@ -346,7 +346,8 @@ my %table=( - wp_obj => "wp-x86_64.o", - cmll_obj => "cmll-x86_64.o cmll_misc.o", - modes_obj => "ghash-x86_64.o aesni-gcm-x86_64.o", -- engines_obj => "e_padlock-x86_64.o" -+ engines_obj => "e_padlock-x86_64.o", -+ chapoly_obj => "chacha20_avx.o poly1305_avx.o chacha20_avx2.o poly1305_avx2.o" - }, - ia64_asm => { - template => 1, -@@ -490,6 +491,7 @@ sub stringtohash { - "wp_obj", - "cmll_obj", - "modes_obj", -+ "chapoly_obj", - "engines_obj", - "perlasm_scheme", - "dso_scheme", -@@ -791,6 +793,7 @@ my $cast_enc="c_enc.o"; - my $rc4_enc="rc4_enc.o rc4_skey.o"; - my $rc5_enc="rc5_enc.o"; - my $cmll_enc="camellia.o cmll_misc.o cmll_cbc.o"; -+my $chapoly_enc=""; - my $processor=""; - my $default_ranlib; - my $perl; -@@ -1268,6 +1271,7 @@ my $rc5_obj = $table{$target}->{rc5_obj}; - my $wp_obj = $table{$target}->{wp_obj}; - my $cmll_obj = $table{$target}->{cmll_obj}; - my $modes_obj = $table{$target}->{modes_obj}; -+my $chapoly_obj = $table{$target}->{chapoly_obj}; - my $engines_obj = $table{$target}->{engines_obj}; - my $perlasm_scheme = $table{$target}->{perlasm_scheme}; - my $dso_scheme = $table{$target}->{dso_scheme}; -@@ -1379,7 +1383,8 @@ if ($no_asm) - { - $cpuid_obj=$bn_obj=$ec_obj= - $des_obj=$aes_obj=$bf_obj=$cast_obj=$rc4_obj=$rc5_obj=$cmll_obj= -- $modes_obj=$sha1_obj=$md5_obj=$rmd160_obj=$wp_obj=$engines_obj=""; -+ $modes_obj=$sha1_obj=$md5_obj=$rmd160_obj=$wp_obj=$chapoly_obj= -+ $engines_obj=""; - $cflags=~s/\-D[BL]_ENDIAN// if ($fips); - $thread_cflags=~s/\-D[BL]_ENDIAN// if ($fips); - } -@@ -1598,7 +1603,14 @@ if ($ec_obj =~ /ecp_nistz256/) - { - $cflags.=" -DECP_NISTZ256_ASM"; - } -- -+if ($chapoly_obj =~ /\.o$/) -+ { -+ $cflags.=" -DCHAPOLY_x86_64_ASM"; -+ } -+else -+ { -+ $chapoly_obj=$chapoly_enc; -+ } - # "Stringify" the C flags string. This permits it to be made part of a string - # and works as well on command lines. - $cflags =~ s/([\\\"])/\\\1/g; -@@ -1726,6 +1738,7 @@ while () - s/^CMLL_ENC=.*$/CMLL_ENC= $cmll_obj/; - s/^MODES_ASM_OBJ.=*$/MODES_ASM_OBJ= $modes_obj/; - s/^ENGINES_ASM_OBJ.=*$/ENGINES_ASM_OBJ= $engines_obj/; -+ s/^CHAPOLY_ENC=.*$/CHAPOLY_ENC= $chapoly_obj/; - s/^PERLASM_SCHEME=.*$/PERLASM_SCHEME= $perlasm_scheme/; - s/^PROCESSOR=.*/PROCESSOR= $processor/; - s/^ARFLAGS=.*/ARFLAGS= $arflags/; -@@ -1783,6 +1796,7 @@ print "SHA1_OBJ_ASM =$sha1_obj\n"; - print "RMD160_OBJ_ASM=$rmd160_obj\n"; - print "CMLL_ENC =$cmll_obj\n"; - print "MODES_OBJ =$modes_obj\n"; -+print "CHAPOLY_ENC =$chapoly_obj\n"; - print "ENGINES_OBJ =$engines_obj\n"; - print "PROCESSOR =$processor\n"; - print "RANLIB =$ranlib\n"; -@@ -2211,6 +2225,7 @@ sub print_table_entry - \$wp_obj = $table{$target}->{wp_obj} - \$cmll_obj = $table{$target}->{cmll_obj} - \$modes_obj = $table{$target}->{modes_obj} -+\$chapoly_obj = $table{$target}->{chapoly_obj} - \$engines_obj = $table{$target}->{engines_obj} - \$perlasm_scheme = $table{$target}->{perlasm_scheme} - \$dso_scheme = $table{$target}->{dso_scheme} -@@ -2249,6 +2264,7 @@ EOF - "rc5_obj", - "wp_obj", - "cmll_obj", -+ "chapoly_obj", - "modes_obj", - "engines_obj", - "perlasm_scheme", -diff --git a/Makefile.org b/Makefile.org -index c56aa86..4a449ab 100644 ---- a/Makefile.org -+++ b/Makefile.org -@@ -95,6 +95,7 @@ DES_ENC= des_enc.o fcrypt_b.o - AES_ENC= aes_core.o aes_cbc.o - BF_ENC= bf_enc.o - CAST_ENC= c_enc.o -+CHAPOLY_ENC= - RC4_ENC= rc4_enc.o - RC5_ENC= rc5_enc.o - MD5_ASM_OBJ= -@@ -143,7 +144,7 @@ SDIRS= \ - bn ec rsa dsa ecdsa dh ecdh dso engine \ - buffer bio stack lhash rand err \ - evp asn1 pem x509 x509v3 conf txt_db pkcs7 pkcs12 comp ocsp ui \ -- cms pqueue ts jpake srp store cmac -+ cms pqueue ts jpake srp store cmac chacha20poly1305 - # keep in mind that the above list is adjusted by ./Configure - # according to no-xxx arguments... - -@@ -234,6 +235,7 @@ BUILDENV= PLATFORM='$(PLATFORM)' PROCESSOR='$(PROCESSOR)' \ - WP_ASM_OBJ='$(WP_ASM_OBJ)' \ - MODES_ASM_OBJ='$(MODES_ASM_OBJ)' \ - ENGINES_ASM_OBJ='$(ENGINES_ASM_OBJ)' \ -+ CHAPOLY_ENC='$(CHAPOLY_ENC)' \ - PERLASM_SCHEME='$(PERLASM_SCHEME)' \ - FIPSLIBDIR='${FIPSLIBDIR}' \ - FIPSCANLIB="$${FIPSCANLIB:-$(FIPSCANLIB)}" \ -diff --git a/apps/speed.c b/apps/speed.c -index 1508076..8dc9b4b 100644 ---- a/apps/speed.c -+++ b/apps/speed.c -@@ -190,7 +190,7 @@ - #endif - - #undef BUFSIZE --#define BUFSIZE (1024*8+1) -+#define BUFSIZE (1024*8+16) - #define MAX_MISALIGNMENT 63 - - static volatile int run = 0; -@@ -207,7 +207,7 @@ static void print_result(int alg, int run_no, int count, double time_used); - static int do_multi(int multi); - #endif - --#define ALGOR_NUM 30 -+#define ALGOR_NUM 31 - #define SIZE_NUM 5 - #define PRIME_NUM 3 - #define RSA_NUM 7 -@@ -224,7 +224,7 @@ static const char *names[ALGOR_NUM] = { - "aes-128 cbc", "aes-192 cbc", "aes-256 cbc", - "camellia-128 cbc", "camellia-192 cbc", "camellia-256 cbc", - "evp", "sha256", "sha512", "whirlpool", -- "aes-128 ige", "aes-192 ige", "aes-256 ige", "ghash" -+ "aes-128 ige", "aes-192 ige", "aes-256 ige", "ghash", "chacha20-poly1305" - }; - - static double results[ALGOR_NUM][SIZE_NUM]; -@@ -405,6 +405,7 @@ OPTIONS speed_options[] = { - #define D_IGE_192_AES 27 - #define D_IGE_256_AES 28 - #define D_GHASH 29 -+#define D_CHAPOLY 30 - static OPT_PAIR doit_choices[] = { - #ifndef OPENSSL_NO_MD2 - {"md2", D_MD2}, -@@ -474,6 +475,7 @@ static OPT_PAIR doit_choices[] = { - {"cast5", D_CBC_CAST}, - #endif - {"ghash", D_GHASH}, -+ {"chacha20-poly1305", D_CHAPOLY}, - {NULL} - }; - -@@ -1069,6 +1071,7 @@ int speed_main(int argc, char **argv) - c[D_IGE_192_AES][0] = count; - c[D_IGE_256_AES][0] = count; - c[D_GHASH][0] = count; -+ c[D_CHAPOLY][0] = count; - - for (i = 1; i < SIZE_NUM; i++) { - long l0, l1; -@@ -1512,6 +1515,21 @@ int speed_main(int argc, char **argv) - CRYPTO_gcm128_release(ctx); - } - #endif -+ if (doit[D_CHAPOLY]) { -+ EVP_CIPHER_CTX ctx; -+ EVP_CIPHER_CTX_init(&ctx); -+ EVP_CipherInit_ex(&ctx,EVP_chacha20_poly1305(),NULL,key32,NULL,1); -+ for (j=0; j $@ -+poly1305_avx.s:asm/poly1305_avx.pl -+ $(PERL) asm/poly1305_avx.pl $(PERLASM_SCHEME) > $@ -+chacha20_avx2.s:asm/chacha20_avx2.pl -+ $(PERL) asm/chacha20_avx2.pl $(PERLASM_SCHEME) > $@ -+poly1305_avx2.s:asm/poly1305_avx2.pl -+ $(PERL) asm/poly1305_avx2.pl $(PERLASM_SCHEME) > $@ -+ -+files: -+ $(PERL) $(TOP)/util/files.pl Makefile >> $(TOP)/MINFO -+ -+links: -+ @$(PERL) $(TOP)/util/mklink.pl ../../include/openssl $(EXHEADER) -+ @$(PERL) $(TOP)/util/mklink.pl ../../test $(TEST) -+ @$(PERL) $(TOP)/util/mklink.pl ../../apps $(APPS) -+ -+install: -+ @[ -n "$(INSTALLTOP)" ] # should be set by top Makefile... -+ @headerlist="$(EXHEADER)"; for i in $$headerlist ; \ -+ do \ -+ (cp $$i $(INSTALL_PREFIX)$(INSTALLTOP)/include/openssl/$$i; \ -+ chmod 644 $(INSTALL_PREFIX)$(INSTALLTOP)/include/openssl/$$i ); \ -+ done; -+ -+tags: -+ ctags $(SRC) -+ -+tests: -+ -+lint: -+ lint -DLINT $(INCLUDES) $(SRC)>fluff -+ -+depend: -+ @[ -n "$(MAKEDEPEND)" ] # should be set by upper Makefile... -+ $(MAKEDEPEND) -- $(CFLAG) $(INCLUDES) $(DEPFLAG) -- $(PROGS) $(LIBSRC) -+ -+dclean: -+ $(PERL) -pe 'if (/^# DO NOT DELETE THIS LINE/) {print; exit(0);}' $(MAKEFILE) >Makefile.new -+ mv -f Makefile.new $(MAKEFILE) -+ -+clean: -+ rm -f *.s *.o *.obj lib tags core .pure .nfs* *.old *.bak fluff -+ -+# DO NOT DELETE THIS LINE -- make depend depends on it. -+ -+chacha20.o: ../../include/openssl/chacha20poly1305.h chacha20.c -+poly1305.o: ../../include/openssl/chacha20poly1305.h poly1305.c -diff --git a/crypto/chacha20poly1305/asm/chacha20_avx.pl b/crypto/chacha20poly1305/asm/chacha20_avx.pl -new file mode 100644 -index 0000000..7b5b763 ---- /dev/null -+++ b/crypto/chacha20poly1305/asm/chacha20_avx.pl -@@ -0,0 +1,388 @@ -+#!/usr/bin/env perl -+ -+############################################################################## -+# # -+# Copyright 2014 Intel Corporation # -+# # -+# Licensed under the Apache License, Version 2.0 (the "License"); # -+# you may not use this file except in compliance with the License. # -+# You may obtain a copy of the License at # -+# # -+# http://www.apache.org/licenses/LICENSE-2.0 # -+# # -+# Unless required by applicable law or agreed to in writing, software # -+# distributed under the License is distributed on an "AS IS" BASIS, # -+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -+# See the License for the specific language governing permissions and # -+# limitations under the License. # -+# # -+############################################################################## -+# # -+# Developers and authors: # -+# Shay Gueron (1, 2), and Vlad Krasnov (1) # -+# (1) Intel Corporation, Israel Development Center # -+# (2) University of Haifa # -+# # -+# Related work: # -+# M. Goll, S. Gueron, "Vectorization on ChaCha Stream Cipher", IEEE # -+# Proceedings of 11th International Conference on Information # -+# Technology: New Generations (ITNG 2014), 612-615 (2014). # -+# M. Goll, S. Gueron, "Vectorization on Poly1305 Message Authentication Code"# -+# to be published. # -+# A. Langley, chacha20poly1305 for the AEAD head # -+# https://git.openssl.org/gitweb/?p=openssl.git;a=commit;h=9a8646510b3d0a48e950748f7a2aaa12ed40d5e0 # -+############################################################################## -+ -+ -+ -+$flavour = shift; -+$output = shift; -+if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } -+ -+$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); -+ -+$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; -+( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or -+( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or -+die "can't locate x86_64-xlate.pl"; -+ -+open OUT,"| \"$^X\" $xlate $flavour $output"; -+*STDOUT=*OUT; -+ -+if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` -+ =~ /GNU assembler version ([2-9]\.[0-9]+)/) { -+ $avx = ($1>=2.19) + ($1>=2.22); -+} -+ -+if ($win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && -+ `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { -+ $avx = ($1>=2.09) + ($1>=2.10); -+} -+ -+if ($win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && -+ `ml64 2>&1` =~ /Version ([0-9]+)\./) { -+ $avx = ($1>=10) + ($1>=11); -+} -+ -+if (`$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9])\.([0-9]+)/) { -+ my $ver = $2 + $3/100.0; # 3.1->3.01, 3.10->3.10 -+ $avx = ($ver>=3.0) + ($ver>=3.01); -+} -+ -+if ($avx>=1) {{ -+ -+sub chacha_qr { -+my ($a,$b,$c,$d,$tmp)=@_; -+$code.=<<___; -+ -+ vpaddd $b, $a, $a # a += b -+ vpxor $a, $d, $d # d ^= a -+ vpshufb .rol16(%rip), $d, $d # d <<<= 16 -+ -+ vpaddd $d, $c, $c # c += d -+ vpxor $c, $b, $b # b ^= c -+ vpslld \$12, $b, $tmp -+ vpsrld \$20, $b, $b -+ vpxor $tmp, $b, $b # b <<<= 12 -+ -+ vpaddd $b, $a, $a # a += b -+ vpxor $a, $d, $d # d ^= a -+ vpshufb .rol8(%rip), $d, $d # d <<<= 8 -+ -+ vpaddd $d, $c, $c # c += d -+ vpxor $c, $b, $b # b ^= c -+ -+ vpslld \$7, $b, $tmp -+ vpsrld \$25, $b, $b -+ vpxor $tmp, $b, $b # b <<<= 7 -+___ -+} -+ -+ -+$code.=<<___; -+.text -+.align 16 -+chacha20_consts: -+.byte 'e','x','p','a','n','d',' ','3','2','-','b','y','t','e',' ','k' -+.rol8: -+.byte 3,0,1,2, 7,4,5,6, 11,8,9,10, 15,12,13,14 -+.rol16: -+.byte 2,3,0,1, 6,7,4,5, 10,11,8,9, 14,15,12,13 -+.avxInc: -+.quad 1,0 -+___ -+ -+{ -+my ($state_4567, $state_89ab, $state_cdef, $tmp, -+ $v0, $v1, $v2, $v3, $v4, $v5, $v6, $v7, -+ $v8, $v9, $v10, $v11)=map("%xmm$_",(0..15)); -+ -+my ($out, $in, $in_len, $key_ptr, $nonce_ptr, $counter, $nr) -+ =("%rdi", "%rsi", "%rdx", "%rcx", "%r8", "%r9", "%rax"); -+ -+$code.=<<___; -+.globl chacha_20_core_avx -+.type chacha_20_core_avx ,\@function,2 -+.align 64 -+chacha_20_core_avx: -+ vzeroupper -+ -+ # Init state -+ vmovdqu 16*0($key_ptr), $state_4567 -+ vmovdqu 16*1($key_ptr), $state_89ab -+ vmovq $counter, $state_cdef -+ vpinsrq \$1, ($nonce_ptr), $state_cdef, $state_cdef -+2: -+ cmp \$3*64, $in_len -+ jb 2f -+ -+ vmovdqa chacha20_consts(%rip), $v0 -+ vmovdqa chacha20_consts(%rip), $v4 -+ vmovdqa chacha20_consts(%rip), $v8 -+ -+ vmovdqa $state_4567, $v1 -+ vmovdqa $state_4567, $v5 -+ vmovdqa $state_4567, $v9 -+ -+ vmovdqa $state_89ab, $v2 -+ vmovdqa $state_89ab, $v6 -+ vmovdqa $state_89ab, $v10 -+ -+ vmovdqa $state_cdef, $v3 -+ vpaddq .avxInc(%rip), $v3, $v7 -+ vpaddq .avxInc(%rip), $v7, $v11 -+ -+ mov \$10, $nr -+ -+ 1: -+___ -+ &chacha_qr($v0,$v1,$v2,$v3,$tmp); -+ &chacha_qr($v4,$v5,$v6,$v7,$tmp); -+ &chacha_qr($v8,$v9,$v10,$v11,$tmp); -+$code.=<<___; -+ vpalignr \$4, $v1, $v1, $v1 -+ vpalignr \$8, $v2, $v2, $v2 -+ vpalignr \$12, $v3, $v3, $v3 -+ vpalignr \$4, $v5, $v5, $v5 -+ vpalignr \$8, $v6, $v6, $v6 -+ vpalignr \$12, $v7, $v7, $v7 -+ vpalignr \$4, $v9, $v9, $v9 -+ vpalignr \$8, $v10, $v10, $v10 -+ vpalignr \$12, $v11, $v11, $v11 -+___ -+ &chacha_qr($v0,$v1,$v2,$v3,$tmp); -+ &chacha_qr($v4,$v5,$v6,$v7,$tmp); -+ &chacha_qr($v8,$v9,$v10,$v11,$tmp); -+$code.=<<___; -+ vpalignr \$12, $v1, $v1, $v1 -+ vpalignr \$8, $v2, $v2, $v2 -+ vpalignr \$4, $v3, $v3, $v3 -+ vpalignr \$12, $v5, $v5, $v5 -+ vpalignr \$8, $v6, $v6, $v6 -+ vpalignr \$4, $v7, $v7, $v7 -+ vpalignr \$12, $v9, $v9, $v9 -+ vpalignr \$8, $v10, $v10, $v10 -+ vpalignr \$4, $v11, $v11, $v11 -+ -+ dec $nr -+ -+ jnz 1b -+ -+ vpaddd chacha20_consts(%rip), $v0, $v0 -+ vpaddd chacha20_consts(%rip), $v4, $v4 -+ vpaddd chacha20_consts(%rip), $v8, $v8 -+ -+ vpaddd $state_4567, $v1, $v1 -+ vpaddd $state_4567, $v5, $v5 -+ vpaddd $state_4567, $v9, $v9 -+ -+ vpaddd $state_89ab, $v2, $v2 -+ vpaddd $state_89ab, $v6, $v6 -+ vpaddd $state_89ab, $v10, $v10 -+ -+ vpaddd $state_cdef, $v3, $v3 -+ vpaddq .avxInc(%rip), $state_cdef, $state_cdef -+ vpaddd $state_cdef, $v7, $v7 -+ vpaddq .avxInc(%rip), $state_cdef, $state_cdef -+ vpaddd $state_cdef, $v11, $v11 -+ vpaddq .avxInc(%rip), $state_cdef, $state_cdef -+ -+ vpxor 16*0($in), $v0, $v0 -+ vpxor 16*1($in), $v1, $v1 -+ vpxor 16*2($in), $v2, $v2 -+ vpxor 16*3($in), $v3, $v3 -+ -+ vmovdqu $v0, 16*0($out) -+ vmovdqu $v1, 16*1($out) -+ vmovdqu $v2, 16*2($out) -+ vmovdqu $v3, 16*3($out) -+ -+ vpxor 16*4($in), $v4, $v4 -+ vpxor 16*5($in), $v5, $v5 -+ vpxor 16*6($in), $v6, $v6 -+ vpxor 16*7($in), $v7, $v7 -+ -+ vmovdqu $v4, 16*4($out) -+ vmovdqu $v5, 16*5($out) -+ vmovdqu $v6, 16*6($out) -+ vmovdqu $v7, 16*7($out) -+ -+ vpxor 16*8($in), $v8, $v8 -+ vpxor 16*9($in), $v9, $v9 -+ vpxor 16*10($in), $v10, $v10 -+ vpxor 16*11($in), $v11, $v11 -+ -+ vmovdqu $v8, 16*8($out) -+ vmovdqu $v9, 16*9($out) -+ vmovdqu $v10, 16*10($out) -+ vmovdqu $v11, 16*11($out) -+ -+ lea 16*12($in), $in -+ lea 16*12($out), $out -+ sub \$16*12, $in_len -+ -+ jmp 2b -+ -+2: -+ cmp \$2*64, $in_len -+ jb 2f -+ -+ vmovdqa chacha20_consts(%rip), $v0 -+ vmovdqa chacha20_consts(%rip), $v4 -+ vmovdqa $state_4567, $v1 -+ vmovdqa $state_4567, $v5 -+ vmovdqa $state_89ab, $v2 -+ vmovdqa $state_89ab, $v6 -+ vmovdqa $state_89ab, $v10 -+ vmovdqa $state_cdef, $v3 -+ vpaddq .avxInc(%rip), $v3, $v7 -+ -+ mov \$10, $nr -+ -+ 1: -+___ -+ &chacha_qr($v0,$v1,$v2,$v3,$tmp); -+ &chacha_qr($v4,$v5,$v6,$v7,$tmp); -+$code.=<<___; -+ vpalignr \$4, $v1, $v1, $v1 -+ vpalignr \$8, $v2, $v2, $v2 -+ vpalignr \$12, $v3, $v3, $v3 -+ vpalignr \$4, $v5, $v5, $v5 -+ vpalignr \$8, $v6, $v6, $v6 -+ vpalignr \$12, $v7, $v7, $v7 -+___ -+ &chacha_qr($v0,$v1,$v2,$v3,$tmp); -+ &chacha_qr($v4,$v5,$v6,$v7,$tmp); -+$code.=<<___; -+ vpalignr \$12, $v1, $v1, $v1 -+ vpalignr \$8, $v2, $v2, $v2 -+ vpalignr \$4, $v3, $v3, $v3 -+ vpalignr \$12, $v5, $v5, $v5 -+ vpalignr \$8, $v6, $v6, $v6 -+ vpalignr \$4, $v7, $v7, $v7 -+ -+ dec $nr -+ -+ jnz 1b -+ -+ vpaddd chacha20_consts(%rip), $v0, $v0 -+ vpaddd chacha20_consts(%rip), $v4, $v4 -+ -+ vpaddd $state_4567, $v1, $v1 -+ vpaddd $state_4567, $v5, $v5 -+ -+ vpaddd $state_89ab, $v2, $v2 -+ vpaddd $state_89ab, $v6, $v6 -+ -+ vpaddd $state_cdef, $v3, $v3 -+ vpaddq .avxInc(%rip), $state_cdef, $state_cdef -+ vpaddd $state_cdef, $v7, $v7 -+ vpaddq .avxInc(%rip), $state_cdef, $state_cdef -+ -+ vpxor 16*0($in), $v0, $v0 -+ vpxor 16*1($in), $v1, $v1 -+ vpxor 16*2($in), $v2, $v2 -+ vpxor 16*3($in), $v3, $v3 -+ -+ vmovdqu $v0, 16*0($out) -+ vmovdqu $v1, 16*1($out) -+ vmovdqu $v2, 16*2($out) -+ vmovdqu $v3, 16*3($out) -+ -+ vpxor 16*4($in), $v4, $v4 -+ vpxor 16*5($in), $v5, $v5 -+ vpxor 16*6($in), $v6, $v6 -+ vpxor 16*7($in), $v7, $v7 -+ -+ vmovdqu $v4, 16*4($out) -+ vmovdqu $v5, 16*5($out) -+ vmovdqu $v6, 16*6($out) -+ vmovdqu $v7, 16*7($out) -+ -+ lea 16*8($in), $in -+ lea 16*8($out), $out -+ sub \$16*8, $in_len -+ -+ jmp 2b -+2: -+ cmp \$64, $in_len -+ jb 2f -+ -+ vmovdqa chacha20_consts(%rip), $v0 -+ vmovdqa $state_4567, $v1 -+ vmovdqa $state_89ab, $v2 -+ vmovdqa $state_cdef, $v3 -+ -+ mov \$10, $nr -+ -+ 1: -+___ -+ &chacha_qr($v0,$v1,$v2,$v3,$tmp); -+$code.=<<___; -+ vpalignr \$4, $v1, $v1, $v1 -+ vpalignr \$8, $v2, $v2, $v2 -+ vpalignr \$12, $v3, $v3, $v3 -+___ -+ &chacha_qr($v0,$v1,$v2,$v3,$tmp); -+$code.=<<___; -+ vpalignr \$12, $v1, $v1, $v1 -+ vpalignr \$8, $v2, $v2, $v2 -+ vpalignr \$4, $v3, $v3, $v3 -+ -+ dec $nr -+ jnz 1b -+ -+ vpaddd chacha20_consts(%rip), $v0, $v0 -+ vpaddd $state_4567, $v1, $v1 -+ vpaddd $state_89ab, $v2, $v2 -+ vpaddd $state_cdef, $v3, $v3 -+ vpaddq .avxInc(%rip), $state_cdef, $state_cdef -+ -+ vpxor 16*0($in), $v0, $v0 -+ vpxor 16*1($in), $v1, $v1 -+ vpxor 16*2($in), $v2, $v2 -+ vpxor 16*3($in), $v3, $v3 -+ -+ vmovdqu $v0, 16*0($out) -+ vmovdqu $v1, 16*1($out) -+ vmovdqu $v2, 16*2($out) -+ vmovdqu $v3, 16*3($out) -+ -+ lea 16*4($in), $in -+ lea 16*4($out), $out -+ sub \$16*4, $in_len -+ jmp 2b -+2: -+ vzeroupper -+ ret -+.size chacha_20_core_avx,.-chacha_20_core_avx -+___ -+} -+}} -+ -+ -+$code =~ s/\`([^\`]*)\`/eval($1)/gem; -+ -+print $code; -+ -+close STDOUT; -diff --git a/crypto/chacha20poly1305/asm/chacha20_avx2.pl b/crypto/chacha20poly1305/asm/chacha20_avx2.pl -new file mode 100644 -index 0000000..31ae721 ---- /dev/null -+++ b/crypto/chacha20poly1305/asm/chacha20_avx2.pl -@@ -0,0 +1,424 @@ -+#!/usr/bin/env perl -+ -+############################################################################## -+# # -+# Copyright 2014 Intel Corporation # -+# # -+# Licensed under the Apache License, Version 2.0 (the "License"); # -+# you may not use this file except in compliance with the License. # -+# You may obtain a copy of the License at # -+# # -+# http://www.apache.org/licenses/LICENSE-2.0 # -+# # -+# Unless required by applicable law or agreed to in writing, software # -+# distributed under the License is distributed on an "AS IS" BASIS, # -+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -+# See the License for the specific language governing permissions and # -+# limitations under the License. # -+# # -+############################################################################## -+# # -+# Developers and authors: # -+# Shay Gueron (1, 2), and Vlad Krasnov (1) # -+# (1) Intel Corporation, Israel Development Center # -+# (2) University of Haifa # -+# # -+# Related work: # -+# M. Goll, S. Gueron, "Vectorization on ChaCha Stream Cipher", IEEE # -+# Proceedings of 11th International Conference on Information # -+# Technology: New Generations (ITNG 2014), 612-615 (2014). # -+# M. Goll, S. Gueron, "Vectorization on Poly1305 Message Authentication Code"# -+# to be published. # -+# A. Langley, chacha20poly1305 for the AEAD head # -+# https://git.openssl.org/gitweb/?p=openssl.git;a=commit;h=9a8646510b3d0a48e950748f7a2aaa12ed40d5e0 # -+############################################################################## -+ -+$flavour = shift; -+$output = shift; -+if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } -+ -+$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); -+ -+$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; -+( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or -+( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or -+die "can't locate x86_64-xlate.pl"; -+ -+open OUT,"| \"$^X\" $xlate $flavour $output"; -+*STDOUT=*OUT; -+ -+if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` -+ =~ /GNU assembler version ([2-9]\.[0-9]+)/) { -+ $avx = ($1>=2.19) + ($1>=2.22); -+} -+ -+if ($win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && -+ `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { -+ $avx = ($1>=2.09) + ($1>=2.10); -+} -+ -+if ($win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && -+ `ml64 2>&1` =~ /Version ([0-9]+)\./) { -+ $avx = ($1>=10) + ($1>=11); -+} -+ -+if (`$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9])\.([0-9]+)/) { -+ my $ver = $2 + $3/100.0; # 3.1->3.01, 3.10->3.10 -+ $avx = ($ver>=3.0) + ($ver>=3.01); -+} -+ -+if ($avx>=2) {{ -+ -+sub chacha_qr { -+my ($a,$b,$c,$d,$tmp)=@_; -+$code.=<<___; -+ -+ vpaddd $b, $a, $a # a += b -+ vpxor $a, $d, $d # d ^= a -+ vpshufb .rol16(%rip), $d, $d # d <<<= 16 -+ -+ vpaddd $d, $c, $c # c += d -+ vpxor $c, $b, $b # b ^= c -+ vpslld \$12, $b, $tmp -+ vpsrld \$20, $b, $b -+ vpxor $tmp, $b, $b # b <<<= 12 -+ -+ vpaddd $b, $a, $a # a += b -+ vpxor $a, $d, $d # d ^= a -+ vpshufb .rol8(%rip), $d, $d # d <<<= 8 -+ -+ vpaddd $d, $c, $c # c += d -+ vpxor $c, $b, $b # b ^= c -+ -+ vpslld \$7, $b, $tmp -+ vpsrld \$25, $b, $b -+ vpxor $tmp, $b, $b # b <<<= 7 -+___ -+} -+ -+ -+$code.=<<___; -+.text -+.align 32 -+chacha20_consts: -+.byte 'e','x','p','a','n','d',' ','3','2','-','b','y','t','e',' ','k' -+.byte 'e','x','p','a','n','d',' ','3','2','-','b','y','t','e',' ','k' -+.rol8: -+.byte 3,0,1,2, 7,4,5,6, 11,8,9,10, 15,12,13,14 -+.byte 3,0,1,2, 7,4,5,6, 11,8,9,10, 15,12,13,14 -+.rol16: -+.byte 2,3,0,1, 6,7,4,5, 10,11,8,9, 14,15,12,13 -+.byte 2,3,0,1, 6,7,4,5, 10,11,8,9, 14,15,12,13 -+.avx2Init: -+.quad 0,0,1,0 -+.avx2Inc: -+.quad 2,0,2,0 -+___ -+ -+{ -+my ($state_4567, $state_89ab, $state_cdef, $tmp, -+ $v0, $v1, $v2, $v3, $v4, $v5, $v6, $v7, -+ $v8, $v9, $v10, $v11)=map("%ymm$_",(0..15)); -+ -+my $state_cdef_xmm="%xmm2"; -+ -+my ($out, $in, $in_len, $key_ptr, $nonce_ptr, $counter, $nr) -+ =("%rdi", "%rsi", "%rdx", "%rcx", "%r8", "%r9", "%rax"); -+ -+$code.=<<___; -+.globl chacha_20_core_avx2 -+.type chacha_20_core_avx2 ,\@function,2 -+.align 64 -+chacha_20_core_avx2: -+ vzeroupper -+ -+ # Init state -+ vbroadcasti128 16*0($key_ptr), $state_4567 -+ vbroadcasti128 16*1($key_ptr), $state_89ab -+ vmovq $counter, $state_cdef_xmm -+ vpinsrq \$1, ($nonce_ptr), $state_cdef_xmm, $state_cdef_xmm -+ vperm2i128 \$0x00, $state_cdef, $state_cdef, $state_cdef -+ vpaddq .avx2Init(%rip), $state_cdef, $state_cdef -+ -+2: -+ cmp \$6*64, $in_len -+ jb 2f -+ -+ vmovdqa chacha20_consts(%rip), $v0 -+ vmovdqa chacha20_consts(%rip), $v4 -+ vmovdqa chacha20_consts(%rip), $v8 -+ -+ vmovdqa $state_4567, $v1 -+ vmovdqa $state_4567, $v5 -+ vmovdqa $state_4567, $v9 -+ -+ vmovdqa $state_89ab, $v2 -+ vmovdqa $state_89ab, $v6 -+ vmovdqa $state_89ab, $v10 -+ -+ vmovdqa $state_cdef, $v3 -+ vpaddq .avx2Inc(%rip), $v3, $v7 -+ vpaddq .avx2Inc(%rip), $v7, $v11 -+ -+ mov \$10, $nr -+ -+ 1: -+___ -+ &chacha_qr($v0,$v1,$v2,$v3,$tmp); -+ &chacha_qr($v4,$v5,$v6,$v7,$tmp); -+ &chacha_qr($v8,$v9,$v10,$v11,$tmp); -+$code.=<<___; -+ vpalignr \$4, $v1, $v1, $v1 -+ vpalignr \$8, $v2, $v2, $v2 -+ vpalignr \$12, $v3, $v3, $v3 -+ vpalignr \$4, $v5, $v5, $v5 -+ vpalignr \$8, $v6, $v6, $v6 -+ vpalignr \$12, $v7, $v7, $v7 -+ vpalignr \$4, $v9, $v9, $v9 -+ vpalignr \$8, $v10, $v10, $v10 -+ vpalignr \$12, $v11, $v11, $v11 -+___ -+ &chacha_qr($v0,$v1,$v2,$v3,$tmp); -+ &chacha_qr($v4,$v5,$v6,$v7,$tmp); -+ &chacha_qr($v8,$v9,$v10,$v11,$tmp); -+$code.=<<___; -+ vpalignr \$12, $v1, $v1, $v1 -+ vpalignr \$8, $v2, $v2, $v2 -+ vpalignr \$4, $v3, $v3, $v3 -+ vpalignr \$12, $v5, $v5, $v5 -+ vpalignr \$8, $v6, $v6, $v6 -+ vpalignr \$4, $v7, $v7, $v7 -+ vpalignr \$12, $v9, $v9, $v9 -+ vpalignr \$8, $v10, $v10, $v10 -+ vpalignr \$4, $v11, $v11, $v11 -+ -+ dec $nr -+ -+ jnz 1b -+ -+ vpaddd chacha20_consts(%rip), $v0, $v0 -+ vpaddd chacha20_consts(%rip), $v4, $v4 -+ vpaddd chacha20_consts(%rip), $v8, $v8 -+ -+ vpaddd $state_4567, $v1, $v1 -+ vpaddd $state_4567, $v5, $v5 -+ vpaddd $state_4567, $v9, $v9 -+ -+ vpaddd $state_89ab, $v2, $v2 -+ vpaddd $state_89ab, $v6, $v6 -+ vpaddd $state_89ab, $v10, $v10 -+ -+ vpaddd $state_cdef, $v3, $v3 -+ vpaddq .avx2Inc(%rip), $state_cdef, $state_cdef -+ vpaddd $state_cdef, $v7, $v7 -+ vpaddq .avx2Inc(%rip), $state_cdef, $state_cdef -+ vpaddd $state_cdef, $v11, $v11 -+ vpaddq .avx2Inc(%rip), $state_cdef, $state_cdef -+ -+ vperm2i128 \$0x02, $v0, $v1, $tmp -+ vpxor 32*0($in), $tmp, $tmp -+ vmovdqu $tmp, 32*0($out) -+ vperm2i128 \$0x02, $v2, $v3, $tmp -+ vpxor 32*1($in), $tmp, $tmp -+ vmovdqu $tmp, 32*1($out) -+ vperm2i128 \$0x13, $v0, $v1, $tmp -+ vpxor 32*2($in), $tmp, $tmp -+ vmovdqu $tmp, 32*2($out) -+ vperm2i128 \$0x13, $v2, $v3, $tmp -+ vpxor 32*3($in), $tmp, $tmp -+ vmovdqu $tmp, 32*3($out) -+ -+ vperm2i128 \$0x02, $v4, $v5, $v0 -+ vperm2i128 \$0x02, $v6, $v7, $v1 -+ vperm2i128 \$0x13, $v4, $v5, $v2 -+ vperm2i128 \$0x13, $v6, $v7, $v3 -+ -+ vpxor 32*4($in), $v0, $v0 -+ vpxor 32*5($in), $v1, $v1 -+ vpxor 32*6($in), $v2, $v2 -+ vpxor 32*7($in), $v3, $v3 -+ -+ vmovdqu $v0, 32*4($out) -+ vmovdqu $v1, 32*5($out) -+ vmovdqu $v2, 32*6($out) -+ vmovdqu $v3, 32*7($out) -+ -+ vperm2i128 \$0x02, $v8, $v9, $v0 -+ vperm2i128 \$0x02, $v10, $v11, $v1 -+ vperm2i128 \$0x13, $v8, $v9, $v2 -+ vperm2i128 \$0x13, $v10, $v11, $v3 -+ -+ vpxor 32*8($in), $v0, $v0 -+ vpxor 32*9($in), $v1, $v1 -+ vpxor 32*10($in), $v2, $v2 -+ vpxor 32*11($in), $v3, $v3 -+ -+ vmovdqu $v0, 32*8($out) -+ vmovdqu $v1, 32*9($out) -+ vmovdqu $v2, 32*10($out) -+ vmovdqu $v3, 32*11($out) -+ -+ lea 64*6($in), $in -+ lea 64*6($out), $out -+ sub \$64*6, $in_len -+ -+ jmp 2b -+ -+2: -+ cmp \$4*64, $in_len -+ jb 2f -+ -+ vmovdqa chacha20_consts(%rip), $v0 -+ vmovdqa chacha20_consts(%rip), $v4 -+ vmovdqa $state_4567, $v1 -+ vmovdqa $state_4567, $v5 -+ vmovdqa $state_89ab, $v2 -+ vmovdqa $state_89ab, $v6 -+ vmovdqa $state_89ab, $v10 -+ vmovdqa $state_cdef, $v3 -+ vpaddq .avx2Inc(%rip), $v3, $v7 -+ -+ mov \$10, $nr -+ -+ 1: -+___ -+ &chacha_qr($v0,$v1,$v2,$v3,$tmp); -+ &chacha_qr($v4,$v5,$v6,$v7,$tmp); -+$code.=<<___; -+ vpalignr \$4, $v1, $v1, $v1 -+ vpalignr \$8, $v2, $v2, $v2 -+ vpalignr \$12, $v3, $v3, $v3 -+ vpalignr \$4, $v5, $v5, $v5 -+ vpalignr \$8, $v6, $v6, $v6 -+ vpalignr \$12, $v7, $v7, $v7 -+___ -+ &chacha_qr($v0,$v1,$v2,$v3,$tmp); -+ &chacha_qr($v4,$v5,$v6,$v7,$tmp); -+$code.=<<___; -+ vpalignr \$12, $v1, $v1, $v1 -+ vpalignr \$8, $v2, $v2, $v2 -+ vpalignr \$4, $v3, $v3, $v3 -+ vpalignr \$12, $v5, $v5, $v5 -+ vpalignr \$8, $v6, $v6, $v6 -+ vpalignr \$4, $v7, $v7, $v7 -+ -+ dec $nr -+ -+ jnz 1b -+ -+ vpaddd chacha20_consts(%rip), $v0, $v0 -+ vpaddd chacha20_consts(%rip), $v4, $v4 -+ -+ vpaddd $state_4567, $v1, $v1 -+ vpaddd $state_4567, $v5, $v5 -+ -+ vpaddd $state_89ab, $v2, $v2 -+ vpaddd $state_89ab, $v6, $v6 -+ -+ vpaddd $state_cdef, $v3, $v3 -+ vpaddq .avx2Inc(%rip), $state_cdef, $state_cdef -+ vpaddd $state_cdef, $v7, $v7 -+ vpaddq .avx2Inc(%rip), $state_cdef, $state_cdef -+ -+ vperm2i128 \$0x02, $v0, $v1, $v8 -+ vperm2i128 \$0x02, $v2, $v3, $v9 -+ vperm2i128 \$0x13, $v0, $v1, $v10 -+ vperm2i128 \$0x13, $v2, $v3, $v11 -+ -+ vpxor 32*0($in), $v8, $v8 -+ vpxor 32*1($in), $v9, $v9 -+ vpxor 32*2($in), $v10, $v10 -+ vpxor 32*3($in), $v11, $v11 -+ -+ vmovdqu $v8, 32*0($out) -+ vmovdqu $v9, 32*1($out) -+ vmovdqu $v10, 32*2($out) -+ vmovdqu $v11, 32*3($out) -+ -+ vperm2i128 \$0x02, $v4, $v5, $v0 -+ vperm2i128 \$0x02, $v6, $v7, $v1 -+ vperm2i128 \$0x13, $v4, $v5, $v2 -+ vperm2i128 \$0x13, $v6, $v7, $v3 -+ -+ vpxor 32*4($in), $v0, $v0 -+ vpxor 32*5($in), $v1, $v1 -+ vpxor 32*6($in), $v2, $v2 -+ vpxor 32*7($in), $v3, $v3 -+ -+ vmovdqu $v0, 32*4($out) -+ vmovdqu $v1, 32*5($out) -+ vmovdqu $v2, 32*6($out) -+ vmovdqu $v3, 32*7($out) -+ -+ lea 64*4($in), $in -+ lea 64*4($out), $out -+ sub \$64*4, $in_len -+ -+ jmp 2b -+2: -+ cmp \$128, $in_len -+ jb 2f -+ -+ vmovdqa chacha20_consts(%rip), $v0 -+ vmovdqa $state_4567, $v1 -+ vmovdqa $state_89ab, $v2 -+ vmovdqa $state_cdef, $v3 -+ -+ mov \$10, $nr -+ -+ 1: -+___ -+ &chacha_qr($v0,$v1,$v2,$v3,$tmp); -+$code.=<<___; -+ vpalignr \$4, $v1, $v1, $v1 -+ vpalignr \$8, $v2, $v2, $v2 -+ vpalignr \$12, $v3, $v3, $v3 -+___ -+ &chacha_qr($v0,$v1,$v2,$v3,$tmp); -+$code.=<<___; -+ vpalignr \$12, $v1, $v1, $v1 -+ vpalignr \$8, $v2, $v2, $v2 -+ vpalignr \$4, $v3, $v3, $v3 -+ -+ dec $nr -+ jnz 1b -+ -+ vpaddd chacha20_consts(%rip), $v0, $v0 -+ vpaddd $state_4567, $v1, $v1 -+ vpaddd $state_89ab, $v2, $v2 -+ vpaddd $state_cdef, $v3, $v3 -+ vpaddq .avx2Inc(%rip), $state_cdef, $state_cdef -+ -+ vperm2i128 \$0x02, $v0, $v1, $v8 -+ vperm2i128 \$0x02, $v2, $v3, $v9 -+ vperm2i128 \$0x13, $v0, $v1, $v10 -+ vperm2i128 \$0x13, $v2, $v3, $v11 -+ -+ vpxor 32*0($in), $v8, $v8 -+ vpxor 32*1($in), $v9, $v9 -+ vpxor 32*2($in), $v10, $v10 -+ vpxor 32*3($in), $v11, $v11 -+ -+ vmovdqu $v8, 32*0($out) -+ vmovdqu $v9, 32*1($out) -+ vmovdqu $v10, 32*2($out) -+ vmovdqu $v11, 32*3($out) -+ -+ lea 64*2($in), $in -+ lea 64*2($out), $out -+ sub \$64*2, $in_len -+ jmp 2b -+2: -+ vzeroupper -+ ret -+.size chacha_20_core_avx2,.-chacha_20_core_avx2 -+___ -+} -+}} -+ -+ -+$code =~ s/\`([^\`]*)\`/eval($1)/gem; -+ -+print $code; -+ -+close STDOUT; -diff --git a/crypto/chacha20poly1305/asm/poly1305_avx.pl b/crypto/chacha20poly1305/asm/poly1305_avx.pl -new file mode 100644 -index 0000000..2d06e41 ---- /dev/null -+++ b/crypto/chacha20poly1305/asm/poly1305_avx.pl -@@ -0,0 +1,717 @@ -+############################################################################## -+# # -+# Copyright 2014 Intel Corporation # -+# # -+# Licensed under the Apache License, Version 2.0 (the "License"); # -+# you may not use this file except in compliance with the License. # -+# You may obtain a copy of the License at # -+# # -+# http://www.apache.org/licenses/LICENSE-2.0 # -+# # -+# Unless required by applicable law or agreed to in writing, software # -+# distributed under the License is distributed on an "AS IS" BASIS, # -+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -+# See the License for the specific language governing permissions and # -+# limitations under the License. # -+# # -+############################################################################## -+# # -+# Developers and authors: # -+# Shay Gueron (1, 2), and Vlad Krasnov (1) # -+# (1) Intel Corporation, Israel Development Center # -+# (2) University of Haifa # -+# # -+############################################################################## -+# state: -+# 0: r[0] || r^2[0] -+# 16: r[1] || r^2[1] -+# 32: r[2] || r^2[2] -+# 48: r[3] || r^2[3] -+# 64: r[4] || r^2[4] -+# 80: r[1]*5 || r^2[1]*5 -+# 96: r[2]*5 || r^2[2]*5 -+#112: r[3]*5 || r^2[3]*5 -+#128: r[4]*5 || r^2[4]*5 -+#144: k -+#160: A0 -+#164: A1 -+#168: A2 -+#172: A3 -+#176: A4 -+#180: END -+ -+$flavour = shift; -+$output = shift; -+if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } -+ -+$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); -+ -+$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; -+( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or -+( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or -+die "can't locate x86_64-xlate.pl"; -+ -+open OUT,"| \"$^X\" $xlate $flavour $output"; -+*STDOUT=*OUT; -+ -+if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` -+ =~ /GNU assembler version ([2-9]\.[0-9]+)/) { -+ $avx = ($1>=2.19) + ($1>=2.22); -+} -+ -+if ($win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && -+ `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { -+ $avx = ($1>=2.09) + ($1>=2.10); -+} -+ -+if ($win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && -+ `ml64 2>&1` =~ /Version ([0-9]+)\./) { -+ $avx = ($1>=10) + ($1>=11); -+} -+ -+if (`$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9])\.([0-9]+)/) { -+ my $ver = $2 + $3/100.0; # 3.1->3.01, 3.10->3.10 -+ $avx = ($ver>=3.0) + ($ver>=3.01); -+} -+ -+if ($avx>=1) {{ -+ -+my ($_r0_, $_r1_, $_r2_, $_r3_, $_r4_, $_r1_x5, $_r2_x5, $_r3_x5, $_r4_x5, $_k_, $_A0_, $_A1_, $_A2_, $_A3_, $_A4_) -+= (0,16,32,48,64,80,96,112,128,144,160,164,168,172,176); -+ -+$code.=<<___; -+.text -+.align 32 -+.LandMask: -+.quad 0x3FFFFFF, 0x3FFFFFF -+.LsetBit: -+.quad 0x1000000, 0x1000000 -+.LrSet: -+.quad 0xFFFFFFC0FFFFFFF, 0xFFFFFFC0FFFFFFF -+.quad 0xFFFFFFC0FFFFFFC, 0xFFFFFFC0FFFFFFC -+.Lone: -+.quad 1,0 -+___ -+ -+ -+{ -+my ($A0, $A1, $A2, $A3, $A4, -+ $r0, $r1, $r2, $r3, $r4, -+ $T0, $T1, $A5, $A6, $A7, $A8)=map("%xmm$_",(0..15)); -+my ($state, $key) -+ =("%rdi", "%rsi"); -+ -+$code.=<<___; -+################################################################################ -+# void poly1305_init_avx(void *state, uint8_t key[32]) -+ -+.globl poly1305_init_avx -+.type poly1305_init_avx, \@function, 2 -+.align 64 -+poly1305_init_avx: -+ vzeroupper -+ # load and convert r -+ vmovq 8*0($key), $r0 -+ vmovq 8*1($key), $T0 -+ vpand .LrSet(%rip), $r0, $r0 -+ vpand .LrSet+16(%rip), $T0, $T0 -+ -+ vpsrlq \$26, $r0, $r1 -+ vpand .LandMask(%rip), $r0, $r0 -+ vpsrlq \$26, $r1, $r2 -+ vpand .LandMask(%rip), $r1, $r1 -+ vpsllq \$12, $T0, $T1 -+ vpxor $T1, $r2, $r2 -+ vpsrlq \$26, $r2, $r3 -+ vpsrlq \$40, $T0, $r4 -+ vpand .LandMask(%rip), $r2, $r2 -+ vpand .LandMask(%rip), $r3, $r3 -+ -+ # SQR R -+ vpmuludq $r0, $r0, $A0 -+ vpmuludq $r1, $r0, $A1 -+ vpmuludq $r2, $r0, $A2 -+ vpmuludq $r3, $r0, $A3 -+ vpmuludq $r4, $r0, $A4 -+ -+ vpsllq \$1, $A1, $A1 -+ vpsllq \$1, $A2, $A2 -+ vpmuludq $r1, $r1, $T0 -+ vpaddq $T0, $A2, $A2 -+ vpmuludq $r2, $r1, $T0 -+ vpaddq $T0, $A3, $A3 -+ vpmuludq $r3, $r1, $T0 -+ vpaddq $T0, $A4, $A4 -+ vpmuludq $r4, $r1, $A5 -+ -+ vpsllq \$1, $A3, $A3 -+ vpsllq \$1, $A4, $A4 -+ vpmuludq $r2, $r2, $T0 -+ vpaddq $T0, $A4, $A4 -+ vpmuludq $r3, $r2, $T0 -+ vpaddq $T0, $A5, $A5 -+ vpmuludq $r4, $r2, $A6 -+ -+ vpsllq \$1, $A5, $A5 -+ vpsllq \$1, $A6, $A6 -+ vpmuludq $r3, $r3, $T0 -+ vpaddq $T0, $A6, $A6 -+ vpmuludq $r4, $r3, $A7 -+ -+ vpsllq \$1, $A7, $A7 -+ vpmuludq $r4, $r4, $A8 -+ -+ # Reduce -+ vpsrlq \$26, $A4, $T0 -+ vpand .LandMask(%rip), $A4, $A4 -+ vpaddq $T0, $A5, $A5 -+ -+ vpsllq \$2, $A5, $T0 -+ vpaddq $T0, $A5, $A5 -+ vpsllq \$2, $A6, $T0 -+ vpaddq $T0, $A6, $A6 -+ vpsllq \$2, $A7, $T0 -+ vpaddq $T0, $A7, $A7 -+ vpsllq \$2, $A8, $T0 -+ vpaddq $T0, $A8, $A8 -+ -+ vpaddq $A5, $A0, $A0 -+ vpaddq $A6, $A1, $A1 -+ vpaddq $A7, $A2, $A2 -+ vpaddq $A8, $A3, $A3 -+ -+ vpsrlq \$26, $A0, $T0 -+ vpand .LandMask(%rip), $A0, $A0 -+ vpaddq $T0, $A1, $A1 -+ vpsrlq \$26, $A1, $T0 -+ vpand .LandMask(%rip), $A1, $A1 -+ vpaddq $T0, $A2, $A2 -+ vpsrlq \$26, $A2, $T0 -+ vpand .LandMask(%rip), $A2, $A2 -+ vpaddq $T0, $A3, $A3 -+ vpsrlq \$26, $A3, $T0 -+ vpand .LandMask(%rip), $A3, $A3 -+ vpaddq $T0, $A4, $A4 -+ -+ vpunpcklqdq $r0, $A0, $r0 -+ vpunpcklqdq $r1, $A1, $r1 -+ vpunpcklqdq $r2, $A2, $r2 -+ vpunpcklqdq $r3, $A3, $r3 -+ vpunpcklqdq $r4, $A4, $r4 -+ -+ vmovdqu $r0, $_r0_($state) -+ vmovdqu $r1, $_r1_($state) -+ vmovdqu $r2, $_r2_($state) -+ vmovdqu $r3, $_r3_($state) -+ vmovdqu $r4, $_r4_($state) -+ -+ vpsllq \$2, $r1, $A1 -+ vpsllq \$2, $r2, $A2 -+ vpsllq \$2, $r3, $A3 -+ vpsllq \$2, $r4, $A4 -+ -+ vpaddq $A1, $r1, $A1 -+ vpaddq $A2, $r2, $A2 -+ vpaddq $A3, $r3, $A3 -+ vpaddq $A4, $r4, $A4 -+ -+ vmovdqu $A1, $_r1_x5($state) -+ vmovdqu $A2, $_r2_x5($state) -+ vmovdqu $A3, $_r3_x5($state) -+ vmovdqu $A4, $_r4_x5($state) -+ # Store k -+ vmovdqu 16*1($key), $T0 -+ vmovdqu $T0, $_k_($state) -+ # Init the MAC value -+ vpxor $T0, $T0, $T0 -+ vmovdqu $T0, $_A0_($state) -+ vmovd $T0, $_A4_($state) -+ vzeroupper -+ ret -+.size poly1305_init_avx,.-poly1305_init_avx -+___ -+} -+ -+{ -+ -+my ($A0, $A1, $A2, $A3, $A4, -+ $T0, $T1, $R0, $R1, $R2, -+ $R3, $R4, $AND_MASK)=map("%xmm$_",(0..12)); -+ -+my ($state, $in, $in_len)=("%rdi", "%rsi", "%rdx"); -+ -+$code.=<<___; -+ -+############################################################################### -+# void* poly1305_update_avx(void* $state, void* in, uint64_t in_len) -+.globl poly1305_update_avx -+.type poly1305_update_avx, \@function, 2 -+.align 64 -+poly1305_update_avx: -+ -+ vzeroupper -+ vmovd $_A0_($state), $A0 -+ vmovd $_A1_($state), $A1 -+ vmovd $_A2_($state), $A2 -+ vmovd $_A3_($state), $A3 -+ vmovd $_A4_($state), $A4 -+ vmovdqa .LandMask(%rip), $AND_MASK -+ # Skip to single block case -+ cmp \$32, $in_len -+ jb 3f -+1: -+ cmp \$16*4, $in_len -+ jb 1f -+ sub \$16*2, $in_len -+ # load the next two blocks -+ vmovdqu 16*0($in), $R2 -+ vmovdqu 16*1($in), $R3 -+ add \$16*2, $in -+ -+ vpunpcklqdq $R3, $R2, $R0 -+ vpunpckhqdq $R3, $R2, $R1 -+ -+ vpsrlq \$26, $R0, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A0, $A0 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpand $AND_MASK, $R2, $R2 -+ vpaddq $R2, $A1, $A1 -+ -+ vpsllq \$12, $R1, $R2 -+ vpxor $R2, $R0, $R0 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A2, $A2 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpsrlq \$40, $R1, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpxor .LsetBit(%rip), $R2, $R2 -+ vpaddq $R0, $A3, $A3 -+ vpaddq $R2, $A4, $A4 -+ -+ # Multiply input by R[0] -+ vbroadcastss $_r0_($state), $T0 -+ vpmuludq $T0, $A0, $R0 -+ vpmuludq $T0, $A1, $R1 -+ vpmuludq $T0, $A2, $R2 -+ vpmuludq $T0, $A3, $R3 -+ vpmuludq $T0, $A4, $R4 -+ # Multiply input by R[1] (and R[1]*5) -+ vbroadcastss $_r1_x5($state), $T0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R0, $R0 -+ vbroadcastss $_r1_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R4, $R4 -+ # Etc -+ vbroadcastss $_r2_x5($state), $T0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R1, $R1 -+ vbroadcastss $_r2_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R4, $R4 -+ -+ vbroadcastss $_r3_x5($state), $T0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R2, $R2 -+ vbroadcastss $_r3_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R4, $R4 -+ -+ vbroadcastss $_r4_x5($state), $T0 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R3, $R3 -+ vbroadcastss $_r4_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R4, $R4 -+ # Reduce -+ vpsrlq \$26, $R3, $T0 -+ vpaddq $T0, $R4, $R4 -+ vpand $AND_MASK, $R3, $R3 -+ -+ vpsrlq \$26, $R4, $T0 -+ vpsllq \$2, $T0, $T1 -+ vpaddq $T1, $T0, $T0 -+ vpaddq $T0, $R0, $R0 -+ vpand $AND_MASK, $R4, $R4 -+ -+ vpsrlq \$26, $R0, $T0 -+ vpand $AND_MASK, $R0, $A0 -+ vpaddq $T0, $R1, $R1 -+ vpsrlq \$26, $R1, $T0 -+ vpand $AND_MASK, $R1, $A1 -+ vpaddq $T0, $R2, $R2 -+ vpsrlq \$26, $R2, $T0 -+ vpand $AND_MASK, $R2, $A2 -+ vpaddq $T0, $R3, $R3 -+ vpsrlq \$26, $R3, $T0 -+ vpand $AND_MASK, $R3, $A3 -+ vpaddq $T0, $R4, $A4 -+ jmp 1b -+1: -+ cmp \$16*2, $in_len -+ jb 1f -+ sub \$16*2, $in_len -+ # load the next two blocks -+ vmovdqu 16*0($in), $R2 -+ vmovdqu 16*1($in), $R3 -+ add \$16*2, $in -+ -+ vpunpcklqdq $R3, $R2, $R0 -+ vpunpckhqdq $R3, $R2, $R1 -+ -+ vpsrlq \$26, $R0, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A0, $A0 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpand $AND_MASK, $R2, $R2 -+ vpaddq $R2, $A1, $A1 -+ -+ vpsllq \$12, $R1, $R2 -+ vpxor $R2, $R0, $R0 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A2, $A2 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpsrlq \$40, $R1, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpxor .LsetBit(%rip), $R2, $R2 -+ vpaddq $R0, $A3, $A3 -+ vpaddq $R2, $A4, $A4 -+ -+ # Multiply input by R[0] -+ vmovdqu $_r0_($state), $T0 -+ vpmuludq $T0, $A0, $R0 -+ vpmuludq $T0, $A1, $R1 -+ vpmuludq $T0, $A2, $R2 -+ vpmuludq $T0, $A3, $R3 -+ vpmuludq $T0, $A4, $R4 -+ # Multiply input by R[1] (and R[1]*5) -+ vmovdqu $_r1_x5($state), $T0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R0, $R0 -+ vmovdqu $_r1_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R4, $R4 -+ # Etc -+ vmovdqu $_r2_x5($state), $T0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R1, $R1 -+ vmovdqu $_r2_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R4, $R4 -+ -+ vmovdqu $_r3_x5($state), $T0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R2, $R2 -+ vmovdqu $_r3_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R4, $R4 -+ -+ vmovdqu $_r4_x5($state), $T0 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R3, $R3 -+ vmovdqu $_r4_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R4, $R4 -+1: -+ vpsrldq \$8, $R0, $A0 -+ vpsrldq \$8, $R1, $A1 -+ vpsrldq \$8, $R2, $A2 -+ vpsrldq \$8, $R3, $A3 -+ vpsrldq \$8, $R4, $A4 -+ -+ vpaddq $R0, $A0, $A0 -+ vpaddq $R1, $A1, $A1 -+ vpaddq $R2, $A2, $A2 -+ vpaddq $R3, $A3, $A3 -+ vpaddq $R4, $A4, $A4 -+ # Reduce -+ vpsrlq \$26, $A3, $T0 -+ vpaddq $T0, $A4, $A4 -+ vpand $AND_MASK, $A3, $A3 -+ vpsrlq \$26, $A4, $T0 -+ vpsllq \$2, $T0, $T1 -+ vpaddq $T1, $T0, $T0 -+ vpaddq $T0, $A0, $A0 -+ vpand $AND_MASK, $A4, $A4 -+ vpsrlq \$26, $A0, $T0 -+ vpand $AND_MASK, $A0, $A0 -+ vpaddq $T0, $A1, $A1 -+ vpsrlq \$26, $A1, $T0 -+ vpand $AND_MASK, $A1, $A1 -+ vpaddq $T0, $A2, $A2 -+ vpsrlq \$26, $A2, $T0 -+ vpand $AND_MASK, $A2, $A2 -+ vpaddq $T0, $A3, $A3 -+ vpsrlq \$26, $A3, $T0 -+ vpand $AND_MASK, $A3, $A3 -+ vpaddq $T0, $A4, $A4 -+3: -+ cmp \$16, $in_len -+ jb 1f -+ -+ # load the next block -+ vmovq 8*0($in), $R0 -+ vmovq 8*1($in), $R1 -+ add \$16, $in -+ sub \$16, $in_len -+ -+ vpsrlq \$26, $R0, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A0, $A0 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpand $AND_MASK, $R2, $R2 -+ vpaddq $R2, $A1, $A1 -+ -+ vpsllq \$12, $R1, $R2 -+ vpxor $R2, $R0, $R0 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A2, $A2 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpsrlq \$40, $R1, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpxor .LsetBit(%rip), $R2, $R2 -+ vpaddq $R0, $A3, $A3 -+ vpaddq $R2, $A4, $A4 -+2: -+ # Multiply input by R[0] -+ vmovq $_r0_+8($state), $T0 -+ vpmuludq $T0, $A0, $R0 -+ vpmuludq $T0, $A1, $R1 -+ vpmuludq $T0, $A2, $R2 -+ vpmuludq $T0, $A3, $R3 -+ vpmuludq $T0, $A4, $R4 -+ # Multiply input by R[1] (and R[1]*5) -+ vmovq $_r1_x5+8($state), $T0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R0, $R0 -+ vmovq $_r1_+8($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R4, $R4 -+ # Etc -+ vmovq $_r2_x5+8($state), $T0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R1, $R1 -+ vmovq $_r2_+8($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R4, $R4 -+ -+ vmovq $_r3_x5+8($state), $T0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R2, $R2 -+ vmovq $_r3_+8($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R4, $R4 -+ -+ vmovq $_r4_x5+8($state), $T0 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R3, $R3 -+ vmovq $_r4_+8($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R4, $R4 -+ -+ # Reduce -+ vpsrlq \$26, $R3, $T0 -+ vpaddq $T0, $R4, $R4 -+ vpand $AND_MASK, $R3, $R3 -+ vpsrlq \$26, $R4, $T0 -+ vpsllq \$2, $T0, $T1 -+ vpaddq $T1, $T0, $T0 -+ vpaddq $T0, $R0, $R0 -+ vpand $AND_MASK, $R4, $R4 -+ vpsrlq \$26, $R0, $T0 -+ vpand $AND_MASK, $R0, $A0 -+ vpaddq $T0, $R1, $R1 -+ vpsrlq \$26, $R1, $T0 -+ vpand $AND_MASK, $R1, $A1 -+ vpaddq $T0, $R2, $R2 -+ vpsrlq \$26, $R2, $T0 -+ vpand $AND_MASK, $R2, $A2 -+ vpaddq $T0, $R3, $R3 -+ vpsrlq \$26, $R3, $T0 -+ vpand $AND_MASK, $R3, $A3 -+ vpaddq $T0, $R4, $A4 -+ -+1: -+ test $in_len, $in_len -+ jz 1f -+ -+ vmovdqa .Lone(%rip), $R0 -+3: -+ dec $in_len -+ vpslldq \$1, $R0, $R0 -+ vpinsrb \$0, ($in, $in_len), $R0, $R0 -+ test $in_len, $in_len -+ jnz 3b -+ -+ vpsrldq \$8, $R0, $R1 -+ vpsrlq \$26, $R0, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A0, $A0 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpand $AND_MASK, $R2, $R2 -+ vpaddq $R2, $A1, $A1 -+ -+ vpsllq \$12, $R1, $R2 -+ vpxor $R2, $R0, $R0 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A2, $A2 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpsrlq \$40, $R1, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A3, $A3 -+ vpaddq $R2, $A4, $A4 -+ xor $in_len, $in_len -+ jmp 2b -+1: -+ vmovd $A0, $_A0_($state) -+ vmovd $A1, $_A1_($state) -+ vmovd $A2, $_A2_($state) -+ vmovd $A3, $_A3_($state) -+ vmovd $A4, $_A4_($state) -+ -+ -+ mov $in, %rax -+ vzeroupper -+ ret -+.size poly1305_update_avx,.-poly1305_update_avx -+############################################################################### -+# void poly1305_finish_avx(void* $state, uint64_t mac[2]); -+.type poly1305_finish_avx,\@function, 2 -+.globl poly1305_finish_avx -+poly1305_finish_avx: -+___ -+my $mac="%rsi"; -+$code.=<<___; -+ vzeroupper -+ vmovd $_A0_($state), $A0 -+ vmovd $_A1_($state), $A1 -+ vmovd $_A2_($state), $A2 -+ vmovd $_A3_($state), $A3 -+ vmovd $_A4_($state), $A4 -+ # Reduce one last time in case there was a carry from 130 bit -+ vpsrlq \$26, $A4, $T0 -+ vpsllq \$2, $T0, $T1 -+ vpaddq $T1, $T0, $T0 -+ vpaddq $T0, $A0, $A0 -+ vpand .LandMask(%rip), $A4, $A4 -+ -+ vpsrlq \$26, $A0, $T0 -+ vpand .LandMask(%rip), $A0, $A0 -+ vpaddq $T0, $A1, $A1 -+ vpsrlq \$26, $A1, $T0 -+ vpand .LandMask(%rip), $A1, $A1 -+ vpaddq $T0, $A2, $A2 -+ vpsrlq \$26, $A2, $T0 -+ vpand .LandMask(%rip), $A2, $A2 -+ vpaddq $T0, $A3, $A3 -+ vpsrlq \$26, $A3, $T0 -+ vpand .LandMask(%rip), $A3, $A3 -+ vpaddq $T0, $A4, $A4 -+ # Convert to normal -+ vpsllq \$26, $A1, $T0 -+ vpxor $T0, $A0, $A0 -+ vpsllq \$52, $A2, $T0 -+ vpxor $T0, $A0, $A0 -+ vpsrlq \$12, $A2, $A1 -+ vpsllq \$14, $A3, $T0 -+ vpxor $T0, $A1, $A1 -+ vpsllq \$40, $A4, $T0 -+ vpxor $T0, $A1, $A1 -+ vmovq $A0, %rax -+ vmovq $A1, %rdx -+ -+ add $_k_($state), %rax -+ adc $_k_+8($state), %rdx -+ mov %rax, ($mac) -+ mov %rdx, 8($mac) -+ vzeroupper -+ ret -+.size poly1305_finish_avx,.-poly1305_finish_avx -+___ -+} -+}} -+ -+$code =~ s/\`([^\`]*)\`/eval($1)/gem; -+print $code; -+close STDOUT; -diff --git a/crypto/chacha20poly1305/asm/poly1305_avx2.pl b/crypto/chacha20poly1305/asm/poly1305_avx2.pl -new file mode 100644 -index 0000000..8134542 ---- /dev/null -+++ b/crypto/chacha20poly1305/asm/poly1305_avx2.pl -@@ -0,0 +1,918 @@ -+############################################################################## -+# # -+# Copyright 2014 Intel Corporation # -+# # -+# Licensed under the Apache License, Version 2.0 (the "License"); # -+# you may not use this file except in compliance with the License. # -+# You may obtain a copy of the License at # -+# # -+# http://www.apache.org/licenses/LICENSE-2.0 # -+# # -+# Unless required by applicable law or agreed to in writing, software # -+# distributed under the License is distributed on an "AS IS" BASIS, # -+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -+# See the License for the specific language governing permissions and # -+# limitations under the License. # -+# # -+############################################################################## -+# # -+# Developers and authors: # -+# Shay Gueron (1, 2), and Vlad Krasnov (1) # -+# (1) Intel Corporation, Israel Development Center # -+# (2) University of Haifa # -+# # -+############################################################################## -+# state: -+# 0: r[0] || r^2[0] -+# 16: r[1] || r^2[1] -+# 32: r[2] || r^2[2] -+# 48: r[3] || r^2[3] -+# 64: r[4] || r^2[4] -+# 80: r[1]*5 || r^2[1]*5 -+# 96: r[2]*5 || r^2[2]*5 -+#112: r[3]*5 || r^2[3]*5 -+#128: r[4]*5 || r^2[4]*5 -+#144: k -+#160: A0 -+#164: A1 -+#168: A2 -+#172: A3 -+#176: A4 -+#180: END -+ -+$flavour = shift; -+$output = shift; -+if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } -+ -+$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); -+ -+$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; -+( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or -+( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or -+die "can't locate x86_64-xlate.pl"; -+ -+open OUT,"| \"$^X\" $xlate $flavour $output"; -+*STDOUT=*OUT; -+ -+if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` -+ =~ /GNU assembler version ([2-9]\.[0-9]+)/) { -+ $avx = ($1>=2.19) + ($1>=2.22); -+} -+ -+if ($win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && -+ `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { -+ $avx = ($1>=2.09) + ($1>=2.10); -+} -+ -+if ($win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && -+ `ml64 2>&1` =~ /Version ([0-9]+)\./) { -+ $avx = ($1>=10) + ($1>=11); -+} -+ -+if (`$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9])\.([0-9]+)/) { -+ my $ver = $2 + $3/100.0; # 3.1->3.01, 3.10->3.10 -+ $avx = ($ver>=3.0) + ($ver>=3.01); -+} -+ -+if ($avx>=1) {{ -+ -+my ($_r0_, $_r1_, $_r2_, $_r3_, $_r4_, $_r1_x5, $_r2_x5, $_r3_x5, $_r4_x5, $_k_, $_A0_, $_A1_, $_A2_, $_A3_, $_A4_) -+= (0,32,64,96,128,160,192,224,256,288,304,308,312,316,320); -+ -+$code.=<<___; -+.text -+.align 32 -+.LandMask: -+.quad 0x3FFFFFF, 0x3FFFFFF, 0x3FFFFFF, 0x3FFFFFF -+.LsetBit: -+.quad 0x1000000, 0x1000000, 0x1000000, 0x1000000 -+.LrSet: -+.quad 0xFFFFFFC0FFFFFFF, 0xFFFFFFC0FFFFFFF, 0xFFFFFFC0FFFFFFF, 0xFFFFFFC0FFFFFFF -+.quad 0xFFFFFFC0FFFFFFC, 0xFFFFFFC0FFFFFFC, 0xFFFFFFC0FFFFFFC, 0xFFFFFFC0FFFFFFC -+ -+.LpermFix: -+.long 6,7,6,7,6,7,6,7 -+.long 4,5,6,7,6,7,6,7 -+.long 2,3,6,7,4,5,6,7 -+.long 0,1,4,5,2,3,6,7 -+___ -+ -+ -+{ -+my ($A0, $A1, $A2, $A3, $A4, -+ $r0, $r1, $r2, $r3, $r4, -+ $T0, $T1, $A5, $A6, $A7, $A8)=map("%xmm$_",(0..15)); -+my ($A0_y, $A1_y, $A2_y, $A3_y, $A4_y, -+ $r0_y, $r1_y, $r2_y, $r3_y, $r4_y)=map("%ymm$_",(0..9)); -+my ($state, $key) -+ =("%rdi", "%rsi"); -+ -+$code.=<<___; -+################################################################################ -+# void poly1305_init_avx2(void *state, uint8_t key[32]) -+ -+.globl poly1305_init_avx2 -+.type poly1305_init_avx2, \@function, 2 -+.align 64 -+poly1305_init_avx2: -+ vzeroupper -+ -+ # Store k -+ vmovdqu 16*1($key), $T0 -+ vmovdqu $T0, $_k_($state) -+ # Init the MAC value -+ vpxor $T0, $T0, $T0 -+ vmovdqu $T0, $_A0_($state) -+ vmovd $T0, $_A4_($state) -+ # load and convert r -+ vmovq 8*0($key), $r0 -+ vmovq 8*1($key), $T0 -+ vpand .LrSet(%rip), $r0, $r0 -+ vpand .LrSet+32(%rip), $T0, $T0 -+ -+ vpsrlq \$26, $r0, $r1 -+ vpand .LandMask(%rip), $r0, $r0 -+ vpsrlq \$26, $r1, $r2 -+ vpand .LandMask(%rip), $r1, $r1 -+ vpsllq \$12, $T0, $T1 -+ vpxor $T1, $r2, $r2 -+ vpsrlq \$26, $r2, $r3 -+ vpsrlq \$40, $T0, $r4 -+ vpand .LandMask(%rip), $r2, $r2 -+ vpand .LandMask(%rip), $r3, $r3 -+ # SQR R -+ vpmuludq $r0, $r0, $A0 -+ vpmuludq $r1, $r0, $A1 -+ vpmuludq $r2, $r0, $A2 -+ vpmuludq $r3, $r0, $A3 -+ vpmuludq $r4, $r0, $A4 -+ -+ vpsllq \$1, $A1, $A1 -+ vpsllq \$1, $A2, $A2 -+ vpmuludq $r1, $r1, $T0 -+ vpaddq $T0, $A2, $A2 -+ vpmuludq $r2, $r1, $T0 -+ vpaddq $T0, $A3, $A3 -+ vpmuludq $r3, $r1, $T0 -+ vpaddq $T0, $A4, $A4 -+ vpmuludq $r4, $r1, $A5 -+ -+ vpsllq \$1, $A3, $A3 -+ vpsllq \$1, $A4, $A4 -+ vpmuludq $r2, $r2, $T0 -+ vpaddq $T0, $A4, $A4 -+ vpmuludq $r3, $r2, $T0 -+ vpaddq $T0, $A5, $A5 -+ vpmuludq $r4, $r2, $A6 -+ -+ vpsllq \$1, $A5, $A5 -+ vpsllq \$1, $A6, $A6 -+ vpmuludq $r3, $r3, $T0 -+ vpaddq $T0, $A6, $A6 -+ vpmuludq $r4, $r3, $A7 -+ -+ vpsllq \$1, $A7, $A7 -+ vpmuludq $r4, $r4, $A8 -+ -+ # Reduce -+ vpsrlq \$26, $A4, $T0 -+ vpand .LandMask(%rip), $A4, $A4 -+ vpaddq $T0, $A5, $A5 -+ -+ vpsllq \$2, $A5, $T0 -+ vpaddq $T0, $A5, $A5 -+ vpsllq \$2, $A6, $T0 -+ vpaddq $T0, $A6, $A6 -+ vpsllq \$2, $A7, $T0 -+ vpaddq $T0, $A7, $A7 -+ vpsllq \$2, $A8, $T0 -+ vpaddq $T0, $A8, $A8 -+ -+ vpaddq $A5, $A0, $A0 -+ vpaddq $A6, $A1, $A1 -+ vpaddq $A7, $A2, $A2 -+ vpaddq $A8, $A3, $A3 -+ -+ vpsrlq \$26, $A0, $T0 -+ vpand .LandMask(%rip), $A0, $A0 -+ vpaddq $T0, $A1, $A1 -+ vpsrlq \$26, $A1, $T0 -+ vpand .LandMask(%rip), $A1, $A1 -+ vpaddq $T0, $A2, $A2 -+ vpsrlq \$26, $A2, $T0 -+ vpand .LandMask(%rip), $A2, $A2 -+ vpaddq $T0, $A3, $A3 -+ vpsrlq \$26, $A3, $T0 -+ vpand .LandMask(%rip), $A3, $A3 -+ vpaddq $T0, $A4, $A4 -+ -+ vpunpcklqdq $r0, $A0, $r0 -+ vpunpcklqdq $r1, $A1, $r1 -+ vpunpcklqdq $r2, $A2, $r2 -+ vpunpcklqdq $r3, $A3, $r3 -+ vpunpcklqdq $r4, $A4, $r4 -+ -+ vmovdqu $r0, $_r0_+16($state) -+ vmovdqu $r1, $_r1_+16($state) -+ vmovdqu $r2, $_r2_+16($state) -+ vmovdqu $r3, $_r3_+16($state) -+ vmovdqu $r4, $_r4_+16($state) -+ -+ vpsllq \$2, $r1, $A1 -+ vpsllq \$2, $r2, $A2 -+ vpsllq \$2, $r3, $A3 -+ vpsllq \$2, $r4, $A4 -+ -+ vpaddq $A1, $r1, $A1 -+ vpaddq $A2, $r2, $A2 -+ vpaddq $A3, $r3, $A3 -+ vpaddq $A4, $r4, $A4 -+ -+ vmovdqu $A1, $_r1_x5+16($state) -+ vmovdqu $A2, $_r2_x5+16($state) -+ vmovdqu $A3, $_r3_x5+16($state) -+ vmovdqu $A4, $_r4_x5+16($state) -+ -+ # Compute r^3 and r^4 -+ vpshufd \$0x44, $r0, $A0 -+ vpshufd \$0x44, $r1, $A1 -+ vpshufd \$0x44, $r2, $A2 -+ vpshufd \$0x44, $r3, $A3 -+ vpshufd \$0x44, $r4, $A4 -+ -+ # Multiply input by R[0] -+ vmovdqu $_r0_+16($state), $T0 -+ vpmuludq $T0, $A0, $r0 -+ vpmuludq $T0, $A1, $r1 -+ vpmuludq $T0, $A2, $r2 -+ vpmuludq $T0, $A3, $r3 -+ vpmuludq $T0, $A4, $r4 -+ # Multiply input by R[1] (and R[1]*5) -+ vmovdqu $_r1_x5+16($state), $T0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $r0, $r0 -+ vmovdqu $_r1_+16($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $r1, $r1 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $r2, $r2 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $r3, $r3 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $r4, $r4 -+ # Etc -+ vmovdqu $_r2_x5+16($state), $T0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $r0, $r0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $r1, $r1 -+ vmovdqu $_r2_+16($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $r2, $r2 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $r3, $r3 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $r4, $r4 -+ -+ vmovdqu $_r3_x5+16($state), $T0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $r0, $r0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $r1, $r1 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $r2, $r2 -+ vmovdqu $_r3_+16($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $r3, $r3 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $r4, $r4 -+ -+ vmovdqu $_r4_x5+16($state), $T0 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $r0, $r0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $r1, $r1 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $r2, $r2 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $r3, $r3 -+ vmovdqu $_r4_+16($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $r4, $r4 -+ # Reduce -+ vpsrlq \$26, $r3, $T0 -+ vpaddq $T0, $r4, $r4 -+ vpand .LandMask(%rip), $r3, $r3 -+ vpsrlq \$26, $r4, $T0 -+ vpsllq \$2, $T0, $T1 -+ vpaddq $T1, $T0, $T0 -+ vpaddq $T0, $r0, $r0 -+ vpand .LandMask(%rip), $r4, $r4 -+ vpsrlq \$26, $r0, $T0 -+ vpand .LandMask(%rip), $r0, $r0 -+ vpaddq $T0, $r1, $r1 -+ vpsrlq \$26, $r1, $T0 -+ vpand .LandMask(%rip), $r1, $r1 -+ vpaddq $T0, $r2, $r2 -+ vpsrlq \$26, $r2, $T0 -+ vpand .LandMask(%rip), $r2, $r2 -+ vpaddq $T0, $r3, $r3 -+ vpsrlq \$26, $r3, $T0 -+ vpand .LandMask(%rip), $r3, $r3 -+ vpaddq $T0, $r4, $r4 -+ -+ vmovdqu $r0, $_r0_($state) -+ vmovdqu $r1, $_r1_($state) -+ vmovdqu $r2, $_r2_($state) -+ vmovdqu $r3, $_r3_($state) -+ vmovdqu $r4, $_r4_($state) -+ -+ vpsllq \$2, $r1, $A1 -+ vpsllq \$2, $r2, $A2 -+ vpsllq \$2, $r3, $A3 -+ vpsllq \$2, $r4, $A4 -+ -+ vpaddq $A1, $r1, $A1 -+ vpaddq $A2, $r2, $A2 -+ vpaddq $A3, $r3, $A3 -+ vpaddq $A4, $r4, $A4 -+ -+ vmovdqu $A1, $_r1_x5($state) -+ vmovdqu $A2, $_r2_x5($state) -+ vmovdqu $A3, $_r3_x5($state) -+ vmovdqu $A4, $_r4_x5($state) -+ -+ ret -+.size poly1305_init_avx2,.-poly1305_init_avx2 -+___ -+} -+ -+{ -+ -+my ($A0, $A1, $A2, $A3, $A4, -+ $T0, $T1, $R0, $R1, $R2, -+ $R3, $R4, $AND_MASK, $PERM_MASK, $SET_MASK)=map("%ymm$_",(0..14)); -+ -+my ($A0_x, $A1_x, $A2_x, $A3_x, $A4_x, -+ $T0_x, $T1_x, $R0_x, $R1_x, $R2_x, -+ $R3_x, $R4_x, $AND_MASK_x, $PERM_MASK_x, $SET_MASK_x)=map("%xmm$_",(0..14)); -+ -+my ($state, $in, $in_len, $hlp, $rsp_save)=("%rdi", "%rsi", "%rdx", "%rcx", "%rax"); -+ -+$code.=<<___; -+ -+############################################################################### -+# void poly1305_update_avx2(void* $state, void* in, uint64_t in_len2) -+.globl poly1305_update_avx2 -+.type poly1305_update_avx2, \@function, 2 -+.align 64 -+poly1305_update_avx2: -+ -+ vmovd $_A0_($state), $A0_x -+ vmovd $_A1_($state), $A1_x -+ vmovd $_A2_($state), $A2_x -+ vmovd $_A3_($state), $A3_x -+ vmovd $_A4_($state), $A4_x -+ -+ vmovdqa .LandMask(%rip), $AND_MASK -+1: -+ cmp \$32*4, $in_len -+ jb 1f -+ sub \$32*2, $in_len -+ -+ # load the next four blocks -+ vmovdqu 32*0($in), $R2 -+ vmovdqu 32*1($in), $R3 -+ add \$32*2, $in -+ -+ vpunpcklqdq $R3, $R2, $R0 -+ vpunpckhqdq $R3, $R2, $R1 -+ -+ vpermq \$0xD8, $R0, $R0 # it is possible to rearrange the precomputations, and save this shuffle -+ vpermq \$0xD8, $R1, $R1 -+ -+ vpsrlq \$26, $R0, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A0, $A0 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpand $AND_MASK, $R2, $R2 -+ vpaddq $R2, $A1, $A1 -+ -+ vpsllq \$12, $R1, $R2 -+ vpxor $R2, $R0, $R0 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A2, $A2 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpsrlq \$40, $R1, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpxor .LsetBit(%rip), $R2, $R2 -+ vpaddq $R0, $A3, $A3 -+ vpaddq $R2, $A4, $A4 -+ -+ # Multiply input by R[0] -+ vpbroadcastq $_r0_($state), $T0 -+ vpmuludq $T0, $A0, $R0 -+ vpmuludq $T0, $A1, $R1 -+ vpmuludq $T0, $A2, $R2 -+ vpmuludq $T0, $A3, $R3 -+ vpmuludq $T0, $A4, $R4 -+ # Multiply input by R[1] (and R[1]*5) -+ vpbroadcastq $_r1_x5($state), $T0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpbroadcastq $_r1_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R4, $R4 -+ # Etc -+ vpbroadcastq $_r2_x5($state), $T0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpbroadcastq $_r2_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R4, $R4 -+ -+ vpbroadcastq $_r3_x5($state), $T0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpbroadcastq $_r3_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R4, $R4 -+ -+ vpbroadcastq $_r4_x5($state), $T0 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpbroadcastq $_r4_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R4, $R4 -+ # Reduce -+ vpsrlq \$26, $R3, $T0 -+ vpaddq $T0, $R4, $R4 -+ vpand $AND_MASK, $R3, $R3 -+ -+ vpsrlq \$26, $R4, $T0 -+ vpsllq \$2, $T0, $T1 -+ vpaddq $T1, $T0, $T0 -+ vpaddq $T0, $R0, $R0 -+ vpand $AND_MASK, $R4, $R4 -+ -+ vpsrlq \$26, $R0, $T0 -+ vpand $AND_MASK, $R0, $A0 -+ vpaddq $T0, $R1, $R1 -+ vpsrlq \$26, $R1, $T0 -+ vpand $AND_MASK, $R1, $A1 -+ vpaddq $T0, $R2, $R2 -+ vpsrlq \$26, $R2, $T0 -+ vpand $AND_MASK, $R2, $A2 -+ vpaddq $T0, $R3, $R3 -+ vpsrlq \$26, $R3, $T0 -+ vpand $AND_MASK, $R3, $A3 -+ vpaddq $T0, $R4, $A4 -+ jmp 1b -+1: -+ -+ cmp \$32*2, $in_len -+ jb 1f -+ sub \$32*2, $in_len -+ # load the next four blocks -+ vmovdqu 32*0($in), $R2 -+ vmovdqu 32*1($in), $R3 -+ add \$32*2, $in -+ -+ vpunpcklqdq $R3, $R2, $R0 -+ vpunpckhqdq $R3, $R2, $R1 -+ -+ vpermq \$0xD8, $R0, $R0 -+ vpermq \$0xD8, $R1, $R1 -+ -+ vpsrlq \$26, $R0, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A0, $A0 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpand $AND_MASK, $R2, $R2 -+ vpaddq $R2, $A1, $A1 -+ -+ vpsllq \$12, $R1, $R2 -+ vpxor $R2, $R0, $R0 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A2, $A2 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpsrlq \$40, $R1, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpxor .LsetBit(%rip), $R2, $R2 -+ vpaddq $R0, $A3, $A3 -+ vpaddq $R2, $A4, $A4 -+ -+ # Multiply input by R[0] -+ vmovdqu $_r0_($state), $T0 -+ vpmuludq $T0, $A0, $R0 -+ vpmuludq $T0, $A1, $R1 -+ vpmuludq $T0, $A2, $R2 -+ vpmuludq $T0, $A3, $R3 -+ vpmuludq $T0, $A4, $R4 -+ # Multiply input by R[1] (and R[1]*5) -+ vmovdqu $_r1_x5($state), $T0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R0, $R0 -+ vmovdqu $_r1_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R4, $R4 -+ # Etc -+ vmovdqu $_r2_x5($state), $T0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R1, $R1 -+ vmovdqu $_r2_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R4, $R4 -+ -+ vmovdqu $_r3_x5($state), $T0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R2, $R2 -+ vmovdqu $_r3_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R4, $R4 -+ -+ vmovdqu $_r4_x5($state), $T0 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R3, $R3 -+ vmovdqu $_r4_($state), $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R4, $R4 -+ # Reduce -+ vpsrlq \$26, $R3, $T0 -+ vpaddq $T0, $R4, $R4 -+ vpand $AND_MASK, $R3, $R3 -+ vpsrlq \$26, $R4, $T0 -+ vpsllq \$2, $T0, $T1 -+ vpaddq $T1, $T0, $T0 -+ vpaddq $T0, $R0, $R0 -+ vpand $AND_MASK, $R4, $R4 -+ vpsrlq \$26, $R0, $T0 -+ vpand $AND_MASK, $R0, $A0 -+ vpaddq $T0, $R1, $R1 -+ vpsrlq \$26, $R1, $T0 -+ vpand $AND_MASK, $R1, $A1 -+ vpaddq $T0, $R2, $R2 -+ vpsrlq \$26, $R2, $T0 -+ vpand $AND_MASK, $R2, $A2 -+ vpaddq $T0, $R3, $R3 -+ vpsrlq \$26, $R3, $T0 -+ vpand $AND_MASK, $R3, $A3 -+ vpaddq $T0, $R4, $A4 -+ -+ vpsrldq \$8, $A0, $R0 -+ vpsrldq \$8, $A1, $R1 -+ vpsrldq \$8, $A2, $R2 -+ vpsrldq \$8, $A3, $R3 -+ vpsrldq \$8, $A4, $R4 -+ -+ vpaddq $R0, $A0, $A0 -+ vpaddq $R1, $A1, $A1 -+ vpaddq $R2, $A2, $A2 -+ vpaddq $R3, $A3, $A3 -+ vpaddq $R4, $A4, $A4 -+ -+ vpermq \$0xAA, $A0, $R0 -+ vpermq \$0xAA, $A1, $R1 -+ vpermq \$0xAA, $A2, $R2 -+ vpermq \$0xAA, $A3, $R3 -+ vpermq \$0xAA, $A4, $R4 -+ -+ vpaddq $R0, $A0, $A0 -+ vpaddq $R1, $A1, $A1 -+ vpaddq $R2, $A2, $A2 -+ vpaddq $R3, $A3, $A3 -+ vpaddq $R4, $A4, $A4 -+1: -+ test $in_len, $in_len -+ jz 5f -+ # In case 1,2 or 3 blocks remain, we want to multiply them correctly -+ vmovq $A0_x, $A0_x -+ vmovq $A1_x, $A1_x -+ vmovq $A2_x, $A2_x -+ vmovq $A3_x, $A3_x -+ vmovq $A4_x, $A4_x -+ -+ mov .LsetBit(%rip), $hlp -+ mov %rsp, $rsp_save -+ test \$15, $in_len -+ jz 1f -+ xor $hlp, $hlp -+ sub \$64, %rsp -+ vpxor $R0, $R0, $R0 -+ vmovdqu $R0, (%rsp) -+ vmovdqu $R0, 32(%rsp) -+3: -+ movb ($in, $hlp), %r8b -+ movb %r8b, (%rsp, $hlp) -+ inc $hlp -+ cmp $hlp, $in_len -+ jne 3b -+ -+ movb \$1, (%rsp, $hlp) -+ xor $hlp, $hlp -+ mov %rsp, $in -+ -+1: -+ -+ cmp \$16, $in_len -+ ja 2f -+ vmovq 8*0($in), $R0_x -+ vmovq 8*1($in), $R1_x -+ vmovq $hlp, $SET_MASK_x -+ vmovdqa .LpermFix(%rip), $PERM_MASK -+ jmp 1f -+2: -+ cmp \$32, $in_len -+ ja 2f -+ vmovdqu 16*0($in), $R2_x -+ vmovdqu 16*1($in), $R3_x -+ vmovq .LsetBit(%rip), $SET_MASK_x -+ vpinsrq \$1, $hlp, $SET_MASK_x, $SET_MASK_x -+ vmovdqa .LpermFix+32(%rip), $PERM_MASK -+ -+ vpunpcklqdq $R3, $R2, $R0 -+ vpunpckhqdq $R3, $R2, $R1 -+ jmp 1f -+2: -+ cmp \$48, $in_len -+ ja 2f -+ vmovdqu 32*0($in), $R2 -+ vmovdqu 32*1($in), $R3_x -+ vmovq .LsetBit(%rip), $SET_MASK_x -+ vpinsrq \$1, $hlp, $SET_MASK_x, $SET_MASK_x -+ vpermq \$0xc4, $SET_MASK, $SET_MASK -+ vmovdqa .LpermFix+64(%rip), $PERM_MASK -+ -+ vpunpcklqdq $R3, $R2, $R0 -+ vpunpckhqdq $R3, $R2, $R1 -+ jmp 1f -+2: -+ vmovdqu 32*0($in), $R2 -+ vmovdqu 32*1($in), $R3 -+ vmovq .LsetBit(%rip), $SET_MASK_x -+ vpinsrq \$1, $hlp, $SET_MASK_x, $SET_MASK_x -+ vpermq \$0x40, $SET_MASK, $SET_MASK -+ vmovdqa .LpermFix+96(%rip), $PERM_MASK -+ -+ vpunpcklqdq $R3, $R2, $R0 -+ vpunpckhqdq $R3, $R2, $R1 -+ -+1: -+ mov $rsp_save, %rsp -+ -+ vpsrlq \$26, $R0, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A0, $A0 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpand $AND_MASK, $R2, $R2 -+ vpaddq $R2, $A1, $A1 -+ -+ vpsllq \$12, $R1, $R2 -+ vpxor $R2, $R0, $R0 -+ vpand $AND_MASK, $R0, $R0 -+ vpaddq $R0, $A2, $A2 -+ -+ vpsrlq \$26, $R2, $R0 -+ vpsrlq \$40, $R1, $R2 -+ vpand $AND_MASK, $R0, $R0 -+ vpxor $SET_MASK, $R2, $R2 -+ vpaddq $R0, $A3, $A3 -+ vpaddq $R2, $A4, $A4 -+ -+ # Multiply input by R[0] -+ vmovdqu $_r0_($state), $T0 -+ vpermd $T0, $PERM_MASK, $T0 -+ vpmuludq $T0, $A0, $R0 -+ vpmuludq $T0, $A1, $R1 -+ vpmuludq $T0, $A2, $R2 -+ vpmuludq $T0, $A3, $R3 -+ vpmuludq $T0, $A4, $R4 -+ # Multiply input by R[1] (and R[1]*5) -+ vmovdqu $_r1_x5($state), $T0 -+ vpermd $T0, $PERM_MASK, $T0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R0, $R0 -+ vmovdqu $_r1_($state), $T0 -+ vpermd $T0, $PERM_MASK, $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R4, $R4 -+ # Etc -+ vmovdqu $_r2_x5($state), $T0 -+ vpermd $T0, $PERM_MASK, $T0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R1, $R1 -+ vmovdqu $_r2_($state), $T0 -+ vpermd $T0, $PERM_MASK, $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R4, $R4 -+ -+ vmovdqu $_r3_x5($state), $T0 -+ vpermd $T0, $PERM_MASK, $T0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R2, $R2 -+ vmovdqu $_r3_($state), $T0 -+ vpermd $T0, $PERM_MASK, $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R3, $R3 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R4, $R4 -+ -+ vmovdqu $_r4_x5($state), $T0 -+ vpermd $T0, $PERM_MASK, $T0 -+ vpmuludq $T0, $A1, $T1 -+ vpaddq $T1, $R0, $R0 -+ vpmuludq $T0, $A2, $T1 -+ vpaddq $T1, $R1, $R1 -+ vpmuludq $T0, $A3, $T1 -+ vpaddq $T1, $R2, $R2 -+ vpmuludq $T0, $A4, $T1 -+ vpaddq $T1, $R3, $R3 -+ vmovdqu $_r4_($state), $T0 -+ vpermd $T0, $PERM_MASK, $T0 -+ vpmuludq $T0, $A0, $T1 -+ vpaddq $T1, $R4, $R4 -+ # Reduce -+ vpsrlq \$26, $R3, $T0 -+ vpaddq $T0, $R4, $R4 -+ vpand $AND_MASK, $R3, $R3 -+ vpsrlq \$26, $R4, $T0 -+ vpsllq \$2, $T0, $T1 -+ vpaddq $T1, $T0, $T0 -+ vpaddq $T0, $R0, $R0 -+ vpand $AND_MASK, $R4, $R4 -+ vpsrlq \$26, $R0, $T0 -+ vpand $AND_MASK, $R0, $A0 -+ vpaddq $T0, $R1, $R1 -+ vpsrlq \$26, $R1, $T0 -+ vpand $AND_MASK, $R1, $A1 -+ vpaddq $T0, $R2, $R2 -+ vpsrlq \$26, $R2, $T0 -+ vpand $AND_MASK, $R2, $A2 -+ vpaddq $T0, $R3, $R3 -+ vpsrlq \$26, $R3, $T0 -+ vpand $AND_MASK, $R3, $A3 -+ vpaddq $T0, $R4, $A4 -+ -+ vpsrldq \$8, $A0, $R0 -+ vpsrldq \$8, $A1, $R1 -+ vpsrldq \$8, $A2, $R2 -+ vpsrldq \$8, $A3, $R3 -+ vpsrldq \$8, $A4, $R4 -+ -+ vpaddq $R0, $A0, $A0 -+ vpaddq $R1, $A1, $A1 -+ vpaddq $R2, $A2, $A2 -+ vpaddq $R3, $A3, $A3 -+ vpaddq $R4, $A4, $A4 -+ -+ vpermq \$0xAA, $A0, $R0 -+ vpermq \$0xAA, $A1, $R1 -+ vpermq \$0xAA, $A2, $R2 -+ vpermq \$0xAA, $A3, $R3 -+ vpermq \$0xAA, $A4, $R4 -+ -+ vpaddq $R0, $A0, $A0 -+ vpaddq $R1, $A1, $A1 -+ vpaddq $R2, $A2, $A2 -+ vpaddq $R3, $A3, $A3 -+ vpaddq $R4, $A4, $A4 -+ -+5: -+ vmovd $A0_x, $_A0_($state) -+ vmovd $A1_x, $_A1_($state) -+ vmovd $A2_x, $_A2_($state) -+ vmovd $A3_x, $_A3_($state) -+ vmovd $A4_x, $_A4_($state) -+ -+ ret -+.size poly1305_update_avx2,.-poly1305_update_avx2 -+############################################################################### -+# void poly1305_finish_avx2(void* $state, uint8_t mac[16]); -+.type poly1305_finish_avx2,\@function,2 -+.globl poly1305_finish_avx2 -+poly1305_finish_avx2: -+___ -+my $mac="%rsi"; -+my ($A0, $A1, $A2, $A3, $A4, $T0, $T1)=map("%xmm$_",(0..6)); -+ -+$code.=<<___; -+ vmovd $_A0_($state), $A0 -+ vmovd $_A1_($state), $A1 -+ vmovd $_A2_($state), $A2 -+ vmovd $_A3_($state), $A3 -+ vmovd $_A4_($state), $A4 -+ # Reduce one last time in case there was a carry from 130 bit -+ vpsrlq \$26, $A4, $T0 -+ vpsllq \$2, $T0, $T1 -+ vpaddq $T1, $T0, $T0 -+ vpaddq $T0, $A0, $A0 -+ vpand .LandMask(%rip), $A4, $A4 -+ -+ vpsrlq \$26, $A0, $T0 -+ vpand .LandMask(%rip), $A0, $A0 -+ vpaddq $T0, $A1, $A1 -+ vpsrlq \$26, $A1, $T0 -+ vpand .LandMask(%rip), $A1, $A1 -+ vpaddq $T0, $A2, $A2 -+ vpsrlq \$26, $A2, $T0 -+ vpand .LandMask(%rip), $A2, $A2 -+ vpaddq $T0, $A3, $A3 -+ vpsrlq \$26, $A3, $T0 -+ vpand .LandMask(%rip), $A3, $A3 -+ vpaddq $T0, $A4, $A4 -+ # Convert to normal -+ vpsllq \$26, $A1, $T0 -+ vpxor $T0, $A0, $A0 -+ vpsllq \$52, $A2, $T0 -+ vpxor $T0, $A0, $A0 -+ vpsrlq \$12, $A2, $A1 -+ vpsllq \$14, $A3, $T0 -+ vpxor $T0, $A1, $A1 -+ vpsllq \$40, $A4, $T0 -+ vpxor $T0, $A1, $A1 -+ vmovq $A0, %rax -+ vmovq $A1, %rdx -+ -+ add $_k_($state), %rax -+ adc $_k_+8($state), %rdx -+ mov %rax, ($mac) -+ mov %rdx, 8($mac) -+ -+ ret -+.size poly1305_finish_avx2,.-poly1305_finish_avx2 -+___ -+} -+}} -+ -+$code =~ s/\`([^\`]*)\`/eval(\$1)/gem; -+print $code; -+close STDOUT; -diff --git a/crypto/chacha20poly1305/chacha20.c b/crypto/chacha20poly1305/chacha20.c -new file mode 100644 -index 0000000..72ab173 ---- /dev/null -+++ b/crypto/chacha20poly1305/chacha20.c -@@ -0,0 +1,157 @@ -+/* Copyright (c) 2014, Google Inc. -+ * -+ * Permission to use, copy, modify, and/or distribute this software for any -+ * purpose with or without fee is hereby granted, provided that the above -+ * copyright notice and this permission notice appear in all copies. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY -+ * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION -+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ -+ -+/* Adapted from the public domain, estream code by D. Bernstein. */ -+ -+#include -+ -+/* sigma contains the ChaCha constants, which happen to be an ASCII string. */ -+static const char sigma[16] = "expand 32-byte k"; -+ -+#define ROTATE(v, n) (((v) << (n)) | ((v) >> (32 - (n)))) -+#define XOR(v, w) ((v) ^ (w)) -+#define PLUS(x, y) ((x) + (y)) -+#define PLUSONE(v) (PLUS((v), 1)) -+ -+#define U32TO8_LITTLE(p, v) \ -+ { \ -+ (p)[0] = (v >> 0) & 0xff; \ -+ (p)[1] = (v >> 8) & 0xff; \ -+ (p)[2] = (v >> 16) & 0xff; \ -+ (p)[3] = (v >> 24) & 0xff; \ -+ } -+ -+#define U8TO32_LITTLE(p) \ -+ (((uint32_t)((p)[0])) | ((uint32_t)((p)[1]) << 8) | \ -+ ((uint32_t)((p)[2]) << 16) | ((uint32_t)((p)[3]) << 24)) -+ -+/* QUARTERROUND updates a, b, c, d with a ChaCha "quarter" round. */ -+#define QUARTERROUND(a,b,c,d) \ -+ x[a] = PLUS(x[a],x[b]); x[d] = ROTATE(XOR(x[d],x[a]),16); \ -+ x[c] = PLUS(x[c],x[d]); x[b] = ROTATE(XOR(x[b],x[c]),12); \ -+ x[a] = PLUS(x[a],x[b]); x[d] = ROTATE(XOR(x[d],x[a]), 8); \ -+ x[c] = PLUS(x[c],x[d]); x[b] = ROTATE(XOR(x[b],x[c]), 7); -+ -+/* chacha_core performs |num_rounds| rounds of ChaCha20 on the input words in -+ * |input| and writes the 64 output bytes to |output|. */ -+static void chacha_core(uint8_t output[64], const uint32_t input[16]) { -+ uint32_t x[16]; -+ int i; -+ -+ memcpy(x, input, sizeof(uint32_t) * 16); -+ for (i = 20; i > 0; i -= 2) { -+ QUARTERROUND(0, 4, 8, 12) -+ QUARTERROUND(1, 5, 9, 13) -+ QUARTERROUND(2, 6, 10, 14) -+ QUARTERROUND(3, 7, 11, 15) -+ QUARTERROUND(0, 5, 10, 15) -+ QUARTERROUND(1, 6, 11, 12) -+ QUARTERROUND(2, 7, 8, 13) -+ QUARTERROUND(3, 4, 9, 14) -+ } -+ -+ for (i = 0; i < 16; ++i) { -+ x[i] = PLUS(x[i], input[i]); -+ } -+ for (i = 0; i < 16; ++i) { -+ U32TO8_LITTLE(output + 4 * i, x[i]); -+ } -+} -+ -+void CRYPTO_chacha_20(uint8_t *out, const uint8_t *in, size_t in_len, -+ const uint8_t key[32], const uint8_t nonce[8], -+ size_t counter) { -+#ifdef CHAPOLY_x86_64_ASM -+ uint8_t buf[256]; -+ size_t buf_size, ctr_msk; -+ void (*core_func)(uint8_t *out, const uint8_t *in, size_t in_len, -+ const uint8_t key[32], const uint8_t nonce[8], -+ size_t counter) = NULL; -+#else -+ uint8_t buf[64]; -+#endif -+ uint32_t input[16]; -+ size_t todo, i; -+ -+#ifdef CHAPOLY_x86_64_ASM -+ -+ if ((OPENSSL_ia32cap_loc()[2] >> 5) & 1) -+ { -+ buf_size = 128; -+ core_func = chacha_20_core_avx2; -+ ctr_msk = -2; -+ } -+ else if ((OPENSSL_ia32cap_loc()[1] >> 28) & 1) -+ { -+ buf_size = 64; -+ core_func = chacha_20_core_avx; -+ ctr_msk = -1; -+ } -+ else goto do_legacy; -+ -+ core_func(out, in, in_len, key, nonce, counter); -+ todo = in_len & (~(-buf_size)); -+ if(todo) -+ { -+ out += in_len&(-buf_size); -+ in += in_len&(-buf_size); -+ counter += (in_len/64) & ctr_msk; -+ memcpy(buf, in, todo); -+ core_func(buf, buf, buf_size, key, nonce, counter); -+ memcpy(out, buf, todo); -+ memset(buf, 0, buf_size); -+ } -+ return; -+ -+do_legacy: -+#endif -+ -+ input[0] = U8TO32_LITTLE(sigma + 0); -+ input[1] = U8TO32_LITTLE(sigma + 4); -+ input[2] = U8TO32_LITTLE(sigma + 8); -+ input[3] = U8TO32_LITTLE(sigma + 12); -+ -+ input[4] = U8TO32_LITTLE(key + 0); -+ input[5] = U8TO32_LITTLE(key + 4); -+ input[6] = U8TO32_LITTLE(key + 8); -+ input[7] = U8TO32_LITTLE(key + 12); -+ -+ input[8] = U8TO32_LITTLE(key + 16); -+ input[9] = U8TO32_LITTLE(key + 20); -+ input[10] = U8TO32_LITTLE(key + 24); -+ input[11] = U8TO32_LITTLE(key + 28); -+ -+ input[12] = counter; -+ input[13] = (uint64_t)counter >> 32; -+ input[14] = U8TO32_LITTLE(nonce + 0); -+ input[15] = U8TO32_LITTLE(nonce + 4); -+ -+ while (in_len > 0) { -+ todo = 64; -+ if (in_len < todo) { -+ todo = in_len; -+ } -+ -+ chacha_core(buf, input); -+ for (i = 0; i < todo; i++) { -+ out[i] = in[i] ^ buf[i]; -+ } -+ -+ out += todo; -+ in += todo; -+ in_len -= todo; -+ -+ ((uint64_t*)input)[6]++; -+ } -+} -diff --git a/crypto/chacha20poly1305/poly1305.c b/crypto/chacha20poly1305/poly1305.c -new file mode 100644 -index 0000000..8b065cd ---- /dev/null -+++ b/crypto/chacha20poly1305/poly1305.c -@@ -0,0 +1,285 @@ -+/* Copyright (c) 2014, Google Inc. -+ * -+ * Permission to use, copy, modify, and/or distribute this software for any -+ * purpose with or without fee is hereby granted, provided that the above -+ * copyright notice and this permission notice appear in all copies. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY -+ * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION -+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ -+ -+/* This implementation of poly1305 is by Andrew Moon -+ * (https://github.com/floodyberry/poly1305-donna) and released as public -+ * domain. */ -+ -+#include -+#include -+ -+#if !defined(B_ENDIAN) -+/* We can assume little-endian. */ -+static uint32_t U8TO32_LE(const uint8_t *m) { -+ uint32_t r; -+ memcpy(&r, m, sizeof(r)); -+ return r; -+} -+ -+static void U32TO8_LE(uint8_t *m, uint32_t v) { memcpy(m, &v, sizeof(v)); } -+#else -+static uint32_t U8TO32_LE(const uint8_t *m) { -+ return (uint32_t)m[0] | (uint32_t)m[1] << 8 | (uint32_t)m[2] << 16 | -+ (uint32_t)m[3] << 24; -+} -+ -+static void U32TO8_LE(uint8_t *m, uint32_t v) { -+ m[0] = v; -+ m[1] = v >> 8; -+ m[2] = v >> 16; -+ m[3] = v >> 24; -+} -+#endif -+ -+static uint64_t mul32x32_64(uint32_t a, uint32_t b) { return (uint64_t)a * b; } -+ -+struct poly1305_state_st { -+ uint32_t r0, r1, r2, r3, r4; -+ uint32_t s1, s2, s3, s4; -+ uint32_t h0, h1, h2, h3, h4; -+ uint8_t buf[16]; -+ unsigned int buf_used; -+ uint8_t key[16]; -+}; -+ -+/* poly1305_blocks updates |state| given some amount of input data. This -+ * function may only be called with a |len| that is not a multiple of 16 at the -+ * end of the data. Otherwise the input must be buffered into 16 byte blocks. */ -+static void poly1305_update(struct poly1305_state_st *state, const uint8_t *in, -+ size_t len) { -+ uint32_t t0, t1, t2, t3; -+ uint64_t t[5]; -+ uint32_t b; -+ uint64_t c; -+ size_t j; -+ uint8_t mp[16]; -+ -+ if (len < 16) { -+ goto poly1305_donna_atmost15bytes; -+ } -+ -+poly1305_donna_16bytes: -+ t0 = U8TO32_LE(in); -+ t1 = U8TO32_LE(in + 4); -+ t2 = U8TO32_LE(in + 8); -+ t3 = U8TO32_LE(in + 12); -+ -+ in += 16; -+ len -= 16; -+ -+ state->h0 += t0 & 0x3ffffff; -+ state->h1 += ((((uint64_t)t1 << 32) | t0) >> 26) & 0x3ffffff; -+ state->h2 += ((((uint64_t)t2 << 32) | t1) >> 20) & 0x3ffffff; -+ state->h3 += ((((uint64_t)t3 << 32) | t2) >> 14) & 0x3ffffff; -+ state->h4 += (t3 >> 8) | (1 << 24); -+ -+poly1305_donna_mul: -+ t[0] = mul32x32_64(state->h0, state->r0) + mul32x32_64(state->h1, state->s4) + -+ mul32x32_64(state->h2, state->s3) + mul32x32_64(state->h3, state->s2) + -+ mul32x32_64(state->h4, state->s1); -+ t[1] = mul32x32_64(state->h0, state->r1) + mul32x32_64(state->h1, state->r0) + -+ mul32x32_64(state->h2, state->s4) + mul32x32_64(state->h3, state->s3) + -+ mul32x32_64(state->h4, state->s2); -+ t[2] = mul32x32_64(state->h0, state->r2) + mul32x32_64(state->h1, state->r1) + -+ mul32x32_64(state->h2, state->r0) + mul32x32_64(state->h3, state->s4) + -+ mul32x32_64(state->h4, state->s3); -+ t[3] = mul32x32_64(state->h0, state->r3) + mul32x32_64(state->h1, state->r2) + -+ mul32x32_64(state->h2, state->r1) + mul32x32_64(state->h3, state->r0) + -+ mul32x32_64(state->h4, state->s4); -+ t[4] = mul32x32_64(state->h0, state->r4) + mul32x32_64(state->h1, state->r3) + -+ mul32x32_64(state->h2, state->r2) + mul32x32_64(state->h3, state->r1) + -+ mul32x32_64(state->h4, state->r0); -+ -+ state->h0 = (uint32_t)t[0] & 0x3ffffff; -+ c = (t[0] >> 26); -+ t[1] += c; -+ state->h1 = (uint32_t)t[1] & 0x3ffffff; -+ b = (uint32_t)(t[1] >> 26); -+ t[2] += b; -+ state->h2 = (uint32_t)t[2] & 0x3ffffff; -+ b = (uint32_t)(t[2] >> 26); -+ t[3] += b; -+ state->h3 = (uint32_t)t[3] & 0x3ffffff; -+ b = (uint32_t)(t[3] >> 26); -+ t[4] += b; -+ state->h4 = (uint32_t)t[4] & 0x3ffffff; -+ b = (uint32_t)(t[4] >> 26); -+ state->h0 += b * 5; -+ -+ if (len >= 16) -+ goto poly1305_donna_16bytes; -+ -+/* final bytes */ -+poly1305_donna_atmost15bytes: -+ if (!len) -+ return; -+ -+ for (j = 0; j < len; j++) -+ mp[j] = in[j]; -+ mp[j++] = 1; -+ for (; j < 16; j++) -+ mp[j] = 0; -+ len = 0; -+ -+ t0 = U8TO32_LE(mp + 0); -+ t1 = U8TO32_LE(mp + 4); -+ t2 = U8TO32_LE(mp + 8); -+ t3 = U8TO32_LE(mp + 12); -+ -+ state->h0 += t0 & 0x3ffffff; -+ state->h1 += ((((uint64_t)t1 << 32) | t0) >> 26) & 0x3ffffff; -+ state->h2 += ((((uint64_t)t2 << 32) | t1) >> 20) & 0x3ffffff; -+ state->h3 += ((((uint64_t)t3 << 32) | t2) >> 14) & 0x3ffffff; -+ state->h4 += (t3 >> 8); -+ -+ goto poly1305_donna_mul; -+} -+ -+void CRYPTO_poly1305_init(poly1305_state *statep, const uint8_t key[32]) { -+ struct poly1305_state_st *state = (struct poly1305_state_st *)statep; -+ uint32_t t0, t1, t2, t3; -+ -+ t0 = U8TO32_LE(key + 0); -+ t1 = U8TO32_LE(key + 4); -+ t2 = U8TO32_LE(key + 8); -+ t3 = U8TO32_LE(key + 12); -+ -+ /* precompute multipliers */ -+ state->r0 = t0 & 0x3ffffff; -+ t0 >>= 26; -+ t0 |= t1 << 6; -+ state->r1 = t0 & 0x3ffff03; -+ t1 >>= 20; -+ t1 |= t2 << 12; -+ state->r2 = t1 & 0x3ffc0ff; -+ t2 >>= 14; -+ t2 |= t3 << 18; -+ state->r3 = t2 & 0x3f03fff; -+ t3 >>= 8; -+ state->r4 = t3 & 0x00fffff; -+ -+ state->s1 = state->r1 * 5; -+ state->s2 = state->r2 * 5; -+ state->s3 = state->r3 * 5; -+ state->s4 = state->r4 * 5; -+ -+ /* init state */ -+ state->h0 = 0; -+ state->h1 = 0; -+ state->h2 = 0; -+ state->h3 = 0; -+ state->h4 = 0; -+ -+ state->buf_used = 0; -+ memcpy(state->key, key + 16, sizeof(state->key)); -+} -+ -+void CRYPTO_poly1305_update(poly1305_state *statep, const uint8_t *in, -+ size_t in_len) { -+ unsigned int i; -+ struct poly1305_state_st *state = (struct poly1305_state_st *)statep; -+ -+ if (state->buf_used) { -+ unsigned int todo = 16 - state->buf_used; -+ if (todo > in_len) -+ todo = in_len; -+ for (i = 0; i < todo; i++) -+ state->buf[state->buf_used + i] = in[i]; -+ state->buf_used += todo; -+ in_len -= todo; -+ in += todo; -+ -+ if (state->buf_used == 16) { -+ poly1305_update(state, state->buf, 16); -+ state->buf_used = 0; -+ } -+ } -+ -+ if (in_len >= 16) { -+ size_t todo = in_len & ~0xf; -+ poly1305_update(state, in, todo); -+ in += todo; -+ in_len &= 0xf; -+ } -+ -+ if (in_len) { -+ for (i = 0; i < in_len; i++) -+ state->buf[i] = in[i]; -+ state->buf_used = in_len; -+ } -+} -+ -+void CRYPTO_poly1305_finish(poly1305_state *statep, uint8_t mac[16]) { -+ struct poly1305_state_st *state = (struct poly1305_state_st *)statep; -+ uint64_t f0, f1, f2, f3; -+ uint32_t g0, g1, g2, g3, g4; -+ uint32_t b, nb; -+ -+ if (state->buf_used) -+ poly1305_update(state, state->buf, state->buf_used); -+ -+ b = state->h0 >> 26; -+ state->h0 = state->h0 & 0x3ffffff; -+ state->h1 += b; -+ b = state->h1 >> 26; -+ state->h1 = state->h1 & 0x3ffffff; -+ state->h2 += b; -+ b = state->h2 >> 26; -+ state->h2 = state->h2 & 0x3ffffff; -+ state->h3 += b; -+ b = state->h3 >> 26; -+ state->h3 = state->h3 & 0x3ffffff; -+ state->h4 += b; -+ b = state->h4 >> 26; -+ state->h4 = state->h4 & 0x3ffffff; -+ state->h0 += b * 5; -+ -+ g0 = state->h0 + 5; -+ b = g0 >> 26; -+ g0 &= 0x3ffffff; -+ g1 = state->h1 + b; -+ b = g1 >> 26; -+ g1 &= 0x3ffffff; -+ g2 = state->h2 + b; -+ b = g2 >> 26; -+ g2 &= 0x3ffffff; -+ g3 = state->h3 + b; -+ b = g3 >> 26; -+ g3 &= 0x3ffffff; -+ g4 = state->h4 + b - (1 << 26); -+ -+ b = (g4 >> 31) - 1; -+ nb = ~b; -+ state->h0 = (state->h0 & nb) | (g0 & b); -+ state->h1 = (state->h1 & nb) | (g1 & b); -+ state->h2 = (state->h2 & nb) | (g2 & b); -+ state->h3 = (state->h3 & nb) | (g3 & b); -+ state->h4 = (state->h4 & nb) | (g4 & b); -+ -+ f0 = ((state->h0) | (state->h1 << 26)) + (uint64_t)U8TO32_LE(&state->key[0]); -+ f1 = ((state->h1 >> 6) | (state->h2 << 20)) + -+ (uint64_t)U8TO32_LE(&state->key[4]); -+ f2 = ((state->h2 >> 12) | (state->h3 << 14)) + -+ (uint64_t)U8TO32_LE(&state->key[8]); -+ f3 = ((state->h3 >> 18) | (state->h4 << 8)) + -+ (uint64_t)U8TO32_LE(&state->key[12]); -+ -+ U32TO8_LE(&mac[0], f0); -+ f1 += (f0 >> 32); -+ U32TO8_LE(&mac[4], f1); -+ f2 += (f1 >> 32); -+ U32TO8_LE(&mac[8], f2); -+ f3 += (f2 >> 32); -+ U32TO8_LE(&mac[12], f3); -+} -diff --git a/crypto/evp/Makefile b/crypto/evp/Makefile -index 7f3cc8c..2abc5cc 100644 ---- a/crypto/evp/Makefile -+++ b/crypto/evp/Makefile -@@ -26,7 +26,8 @@ LIBSRC= encode.c digest.c evp_enc.c evp_key.c evp_acnf.c evp_cnf.c \ - c_all.c c_allc.c c_alld.c evp_lib.c bio_ok.c \ - evp_pkey.c evp_pbe.c p5_crpt.c p5_crpt2.c scrypt.c \ - e_old.c pmeth_lib.c pmeth_fn.c pmeth_gn.c m_sigver.c \ -- e_aes_cbc_hmac_sha1.c e_aes_cbc_hmac_sha256.c e_rc4_hmac_md5.c -+ e_aes_cbc_hmac_sha1.c e_aes_cbc_hmac_sha256.c e_rc4_hmac_md5.c \ -+ e_chacha20poly1305.c - - LIBOBJ= encode.o digest.o evp_enc.o evp_key.o evp_acnf.o evp_cnf.o \ - e_des.o e_bf.o e_idea.o e_des3.o e_camellia.o\ -@@ -39,7 +40,8 @@ LIBOBJ= encode.o digest.o evp_enc.o evp_key.o evp_acnf.o evp_cnf.o \ - c_all.o c_allc.o c_alld.o evp_lib.o bio_ok.o \ - evp_pkey.o evp_pbe.o p5_crpt.o p5_crpt2.o scrypt.o \ - e_old.o pmeth_lib.o pmeth_fn.o pmeth_gn.o m_sigver.o \ -- e_aes_cbc_hmac_sha1.o e_aes_cbc_hmac_sha256.o e_rc4_hmac_md5.o -+ e_aes_cbc_hmac_sha1.o e_aes_cbc_hmac_sha256.o e_rc4_hmac_md5.o \ -+ e_chacha20poly1305.o - - SRC= $(LIBSRC) - -@@ -250,6 +252,7 @@ e_cast.o: ../../include/openssl/opensslv.h ../../include/openssl/ossl_typ.h - e_cast.o: ../../include/openssl/safestack.h ../../include/openssl/stack.h - e_cast.o: ../../include/openssl/symhacks.h ../include/internal/cryptlib.h - e_cast.o: e_cast.c evp_locl.h -+e_chacha20poly1305.o: ../../include/openssl/chacha20poly1305.h e_chacha20poly1305.c - e_des.o: ../../e_os.h ../../include/openssl/asn1.h ../../include/openssl/bio.h - e_des.o: ../../include/openssl/buffer.h ../../include/openssl/crypto.h - e_des.o: ../../include/openssl/des.h ../../include/openssl/e_os2.h -diff --git a/crypto/evp/e_chacha20poly1305.c b/crypto/evp/e_chacha20poly1305.c -new file mode 100644 -index 0000000..0cb2af7 ---- /dev/null -+++ b/crypto/evp/e_chacha20poly1305.c -@@ -0,0 +1,323 @@ -+/* ==================================================================== -+ * Copyright (c) 2001-2014 The OpenSSL Project. All rights reserved. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions -+ * are met: -+ * -+ * 1. Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * 2. Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in -+ * the documentation and/or other materials provided with the -+ * distribution. -+ * -+ * 3. All advertising materials mentioning features or use of this -+ * software must display the following acknowledgment: -+ * "This product includes software developed by the OpenSSL Project -+ * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" -+ * -+ * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to -+ * endorse or promote products derived from this software without -+ * prior written permission. For written permission, please contact -+ * openssl-core@openssl.org. -+ * -+ * 5. Products derived from this software may not be called "OpenSSL" -+ * nor may "OpenSSL" appear in their names without prior written -+ * permission of the OpenSSL Project. -+ * -+ * 6. Redistributions of any form whatsoever must retain the following -+ * acknowledgment: -+ * "This product includes software developed by the OpenSSL Project -+ * for use in the OpenSSL Toolkit (http://www.openssl.org/)" -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY -+ * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR -+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED -+ * OF THE POSSIBILITY OF SUCH DAMAGE. -+ * ==================================================================== -+ * -+ */ -+ -+#include -+#ifndef OPENSSL_NO_CHACHA_POLY -+#include -+#include -+#include -+#include "evp_locl.h" -+#include -+ -+typedef struct -+ { -+ uint8_t key[32]; -+ /* uint8_t salt[4] */; -+ uint8_t nonce[8]; -+ poly1305_state poly_state; -+ size_t aad_l; -+ size_t ct_l; -+ int valid; -+#ifdef CHAPOLY_x86_64_ASM -+ void (*poly1305_init_ptr)(poly1305_state *, const uint8_t *); -+ void (*poly1305_update_ptr)(poly1305_state *, const uint8_t *, size_t); -+ void (*poly1305_finish_ptr)(poly1305_state *, uint8_t *); -+ #define poly_init aead_ctx->poly1305_init_ptr -+ #define poly_update poly1305_update_wrapper -+ #define poly_finish poly1305_finish_wrapper -+ #define FILL_BUFFER ((size_t)128) -+ uint8_t poly_buffer[FILL_BUFFER]; -+ uint8_t chacha_buffer[FILL_BUFFER]; -+ uint8_t poly_buffer_used; -+ uint8_t chacha_used; -+#else -+ #define poly_init CRYPTO_poly1305_init -+ #define poly_update(c,i,l) CRYPTO_poly1305_update(&c->poly_state,i,l) -+ #define poly_finish(c,m) CRYPTO_poly1305_finish(&c->poly_state,m) -+#endif -+ } EVP_CHACHA20_POLY1305_CTX; -+ -+#ifdef CHAPOLY_x86_64_ASM -+static void poly1305_update_wrapper(EVP_CHACHA20_POLY1305_CTX *ctx, const uint8_t *in, size_t in_len) -+ { -+ int todo; -+ /* Attempt to fill as many bytes as possible before calling the update function */ -+ if(in_len < FILL_BUFFER || ctx->poly_buffer_used) -+ { -+ todo = FILL_BUFFER - ctx->poly_buffer_used; -+ todo = in_len < todo? in_len : todo; -+ memcpy(ctx->poly_buffer + ctx->poly_buffer_used, in, todo); -+ ctx->poly_buffer_used += todo; -+ in += todo; -+ in_len -= todo; -+ if(ctx->poly_buffer_used == FILL_BUFFER) -+ { -+ ctx->poly1305_update_ptr(&ctx->poly_state, ctx->poly_buffer, FILL_BUFFER); -+ ctx->poly_buffer_used = 0; -+ } -+ } -+ if(in_len >= FILL_BUFFER) -+ { -+ ctx->poly1305_update_ptr(&ctx->poly_state, in, in_len&(-FILL_BUFFER)); -+ in += in_len&(-FILL_BUFFER); -+ in_len &= (FILL_BUFFER-1); -+ } -+ if(in_len) -+ { -+ memcpy(ctx->poly_buffer, in, in_len); -+ ctx->poly_buffer_used = in_len; -+ } -+ } -+ -+static void poly1305_finish_wrapper(EVP_CHACHA20_POLY1305_CTX *ctx, uint8_t mac[16]) -+ { -+ if(ctx->poly_buffer_used) -+ { -+ if(ctx->poly_buffer_used % 16) -+ { -+ memset(ctx->poly_buffer + ctx->poly_buffer_used, 0, 16 - (ctx->poly_buffer_used%16)); -+ } -+ ctx->poly1305_update_ptr(&ctx->poly_state, ctx->poly_buffer, ctx->poly_buffer_used); -+ } -+ ctx->poly1305_finish_ptr(&ctx->poly_state, mac); -+ memset(ctx->poly_buffer, 0 ,FILL_BUFFER); -+ } -+#endif -+ -+static int EVP_chacha20_poly1305_init(EVP_CIPHER_CTX *ctx, const unsigned char *key, const unsigned char *iv, int enc) -+ { -+ EVP_CHACHA20_POLY1305_CTX *aead_ctx = ctx->cipher_data; -+ /* simply copy the chacha key and iv*/ -+ memcpy(aead_ctx->key, key, 32); -+ /* memcpy(aead_ctx->salt, iv, 4); */ -+ aead_ctx->valid = 0; -+ return 1; -+ } -+ -+static int EVP_chacha20_poly1305_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl) -+ { -+ EVP_CHACHA20_POLY1305_CTX *aead_ctx = ctx->cipher_data; -+ uint8_t poly_block[16]; -+ uint64_t cl; -+ if(!aead_ctx->valid) -+ return 0; -+ if (inl < 16) -+ return -1; -+ /* Fix for MAC */ -+ inl -= 16; -+ /* Encryption */ -+ if(ctx->encrypt) -+ { -+#ifdef FILL_BUFFER -+ /* we can use the buffer we already accumulated during the parallel computation in init */ -+ if(inl<=FILL_BUFFER-64) -+ { -+ int i; -+ for(i=0; ichacha_buffer[i+64]; -+ } -+ else -+#endif -+ CRYPTO_chacha_20(out, in, inl, aead_ctx->key, aead_ctx->nonce, 1); -+ poly_update(aead_ctx, out, inl); -+ aead_ctx->ct_l += inl; -+ cl = aead_ctx->ct_l; -+ poly_update(aead_ctx, (uint8_t*)&cl, sizeof(cl)); -+ poly_finish(aead_ctx, &out[inl]); -+ aead_ctx->valid = 0; -+ return inl+16; -+ } -+ /* Decryption */ -+ else -+ { -+ /* Fix to accommodate for the MAC */ -+ poly_update(aead_ctx, in, inl); -+#ifdef FILL_BUFFER -+ /* we can use the buffer we already accumulated during the parallel computation in init */ -+ if(inl<=FILL_BUFFER-64) -+ { -+ int i; -+ for(i=0; ichacha_buffer[i+64]; -+ } -+ else -+#endif -+ CRYPTO_chacha_20(out, in, inl, aead_ctx->key, aead_ctx->nonce, 1); -+ aead_ctx->ct_l += inl; -+ cl = aead_ctx->ct_l; -+ poly_update(aead_ctx, (uint8_t*)&cl, sizeof(cl)); -+ poly_finish(aead_ctx, poly_block); -+ -+ uint64_t cmp = ((uint64_t*)poly_block)[0] ^ ((uint64_t*)(in + inl))[0]; -+ cmp |= ((uint64_t*)poly_block)[1] ^ ((uint64_t*)(in + inl))[1]; -+ -+ /*if (memcmp(poly_block, in + inl, POLY1305_MAC_LEN)) */ -+ if (cmp) -+ { -+ OPENSSL_cleanse(out, inl); -+ aead_ctx->valid = 0; -+ return -1; -+ } -+ aead_ctx->valid = 0; -+ return inl; -+ } -+ return 0; -+ } -+ -+static int EVP_chacha20_poly1305_cleanup(EVP_CIPHER_CTX *ctx) -+ { -+ return 1; -+ } -+ -+static int EVP_chacha20_poly1305_ctrl(EVP_CIPHER_CTX *ctx, int type, int arg, void *ptr) -+ { -+ EVP_CHACHA20_POLY1305_CTX *aead_ctx = ctx->cipher_data; -+#ifndef FILL_BUFFER -+ uint8_t poly1305_key[32]; -+#endif -+ uint8_t aad[13 + 8]; -+ uint64_t thirteen = 13; -+ -+ switch(type) -+ { -+ case EVP_CTRL_AEAD_TLS1_AAD: -+ if(arg!=13) -+ return 0; -+ /* Initialize poly keys */ -+#ifndef FILL_BUFFER -+ memset(poly1305_key, 0, sizeof(poly1305_key)); -+#else -+ memset(aead_ctx->chacha_buffer, 0, FILL_BUFFER); -+#endif -+ /* Salt is the IV (not in draft) */ -+ /* memcpy(aead_ctx->nonce, aead_ctx->salt, 4); */ -+ /* Take sequence number from AAD */ -+ /* memcpy(&aead_ctx->nonce[4], ptr, 8); */ -+ memcpy(aead_ctx->nonce, ptr, 8); -+ -+#ifdef CHAPOLY_x86_64_ASM -+ aead_ctx->poly_buffer_used = 0; -+ if((OPENSSL_ia32cap_loc()[2] >> 5) & 1) /* AVX2 */ -+ { -+ aead_ctx->poly1305_init_ptr = poly1305_init_avx2; -+ aead_ctx->poly1305_update_ptr = poly1305_update_avx2; -+ aead_ctx->poly1305_finish_ptr = poly1305_finish_avx2; -+ } -+ else if ((OPENSSL_ia32cap_loc()[1] >> 28) & 1) /* AVX */ -+ { -+ aead_ctx->poly1305_init_ptr = poly1305_init_avx; -+ aead_ctx->poly1305_update_ptr = poly1305_update_avx; -+ aead_ctx->poly1305_finish_ptr = poly1305_finish_avx; -+ } -+ else /*C*/ -+ { -+ aead_ctx->poly1305_init_ptr = CRYPTO_poly1305_init; -+ aead_ctx->poly1305_update_ptr = CRYPTO_poly1305_update; -+ aead_ctx->poly1305_finish_ptr = CRYPTO_poly1305_finish; -+ } -+ -+#endif -+#ifndef FILL_BUFFER -+ CRYPTO_chacha_20(poly1305_key, poly1305_key, sizeof(poly1305_key), aead_ctx->key, aead_ctx->nonce, 0); -+ poly_init(&aead_ctx->poly_state, poly1305_key); -+#else -+ CRYPTO_chacha_20(aead_ctx->chacha_buffer, aead_ctx->chacha_buffer, FILL_BUFFER, aead_ctx->key, aead_ctx->nonce, 0); -+ poly_init(&aead_ctx->poly_state, aead_ctx->chacha_buffer); -+ aead_ctx->chacha_used = 64; /* We keep 64 byte for future use, to accelerate for very short messages */ -+#endif -+ aead_ctx->aad_l = 0; -+ aead_ctx->ct_l = 0; -+ /* Absorb AAD */ -+ memcpy(aad, ptr, arg); -+ memcpy(&aad[arg], &thirteen, sizeof(thirteen)); -+ /* If decrypting fix length for tag */ -+ if (!ctx->encrypt) -+ { -+ unsigned int len=aad[arg-2]<<8|aad[arg-1]; -+ len -= POLY1305_MAC_LEN; -+ aad[arg-2] = len>>8; -+ aad[arg-1] = len & 0xff; -+ } -+ poly_update(aead_ctx, aad, arg + sizeof(thirteen)); -+ /* aead_ctx->aad_l += arg; */ -+ aead_ctx->valid = 1; -+ return POLY1305_MAC_LEN; -+ break; -+ default: -+ return 0; -+ break; -+ } -+ return 0; -+ } -+ -+#define CUSTOM_FLAGS (\ -+ EVP_CIPH_CUSTOM_IV | EVP_CIPH_FLAG_CUSTOM_CIPHER \ -+ | EVP_CIPH_ALWAYS_CALL_INIT \ -+ | EVP_CIPH_CUSTOM_COPY) -+ -+static const EVP_CIPHER chacha20_poly1305 = { -+ NID_chacha20_poly1305, /* nid */ -+ 1, /* block size, sorta */ -+ 32, /* key len */ -+ 0, /* iv len */ -+ CUSTOM_FLAGS|EVP_CIPH_FLAG_AEAD_CIPHER, /* flags */ -+ EVP_chacha20_poly1305_init, -+ EVP_chacha20_poly1305_cipher, -+ EVP_chacha20_poly1305_cleanup, -+ sizeof(EVP_CHACHA20_POLY1305_CTX), /* ctx size */ -+ NULL, NULL, -+ EVP_chacha20_poly1305_ctrl, -+ NULL -+ }; -+ -+const EVP_CIPHER *EVP_chacha20_poly1305(void) -+{ return &chacha20_poly1305; } -+ -+#endif -diff --git a/crypto/objects/obj_dat.h b/crypto/objects/obj_dat.h -index 6210784..ae335ac 100644 ---- a/crypto/objects/obj_dat.h -+++ b/crypto/objects/obj_dat.h -@@ -62,9 +62,9 @@ - * [including the GNU Public Licence.] - */ - --#define NUM_NID 1009 --#define NUM_SN 1002 --#define NUM_LN 1002 -+#define NUM_NID 1010 -+#define NUM_SN 1003 -+#define NUM_LN 1003 - #define NUM_OBJ 936 - - static const unsigned char lvalues[6604]={ -@@ -2658,6 +2658,8 @@ static const ASN1_OBJECT nid_objs[NUM_NID]={ - &(lvalues[6593]),0}, - {"issuerSignTool","Signing Tool of Issuer",NID_issuerSignTool,5, - &(lvalues[6598]),0}, -+{"id-chacha20-poly1305","chacha20-poly1305",NID_chacha20_poly1305,0, -+ NULL,0}, - }; - - static const unsigned int sn_objs[NUM_SN]={ -@@ -3119,6 +3121,7 @@ static const unsigned int sn_objs[NUM_SN]={ - 362, /* "id-cct-PKIResponse" */ - 360, /* "id-cct-crs" */ - 81, /* "id-ce" */ -+1009, /* "id-chacha20-poly1305" */ - 680, /* "id-characteristic-two-basis" */ - 263, /* "id-cmc" */ - 334, /* "id-cmc-addExtensions" */ -@@ -3955,6 +3958,7 @@ static const unsigned int ln_objs[NUM_LN]={ - 677, /* "certicom-arc" */ - 517, /* "certificate extensions" */ - 883, /* "certificateRevocationList" */ -+1009, /* "chacha20-poly1305" */ - 54, /* "challengePassword" */ - 407, /* "characteristic-two-field" */ - 395, /* "clearance" */ -diff --git a/include/openssl/chacha20poly1305.h b/include/openssl/chacha20poly1305.h -new file mode 100644 -index 0000000..bcabbb4 ---- /dev/null -+++ b/include/openssl/chacha20poly1305.h -@@ -0,0 +1,63 @@ -+#ifndef OPENSSL_HEADER_POLY1305_H -+#define OPENSSL_HEADER_POLY1305_H -+ -+#include -+#include -+#include -+#include "crypto.h" -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+#define POLY1305_MAC_LEN (16) -+ -+typedef unsigned char poly1305_state[512]; -+ -+ -+/* CRYPTO_poly1305_init sets up |state| so that it can be used to calculate an -+ * authentication tag with the one-time key |key|. Note that |key| is a -+ * one-time key and therefore there is no `reset' method because that would -+ * enable several messages to be authenticated with the same key. */ -+void CRYPTO_poly1305_init(poly1305_state* state, const uint8_t key[32]); -+ -+/* CRYPTO_poly1305_update processes |in_len| bytes from |in|. It can be called -+ * zero or more times after poly1305_init. */ -+void CRYPTO_poly1305_update(poly1305_state* state, const uint8_t* in, -+ size_t in_len); -+ -+/* CRYPTO_poly1305_finish completes the poly1305 calculation and writes a 16 -+ * byte authentication tag to |mac|. */ -+void CRYPTO_poly1305_finish(poly1305_state* state, uint8_t mac[16]); -+ -+/* CRYPTO_chacha_20 encrypts |in_len| bytes from |in| with the given key and -+ * nonce and writes the result to |out|, which may be equal to |in|. The -+ * initial block counter is specified by |counter|. */ -+void CRYPTO_chacha_20(uint8_t *out, const uint8_t *in, size_t in_len, -+ const uint8_t key[32], const uint8_t nonce[8], -+ size_t counter); -+ -+#ifdef CHAPOLY_x86_64_ASM -+void poly1305_init_avx(poly1305_state* state, const uint8_t key[32]); -+void poly1305_update_avx(poly1305_state* state, const uint8_t *in, size_t in_len); -+void poly1305_finish_avx(poly1305_state* state, uint8_t mac[16]); -+ -+void poly1305_init_avx2(poly1305_state* state, const uint8_t key[32]); -+void poly1305_update_avx2(poly1305_state* state, const uint8_t *in, size_t in_len); -+void poly1305_finish_avx2(poly1305_state* state, uint8_t mac[16]); -+ -+void chacha_20_core_avx(uint8_t *out, const uint8_t *in, size_t in_len, -+ const uint8_t key[32], const uint8_t nonce[8], -+ size_t counter); -+ -+void chacha_20_core_avx2(uint8_t *out, const uint8_t *in, size_t in_len, -+ const uint8_t key[32], const uint8_t nonce[8], -+ size_t counter); -+#endif -+ -+ -+#if defined(__cplusplus) -+} /* extern C */ -+#endif -+ -+#endif /* OPENSSL_HEADER_POLY1305_H */ -diff --git a/include/openssl/evp.h b/include/openssl/evp.h -index f92f109..919d66f 100644 ---- a/include/openssl/evp.h -+++ b/include/openssl/evp.h -@@ -937,6 +937,7 @@ const EVP_CIPHER *EVP_camellia_256_cfb128(void); - const EVP_CIPHER *EVP_camellia_256_ofb(void); - const EVP_CIPHER *EVP_camellia_256_ctr(void); - # endif -+const EVP_CIPHER *EVP_chacha20_poly1305(void); - - # ifndef OPENSSL_NO_SEED - const EVP_CIPHER *EVP_seed_ecb(void); -diff --git a/include/openssl/obj_mac.h b/include/openssl/obj_mac.h -index fdf5b62..7730c42 100644 ---- a/include/openssl/obj_mac.h -+++ b/include/openssl/obj_mac.h -@@ -4423,3 +4423,7 @@ - #define SN_id_scrypt "id-scrypt" - #define NID_id_scrypt 973 - #define OBJ_id_scrypt 1L,3L,6L,1L,4L,1L,11591L,4L,11L -+ -+#define SN_chacha20_poly1305 "id-chacha20-poly1305" -+#define LN_chacha20_poly1305 "chacha20-poly1305" -+#define NID_chacha20_poly1305 1009 -diff --git a/include/openssl/ssl.h b/include/openssl/ssl.h -index 1334eb6..c6b3b3b 100644 ---- a/include/openssl/ssl.h -+++ b/include/openssl/ssl.h -@@ -250,6 +250,7 @@ extern "C" { - # define SSL_TXT_CAMELLIA128 "CAMELLIA128" - # define SSL_TXT_CAMELLIA256 "CAMELLIA256" - # define SSL_TXT_CAMELLIA "CAMELLIA" -+# define SSL_TXT_CHACHA20 "CHACHA20" - - # define SSL_TXT_MD5 "MD5" - # define SSL_TXT_SHA1 "SHA1" -diff --git a/include/openssl/tls1.h b/include/openssl/tls1.h -index 6adfcf3..83d0d64 100644 ---- a/include/openssl/tls1.h -+++ b/include/openssl/tls1.h -@@ -659,6 +659,11 @@ SSL_CTX_callback_ctrl(ssl,SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB,(void (*)(void))cb) - # define TLS1_CK_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 0x0300C09A - # define TLS1_CK_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 0x0300C09B - -+/* ChaCha20-Poly1305 ciphersuites draft-agl-tls-chacha20poly1305-01 */ -+# define TLS1_CK_ECDHE_RSA_CHACHA20_POLY1305 0x0300CC13 -+# define TLS1_CK_ECDHE_ECDSA_CHACHA20_POLY1305 0x0300CC14 -+# define TLS1_CK_DHE_RSA_CHACHA20_POLY1305 0x0300CC15 -+ - /* - * XXX Backward compatibility alert: Older versions of OpenSSL gave some DHE - * ciphers names with "EDH" instead of "DHE". Going forward, we should be -@@ -925,6 +930,11 @@ SSL_CTX_callback_ctrl(ssl,SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB,(void (*)(void))cb) - # define TLS1_TXT_ECDH_RSA_WITH_CAMELLIA_128_CBC_SHA256 "ECDH-RSA-CAMELLIA128-SHA256" - # define TLS1_TXT_ECDH_RSA_WITH_CAMELLIA_256_CBC_SHA384 "ECDH-RSA-CAMELLIA256-SHA384" - -+/* ChaCha20-Poly1305 ciphersuites draft-agl-tls-chacha20poly1305-01 */ -+#define TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305 "ECDHE-RSA-CHACHA20-POLY1305" -+#define TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 "ECDHE-ECDSA-CHACHA20-POLY1305" -+#define TLS1_TXT_DHE_RSA_WITH_CHACHA20_POLY1305 "DHE-RSA-CHACHA20-POLY1305" -+ - # define TLS_CT_RSA_SIGN 1 - # define TLS_CT_DSS_SIGN 2 - # define TLS_CT_RSA_FIXED_DH 3 -diff --git a/ssl/s3_lib.c b/ssl/s3_lib.c -index bb090ef..0463bf1 100644 ---- a/ssl/s3_lib.c -+++ b/ssl/s3_lib.c -@@ -3770,6 +3770,54 @@ OPENSSL_GLOBAL const SSL_CIPHER ssl3_ciphers[] = { - 256, - }, - -+ /* Chacha20-Poly1305 draft cipher suites */ -+#if !defined(OPENSSL_NO_CHACHA_POLY) -+ { -+ 1, -+ TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305, -+ TLS1_CK_ECDHE_RSA_CHACHA20_POLY1305, -+ SSL_kEECDH, -+ SSL_aRSA, -+ SSL_CHACHA20POLY1305, -+ SSL_AEAD, -+ SSL_TLSV1_2, -+ SSL_NOT_EXP|SSL_HIGH, -+ SSL_HANDSHAKE_MAC_SHA256|TLS1_PRF_SHA256, -+ 256, -+ 0, -+ }, -+ -+ { -+ 1, -+ TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, -+ TLS1_CK_ECDHE_ECDSA_CHACHA20_POLY1305, -+ SSL_kEECDH, -+ SSL_aECDSA, -+ SSL_CHACHA20POLY1305, -+ SSL_AEAD, -+ SSL_TLSV1_2, -+ SSL_NOT_EXP|SSL_HIGH, -+ SSL_HANDSHAKE_MAC_SHA256|TLS1_PRF_SHA256, -+ 256, -+ 0, -+ }, -+ -+ { -+ 1, -+ TLS1_TXT_DHE_RSA_WITH_CHACHA20_POLY1305, -+ TLS1_CK_DHE_RSA_CHACHA20_POLY1305, -+ SSL_kEDH, -+ SSL_aRSA, -+ SSL_CHACHA20POLY1305, -+ SSL_AEAD, -+ SSL_TLSV1_2, -+ SSL_NOT_EXP|SSL_HIGH, -+ SSL_HANDSHAKE_MAC_SHA256|TLS1_PRF_SHA256, -+ 256, -+ 0, -+ }, -+#endif -+ - /* end of list */ - }; - -@@ -4796,6 +4844,7 @@ SSL_CIPHER *ssl3_choose_cipher(SSL *s, STACK_OF(SSL_CIPHER) *clnt, - STACK_OF(SSL_CIPHER) *prio, *allow; - int i, ii, ok; - unsigned long alg_k, alg_a, mask_k, mask_a, emask_k, emask_a; -+ int use_chacha = 0; - - /* Let's see which ciphers we can support */ - -@@ -4828,9 +4877,16 @@ SSL_CIPHER *ssl3_choose_cipher(SSL *s, STACK_OF(SSL_CIPHER) *clnt, - if (s->options & SSL_OP_CIPHER_SERVER_PREFERENCE || tls1_suiteb(s)) { - prio = srvr; - allow = clnt; -+ /* Use ChaCha20+Poly1305 iff it's client's most preferred cipher suite */ -+ if (sk_SSL_CIPHER_num(clnt) > 0) { -+ c = sk_SSL_CIPHER_value(clnt, 0); -+ if (c->algorithm_enc == SSL_CHACHA20POLY1305) -+ use_chacha = 1; -+ } - } else { - prio = clnt; - allow = srvr; -+ use_chacha = 1; - } - - tls1_set_cert_validity(s); -@@ -4842,6 +4898,11 @@ SSL_CIPHER *ssl3_choose_cipher(SSL *s, STACK_OF(SSL_CIPHER) *clnt, - if ((c->algorithm_ssl & SSL_TLSV1_2) && !SSL_USE_TLS1_2_CIPHERS(s)) - continue; - -+ /* Skip ChaCha unless top client priority */ -+ if ((c->algorithm_enc == SSL_CHACHA20POLY1305) && -+ !use_chacha) -+ continue; -+ - ssl_set_masks(s, c); - mask_k = s->s3->tmp.mask_k; - mask_a = s->s3->tmp.mask_a; -diff --git a/ssl/ssl_algs.c b/ssl/ssl_algs.c -index f4827fd..56d1b9c 100644 ---- a/ssl/ssl_algs.c -+++ b/ssl/ssl_algs.c -@@ -102,6 +102,9 @@ int SSL_library_init(void) - EVP_add_cipher(EVP_camellia_128_cbc()); - EVP_add_cipher(EVP_camellia_256_cbc()); - #endif -+#ifndef OPENSSL_NO_CHACHA_POLY -+ EVP_add_cipher(EVP_chacha20_poly1305()); -+#endif - - #ifndef OPENSSL_NO_SEED - EVP_add_cipher(EVP_seed_cbc()); -diff --git a/ssl/ssl_ciph.c b/ssl/ssl_ciph.c -index 12dac04..68646ca 100644 ---- a/ssl/ssl_ciph.c -+++ b/ssl/ssl_ciph.c -@@ -150,25 +150,26 @@ - #endif - #include "ssl_locl.h" - --#define SSL_ENC_DES_IDX 0 --#define SSL_ENC_3DES_IDX 1 --#define SSL_ENC_RC4_IDX 2 --#define SSL_ENC_RC2_IDX 3 --#define SSL_ENC_IDEA_IDX 4 --#define SSL_ENC_NULL_IDX 5 --#define SSL_ENC_AES128_IDX 6 --#define SSL_ENC_AES256_IDX 7 --#define SSL_ENC_CAMELLIA128_IDX 8 --#define SSL_ENC_CAMELLIA256_IDX 9 --#define SSL_ENC_GOST89_IDX 10 --#define SSL_ENC_SEED_IDX 11 --#define SSL_ENC_AES128GCM_IDX 12 --#define SSL_ENC_AES256GCM_IDX 13 --#define SSL_ENC_AES128CCM_IDX 14 --#define SSL_ENC_AES256CCM_IDX 15 --#define SSL_ENC_AES128CCM8_IDX 16 --#define SSL_ENC_AES256CCM8_IDX 17 --#define SSL_ENC_NUM_IDX 18 -+#define SSL_ENC_DES_IDX 0 -+#define SSL_ENC_3DES_IDX 1 -+#define SSL_ENC_RC4_IDX 2 -+#define SSL_ENC_RC2_IDX 3 -+#define SSL_ENC_IDEA_IDX 4 -+#define SSL_ENC_NULL_IDX 5 -+#define SSL_ENC_AES128_IDX 6 -+#define SSL_ENC_AES256_IDX 7 -+#define SSL_ENC_CAMELLIA128_IDX 8 -+#define SSL_ENC_CAMELLIA256_IDX 9 -+#define SSL_ENC_GOST89_IDX 10 -+#define SSL_ENC_SEED_IDX 11 -+#define SSL_ENC_AES128GCM_IDX 12 -+#define SSL_ENC_AES256GCM_IDX 13 -+#define SSL_ENC_AES128CCM_IDX 14 -+#define SSL_ENC_AES256CCM_IDX 15 -+#define SSL_ENC_AES128CCM8_IDX 16 -+#define SSL_ENC_AES256CCM8_IDX 17 -+#define SSL_ENC_CHACHA20POLY1305_IDX 18 -+#define SSL_ENC_NUM_IDX 19 - - /* NB: make sure indices in these tables match values above */ - -@@ -196,7 +197,8 @@ static const ssl_cipher_table ssl_cipher_table_cipher[SSL_ENC_NUM_IDX] = { - {SSL_AES128CCM, NID_aes_128_ccm}, /* SSL_ENC_AES128CCM_IDX 14 */ - {SSL_AES256CCM, NID_aes_256_ccm}, /* SSL_ENC_AES256CCM_IDX 15 */ - {SSL_AES128CCM8, NID_aes_128_ccm}, /* SSL_ENC_AES128CCM8_IDX 16 */ -- {SSL_AES256CCM8, NID_aes_256_ccm} /* SSL_ENC_AES256CCM8_IDX 17 */ -+ {SSL_AES256CCM8, NID_aes_256_ccm}, /* SSL_ENC_AES256CCM8_IDX 17 */ -+ {SSL_CHACHA20POLY1305, NID_chacha20_poly1305} /* SSL_ENC_CHACHA20POLY1305_IDX 18 */ - }; - - static const EVP_CIPHER *ssl_cipher_methods[SSL_ENC_NUM_IDX] = { -@@ -378,6 +380,7 @@ static const SSL_CIPHER cipher_aliases[] = { - {0, SSL_TXT_CAMELLIA256, 0, 0, 0, SSL_CAMELLIA256, 0, 0, 0, 0, 0, 0}, - {0, SSL_TXT_CAMELLIA, 0, 0, 0, SSL_CAMELLIA128 | SSL_CAMELLIA256, 0, 0, 0, - 0, 0, 0}, -+ {0, SSL_TXT_CHACHA20, 0, 0, 0, SSL_CHACHA20POLY1305, 0, 0, 0, 0, 0, 0}, - - /* MAC aliases */ - {0, SSL_TXT_MD5, 0, 0, 0, 0, SSL_MD5, 0, 0, 0, 0, 0}, -@@ -1745,6 +1748,9 @@ char *SSL_CIPHER_description(const SSL_CIPHER *cipher, char *buf, int len) - case SSL_eGOST2814789CNT: - enc = "GOST89(256)"; - break; -+ case SSL_CHACHA20POLY1305: -+ enc = "CHACHA20-POLY1305(256)"; -+ break; - default: - enc = "unknown"; - break; -diff --git a/ssl/ssl_locl.h b/ssl/ssl_locl.h -index 083ced3..e9cb277 100644 ---- a/ssl/ssl_locl.h -+++ b/ssl/ssl_locl.h -@@ -363,6 +363,7 @@ - # define SSL_AES256CCM 0x00008000L - # define SSL_AES128CCM8 0x00010000L - # define SSL_AES256CCM8 0x00020000L -+# define SSL_CHACHA20POLY1305 0x00040000L - - # define SSL_AES (SSL_AES128|SSL_AES256|SSL_AES128GCM|SSL_AES256GCM|SSL_AES128CCM|SSL_AES256CCM|SSL_AES128CCM8|SSL_AES256CCM8) - # define SSL_CAMELLIA (SSL_CAMELLIA128|SSL_CAMELLIA256) -diff --git a/test/Makefile b/test/Makefile -index 6f32758..b7884a9 100644 ---- a/test/Makefile -+++ b/test/Makefile -@@ -78,6 +78,7 @@ SSLVERTOLTEST= sslvertoltest - SSLEXTENSIONTEST= sslextensiontest - SSLSESSIONTICKTEST= sslsessionticktest - SSLSKEWITH0PTEST= sslskewith0ptest -+CHAPOLYTEST= chapolytest - - TESTS= alltests - -@@ -97,7 +98,8 @@ EXE= $(NPTEST)$(EXE_EXT) $(BNTEST)$(EXE_EXT) $(ECTEST)$(EXE_EXT) \ - $(SRPTEST)$(EXE_EXT) $(V3NAMETEST)$(EXE_EXT) \ - $(HEARTBEATTEST)$(EXE_EXT) $(P5_CRPT2_TEST)$(EXE_EXT) \ - $(CONSTTIMETEST)$(EXE_EXT) $(VERIFYEXTRATEST)$(EXE_EXT) \ -- $(CLIENTHELLOTEST)$(EXE_EXT) $(PACKETTEST)$(EXE_EXT) -+ $(CLIENTHELLOTEST)$(EXE_EXT) $(PACKETTEST)$(EXE_EXT) \ -+ $(CHAPOLYTEST)$(EXE_EXT) - - # $(METHTEST)$(EXE_EXT) - -@@ -113,7 +115,7 @@ OBJ= $(NPTEST).o $(BNTEST).o $(ECTEST).o \ - $(EVPTEST).o $(EVPEXTRATEST).o $(IGETEST).o $(JPAKETEST).o $(V3NAMETEST).o \ - $(GOST2814789TEST).o $(HEARTBEATTEST).o $(P5_CRPT2_TEST).o \ - $(CONSTTIMETEST).o $(VERIFYEXTRATEST).o $(CLIENTHELLOTEST).o \ -- $(PACKETTEST).o testutil.o -+ $(PACKETTEST).o $(CHAPOLYTEST).o testutil.o - - SRC= $(NPTEST).c $(BNTEST).c $(ECTEST).c \ - $(ECDSATEST).c $(ECDHTEST).c $(GMDIFFTEST).c $(PBELUTEST).c $(IDEATEST).c \ -@@ -126,7 +128,7 @@ SRC= $(NPTEST).c $(BNTEST).c $(ECTEST).c \ - $(EVPTEST).c $(EVPEXTRATEST).c $(IGETEST).c $(JPAKETEST).c $(V3NAMETEST).c \ - $(GOST2814789TEST).c $(HEARTBEATTEST).c $(P5_CRPT2_TEST).c \ - $(CONSTTIMETEST).c $(VERIFYEXTRATEST).c $(CLIENTHELLOTEST).c \ -- $(PACKETTEST).c testutil.c -+ $(PACKETTEST).c $(CHAPOLYTEST).c testutil.c - - HEADER= testutil.h - -@@ -233,6 +235,9 @@ $(SHA256TEST)$(EXE_EXT): $(SHA256TEST).o $(DLIBCRYPTO) - $(SHA512TEST)$(EXE_EXT): $(SHA512TEST).o $(DLIBCRYPTO) - @target=$(SHA512TEST); $(BUILD_CMD) - -+$(CHAPOLYTEST)$(EXE_EXT): $(CHAPOLYTEST).o $(DLIBCRYPTO) -+ @target=$(CHAPOLYTEST); $(BUILD_CMD) -+ - FIPS_BUILD_CMD=shlib_target=; if [ -n "$(SHARED_LIBS)" ]; then \ - shlib_target="$(SHLIB_TARGET)"; \ - fi; \ -@@ -717,3 +722,4 @@ wp_test.o: ../include/openssl/opensslconf.h ../include/openssl/opensslv.h - wp_test.o: ../include/openssl/ossl_typ.h ../include/openssl/safestack.h - wp_test.o: ../include/openssl/stack.h ../include/openssl/symhacks.h - wp_test.o: ../include/openssl/whrlpool.h wp_test.c -+chapoly_test.o: ../include/openssl/chacha20poly1305.h chapoly_test.c -diff --git a/test/chapolytest.c b/test/chapolytest.c -new file mode 100644 -index 0000000..56e713e ---- /dev/null -+++ b/test/chapolytest.c -@@ -0,0 +1,287 @@ -+/* ==================================================================== -+ * Copyright (c) 2011-2013 The OpenSSL Project. All rights reserved. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions -+ * are met: -+ * -+ * 1. Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * 2. Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in -+ * the documentation and/or other materials provided with the -+ * distribution. -+ * -+ * 3. All advertising materials mentioning features or use of this -+ * software must display the following acknowledgment: -+ * "This product includes software developed by the OpenSSL Project -+ * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)" -+ * -+ * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to -+ * endorse or promote products derived from this software without -+ * prior written permission. For written permission, please contact -+ * licensing@OpenSSL.org. -+ * -+ * 5. Products derived from this software may not be called "OpenSSL" -+ * nor may "OpenSSL" appear in their names without prior written -+ * permission of the OpenSSL Project. -+ * -+ * 6. Redistributions of any form whatsoever must retain the following -+ * acknowledgment: -+ * "This product includes software developed by the OpenSSL Project -+ * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)" -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY -+ * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR -+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED -+ * OF THE POSSIBILITY OF SUCH DAMAGE. -+ * ==================================================================== -+ */ -+ -+ -+#include -+#include -+#include -+#include -+ -+#include -+ -+struct chacha_test { -+ const char *keyhex; -+ const char *noncehex; -+ const char *outhex; -+}; -+ -+struct poly1305_test -+ { -+ const char *inputhex; -+ const char *keyhex; -+ const char *outhex; -+ }; -+ -+static const struct chacha_test chacha_tests[] = { -+ { -+ "0000000000000000000000000000000000000000000000000000000000000000", -+ "0000000000000000", -+ "76b8e0ada0f13d90405d6ae55386bd28bdd219b8a08ded1aa836efcc8b770dc7da41597c5157488d7724e03fb8d84a376a43b8f41518a11cc387b669b2ee6586", -+ }, -+ { -+ "0000000000000000000000000000000000000000000000000000000000000001", -+ "0000000000000000", -+ "4540f05a9f1fb296d7736e7b208e3c96eb4fe1834688d2604f450952ed432d41bbe2a0b6ea7566d2a5d1e7e20d42af2c53d792b1c43fea817e9ad275ae546963", -+ }, -+ { -+ "0000000000000000000000000000000000000000000000000000000000000000", -+ "0000000000000001", -+ "de9cba7bf3d69ef5e786dc63973f653a0b49e015adbff7134fcb7df137821031e85a050278a7084527214f73efc7fa5b5277062eb7a0433e445f41e31afab757", -+ }, -+ { -+ "0000000000000000000000000000000000000000000000000000000000000000", -+ "0100000000000000", -+ "ef3fdfd6c61578fbf5cf35bd3dd33b8009631634d21e42ac33960bd138e50d32111e4caf237ee53ca8ad6426194a88545ddc497a0b466e7d6bbdb0041b2f586b", -+ }, -+ { -+ "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", -+ "0001020304050607", -+ "f798a189f195e66982105ffb640bb7757f579da31602fc93ec01ac56f85ac3c134a4547b733b46413042c9440049176905d3be59ea1c53f15916155c2be8241a38008b9a26bc35941e2444177c8ade6689de95264986d95889fb60e84629c9bd9a5acb1cc118be563eb9b3a4a472f82e09a7e778492b562ef7130e88dfe031c79db9d4f7c7a899151b9a475032b63fc385245fe054e3dd5a97a5f576fe064025d3ce042c566ab2c507b138db853e3d6959660996546cc9c4a6eafdc777c040d70eaf46f76dad3979e5c5360c3317166a1c894c94a371876a94df7628fe4eaaf2ccb27d5aaae0ad7ad0f9d4b6ad3b54098746d4524d38407a6deb", -+ }, -+}; -+ -+static const struct poly1305_test poly1305_tests[] = { -+ { -+ "", -+ "c8afaac331ee372cd6082de134943b174710130e9f6fea8d72293850a667d86c", -+ "4710130e9f6fea8d72293850a667d86c", -+ }, -+ { -+ "48656c6c6f20776f726c6421", -+ "746869732069732033322d62797465206b657920666f7220506f6c7931333035", -+ "a6f745008f81c916a20dcc74eef2b2f0", -+ }, -+ { -+ "0000000000000000000000000000000000000000000000000000000000000000", -+ "746869732069732033322d62797465206b657920666f7220506f6c7931333035", -+ "49ec78090e481ec6c26b33b91ccc0307", -+ }, -+}; -+ -+static unsigned char hex_digit(char h) -+ { -+ if (h >= '0' && h <= '9') -+ return h - '0'; -+ else if (h >= 'a' && h <= 'f') -+ return h - 'a' + 10; -+ else if (h >= 'A' && h <= 'F') -+ return h - 'A' + 10; -+ else -+ abort(); -+ } -+ -+static void hex_decode(unsigned char *out, const char* hex) -+ { -+ size_t j = 0; -+ -+ while (*hex != 0) -+ { -+ unsigned char v = hex_digit(*hex++); -+ v <<= 4; -+ v |= hex_digit(*hex++); -+ out[j++] = v; -+ } -+ } -+ -+static void hexdump(unsigned char *a, size_t len) -+ { -+ size_t i; -+ -+ for (i = 0; i < len; i++) -+ printf("%02x", a[i]); -+ } -+ -+/* misalign returns a pointer that points 0 to 15 bytes into |in| such that the -+ * returned pointer has alignment 1 mod 16. */ -+static void* misalign(void* in) -+ { -+ intptr_t x = (intptr_t) in; -+ x += (17 - (x % 16)) % 16; -+ return (void*) x; -+ } -+ -+int main() -+ { -+ unsigned num_tests = -+ sizeof(chacha_tests) / sizeof(struct chacha_test); -+ unsigned i; -+ unsigned char key_bytes[32 + 16]; -+ unsigned char nonce_bytes[8 + 16] = {0}; -+ -+ -+ for (i = 0; i < num_tests; i++) -+ { -+ unsigned char *key = misalign(key_bytes); -+ unsigned char *nonce = misalign(nonce_bytes); -+ -+ printf("ChaCha20 test #%d\n", i); -+ const struct chacha_test *test = &chacha_tests[i]; -+ unsigned char *expected, *out_bytes, *zero_bytes, *out, *zeros; -+ size_t len = strlen(test->outhex); -+ -+ if (strlen(test->keyhex) != 32*2 || -+ strlen(test->noncehex) != 8*2 || -+ (len & 1) == 1) -+ return 1; -+ -+ len /= 2; -+ -+ hex_decode(key, test->keyhex); -+ hex_decode(nonce, test->noncehex); -+ -+ expected = malloc(len); -+ out_bytes = malloc(len+16); -+ zero_bytes = malloc(len+16); -+ /* Attempt to test unaligned inputs. */ -+ out = misalign(out_bytes); -+ zeros = misalign(zero_bytes); -+ memset(zeros, 0, len); -+ -+ hex_decode(expected, test->outhex); -+ CRYPTO_chacha_20(out, zeros, len, key, nonce, 0); -+ -+ if (memcmp(out, expected, len) != 0) -+ { -+ printf("ChaCha20 test #%d failed.\n", i); -+ printf("got: "); -+ hexdump(out, len); -+ printf("\nexpected: "); -+ hexdump(expected, len); -+ printf("\n"); -+ return 1; -+ } -+ -+ /* The last test has a large output. We test whether the -+ * counter works as expected by skipping the first 64 bytes of -+ * it. */ -+ if (i == num_tests - 1) -+ { -+ CRYPTO_chacha_20(out, zeros, len - 64, key, nonce, 1); -+ if (memcmp(out, expected + 64, len - 64) != 0) -+ { -+ printf("ChaCha20 skip test failed.\n"); -+ return 1; -+ } -+ } -+ -+ free(expected); -+ free(zero_bytes); -+ free(out_bytes); -+ } -+ num_tests = -+ sizeof(poly1305_tests) / sizeof(struct poly1305_test); -+ unsigned char key[32], out[16], expected[16]; -+ poly1305_state poly1305; -+ -+ for (i = 0; i < num_tests; i++) -+ { -+ printf("Poly1305 test #%d\n", i); -+ const struct poly1305_test *test = &poly1305_tests[i]; -+ unsigned char *in; -+ size_t inlen = strlen(test->inputhex); -+ -+ if (strlen(test->keyhex) != sizeof(key)*2 || -+ strlen(test->outhex) != sizeof(out)*2 || -+ (inlen & 1) == 1) -+ return 1; -+ -+ inlen /= 2; -+ -+ hex_decode(key, test->keyhex); -+ hex_decode(expected, test->outhex); -+ -+ in = malloc(inlen); -+ -+ hex_decode(in, test->inputhex); -+ -+#ifdef CHAPOLY_x86_64_ASM -+ if((OPENSSL_ia32cap_loc()[1] >> 5) & 1) { -+ poly1305_init_avx2(&poly1305, key); -+ poly1305_update_avx2(&poly1305, in, inlen); -+ poly1305_finish_avx2(&poly1305, out); -+ } -+ else if ((OPENSSL_ia32cap_loc()[0] >> 60) & 1) { -+ poly1305_init_avx(&poly1305, key); -+ poly1305_update_avx(&poly1305, in, inlen); -+ poly1305_finish_avx(&poly1305, out); -+ } -+ else -+#endif -+ { -+ CRYPTO_poly1305_init(&poly1305, key); -+ CRYPTO_poly1305_update(&poly1305, in, inlen); -+ CRYPTO_poly1305_finish(&poly1305, out); -+ } -+ if (memcmp(out, expected, sizeof(expected)) != 0) -+ { -+ printf("Poly1305 test #%d failed.\n", i); -+ printf("got: "); -+ hexdump(out, sizeof(out)); -+ printf("\nexpected: "); -+ hexdump(expected, sizeof(expected)); -+ printf("\n"); -+ return 1; -+ } -+ -+ free(in); -+ } -+ -+ printf("PASS\n"); -+ return 0; -+ } -diff --git a/test/recipes/05-test_chacha20poly1305.t b/test/recipes/05-test_chacha20poly1305.t -new file mode 100644 -index 0000000..5f5069d ---- /dev/null -+++ b/test/recipes/05-test_chacha20poly1305.t -@@ -0,0 +1,5 @@ -+#! /usr/bin/perl -+ -+use OpenSSL::Test::Simple; -+ -+simple_test("test_chacha20poly1305", "chapolytest", "chacha20-poly1305"); --- -1.9.1 - diff --git a/patches/openssl__chacha20_poly1305_draft_and_rfc_ossl102j.patch b/patches/openssl__chacha20_poly1305_draft_and_rfc_ossl102j.patch new file mode 100644 index 0000000..cdb7673 --- /dev/null +++ b/patches/openssl__chacha20_poly1305_draft_and_rfc_ossl102j.patch @@ -0,0 +1,4718 @@ +From dcf9b5698b8658c9248327b3fdb280090c5c78ec Mon Sep 17 00:00:00 2001 +From: vkrasnov +Date: Tue, 4 Oct 2016 15:47:32 -0700 +Subject: [PATCH] ChaCha20-Poly1305 draft and RFC cipher suites for OpenSSL + 1.0.2j + +--- + Configure | 44 +- + Makefile.org | 4 +- + crypto/chacha20_poly1305/Makefile | 89 + + .../asm/chacha20_poly1305_x86_64.pl | 2299 ++++++++++++++++++++ + crypto/chacha20_poly1305/asm/chacha20_x86_64.pl | 415 ++++ + crypto/chacha20_poly1305/asm/poly1305_x86_64.pl | 280 +++ + crypto/chacha20_poly1305/chacha20.c | 142 ++ + crypto/chacha20_poly1305/chacha20poly1305.h | 64 + + crypto/chacha20_poly1305/poly1305.c | 355 +++ + crypto/evp/Makefile | 8 +- + crypto/evp/c_allc.c | 5 + + crypto/evp/e_chacha20_poly1305.c | 362 +++ + crypto/evp/evp.h | 5 + + crypto/objects/obj_dat.h | 13 +- + crypto/objects/obj_mac.h | 8 + + crypto/objects/obj_mac.num | 2 + + crypto/objects/objects.txt | 2 + + ssl/s3_lib.c | 128 +- + ssl/ssl.h | 2 + + ssl/ssl_ciph.c | 31 +- + ssl/ssl_locl.h | 2 + + ssl/tls1.h | 26 + + 22 files changed, 4260 insertions(+), 26 deletions(-) + create mode 100644 crypto/chacha20_poly1305/Makefile + create mode 100755 crypto/chacha20_poly1305/asm/chacha20_poly1305_x86_64.pl + create mode 100644 crypto/chacha20_poly1305/asm/chacha20_x86_64.pl + create mode 100644 crypto/chacha20_poly1305/asm/poly1305_x86_64.pl + create mode 100644 crypto/chacha20_poly1305/chacha20.c + create mode 100644 crypto/chacha20_poly1305/chacha20poly1305.h + create mode 100644 crypto/chacha20_poly1305/poly1305.c + create mode 100644 crypto/evp/e_chacha20_poly1305.c + +diff --git a/Configure b/Configure +index c39f71a..f5f7c06 100755 +--- a/Configure ++++ b/Configure +@@ -150,25 +150,25 @@ my $tlib="-lnsl -lsocket"; + my $bits1="THIRTY_TWO_BIT "; + my $bits2="SIXTY_FOUR_BIT "; + +-my $x86_asm="x86cpuid.o:bn-586.o co-586.o x86-mont.o x86-gf2m.o::des-586.o crypt586.o:aes-586.o vpaes-x86.o aesni-x86.o:bf-586.o:md5-586.o:sha1-586.o sha256-586.o sha512-586.o:cast-586.o:rc4-586.o:rmd-586.o:rc5-586.o:wp_block.o wp-mmx.o:cmll-x86.o:ghash-x86.o:"; ++my $x86_asm="x86cpuid.o:bn-586.o co-586.o x86-mont.o x86-gf2m.o::des-586.o crypt586.o:aes-586.o vpaes-x86.o aesni-x86.o:bf-586.o:md5-586.o:sha1-586.o sha256-586.o sha512-586.o:cast-586.o:rc4-586.o:rmd-586.o:rc5-586.o:wp_block.o wp-mmx.o:cmll-x86.o:ghash-x86.o::"; + + my $x86_elf_asm="$x86_asm:elf"; + +-my $x86_64_asm="x86_64cpuid.o:x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o rsaz_exp.o rsaz-x86_64.o rsaz-avx2.o:ecp_nistz256.o ecp_nistz256-x86_64.o::aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o aesni-sha256-x86_64.o aesni-mb-x86_64.o::md5-x86_64.o:sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o sha1-mb-x86_64.o sha256-mb-x86_64.o::rc4-x86_64.o rc4-md5-x86_64.o:::wp-x86_64.o:cmll-x86_64.o cmll_misc.o:ghash-x86_64.o aesni-gcm-x86_64.o:"; +-my $ia64_asm="ia64cpuid.o:bn-ia64.o ia64-mont.o:::aes_core.o aes_cbc.o aes-ia64.o::md5-ia64.o:sha1-ia64.o sha256-ia64.o sha512-ia64.o::rc4-ia64.o rc4_skey.o:::::ghash-ia64.o::void"; +-my $sparcv9_asm="sparcv9cap.o sparccpuid.o:bn-sparcv9.o sparcv9-mont.o sparcv9a-mont.o vis3-mont.o sparct4-mont.o sparcv9-gf2m.o::des_enc-sparc.o fcrypt_b.o dest4-sparcv9.o:aes_core.o aes_cbc.o aes-sparcv9.o aest4-sparcv9.o::md5-sparcv9.o:sha1-sparcv9.o sha256-sparcv9.o sha512-sparcv9.o::::::camellia.o cmll_misc.o cmll_cbc.o cmllt4-sparcv9.o:ghash-sparcv9.o::void"; +-my $sparcv8_asm=":sparcv8.o::des_enc-sparc.o fcrypt_b.o:::::::::::::void"; +-my $alpha_asm="alphacpuid.o:bn_asm.o alpha-mont.o::::::sha1-alpha.o:::::::ghash-alpha.o::void"; +-my $mips64_asm=":bn-mips.o mips-mont.o:::aes_cbc.o aes-mips.o:::sha1-mips.o sha256-mips.o sha512-mips.o::::::::"; ++my $x86_64_asm="x86_64cpuid.o:x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o rsaz_exp.o rsaz-x86_64.o rsaz-avx2.o:ecp_nistz256.o ecp_nistz256-x86_64.o::aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o aesni-sha256-x86_64.o aesni-mb-x86_64.o::md5-x86_64.o:sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o sha1-mb-x86_64.o sha256-mb-x86_64.o::rc4-x86_64.o rc4-md5-x86_64.o:::wp-x86_64.o:cmll-x86_64.o cmll_misc.o:ghash-x86_64.o aesni-gcm-x86_64.o:chacha20_poly1305_x86_64.o poly1305_x86_64.o chacha20_x86_64.o:"; ++my $ia64_asm="ia64cpuid.o:bn-ia64.o ia64-mont.o:::aes_core.o aes_cbc.o aes-ia64.o::md5-ia64.o:sha1-ia64.o sha256-ia64.o sha512-ia64.o::rc4-ia64.o rc4_skey.o:::::ghash-ia64.o:::void"; ++my $sparcv9_asm="sparcv9cap.o sparccpuid.o:bn-sparcv9.o sparcv9-mont.o sparcv9a-mont.o vis3-mont.o sparct4-mont.o sparcv9-gf2m.o::des_enc-sparc.o fcrypt_b.o dest4-sparcv9.o:aes_core.o aes_cbc.o aes-sparcv9.o aest4-sparcv9.o::md5-sparcv9.o:sha1-sparcv9.o sha256-sparcv9.o sha512-sparcv9.o::::::camellia.o cmll_misc.o cmll_cbc.o cmllt4-sparcv9.o:ghash-sparcv9.o:::void"; ++my $sparcv8_asm=":sparcv8.o::des_enc-sparc.o fcrypt_b.o::::::::::::::void"; ++my $alpha_asm="alphacpuid.o:bn_asm.o alpha-mont.o::::::sha1-alpha.o:::::::ghash-alpha.o:::void"; ++my $mips64_asm=":bn-mips.o mips-mont.o:::aes_cbc.o aes-mips.o:::sha1-mips.o sha256-mips.o sha512-mips.o:::::::::"; + my $mips32_asm=$mips64_asm; $mips32_asm =~ s/\s*sha512\-mips\.o//; +-my $s390x_asm="s390xcap.o s390xcpuid.o:bn-s390x.o s390x-mont.o s390x-gf2m.o:::aes-s390x.o aes-ctr.o aes-xts.o:::sha1-s390x.o sha256-s390x.o sha512-s390x.o::rc4-s390x.o:::::ghash-s390x.o:"; +-my $armv4_asm="armcap.o armv4cpuid.o:bn_asm.o armv4-mont.o armv4-gf2m.o:::aes_cbc.o aes-armv4.o bsaes-armv7.o aesv8-armx.o:::sha1-armv4-large.o sha256-armv4.o sha512-armv4.o:::::::ghash-armv4.o ghashv8-armx.o::void"; +-my $aarch64_asm="armcap.o arm64cpuid.o mem_clr.o::::aes_core.o aes_cbc.o aesv8-armx.o:::sha1-armv8.o sha256-armv8.o sha512-armv8.o:::::::ghashv8-armx.o:"; +-my $parisc11_asm="pariscid.o:bn_asm.o parisc-mont.o:::aes_core.o aes_cbc.o aes-parisc.o:::sha1-parisc.o sha256-parisc.o sha512-parisc.o::rc4-parisc.o:::::ghash-parisc.o::32"; +-my $parisc20_asm="pariscid.o:pa-risc2W.o parisc-mont.o:::aes_core.o aes_cbc.o aes-parisc.o:::sha1-parisc.o sha256-parisc.o sha512-parisc.o::rc4-parisc.o:::::ghash-parisc.o::64"; +-my $ppc64_asm="ppccpuid.o ppccap.o:bn-ppc.o ppc-mont.o ppc64-mont.o:::aes_core.o aes_cbc.o aes-ppc.o vpaes-ppc.o aesp8-ppc.o:::sha1-ppc.o sha256-ppc.o sha512-ppc.o sha256p8-ppc.o sha512p8-ppc.o:::::::ghashp8-ppc.o:"; ++my $s390x_asm="s390xcap.o s390xcpuid.o:bn-s390x.o s390x-mont.o s390x-gf2m.o:::aes-s390x.o aes-ctr.o aes-xts.o:::sha1-s390x.o sha256-s390x.o sha512-s390x.o::rc4-s390x.o:::::ghash-s390x.o::"; ++my $armv4_asm="armcap.o armv4cpuid.o:bn_asm.o armv4-mont.o armv4-gf2m.o:::aes_cbc.o aes-armv4.o bsaes-armv7.o aesv8-armx.o:::sha1-armv4-large.o sha256-armv4.o sha512-armv4.o:::::::ghash-armv4.o ghashv8-armx.o:::void"; ++my $aarch64_asm="armcap.o arm64cpuid.o mem_clr.o::::aes_core.o aes_cbc.o aesv8-armx.o:::sha1-armv8.o sha256-armv8.o sha512-armv8.o:::::::ghashv8-armx.o::"; ++my $parisc11_asm="pariscid.o:bn_asm.o parisc-mont.o:::aes_core.o aes_cbc.o aes-parisc.o:::sha1-parisc.o sha256-parisc.o sha512-parisc.o::rc4-parisc.o:::::ghash-parisc.o:::32"; ++my $parisc20_asm="pariscid.o:pa-risc2W.o parisc-mont.o:::aes_core.o aes_cbc.o aes-parisc.o:::sha1-parisc.o sha256-parisc.o sha512-parisc.o::rc4-parisc.o:::::ghash-parisc.o:::64"; ++my $ppc64_asm="ppccpuid.o ppccap.o:bn-ppc.o ppc-mont.o ppc64-mont.o:::aes_core.o aes_cbc.o aes-ppc.o vpaes-ppc.o aesp8-ppc.o:::sha1-ppc.o sha256-ppc.o sha512-ppc.o sha256p8-ppc.o sha512p8-ppc.o:::::::ghashp8-ppc.o::"; + my $ppc32_asm=$ppc64_asm; +-my $no_asm="::::::::::::::::void"; ++my $no_asm=":::::::::::::::::void"; + + # As for $BSDthreads. Idea is to maintain "collective" set of flags, + # which would cover all BSD flavors. -pthread applies to them all, +@@ -179,7 +179,7 @@ my $no_asm="::::::::::::::::void"; + # seems to be sufficient? + my $BSDthreads="-pthread -D_THREAD_SAFE -D_REENTRANT"; + +-#config-string $cc : $cflags : $unistd : $thread_cflag : $sys_id : $lflags : $bn_ops : $cpuid_obj : $bn_obj : $ec_obj : $des_obj : $aes_obj : $bf_obj : $md5_obj : $sha1_obj : $cast_obj : $rc4_obj : $rmd160_obj : $rc5_obj : $wp_obj : $cmll_obj : $modes_obj : $engines_obj : $dso_scheme : $shared_target : $shared_cflag : $shared_ldflag : $shared_extension : $ranlib : $arflags : $multilib ++#config-string $cc : $cflags : $unistd : $thread_cflag : $sys_id : $lflags : $bn_ops : $cpuid_obj : $bn_obj : $ec_obj : $des_obj : $aes_obj : $bf_obj : $md5_obj : $sha1_obj : $cast_obj : $rc4_obj : $rmd160_obj : $rc5_obj : $wp_obj : $cmll_obj : $modes_obj : $chapoly_obj : $engines_obj : $dso_scheme : $shared_target : $shared_cflag : $shared_ldflag : $shared_extension : $ranlib : $arflags : $multilib + + my %table=( + # File 'TABLE' (created by 'make TABLE') contains the data from this list, +@@ -713,6 +713,7 @@ my $idx_rc5_obj = $idx++; + my $idx_wp_obj = $idx++; + my $idx_cmll_obj = $idx++; + my $idx_modes_obj = $idx++; ++my $idx_chapoly_obj = $idx++; + my $idx_engines_obj = $idx++; + my $idx_perlasm_scheme = $idx++; + my $idx_dso_scheme = $idx++; +@@ -1239,6 +1240,7 @@ my $rc5_obj = $fields[$idx_rc5_obj]; + my $wp_obj = $fields[$idx_wp_obj]; + my $cmll_obj = $fields[$idx_cmll_obj]; + my $modes_obj = $fields[$idx_modes_obj]; ++my $chapoly_obj= $fields[$idx_chapoly_obj]; + my $engines_obj = $fields[$idx_engines_obj]; + my $perlasm_scheme = $fields[$idx_perlasm_scheme]; + my $dso_scheme = $fields[$idx_dso_scheme]; +@@ -1407,7 +1409,8 @@ if ($no_asm) + { + $cpuid_obj=$bn_obj=$ec_obj= + $des_obj=$aes_obj=$bf_obj=$cast_obj=$rc4_obj=$rc5_obj=$cmll_obj= +- $modes_obj=$sha1_obj=$md5_obj=$rmd160_obj=$wp_obj=$engines_obj=""; ++ $modes_obj=$sha1_obj=$md5_obj=$rmd160_obj=$wp_obj=$engines_obj= ++ $chapoly_obj=""; + } + + if (!$no_shared) +@@ -1622,6 +1625,10 @@ if ($ec_obj =~ /ecp_nistz256/) + { + $cflags.=" -DECP_NISTZ256_ASM"; + } ++if ($chapoly_obj =~ /chacha20_poly1305/) ++ { ++ $cflags.=" -DCHAPOLY_ASM"; ++ } + + # "Stringify" the C flags string. This permits it to be made part of a string + # and works as well on command lines. +@@ -1751,6 +1758,7 @@ while () + s/^WP_ASM_OBJ=.*$/WP_ASM_OBJ= $wp_obj/; + s/^CMLL_ENC=.*$/CMLL_ENC= $cmll_obj/; + s/^MODES_ASM_OBJ.=*$/MODES_ASM_OBJ= $modes_obj/; ++ s/^CHAPOLY_ASM=.*$/CHAPOLY_ASM= $chapoly_obj/; + s/^ENGINES_ASM_OBJ.=*$/ENGINES_ASM_OBJ= $engines_obj/; + s/^PERLASM_SCHEME=.*$/PERLASM_SCHEME= $perlasm_scheme/; + s/^PROCESSOR=.*/PROCESSOR= $processor/; +@@ -1812,6 +1820,7 @@ print "SHA1_OBJ_ASM =$sha1_obj\n"; + print "RMD160_OBJ_ASM=$rmd160_obj\n"; + print "CMLL_ENC =$cmll_obj\n"; + print "MODES_OBJ =$modes_obj\n"; ++print "CHAPOLY_ASM =$chapoly_obj\n"; + print "ENGINES_OBJ =$engines_obj\n"; + print "PROCESSOR =$processor\n"; + print "RANLIB =$ranlib\n"; +@@ -2211,7 +2220,7 @@ sub print_table_entry + my ($cc, $cflags, $unistd, $thread_cflag, $sys_id, $lflags, + $bn_ops, $cpuid_obj, $bn_obj, $ec_obj, $des_obj, $aes_obj, $bf_obj, + $md5_obj, $sha1_obj, $cast_obj, $rc4_obj, $rmd160_obj, +- $rc5_obj, $wp_obj, $cmll_obj, $modes_obj, $engines_obj, ++ $rc5_obj, $wp_obj, $cmll_obj, $modes_obj, $chapoly_obj, $engines_obj, + $perlasm_scheme, $dso_scheme, $shared_target, $shared_cflag, + $shared_ldflag, $shared_extension, $ranlib, $arflags, $multilib)= + split(/\s*:\s*/,$table{$target} . ":" x 30 , -1); +@@ -2241,6 +2250,7 @@ sub print_table_entry + \$wp_obj = $wp_obj + \$cmll_obj = $cmll_obj + \$modes_obj = $modes_obj ++\$chapoly_obj = $chapoly_obj + \$engines_obj = $engines_obj + \$perlasm_scheme = $perlasm_scheme + \$dso_scheme = $dso_scheme +diff --git a/Makefile.org b/Makefile.org +index 2377f50..1f20a61 100644 +--- a/Makefile.org ++++ b/Makefile.org +@@ -103,6 +103,7 @@ WP_ASM_OBJ= + CMLL_ENC= + MODES_ASM_OBJ= + ENGINES_ASM_OBJ= ++CHAPOLY_ASM= + PERLASM_SCHEME= + + # KRB5 stuff +@@ -149,7 +150,7 @@ SDIRS= \ + bn ec rsa dsa ecdsa dh ecdh dso engine \ + buffer bio stack lhash rand err \ + evp asn1 pem x509 x509v3 conf txt_db pkcs7 pkcs12 comp ocsp ui krb5 \ +- cms pqueue ts jpake srp store cmac ++ cms pqueue ts jpake srp store cmac chacha20_poly1305 + # keep in mind that the above list is adjusted by ./Configure + # according to no-xxx arguments... + +@@ -240,6 +241,7 @@ BUILDENV= LC_ALL=C PLATFORM='$(PLATFORM)' PROCESSOR='$(PROCESSOR)'\ + FIPSLIBDIR='${FIPSLIBDIR}' \ + FIPSDIR='${FIPSDIR}' \ + FIPSCANLIB="$${FIPSCANLIB:-$(FIPSCANLIB)}" \ ++ CHAPOLY_ASM='$(CHAPOLY_ASM)' \ + THIS=$${THIS:-$@} MAKEFILE=Makefile MAKEOVERRIDES= + # MAKEOVERRIDES= effectively "equalizes" GNU-ish and SysV-ish make flavors, + # which in turn eliminates ambiguities in variable treatment with -e. +diff --git a/crypto/chacha20_poly1305/Makefile b/crypto/chacha20_poly1305/Makefile +new file mode 100644 +index 0000000..87f4ba3 +--- /dev/null ++++ b/crypto/chacha20_poly1305/Makefile +@@ -0,0 +1,89 @@ ++# ++# crypto/chacha20poly1305/Makefile ++# ++ ++DIR= chacha20poly1305 ++TOP= ../.. ++CC= cc ++INCLUDES= -I.. -I$(TOP) -I../../include ++CFLAG=-g ++MAKEFILE= Makefile ++AR= ar r ++ ++CFLAGS= $(INCLUDES) $(CFLAG) ++ASFLAGS= $(INCLUDES) $(ASFLAG) ++AFLAGS= $(ASFLAGS) ++ ++GENERAL=Makefile ++TEST= ++APPS= ++ ++LIB=$(TOP)/libcrypto.a ++LIBSRC= chacha20.c poly1305.c ++LIBOBJ= chacha20.o poly1305.o $(CHAPOLY_ASM) ++ ++SRC= $(LIBSRC) ++ ++EXHEADER= chacha20poly1305.h ++HEADER= $(EXHEADER) ++ ++ALL= $(GENERAL) $(SRC) $(HEADER) ++ ++top: ++ (cd ../..; $(MAKE) DIRS=crypto SDIRS=$(DIR) sub_all) ++ ++all: lib ++ ++lib: $(LIBOBJ) ++ $(AR) $(LIB) $(LIBOBJ) ++ $(RANLIB) $(LIB) || echo Never mind. ++ @touch lib ++ ++chacha20_poly1305_x86_64.s: asm/chacha20_poly1305_x86_64.pl ++ $(PERL) asm/chacha20_poly1305_x86_64.pl $(PERLASM_SCHEME) > $@ ++ ++poly1305_x86_64.s: asm/poly1305_x86_64.pl ++ $(PERL) asm/poly1305_x86_64.pl $(PERLASM_SCHEME) > $@ ++ ++chacha20_x86_64.s: asm/chacha20_x86_64.pl ++ $(PERL) asm/chacha20_x86_64.pl $(PERLASM_SCHEME) > $@ ++ ++files: ++ $(PERL) $(TOP)/util/files.pl Makefile >> $(TOP)/MINFO ++ ++links: ++ @$(PERL) $(TOP)/util/mklink.pl ../../include/openssl $(EXHEADER) ++ @$(PERL) $(TOP)/util/mklink.pl ../../test $(TEST) ++ @$(PERL) $(TOP)/util/mklink.pl ../../apps $(APPS) ++ ++install: ++ @[ -n "$(INSTALLTOP)" ] # should be set by top Makefile... ++ @headerlist="$(EXHEADER)"; for i in $$headerlist ; \ ++ do \ ++ (cp $$i $(INSTALL_PREFIX)$(INSTALLTOP)/include/openssl/$$i; \ ++ chmod 644 $(INSTALL_PREFIX)$(INSTALLTOP)/include/openssl/$$i ); \ ++ done; ++ ++tags: ++ ctags $(SRC) ++ ++tests: ++ ++lint: ++ lint -DLINT $(INCLUDES) $(SRC)>fluff ++ ++depend: ++ @[ -n "$(MAKEDEPEND)" ] # should be set by upper Makefile... ++ $(MAKEDEPEND) -- $(CFLAG) $(INCLUDES) $(DEPFLAG) -- $(PROGS) $(LIBSRC) ++ ++dclean: ++ $(PERL) -pe 'if (/^# DO NOT DELETE THIS LINE/) {print; exit(0);}' $(MAKEFILE) >Makefile.new ++ mv -f Makefile.new $(MAKEFILE) ++ ++clean: ++ rm -f *.s *.o *.obj lib tags core .pure .nfs* *.old *.bak fluff ++ ++# DO NOT DELETE THIS LINE -- make depend depends on it. ++ ++chacha20.o: ../../include/openssl/chacha20poly1305.h chacha20.c ++poly1305.o: ../../include/openssl/chacha20poly1305.h poly1305.c +diff --git a/crypto/chacha20_poly1305/asm/chacha20_poly1305_x86_64.pl b/crypto/chacha20_poly1305/asm/chacha20_poly1305_x86_64.pl +new file mode 100755 +index 0000000..ef90831 +--- /dev/null ++++ b/crypto/chacha20_poly1305/asm/chacha20_poly1305_x86_64.pl +@@ -0,0 +1,2299 @@ ++#!/usr/bin/env perl ++ ++############################################################################## ++# # ++# Copyright 2016 CloudFlare LTD # ++# # ++# Licensed under the Apache License, Version 2.0 (the "License"); # ++# you may not use this file except in compliance with the License. # ++# You may obtain a copy of the License at # ++# # ++# http://www.apache.org/licenses/LICENSE-2.0 # ++# # ++# Unless required by applicable law or agreed to in writing, software # ++# distributed under the License is distributed on an "AS IS" BASIS, # ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # ++# See the License for the specific language governing permissions and # ++# limitations under the License. # ++# # ++############################################################################## ++# # ++# Author: Vlad Krasnov # ++# # ++############################################################################## ++ ++$flavour = shift; ++$output = shift; ++if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } ++ ++$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); ++ ++$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; ++( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or ++( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or ++die "can't locate x86_64-xlate.pl"; ++ ++open OUT,"| \"$^X\" $xlate $flavour $output"; ++*STDOUT=*OUT; ++ ++if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` ++ =~ /GNU assembler version ([2-9]\.[0-9]+)/) { ++ $avx = ($1>=2.19) + ($1>=2.22); ++} ++ ++if ($win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && ++ `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { ++ $avx = ($1>=2.09) + ($1>=2.10); ++} ++ ++if ($win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && ++ `ml64 2>&1` =~ /Version ([0-9]+)\./) { ++ $avx = ($1>=10) + ($1>=11); ++} ++ ++if (`$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9])\.([0-9]+)/) { ++ my $ver = $2 + $3/100.0; # 3.1->3.01, 3.10->3.10 ++ $avx = ($ver>=3.0) + ($ver>=3.01); ++} ++ ++$code.=<<___; ++.text ++.extern OPENSSL_ia32cap_P ++.align 64 ++.chacha20_consts: ++.byte 'e','x','p','a','n','d',' ','3','2','-','b','y','t','e',' ','k' ++.byte 'e','x','p','a','n','d',' ','3','2','-','b','y','t','e',' ','k' ++.rol8: ++.byte 3,0,1,2, 7,4,5,6, 11,8,9,10, 15,12,13,14 ++.byte 3,0,1,2, 7,4,5,6, 11,8,9,10, 15,12,13,14 ++.rol16: ++.byte 2,3,0,1, 6,7,4,5, 10,11,8,9, 14,15,12,13 ++.byte 2,3,0,1, 6,7,4,5, 10,11,8,9, 14,15,12,13 ++.avx2_init: ++.long 0,0,0,0 ++.sse_inc: ++.long 1,0,0,0 ++.avx2_inc: ++.long 2,0,0,0,2,0,0,0 ++.clamp: ++.quad 0x0FFFFFFC0FFFFFFF, 0x0FFFFFFC0FFFFFFC ++.quad 0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF ++.align 16 ++.and_masks: ++.byte 0xff,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 ++.byte 0xff,0xff,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 ++.byte 0xff,0xff,0xff,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 ++.byte 0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 ++.byte 0xff,0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 ++.byte 0xff,0xff,0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 ++.byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 ++.byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 ++.byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00,0x00,0x00,0x00 ++.byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00,0x00,0x00 ++.byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00,0x00 ++.byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00 ++.byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x00,0x00,0x00 ++.byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x00,0x00 ++.byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x00 ++___ ++ ++my ($oup,$inp,$inl,$adp,$keyp,$itr1,$itr2)=("%rdi","%rsi","%rbx","%rcx","%r9","%rcx","%r8"); ++my ($acc0,$acc1,$acc2)=map("%r$_",(10..12)); ++my ($t0,$t1,$t2,$t3)=("%r13","%r14","%r15","%r9"); ++my ($A0,$A1,$A2,$A3,$B0,$B1,$B2,$B3,$C0,$C1,$C2,$C3,$D0,$D1,$D2,$D3)=map("%xmm$_",(0..15)); ++my ($T0,$T1,$T2,$T3)=($A3,$B3,$C3,$D3); ++my $r_store="0*16(%rbp)"; ++my $s_store="1*16(%rbp)"; ++my $len_store="2*16(%rbp)"; ++my $state1_store="3*16(%rbp)"; ++my $state2_store="4*16(%rbp)"; ++my $tmp_store="5*16(%rbp)"; ++my $ctr0_store="6*16(%rbp)"; ++my $ctr1_store="7*16(%rbp)"; ++my $ctr2_store="8*16(%rbp)"; ++my $ctr3_store="9*16(%rbp)"; ++ ++sub chacha_qr { ++my ($a,$b,$c,$d,$t,$dir)=@_; ++$code.="movdqa $t, $tmp_store\n" if ($dir =~ /store/); ++$code.="paddd $b, $a ++ pxor $a, $d ++ pshufb .rol16(%rip), $d ++ paddd $d, $c ++ pxor $c, $b ++ movdqa $b, $t ++ pslld \$12, $t ++ psrld \$20, $b ++ pxor $t, $b ++ paddd $b, $a ++ pxor $a, $d ++ pshufb .rol8(%rip), $d ++ paddd $d, $c ++ pxor $c, $b ++ movdqa $b, $t ++ pslld \$7, $t ++ psrld \$25, $b ++ pxor $t, $b\n"; ++$code.="palignr \$4, $b, $b ++ palignr \$8, $c, $c ++ palignr \$12, $d, $d\n" if ($dir =~ /left/); ++$code.="palignr \$12, $b, $b ++ palignr \$8, $c, $c ++ palignr \$4, $d, $d\n" if ($dir =~ /right/); ++$code.="movdqa $tmp_store, $t\n" if ($dir =~ /load/); ++} ++ ++sub poly_add { ++my ($src)=@_; ++$code.="add $src, $acc0 ++ adc 8+$src, $acc1 ++ adc \$1, $acc2\n"; ++} ++ ++sub poly_stage1 { ++$code.="mov 0+$r_store, %rax ++ mov %rax, $t2 ++ mul $acc0 ++ mov %rax, $t0 ++ mov %rdx, $t1 ++ mov 0+$r_store, %rax ++ mul $acc1 ++ imul $acc2, $t2 ++ add %rax, $t1 ++ adc %rdx, $t2\n"; ++} ++ ++sub poly_stage2 { ++$code.="mov 8+$r_store, %rax ++ mov %rax, $t3 ++ mul $acc0 ++ add %rax, $t1 ++ adc \$0, %rdx ++ mov %rdx, $acc0 ++ mov 8+$r_store, %rax ++ mul $acc1 ++ add %rax, $t2 ++ adc \$0, %rdx\n"; ++} ++ ++sub poly_stage3 { ++$code.="imul $acc2, $t3 ++ add $acc0, $t2 ++ adc %rdx, $t3\n"; ++} ++ ++sub poly_reduce_stage { ++$code.="mov $t0, $acc0 ++ mov $t1, $acc1 ++ mov $t2, $acc2 ++ and \$3, $acc2 ++ mov $t2, $t0 ++ and \$-4, $t0 ++ mov $t3, $t1 ++ shrd \$2, $t3, $t2 ++ shr \$2, $t3 ++ add $t0, $acc0 ++ adc $t1, $acc1 ++ adc \$0, $acc2 ++ add $t2, $acc0 ++ adc $t3, $acc1 ++ adc \$0, $acc2\n"; ++} ++ ++sub poly_mul { ++ &poly_stage1(); ++ &poly_stage2(); ++ &poly_stage3(); ++ &poly_reduce_stage(); ++} ++ ++sub prep_state { ++my ($n)=@_; ++$code.="movdqa .chacha20_consts(%rip), $A0 ++ movdqa $state1_store, $B0 ++ movdqa $state2_store, $C0\n"; ++$code.="movdqa $A0, $A1 ++ movdqa $B0, $B1 ++ movdqa $C0, $C1\n" if ($n ge 2); ++$code.="movdqa $A0, $A2 ++ movdqa $B0, $B2 ++ movdqa $C0, $C2\n" if ($n ge 3); ++$code.="movdqa $A0, $A3 ++ movdqa $B0, $B3 ++ movdqa $C0, $C3\n" if ($n ge 4); ++$code.="movdqa $ctr0_store, $D0 ++ paddd .sse_inc(%rip), $D0 ++ movdqa $D0, $ctr0_store\n" if ($n eq 1); ++$code.="movdqa $ctr0_store, $D1 ++ paddd .sse_inc(%rip), $D1 ++ movdqa $D1, $D0 ++ paddd .sse_inc(%rip), $D0 ++ movdqa $D0, $ctr0_store ++ movdqa $D1, $ctr1_store\n" if ($n eq 2); ++$code.="movdqa $ctr0_store, $D2 ++ paddd .sse_inc(%rip), $D2 ++ movdqa $D2, $D1 ++ paddd .sse_inc(%rip), $D1 ++ movdqa $D1, $D0 ++ paddd .sse_inc(%rip), $D0 ++ movdqa $D0, $ctr0_store ++ movdqa $D1, $ctr1_store ++ movdqa $D2, $ctr2_store\n" if ($n eq 3); ++$code.="movdqa $ctr0_store, $D3 ++ paddd .sse_inc(%rip), $D3 ++ movdqa $D3, $D2 ++ paddd .sse_inc(%rip), $D2 ++ movdqa $D2, $D1 ++ paddd .sse_inc(%rip), $D1 ++ movdqa $D1, $D0 ++ paddd .sse_inc(%rip), $D0 ++ movdqa $D0, $ctr0_store ++ movdqa $D1, $ctr1_store ++ movdqa $D2, $ctr2_store ++ movdqa $D3, $ctr3_store\n" if ($n eq 4); ++} ++ ++sub finalize_state { ++my ($n)=@_; ++$code.="paddd .chacha20_consts(%rip), $A3 ++ paddd $state1_store, $B3 ++ paddd $state2_store, $C3 ++ paddd $ctr3_store, $D3\n" if ($n eq 4); ++$code.="paddd .chacha20_consts(%rip), $A2 ++ paddd $state1_store, $B2 ++ paddd $state2_store, $C2 ++ paddd $ctr2_store, $D2\n" if ($n ge 3); ++$code.="paddd .chacha20_consts(%rip), $A1 ++ paddd $state1_store, $B1 ++ paddd $state2_store, $C1 ++ paddd $ctr1_store, $D1\n" if ($n ge 2); ++$code.="paddd .chacha20_consts(%rip), $A0 ++ paddd $state1_store, $B0 ++ paddd $state2_store, $C0 ++ paddd $ctr0_store, $D0\n"; ++} ++ ++sub xor_stream { ++my ($A, $B, $C, $D, $offset)=@_; ++$code.="movdqu 0*16 + $offset($inp), $A3 ++ movdqu 1*16 + $offset($inp), $B3 ++ movdqu 2*16 + $offset($inp), $C3 ++ movdqu 3*16 + $offset($inp), $D3 ++ pxor $A3, $A ++ pxor $B3, $B ++ pxor $C3, $C ++ pxor $D, $D3 ++ movdqu $A, 0*16 + $offset($oup) ++ movdqu $B, 1*16 + $offset($oup) ++ movdqu $C, 2*16 + $offset($oup) ++ movdqu $D3, 3*16 + $offset($oup)\n"; ++} ++ ++sub xor_stream_using_temp { ++my ($A, $B, $C, $D, $offset, $temp)=@_; ++$code.="movdqa $temp, $tmp_store ++ movdqu 0*16 + $offset($inp), $temp ++ pxor $A, $temp ++ movdqu $temp, 0*16 + $offset($oup) ++ movdqu 1*16 + $offset($inp), $temp ++ pxor $B, $temp ++ movdqu $temp, 1*16 + $offset($oup) ++ movdqu 2*16 + $offset($inp), $temp ++ pxor $C, $temp ++ movdqu $temp, 2*16 + $offset($oup) ++ movdqu 3*16 + $offset($inp), $temp ++ pxor $D, $temp ++ movdqu $temp, 3*16 + $offset($oup)\n"; ++} ++ ++sub gen_chacha_round { ++my ($rot1, $rot2, $shift)=@_; ++my $round=""; ++$round.="movdqa $C0, $tmp_store\n" if ($rot1 eq 20); ++$round.="movdqa $rot2, $C0 ++ paddd $B3, $A3 ++ paddd $B2, $A2 ++ paddd $B1, $A1 ++ paddd $B0, $A0 ++ pxor $A3, $D3 ++ pxor $A2, $D2 ++ pxor $A1, $D1 ++ pxor $A0, $D0 ++ pshufb $C0, $D3 ++ pshufb $C0, $D2 ++ pshufb $C0, $D1 ++ pshufb $C0, $D0 ++ movdqa $tmp_store, $C0 ++ paddd $D3, $C3 ++ paddd $D2, $C2 ++ paddd $D1, $C1 ++ paddd $D0, $C0 ++ pxor $C3, $B3 ++ pxor $C2, $B2 ++ pxor $C1, $B1 ++ pxor $C0, $B0 ++ movdqa $C0, $tmp_store ++ movdqa $B3, $C0 ++ psrld \$$rot1, $C0 ++ pslld \$32-$rot1, $B3 ++ pxor $C0, $B3 ++ movdqa $B2, $C0 ++ psrld \$$rot1, $C0 ++ pslld \$32-$rot1, $B2 ++ pxor $C0, $B2 ++ movdqa $B1, $C0 ++ psrld \$$rot1, $C0 ++ pslld \$32-$rot1, $B1 ++ pxor $C0, $B1 ++ movdqa $B0, $C0 ++ psrld \$$rot1, $C0 ++ pslld \$32-$rot1, $B0 ++ pxor $C0, $B0\n"; ++($s1,$s2,$s3)=(4,8,12) if ($shift =~ /left/); ++($s1,$s2,$s3)=(12,8,4) if ($shift =~ /right/); ++$round.="movdqa $tmp_store, $C0 ++ palignr \$$s1, $B3, $B3 ++ palignr \$$s2, $C3, $C3 ++ palignr \$$s3, $D3, $D3 ++ palignr \$$s1, $B2, $B2 ++ palignr \$$s2, $C2, $C2 ++ palignr \$$s3, $D2, $D2 ++ palignr \$$s1, $B1, $B1 ++ palignr \$$s2, $C1, $C1 ++ palignr \$$s3, $D1, $D1 ++ palignr \$$s1, $B0, $B0 ++ palignr \$$s2, $C0, $C0 ++ palignr \$$s3, $D0, $D0\n" ++if (($shift =~ /left/) || ($shift =~ /right/)); ++return $round; ++}; ++ ++$chacha_body = &gen_chacha_round(20, ".rol16(%rip)") . ++ &gen_chacha_round(25, ".rol8(%rip)", "left") . ++ &gen_chacha_round(20, ".rol16(%rip)") . ++ &gen_chacha_round(25, ".rol8(%rip)", "right"); ++ ++my @loop_body = split /\n/, $chacha_body; ++ ++sub emit_body { ++my ($n)=@_; ++ for (my $i=0; $i < $n; $i++) { ++ $code=$code.shift(@loop_body)."\n"; ++ }; ++} ++ ++{ ++################################################################################ ++# void poly_hash_ad_internal(); ++$code.=" ++.type poly_hash_ad_internal,\@function,2 ++.align 64 ++poly_hash_ad_internal: ++ xor $acc0, $acc0 ++ xor $acc1, $acc1 ++ xor $acc2, $acc2 ++ cmp \$13, $itr2 ++ jne hash_ad_loop ++poly_fast_tls_ad: ++ # Special treatment for the TLS case of 13 bytes ++ mov ($adp), $acc0 ++ mov 5($adp), $acc1 ++ shr \$24, $acc1 ++ mov \$1, $acc2\n"; ++ &poly_mul(); $code.=" ++ ret ++hash_ad_loop: ++ # Hash in 16 byte chunk ++ cmp \$16, $itr2 ++ jb hash_ad_tail\n"; ++ &poly_add("0($adp)"); ++ &poly_mul(); $code.=" ++ lea (1*16)($adp), $adp ++ sub \$16, $itr2 ++ jmp hash_ad_loop ++hash_ad_tail: ++ cmp \$0, $itr2 ++ je 1f ++ # Hash last < 16 byte tail ++ xor $t0, $t0 ++ xor $t1, $t1 ++ xor $t2, $t2 ++ add $itr2, $adp ++hash_ad_tail_loop: ++ shld \$8, $t0, $t1 ++ shl \$8, $t0 ++ movzxb -1($adp), $t2 ++ xor $t2, $t0 ++ dec $adp ++ dec $itr2 ++ jne hash_ad_tail_loop ++ ++ add $t0, $acc0 ++ adc $t1, $acc1 ++ adc \$1, $acc2\n"; ++ &poly_mul(); $code.=" ++ # Finished AD ++1: ++ ret ++.size poly_hash_ad_internal, .-poly_hash_ad_internal\n"; ++} ++ ++{ ++################################################################################ ++# int chacha20_poly1305_open(uint8_t *pt, uint8_t *ct, size_t len_in, uint8_t *ad, size_t len_ad, uint8_t *keyp); ++$code.=" ++.globl chacha20_poly1305_open ++.type chacha20_poly1305_open,\@function,2 ++.align 64 ++chacha20_poly1305_open: ++ push %rbp ++ push %rbx ++ push %r12 ++ push %r13 ++ push %r14 ++ push %r15 ++ sub \$288 + 32, %rsp ++ lea 32(%rsp), %rbp ++ and \$-32, %rbp ++ mov %rdx, 8+$len_store ++ mov %r8, 0+$len_store ++ mov %rdx, $inl\n"; $code.=" ++ mov OPENSSL_ia32cap_P+8(%rip), %eax ++ test \$`1<<5`, %eax ++ jnz chacha20_poly1305_open_avx2\n" if ($avx>1); ++$code.=" ++ cmp \$128, $inl ++ jbe open_sse_128 ++ # For long buffers, prepare the poly key first ++ movdqa .chacha20_consts(%rip), $A0 ++ movdqu 0*16($keyp), $B0 ++ movdqu 1*16($keyp), $C0 ++ movdqu 2*16($keyp), $D0 ++ movdqa $D0, $T1 ++ # Store on stack, to free keyp ++ movdqa $B0, $state1_store ++ movdqa $C0, $state2_store ++ movdqa $D0, $ctr0_store ++ mov \$10, $acc0 ++1: \n"; ++ &chacha_qr($A0,$B0,$C0,$D0,$T0,"left"); ++ &chacha_qr($A0,$B0,$C0,$D0,$T0,"right"); $code.=" ++ dec $acc0 ++ jne 1b ++ # A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded ++ paddd .chacha20_consts(%rip), $A0 ++ paddd $state1_store, $B0 ++ # Clamp and store the key ++ pand .clamp(%rip), $A0 ++ movdqa $A0, $r_store ++ movdqa $B0, $s_store ++ # Hash ++ mov %r8, $itr2 ++ call poly_hash_ad_internal ++open_sse_main_loop: ++ cmp \$16*16, $inl ++ jb 2f ++ # Load state, increment counter blocks\n"; ++ &prep_state(4); $code.=" ++ # There are 10 ChaCha20 iterations of 2QR each, so for 6 iterations we ++ # hash 2 blocks, and for the remaining 4 only 1 block - for a total of 16 ++ mov \$4, $itr1 ++ mov $inp, $itr2 ++1: \n"; ++ &emit_body(20); ++ &poly_add("0($itr2)"); $code.=" ++ lea 2*8($itr2), $itr2\n"; ++ &emit_body(20); ++ &poly_stage1(); ++ &emit_body(20); ++ &poly_stage2(); ++ &emit_body(20); ++ &poly_stage3(); ++ &emit_body(20); ++ &poly_reduce_stage(); ++ foreach $l (@loop_body) {$code.=$l."\n";} ++ @loop_body = split /\n/, $chacha_body; $code.=" ++ dec $itr1 ++ jge 1b\n"; ++ &poly_add("0($itr2)"); ++ &poly_mul(); $code.=" ++ lea 2*8($itr2), $itr2 ++ cmp \$-6, $itr1 ++ jg 1b\n"; ++ &finalize_state(4); ++ &xor_stream_using_temp($A3, $B3, $C3, $D3, "0*16", $D0); ++ &xor_stream($A2, $B2, $C2, $D2, "4*16"); ++ &xor_stream($A1, $B1, $C1, $D1, "8*16"); ++ &xor_stream($A0, $B0, $C0, $tmp_store, "12*16"); $code.=" ++ lea 16*16($inp), $inp ++ lea 16*16($oup), $oup ++ sub \$16*16, $inl ++ jmp open_sse_main_loop ++2: ++ # Handle the various tail sizes efficiently ++ test $inl, $inl ++ jz open_sse_finalize ++ cmp \$4*16, $inl ++ ja 3f\n"; ++############################################################################### ++ # At most 64 bytes are left ++ &prep_state(1); $code.=" ++ xor $itr2, $itr2 ++ mov $inl, $itr1 ++ cmp \$16, $itr1 ++ jb 2f ++1: \n"; ++ &poly_add("0($inp, $itr2)"); ++ &poly_mul(); $code.=" ++ sub \$16, $itr1 ++2: ++ add \$16, $itr2\n"; ++ &chacha_qr($A0,$B0,$C0,$D0,$T0,"left"); ++ &chacha_qr($A0,$B0,$C0,$D0,$T0,"right"); $code.=" ++ cmp \$16, $itr1 ++ jae 1b ++ cmp \$10*16, $itr2 ++ jne 2b\n"; ++ &finalize_state(1); $code.=" ++ jmp open_sse_tail_64_dec_loop ++3: ++ cmp \$8*16, $inl ++ ja 3f\n"; ++############################################################################### ++ # 65 - 128 bytes are left ++ &prep_state(2); $code.=" ++ mov $inl, $itr1 ++ and \$-16, $itr1 ++ xor $itr2, $itr2 ++1: \n"; ++ &poly_add("0($inp, $itr2)"); ++ &poly_mul(); $code.=" ++2: ++ add \$16, $itr2\n"; ++ &chacha_qr($A0,$B0,$C0,$D0,$T0,"left"); ++ &chacha_qr($A1,$B1,$C1,$D1,$T0,"left"); ++ &chacha_qr($A0,$B0,$C0,$D0,$T0,"right"); ++ &chacha_qr($A1,$B1,$C1,$D1,$T0,"right");$code.=" ++ cmp $itr1, $itr2 ++ jb 1b ++ cmp \$10*16, $itr2 ++ jne 2b\n"; ++ &finalize_state(2); ++ &xor_stream($A1, $B1, $C1, $D1, "0*16"); $code.=" ++ sub \$4*16, $inl ++ lea 4*16($inp), $inp ++ lea 4*16($oup), $oup ++ jmp open_sse_tail_64_dec_loop ++3: ++ cmp \$12*16, $inl ++ ja 3f\n"; ++############################################################################### ++ # 129 - 192 bytes are left ++ &prep_state(3); $code.=" ++ mov $inl, $itr1 ++ mov \$10*16, $itr2 ++ cmp \$10*16, $itr1 ++ cmovg $itr2, $itr1 ++ and \$-16, $itr1 ++ xor $itr2, $itr2 ++1: \n"; ++ &poly_add("0($inp, $itr2)"); ++ &poly_mul(); $code.=" ++2: ++ add \$16, $itr2\n"; ++ &chacha_qr($A0,$B0,$C0,$D0,$T0,"left"); ++ &chacha_qr($A1,$B1,$C1,$D1,$T0,"left"); ++ &chacha_qr($A2,$B2,$C2,$D2,$T0,"left"); ++ &chacha_qr($A0,$B0,$C0,$D0,$T0,"right"); ++ &chacha_qr($A1,$B1,$C1,$D1,$T0,"right"); ++ &chacha_qr($A2,$B2,$C2,$D2,$T0,"right"); $code.=" ++ cmp $itr1, $itr2 ++ jb 1b ++ cmp \$10*16, $itr2 ++ jne 2b ++ cmp \$11*16, $inl ++ jb 1f\n"; ++ &poly_add("10*16($inp)"); ++ &poly_mul(); $code.=" ++ cmp \$12*16, $inl ++ jb 1f\n"; ++ &poly_add("11*16($inp)"); ++ &poly_mul(); $code.=" ++1: \n"; ++ &finalize_state(3); ++ &xor_stream($A2, $B2, $C2, $D2, "0*16"); ++ &xor_stream($A1, $B1, $C1, $D1, "4*16"); $code.=" ++ sub \$8*16, $inl ++ lea 8*16($inp), $inp ++ lea 8*16($oup), $oup ++ jmp open_sse_tail_64_dec_loop ++3: ++###############################################################################\n"; ++ # 193 - 255 bytes are left ++ &prep_state(4); $code.=" ++ xor $itr2, $itr2 ++1: \n"; ++ &poly_add("0($inp, $itr2)"); ++ &chacha_qr($A0,$B0,$C0,$D0,$C3,"store_left"); ++ &chacha_qr($A1,$B1,$C1,$D1,$C3,"left"); ++ &chacha_qr($A2,$B2,$C2,$D2,$C3,"left_load"); ++ &poly_stage1(); ++ &chacha_qr($A3,$B3,$C3,$D3,$C1,"store_left_load"); ++ &poly_stage2(); ++ &chacha_qr($A0,$B0,$C0,$D0,$C3,"store_right"); ++ &chacha_qr($A1,$B1,$C1,$D1,$C3,"right"); ++ &poly_stage3(); ++ &chacha_qr($A2,$B2,$C2,$D2,$C3,"right_load"); ++ &poly_reduce_stage(); ++ &chacha_qr($A3,$B3,$C3,$D3,$C1,"store_right_load"); $code.=" ++ add \$16, $itr2 ++ cmp \$10*16, $itr2 ++ jb 1b ++ mov $inl, $itr1 ++ and \$-16, $itr1 ++1: \n"; ++ &poly_add("0($inp, $itr2)"); ++ &poly_mul(); $code.=" ++ add \$16, $itr2 ++ cmp $itr1, $itr2 ++ jb 1b\n"; ++ &finalize_state(4); ++ &xor_stream_using_temp($A3, $B3, $C3, $D3, "0*16", $D0); ++ &xor_stream($A2, $B2, $C2, $D2, "4*16"); ++ &xor_stream($A1, $B1, $C1, $D1, "8*16"); $code.=" ++ movdqa $tmp_store, $D0 ++ sub \$12*16, $inl ++ lea 12*16($inp), $inp ++ lea 12*16($oup), $oup ++############################################################################### ++ # Decrypt the remaining data, 16B at a time, using existing stream ++open_sse_tail_64_dec_loop: ++ cmp \$16, $inl ++ jb 1f ++ sub \$16, $inl ++ movdqu ($inp), $T0 ++ pxor $T0, $A0 ++ movdqu $A0, ($oup) ++ lea 16($inp), $inp ++ lea 16($oup), $oup ++ movdqa $B0, $A0 ++ movdqa $C0, $B0 ++ movdqa $D0, $C0 ++ jmp open_sse_tail_64_dec_loop ++1: ++ movdqa $A0, $A1 ++ # Decrypt up to 16B ++open_sse_tail_16: ++ test $inl, $inl ++ jz open_sse_finalize ++ # We can safely load the CT from the end, because it is padded with the MAC ++ mov $inl, $itr2 ++ shl \$4, $itr2 ++ lea .and_masks(%rip), $t0 ++ movdqu ($inp), $T0 ++ add $inl, $inp ++ pand -16($t0, $itr2), $T0 ++ movq $T0, $t0 ++ pextrq \$1, $T0, $t1 ++ pxor $A1, $T0 ++ # We can only store 1 byte at a time, since plaintext can be shorter than 16 bytes ++2: ++ pextrb \$0, $T0, ($oup) ++ psrldq \$1, $T0 ++ inc $oup ++ dec $inl ++ jne 2b ++ ++ add $t0, $acc0 ++ adc $t1, $acc1 ++ adc \$1, $acc2\n"; ++ &poly_mul(); $code.=" ++ ++open_sse_finalize:\n"; ++ &poly_add($len_store); ++ &poly_mul(); $code.=" ++ # Final reduce ++ mov $acc0, $t0 ++ mov $acc1, $t1 ++ mov $acc2, $t2 ++ sub \$-5, $acc0 ++ sbb \$-1, $acc1 ++ sbb \$3, $acc2 ++ cmovc $t0, $acc0 ++ cmovc $t1, $acc1 ++ cmovc $t2, $acc2 ++ # Add in s part of the key ++ add 0+$s_store, $acc0 ++ adc 8+$s_store, $acc1 ++ # Constant time compare ++ xor %rax, %rax ++ mov \$1, %rdx ++ xor 0*8($inp), $acc0 ++ xor 1*8($inp), $acc1 ++ or $acc1, $acc0 ++ cmovz %rdx, %rax ++ ++ add \$288 + 32, %rsp ++ pop %r15 ++ pop %r14 ++ pop %r13 ++ pop %r12 ++ pop %rbx ++ pop %rbp ++ ret ++############################################################################### ++open_sse_128: ++ movdqu .chacha20_consts(%rip), $A0\nmovdqa $A0, $A1\nmovdqa $A0, $A2 ++ movdqu 0*16($keyp), $B0\nmovdqa $B0, $B1\nmovdqa $B0, $B2 ++ movdqu 1*16($keyp), $C0\nmovdqa $C0, $C1\nmovdqa $C0, $C2 ++ movdqu 2*16($keyp), $D0 ++ movdqa $D0, $D1\npaddd .sse_inc(%rip), $D1 ++ movdqa $D1, $D2\npaddd .sse_inc(%rip), $D2 ++ movdqa $B0, $T1\nmovdqa $C0, $T2\nmovdqa $D1, $T3 ++ mov \$10, $acc0 ++1: \n"; ++ &chacha_qr($A0,$B0,$C0,$D0,$T0,"left"); ++ &chacha_qr($A1,$B1,$C1,$D1,$T0,"left"); ++ &chacha_qr($A2,$B2,$C2,$D2,$T0,"left"); ++ &chacha_qr($A0,$B0,$C0,$D0,$T0,"right"); ++ &chacha_qr($A1,$B1,$C1,$D1,$T0,"right"); ++ &chacha_qr($A2,$B2,$C2,$D2,$T0,"right"); $code.=" ++ dec $acc0 ++ jnz 1b ++ paddd .chacha20_consts(%rip), $A0 ++ paddd .chacha20_consts(%rip), $A1 ++ paddd .chacha20_consts(%rip), $A2 ++ paddd $T1, $B0\npaddd $T1, $B1\npaddd $T1, $B2 ++ paddd $T2, $C1\npaddd $T2, $C2 ++ paddd $T3, $D1 ++ paddd .sse_inc(%rip), $T3 ++ paddd $T3, $D2 ++ # Clamp and store the key ++ pand .clamp(%rip), $A0 ++ movdqa $A0, $r_store ++ movdqa $B0, $s_store ++ # Hash ++ mov %r8, $itr2 ++ call poly_hash_ad_internal ++1: ++ cmp \$16, $inl ++ jb open_sse_tail_16 ++ sub \$16, $inl\n"; ++ # Load for hashing ++ &poly_add("0*8($inp)"); $code.=" ++ # Load for decryption ++ movdqu 0*16($inp), $T0 ++ pxor $T0, $A1 ++ movdqu $A1, 0*16($oup) ++ lea 1*16($inp), $inp ++ lea 1*16($oup), $oup\n"; ++ &poly_mul(); $code.=" ++ # Shift the stream left ++ movdqa $B1, $A1 ++ movdqa $C1, $B1 ++ movdqa $D1, $C1 ++ movdqa $A2, $D1 ++ movdqa $B2, $A2 ++ movdqa $C2, $B2 ++ movdqa $D2, $C2 ++ jmp 1b ++ jmp open_sse_tail_16 ++.size chacha20_poly1305_open, .-chacha20_poly1305_open ++################################################################################ ++################################################################################ ++# void chacha20_poly1305_seal(uint8_t *pt, uint8_t *ct, size_t len_in, uint8_t *ad, size_t len_ad, uint8_t *keyp); ++.globl chacha20_poly1305_seal ++.type chacha20_poly1305_seal,\@function,2 ++.align 64 ++chacha20_poly1305_seal: ++ push %rbp ++ push %rbx ++ push %r12 ++ push %r13 ++ push %r14 ++ push %r15 ++ sub \$288 + 32, %rsp ++ lea 32(%rsp), %rbp ++ and \$-32, %rbp ++ mov %rdx, 8+$len_store ++ mov %r8, 0+$len_store ++ mov %rdx, $inl\n"; $code.=" ++ mov OPENSSL_ia32cap_P+8(%rip), %eax ++ test \$`1<<5`, %eax ++ jnz chacha20_poly1305_seal_avx2\n" if ($avx>1); ++$code.=" ++ cmp \$128, $inl ++ jbe seal_sse_128 ++ # For longer buffers, prepare the poly key + some stream ++ movdqa .chacha20_consts(%rip), $A0 ++ movdqu 0*16($keyp), $B0 ++ movdqu 1*16($keyp), $C0 ++ movdqu 2*16($keyp), $D0 ++ movdqa $A0, $A1 ++ movdqa $A0, $A2 ++ movdqa $A0, $A3 ++ movdqa $B0, $B1 ++ movdqa $B0, $B2 ++ movdqa $B0, $B3 ++ movdqa $C0, $C1 ++ movdqa $C0, $C2 ++ movdqa $C0, $C3 ++ movdqa $D0, $D3 ++ paddd .sse_inc(%rip), $D0 ++ movdqa $D0, $D2 ++ paddd .sse_inc(%rip), $D0 ++ movdqa $D0, $D1 ++ paddd .sse_inc(%rip), $D0 ++ # Store on stack ++ movdqa $B0, $state1_store ++ movdqa $C0, $state2_store ++ movdqa $D0, $ctr0_store ++ movdqa $D1, $ctr1_store ++ movdqa $D2, $ctr2_store ++ movdqa $D3, $ctr3_store ++ mov \$10, $acc0 ++1: \n"; ++ foreach $l (@loop_body) {$code.=$l."\n";} ++ @loop_body = split /\n/, $chacha_body; $code.=" ++ dec $acc0 ++ jnz 1b\n"; ++ &finalize_state(4); $code.=" ++ # Clamp and store the key ++ pand .clamp(%rip), $A3 ++ movdqa $A3, $r_store ++ movdqa $B3, $s_store ++ # Hash ++ mov %r8, $itr2 ++ call poly_hash_ad_internal\n"; ++ &xor_stream($A2,$B2,$C2,$D2,"0*16"); ++ &xor_stream($A1,$B1,$C1,$D1,"4*16"); $code.=" ++ cmp \$12*16, $inl ++ ja 1f ++ mov \$8*16, $itr1 ++ sub \$8*16, $inl ++ lea 8*16($inp), $inp ++ jmp seal_sse_128_seal_hash ++1: \n"; ++ &xor_stream($A0, $B0, $C0, $D0, "8*16"); $code.=" ++ mov \$12*16, $itr1 ++ sub \$12*16, $inl ++ lea 12*16($inp), $inp ++ mov \$2, $itr1 ++ mov \$8, $itr2 ++ cmp \$4*16, $inl ++ jbe seal_sse_tail_64 ++ cmp \$8*16, $inl ++ jbe seal_sse_tail_128 ++ cmp \$12*16, $inl ++ jbe seal_sse_tail_192 ++ ++1: \n"; ++ # The main loop ++ &prep_state(4); $code.=" ++2: \n"; ++ &emit_body(20); ++ &poly_add("0($oup)"); ++ &emit_body(20); ++ &poly_stage1(); ++ &emit_body(20); ++ &poly_stage2(); ++ &emit_body(20); ++ &poly_stage3(); ++ &emit_body(20); ++ &poly_reduce_stage(); ++ foreach $l (@loop_body) {$code.=$l."\n";} ++ @loop_body = split /\n/, $chacha_body; $code.=" ++ lea 16($oup), $oup ++ dec $itr2 ++ jge 2b\n"; ++ &poly_add("0*8($oup)"); ++ &poly_mul(); $code.=" ++ lea 16($oup), $oup ++ dec $itr1 ++ jg 2b\n"; ++ ++ &finalize_state(4);$code.=" ++ movdqa $D2, $tmp_store\n"; ++ &xor_stream_using_temp($A3,$B3,$C3,$D3,0*16,$D2); $code.=" ++ movdqa $tmp_store, $D2\n"; ++ &xor_stream($A2,$B2,$C2,$D2, 4*16); ++ &xor_stream($A1,$B1,$C1,$D1, 8*16); $code.=" ++ cmp \$16*16, $inl ++ ja 3f ++ ++ mov \$12*16, $itr1 ++ sub \$12*16, $inl ++ lea 12*16($inp), $inp ++ jmp seal_sse_128_seal_hash ++3: \n"; ++ &xor_stream($A0,$B0,$C0,$D0,"12*16"); $code.=" ++ lea 16*16($inp), $inp ++ sub \$16*16, $inl ++ mov \$6, $itr1 ++ mov \$4, $itr2 ++ cmp \$12*16, $inl ++ jg 1b ++ mov $inl, $itr1 ++ test $inl, $inl ++ je seal_sse_128_seal_hash ++ mov \$6, $itr1 ++ cmp \$4*16, $inl ++ jg 3f ++############################################################################### ++seal_sse_tail_64:\n"; ++ &prep_state(1); $code.=" ++1: \n"; ++ &poly_add("0($oup)"); ++ &poly_mul(); $code.=" ++ lea 16($oup), $oup ++2: \n"; ++ &chacha_qr($A0,$B0,$C0,$D0,$T0,"left"); ++ &chacha_qr($A0,$B0,$C0,$D0,$T0,"right"); ++ &poly_add("0($oup)"); ++ &poly_mul(); $code.=" ++ lea 16($oup), $oup ++ dec $itr1 ++ jg 1b ++ dec $itr2 ++ jge 2b\n"; ++ &finalize_state(1); $code.=" ++ jmp seal_sse_128_seal ++3: ++ cmp \$8*16, $inl ++ jg 3f ++############################################################################### ++seal_sse_tail_128:\n"; ++ &prep_state(2); $code.=" ++1: \n"; ++ &poly_add("0($oup)"); ++ &poly_mul(); $code.=" ++ lea 16($oup), $oup ++2: \n"; ++ &chacha_qr($A0,$B0,$C0,$D0,$T0,"left"); ++ &chacha_qr($A1,$B1,$C1,$D1,$T0,"left"); ++ &poly_add("0($oup)"); ++ &poly_mul(); ++ &chacha_qr($A0,$B0,$C0,$D0,$T0,"right"); ++ &chacha_qr($A1,$B1,$C1,$D1,$T0,"right"); $code.=" ++ lea 16($oup), $oup ++ dec $itr1 ++ jg 1b ++ dec $itr2 ++ jge 2b\n"; ++ &finalize_state(2); ++ &xor_stream($A1,$B1,$C1,$D1,0*16); $code.=" ++ mov \$4*16, $itr1 ++ sub \$4*16, $inl ++ lea 4*16($inp), $inp ++ jmp seal_sse_128_seal_hash ++3: ++############################################################################### ++seal_sse_tail_192:\n"; ++ &prep_state(3); $code.=" ++1: \n"; ++ &poly_add("0($oup)"); ++ &poly_mul(); $code.=" ++ lea 16($oup), $oup ++2: \n"; ++ &chacha_qr($A0,$B0,$C0,$D0,$T0,"left"); ++ &chacha_qr($A1,$B1,$C1,$D1,$T0,"left"); ++ &chacha_qr($A2,$B2,$C2,$D2,$T0,"left"); ++ &poly_add("0($oup)"); ++ &poly_mul(); ++ &chacha_qr($A0,$B0,$C0,$D0,$T0,"right"); ++ &chacha_qr($A1,$B1,$C1,$D1,$T0,"right"); ++ &chacha_qr($A2,$B2,$C2,$D2,$T0,"right"); $code.=" ++ lea 16($oup), $oup ++ dec $itr1 ++ jg 1b ++ dec $itr2 ++ jge 2b\n"; ++ &finalize_state(3); ++ &xor_stream($A2,$B2,$C2,$D2,0*16); ++ &xor_stream($A1,$B1,$C1,$D1,4*16); $code.=" ++ mov \$8*16, $itr1 ++ sub \$8*16, $inl ++ lea 8*16($inp), $inp ++############################################################################### ++seal_sse_128_seal_hash: ++ cmp \$16, $itr1 ++ jb seal_sse_128_seal\n"; ++ &poly_add("0($oup)"); ++ &poly_mul(); $code.=" ++ sub \$16, $itr1 ++ lea 16($oup), $oup ++ jmp seal_sse_128_seal_hash ++ ++seal_sse_128_seal: ++ cmp \$16, $inl ++ jb seal_sse_tail_16 ++ sub \$16, $inl ++ # Load for decryption ++ movdqu 0*16($inp), $T0 ++ pxor $T0, $A0 ++ movdqu $A0, 0*16($oup) ++ # Then hash ++ add 0*8($oup), $acc0 ++ adc 1*8($oup), $acc1 ++ adc \$1, $acc2 ++ lea 1*16($inp), $inp ++ lea 1*16($oup), $oup\n"; ++ &poly_mul(); $code.=" ++ # Shift the stream left ++ movdqa $B0, $A0 ++ movdqa $C0, $B0 ++ movdqa $D0, $C0 ++ movdqa $A1, $D0 ++ movdqa $B1, $A1 ++ movdqa $C1, $B1 ++ movdqa $D1, $C1 ++ jmp seal_sse_128_seal ++ ++seal_sse_tail_16: ++ test $inl, $inl ++ jz seal_sse_finalize ++ # We can only load the PT one byte at a time to avoid buffer overread ++ mov $inl, $itr2 ++ shl \$4, $itr2 ++ lea .and_masks(%rip), $t0 ++ mov $inl, $itr1 ++ lea -1($inp, $inl), $inp ++ pxor $T3, $T3 ++1: ++ pslldq \$1, $T3 ++ pinsrb \$0, ($inp), $T3 ++ lea -1($inp), $inp ++ dec $itr1 ++ jne 1b ++ pxor $A0, $T3 ++ movdqu $T3, ($oup) ++ pand -16($t0, $itr2), $T3 ++ movq $T3, $t0 ++ pextrq \$1, $T3, $t1 ++ add $t0, $acc0 ++ adc $t1, $acc1 ++ adc \$1, $acc2 ++ lea ($inl, $oup), $oup\n"; ++ &poly_mul(); $code.=" ++seal_sse_finalize:\n"; ++ &poly_add($len_store); ++ &poly_mul(); $code.=" ++ # Final reduce ++ mov $acc0, $t0 ++ mov $acc1, $t1 ++ mov $acc2, $t2 ++ sub \$-5, $acc0 ++ sbb \$-1, $acc1 ++ sbb \$3, $acc2 ++ cmovc $t0, $acc0 ++ cmovc $t1, $acc1 ++ cmovc $t2, $acc2 ++ # Add in s part of the key ++ add 0+$s_store, $acc0 ++ adc 8+$s_store, $acc1 ++ mov $acc0, 0*8($oup) ++ mov $acc1, 1*8($oup) ++ add \$288 + 32, %rsp ++ pop %r15 ++ pop %r14 ++ pop %r13 ++ pop %r12 ++ pop %rbx ++ pop %rbp ++ ret ++################################################################################ ++seal_sse_128: ++ movdqu .chacha20_consts(%rip), $A0\nmovdqa $A0, $A1\nmovdqa $A0, $A2 ++ movdqu 0*16($keyp), $B0\nmovdqa $B0, $B1\nmovdqa $B0, $B2 ++ movdqu 1*16($keyp), $C0\nmovdqa $C0, $C1\nmovdqa $C0, $C2 ++ movdqu 2*16($keyp), $D2 ++ movdqa $D2, $D0\npaddd .sse_inc(%rip), $D0 ++ movdqa $D0, $D1\npaddd .sse_inc(%rip), $D1 ++ movdqa $B0, $T1\nmovdqa $C0, $T2\nmovdqa $D0, $T3 ++ mov \$10, $acc0 ++1:\n"; ++ &chacha_qr($A0,$B0,$C0,$D0,$T0,"left"); ++ &chacha_qr($A1,$B1,$C1,$D1,$T0,"left"); ++ &chacha_qr($A2,$B2,$C2,$D2,$T0,"left"); ++ &chacha_qr($A0,$B0,$C0,$D0,$T0,"right"); ++ &chacha_qr($A1,$B1,$C1,$D1,$T0,"right"); ++ &chacha_qr($A2,$B2,$C2,$D2,$T0,"right"); $code.=" ++ dec $acc0 ++ jnz 1b ++ paddd .chacha20_consts(%rip), $A0 ++ paddd .chacha20_consts(%rip), $A1 ++ paddd .chacha20_consts(%rip), $A2 ++ paddd $T1, $B0\npaddd $T1, $B1\npaddd $T1, $B2 ++ paddd $T2, $C0\npaddd $T2, $C1 ++ paddd $T3, $D0 ++ paddd .sse_inc(%rip), $T3 ++ paddd $T3, $D1 ++ # Clamp and store the key ++ pand .clamp(%rip), $A2 ++ movdqa $A2, $r_store ++ movdqa $B2, $s_store ++ # Hash ++ mov %r8, $itr2 ++ call poly_hash_ad_internal ++ jmp seal_sse_128_seal ++.size chacha20_poly1305_seal, .-chacha20_poly1305_seal\n"; ++} ++ ++if ($avx>1) { ++ ++($A0,$A1,$A2,$A3,$B0,$B1,$B2,$B3,$C0,$C1,$C2,$C3,$D0,$D1,$D2,$D3)=map("%ymm$_",(0..15)); ++my ($A0x,$A1x,$A2x,$A3x,$B0x,$B1x,$B2x,$B3x,$C0x,$C1x,$C2x,$C3x,$D0x,$D1x,$D2x,$D3x)=map("%xmm$_",(0..15)); ++($T0,$T1,$T2,$T3)=($A3,$B3,$C3,$D3); ++$state1_store="2*32(%rbp)"; ++$state2_store="3*32(%rbp)"; ++$tmp_store="4*32(%rbp)"; ++$ctr0_store="5*32(%rbp)"; ++$ctr1_store="6*32(%rbp)"; ++$ctr2_store="7*32(%rbp)"; ++$ctr3_store="8*32(%rbp)"; ++ ++sub chacha_qr_avx2 { ++my ($a,$b,$c,$d,$t,$dir)=@_; ++$code.=<<___ if ($dir =~ /store/); ++ vmovdqa $t, $tmp_store ++___ ++$code.=<<___; ++ vpaddd $b, $a, $a ++ vpxor $a, $d, $d ++ vpshufb .rol16(%rip), $d, $d ++ vpaddd $d, $c, $c ++ vpxor $c, $b, $b ++ vpsrld \$20, $b, $t ++ vpslld \$12, $b, $b ++ vpxor $t, $b, $b ++ vpaddd $b, $a, $a ++ vpxor $a, $d, $d ++ vpshufb .rol8(%rip), $d, $d ++ vpaddd $d, $c, $c ++ vpxor $c, $b, $b ++ vpslld \$7, $b, $t ++ vpsrld \$25, $b, $b ++ vpxor $t, $b, $b ++___ ++$code.=<<___ if ($dir =~ /left/); ++ vpalignr \$12, $d, $d, $d ++ vpalignr \$8, $c, $c, $c ++ vpalignr \$4, $b, $b, $b ++___ ++$code.=<<___ if ($dir =~ /right/); ++ vpalignr \$4, $d, $d, $d ++ vpalignr \$8, $c, $c, $c ++ vpalignr \$12, $b, $b, $b ++___ ++$code.=<<___ if ($dir =~ /load/); ++ vmovdqa $tmp_store, $t ++___ ++} ++ ++sub prep_state_avx2 { ++my ($n)=@_; ++$code.=<<___; ++ vmovdqa .chacha20_consts(%rip), $A0 ++ vmovdqa $state1_store, $B0 ++ vmovdqa $state2_store, $C0 ++___ ++$code.=<<___ if ($n ge 2); ++ vmovdqa $A0, $A1 ++ vmovdqa $B0, $B1 ++ vmovdqa $C0, $C1 ++___ ++$code.=<<___ if ($n ge 3); ++ vmovdqa $A0, $A2 ++ vmovdqa $B0, $B2 ++ vmovdqa $C0, $C2 ++___ ++$code.=<<___ if ($n ge 4); ++ vmovdqa $A0, $A3 ++ vmovdqa $B0, $B3 ++ vmovdqa $C0, $C3 ++___ ++$code.=<<___ if ($n eq 1); ++ vmovdqa .avx2_inc(%rip), $D0 ++ vpaddd $ctr0_store, $D0, $D0 ++ vmovdqa $D0, $ctr0_store ++___ ++$code.=<<___ if ($n eq 2); ++ vmovdqa .avx2_inc(%rip), $D0 ++ vpaddd $ctr0_store, $D0, $D1 ++ vpaddd $D1, $D0, $D0 ++ vmovdqa $D0, $ctr0_store ++ vmovdqa $D1, $ctr1_store ++___ ++$code.=<<___ if ($n eq 3); ++ vmovdqa .avx2_inc(%rip), $D0 ++ vpaddd $ctr0_store, $D0, $D2 ++ vpaddd $D2, $D0, $D1 ++ vpaddd $D1, $D0, $D0 ++ vmovdqa $D0, $ctr0_store ++ vmovdqa $D1, $ctr1_store ++ vmovdqa $D2, $ctr2_store ++___ ++$code.=<<___ if ($n eq 4); ++ vmovdqa .avx2_inc(%rip), $D0 ++ vpaddd $ctr0_store, $D0, $D3 ++ vpaddd $D3, $D0, $D2 ++ vpaddd $D2, $D0, $D1 ++ vpaddd $D1, $D0, $D0 ++ vmovdqa $D3, $ctr3_store ++ vmovdqa $D2, $ctr2_store ++ vmovdqa $D1, $ctr1_store ++ vmovdqa $D0, $ctr0_store ++___ ++} ++ ++sub finalize_state_avx2 { ++my ($n)=@_; ++$code.=<<___ if ($n eq 4); ++ vpaddd .chacha20_consts(%rip), $A3, $A3 ++ vpaddd $state1_store, $B3, $B3 ++ vpaddd $state2_store, $C3, $C3 ++ vpaddd $ctr3_store, $D3, $D3 ++___ ++$code.=<<___ if ($n ge 3); ++ vpaddd .chacha20_consts(%rip), $A2, $A2 ++ vpaddd $state1_store, $B2, $B2 ++ vpaddd $state2_store, $C2, $C2 ++ vpaddd $ctr2_store, $D2, $D2 ++___ ++$code.=<<___ if ($n ge 2); ++ vpaddd .chacha20_consts(%rip), $A1, $A1 ++ vpaddd $state1_store, $B1, $B1 ++ vpaddd $state2_store, $C1, $C1 ++ vpaddd $ctr1_store, $D1, $D1 ++___ ++$code.=<<___; ++ vpaddd .chacha20_consts(%rip), $A0, $A0 ++ vpaddd $state1_store, $B0, $B0 ++ vpaddd $state2_store, $C0, $C0 ++ vpaddd $ctr0_store, $D0, $D0 ++___ ++} ++ ++sub xor_stream_avx2 { ++my ($A, $B, $C, $D, $offset, $hlp)=@_; ++$code.=<<___; ++ vperm2i128 \$0x02, $A, $B, $hlp ++ vperm2i128 \$0x13, $A, $B, $B ++ vperm2i128 \$0x02, $C, $D, $A ++ vperm2i128 \$0x13, $C, $D, $C ++ vpxor 0*32+$offset($inp), $hlp, $hlp ++ vpxor 1*32+$offset($inp), $A, $A ++ vpxor 2*32+$offset($inp), $B, $B ++ vpxor 3*32+$offset($inp), $C, $C ++ vmovdqu $hlp, 0*32+$offset($oup) ++ vmovdqu $A, 1*32+$offset($oup) ++ vmovdqu $B, 2*32+$offset($oup) ++ vmovdqu $C, 3*32+$offset($oup) ++___ ++} ++ ++sub finish_stream_avx2 { ++my ($A, $B, $C, $D, $hlp)=@_; ++$code.=<<___; ++ vperm2i128 \$0x13, $A, $B, $hlp ++ vperm2i128 \$0x02, $A, $B, $A ++ vperm2i128 \$0x02, $C, $D, $B ++ vperm2i128 \$0x13, $C, $D, $D ++ vmovdqa $hlp, $C ++___ ++} ++ ++sub poly_stage1_mulx { ++$code.=<<___; ++ mov 0+$r_store, %rdx ++ mov %rdx, $t2 ++ mulx $acc0, $t0, $t1 ++ mulx $acc1, %rax, %rdx ++ imul $acc2, $t2 ++ add %rax, $t1 ++ adc %rdx, $t2 ++___ ++} ++ ++sub poly_stage2_mulx { ++$code.=<<___; ++ mov 8+$r_store, %rdx ++ mulx $acc0, $acc0, %rax ++ add $acc0, $t1 ++ mulx $acc1, $acc1, $t3 ++ adc $acc1, $t2 ++ adc \$0, $t3 ++ imul $acc2, %rdx ++___ ++} ++ ++sub poly_stage3_mulx { ++$code.=<<___; ++ add %rax, $t2 ++ adc %rdx, $t3 ++___ ++} ++ ++sub poly_mul_mulx { ++ &poly_stage1_mulx(); ++ &poly_stage2_mulx(); ++ &poly_stage3_mulx(); ++ &poly_reduce_stage(); ++} ++ ++sub gen_chacha_round_avx2 { ++my ($rot1, $rot2, $shift)=@_; ++my $round=""; ++$round=$round ."vmovdqa $C0, $tmp_store\n" if ($rot1 eq 20); ++$round=$round ."vmovdqa $rot2, $C0 ++ vpaddd $B3, $A3, $A3 ++ vpaddd $B2, $A2, $A2 ++ vpaddd $B1, $A1, $A1 ++ vpaddd $B0, $A0, $A0 ++ vpxor $A3, $D3, $D3 ++ vpxor $A2, $D2, $D2 ++ vpxor $A1, $D1, $D1 ++ vpxor $A0, $D0, $D0 ++ vpshufb $C0, $D3, $D3 ++ vpshufb $C0, $D2, $D2 ++ vpshufb $C0, $D1, $D1 ++ vpshufb $C0, $D0, $D0 ++ vmovdqa $tmp_store, $C0 ++ vpaddd $D3, $C3, $C3 ++ vpaddd $D2, $C2, $C2 ++ vpaddd $D1, $C1, $C1 ++ vpaddd $D0, $C0, $C0 ++ vpxor $C3, $B3, $B3 ++ vpxor $C2, $B2, $B2 ++ vpxor $C1, $B1, $B1 ++ vpxor $C0, $B0, $B0 ++ vmovdqa $C0, $tmp_store ++ vpsrld \$$rot1, $B3, $C0 ++ vpslld \$32-$rot1, $B3, $B3 ++ vpxor $C0, $B3, $B3 ++ vpsrld \$$rot1, $B2, $C0 ++ vpslld \$32-$rot1, $B2, $B2 ++ vpxor $C0, $B2, $B2 ++ vpsrld \$$rot1, $B1, $C0 ++ vpslld \$32-$rot1, $B1, $B1 ++ vpxor $C0, $B1, $B1 ++ vpsrld \$$rot1, $B0, $C0 ++ vpslld \$32-$rot1, $B0, $B0 ++ vpxor $C0, $B0, $B0\n"; ++($s1,$s2,$s3)=(4,8,12) if ($shift =~ /left/); ++($s1,$s2,$s3)=(12,8,4) if ($shift =~ /right/); ++$round=$round ."vmovdqa $tmp_store, $C0 ++ vpalignr \$$s1, $B3, $B3, $B3 ++ vpalignr \$$s2, $C3, $C3, $C3 ++ vpalignr \$$s3, $D3, $D3, $D3 ++ vpalignr \$$s1, $B2, $B2, $B2 ++ vpalignr \$$s2, $C2, $C2, $C2 ++ vpalignr \$$s3, $D2, $D2, $D2 ++ vpalignr \$$s1, $B1, $B1, $B1 ++ vpalignr \$$s2, $C1, $C1, $C1 ++ vpalignr \$$s3, $D1, $D1, $D1 ++ vpalignr \$$s1, $B0, $B0, $B0 ++ vpalignr \$$s2, $C0, $C0, $C0 ++ vpalignr \$$s3, $D0, $D0, $D0\n" ++if (($shift =~ /left/) || ($shift =~ /right/)); ++return $round; ++}; ++ ++$chacha_body = &gen_chacha_round_avx2(20, ".rol16(%rip)") . ++ &gen_chacha_round_avx2(25, ".rol8(%rip)", "left") . ++ &gen_chacha_round_avx2(20, ".rol16(%rip)") . ++ &gen_chacha_round_avx2(25, ".rol8(%rip)", "right"); ++ ++@loop_body = split /\n/, $chacha_body; ++ ++$code.=" ++############################################################################### ++.type chacha20_poly1305_open_avx2,\@function,2 ++.align 64 ++chacha20_poly1305_open_avx2: ++ vzeroupper ++ vmovdqa .chacha20_consts(%rip), $A0 ++ vbroadcasti128 0*16($keyp), $B0 ++ vbroadcasti128 1*16($keyp), $C0 ++ vbroadcasti128 2*16($keyp), $D0 ++ vpaddd .avx2_init(%rip), $D0, $D0 ++ cmp \$6*32, $inl ++ jbe open_avx2_192 ++ cmp \$10*32, $inl ++ jbe open_avx2_320 ++ ++ vmovdqa $B0, $state1_store ++ vmovdqa $C0, $state2_store ++ vmovdqa $D0, $ctr0_store ++ mov \$10, $acc0 ++1: \n"; ++ &chacha_qr_avx2($A0,$B0,$C0,$D0,$T0,"left"); ++ &chacha_qr_avx2($A0,$B0,$C0,$D0,$T0,"right"); $code.=" ++ dec $acc0 ++ jne 1b ++ vpaddd .chacha20_consts(%rip), $A0, $A0 ++ vpaddd $state1_store, $B0, $B0 ++ vpaddd $state2_store, $C0, $C0 ++ vpaddd $ctr0_store, $D0, $D0 ++ ++ vperm2i128 \$0x02, $A0, $B0, $T0 ++ # Clamp and store key ++ vpand .clamp(%rip), $T0, $T0 ++ vmovdqa $T0, $r_store ++ # Stream for the first 64 bytes ++ vperm2i128 \$0x13, $A0, $B0, $A0 ++ vperm2i128 \$0x13, $C0, $D0, $B0 ++ # Hash AD + first 64 bytes ++ mov %r8, $itr2 ++ call poly_hash_ad_internal ++ xor $itr1, $itr1 ++ # Hash first 64 bytes ++1: \n"; ++ &poly_add("0($inp, $itr1)"); ++ &poly_mul(); $code.=" ++ add \$16, $itr1 ++ cmp \$2*32, $itr1 ++ jne 1b ++ # Decrypt first 64 bytes ++ vpxor 0*32($inp), $A0, $A0 ++ vpxor 1*32($inp), $B0, $B0 ++ vmovdqu $A0, 0*32($oup) ++ vmovdqu $B0, 1*32($oup) ++ lea 2*32($inp), $inp ++ lea 2*32($oup), $oup ++ sub \$2*32, $inl ++1: ++ # Hash and decrypt 512 bytes each iteration ++ cmp \$16*32, $inl ++ jb 3f\n"; ++ &prep_state_avx2(4); $code.=" ++ xor $itr1, $itr1 ++2: \n"; ++ &poly_add("0*8($inp, $itr1)"); ++ &emit_body(10); ++ &poly_stage1_mulx(); ++ &emit_body(9); ++ &poly_stage2_mulx(); ++ &emit_body(12); ++ &poly_stage3_mulx(); ++ &emit_body(10); ++ &poly_reduce_stage(); ++ &emit_body(9); ++ &poly_add("2*8($inp, $itr1)"); ++ &emit_body(8); ++ &poly_stage1_mulx(); ++ &emit_body(18); ++ &poly_stage2_mulx(); ++ &emit_body(18); ++ &poly_stage3_mulx(); ++ &emit_body(9); ++ &poly_reduce_stage(); ++ &emit_body(8); ++ &poly_add("4*8($inp, $itr1)"); $code.=" ++ lea 6*8($itr1), $itr1\n"; ++ &emit_body(18); ++ &poly_stage1_mulx(); ++ &emit_body(8); ++ &poly_stage2_mulx(); ++ &emit_body(8); ++ &poly_stage3_mulx(); ++ &emit_body(18); ++ &poly_reduce_stage(); ++ foreach $l (@loop_body) {$code.=$l."\n";} ++ @loop_body = split /\n/, $chacha_body; $code.=" ++ cmp \$10*6*8, $itr1 ++ jne 2b\n"; ++ &finalize_state_avx2(4); $code.=" ++ vmovdqa $A0, $tmp_store\n"; ++ &poly_add("10*6*8($inp)"); ++ &xor_stream_avx2($A3, $B3, $C3, $D3, 0*32, $A0); $code.=" ++ vmovdqa $tmp_store, $A0\n"; ++ &poly_mul(); ++ &xor_stream_avx2($A2, $B2, $C2, $D2, 4*32, $A3); ++ &poly_add("10*6*8+2*8($inp)"); ++ &xor_stream_avx2($A1, $B1, $C1, $D1, 8*32, $A3); ++ &poly_mul(); ++ &xor_stream_avx2($A0, $B0, $C0, $D0, 12*32, $A3); $code.=" ++ lea 16*32($inp), $inp ++ lea 16*32($oup), $oup ++ sub \$16*32, $inl ++ jmp 1b ++3: ++ test $inl, $inl ++ vzeroupper ++ je open_sse_finalize ++3: ++ cmp \$4*32, $inl ++ ja 3f\n"; ++############################################################################### ++ # 1-128 bytes left ++ &prep_state_avx2(1); $code.=" ++ xor $itr2, $itr2 ++ mov $inl, $itr1 ++ and \$-16, $itr1 ++ test $itr1, $itr1 ++ je 2f ++1: \n"; ++ &poly_add("0*8($inp, $itr2)"); ++ &poly_mul(); $code.=" ++2: ++ add \$16, $itr2\n"; ++ &chacha_qr_avx2($A0,$B0,$C0,$D0,$T0,"left"); ++ &chacha_qr_avx2($A0,$B0,$C0,$D0,$T0,"right"); $code.=" ++ cmp $itr1, $itr2 ++ jb 1b ++ cmp \$160, $itr2 ++ jne 2b\n"; ++ &finalize_state_avx2(1); ++ &finish_stream_avx2($A0,$B0,$C0,$D0,$T0); $code.=" ++ jmp open_avx2_tail_loop ++3: ++ cmp \$8*32, $inl ++ ja 3f\n"; ++############################################################################### ++ # 129-256 bytes left ++ &prep_state_avx2(2); $code.=" ++ mov $inl, $tmp_store ++ mov $inl, $itr1 ++ sub \$4*32, $itr1 ++ shr \$4, $itr1 ++ mov \$10, $itr2 ++ cmp \$10, $itr1 ++ cmovg $itr2, $itr1 ++ mov $inp, $inl ++ xor $itr2, $itr2 ++1: \n"; ++ &poly_add("0*8($inl)"); ++ &poly_mul_mulx(); $code.=" ++ lea 16($inl), $inl ++2: \n"; ++ &chacha_qr_avx2($A0,$B0,$C0,$D0,$T0,"left"); ++ &chacha_qr_avx2($A1,$B1,$C1,$D1,$T0,"left"); $code.=" ++ inc $itr2\n"; ++ &chacha_qr_avx2($A0,$B0,$C0,$D0,$T0,"right"); ++ &chacha_qr_avx2($A1,$B1,$C1,$D1,$T0,"right"); ++ &chacha_qr_avx2($A2,$B2,$C2,$D2,$T0,"right"); $code.=" ++ cmp $itr1, $itr2 ++ jb 1b ++ cmp \$10, $itr2 ++ jne 2b ++ mov $inl, $itr2 ++ sub $inp, $inl ++ mov $inl, $itr1 ++ mov $tmp_store, $inl ++1: ++ add \$16, $itr1 ++ cmp $inl, $itr1 ++ jg 1f\n"; ++ &poly_add("0*8($itr2)"); ++ &poly_mul_mulx(); $code.=" ++ lea 16($itr2), $itr2 ++ jmp 1b ++1: \n"; ++ &finalize_state_avx2(2); ++ &xor_stream_avx2($A1, $B1, $C1, $D1, 0*32, $T0); ++ &finish_stream_avx2($A0, $B0, $C0, $D0, $T0); $code.=" ++ lea 4*32($inp), $inp ++ lea 4*32($oup), $oup ++ sub \$4*32, $inl ++ jmp open_avx2_tail_loop ++3: ++ cmp \$12*32, $inl ++ ja 3f\n"; ++############################################################################### ++ # 257-383 bytes left ++ &prep_state_avx2(3); $code.=" ++ mov $inl, $tmp_store ++ mov $inl, $itr1 ++ sub \$8*32, $itr1 ++ shr \$4, $itr1 ++ add \$6, $itr1 ++ mov \$10, $itr2 ++ cmp \$10, $itr1 ++ cmovg $itr2, $itr1 ++ mov $inp, $inl ++ xor $itr2, $itr2 ++1: \n"; ++ &poly_add("0*8($inl)"); ++ &poly_mul_mulx(); $code.=" ++ lea 16($inl), $inl ++2: \n"; ++ &chacha_qr_avx2($A2,$B2,$C2,$D2,$T0,"left"); ++ &chacha_qr_avx2($A1,$B1,$C1,$D1,$T0,"left"); ++ &chacha_qr_avx2($A0,$B0,$C0,$D0,$T0,"left"); ++ &poly_add("0*8($inl)"); ++ &poly_mul(); $code.=" ++ lea 16($inl), $inl ++ inc $itr2\n"; ++ &chacha_qr_avx2($A2,$B2,$C2,$D2,$T0,"right"); ++ &chacha_qr_avx2($A1,$B1,$C1,$D1,$T0,"right"); ++ &chacha_qr_avx2($A0,$B0,$C0,$D0,$T0,"right"); $code.=" ++ cmp $itr1, $itr2 ++ jb 1b ++ cmp \$10, $itr2 ++ jne 2b ++ mov $inl, $itr2 ++ sub $inp, $inl ++ mov $inl, $itr1 ++ mov $tmp_store, $inl ++1: ++ add \$16, $itr1 ++ cmp $inl, $itr1 ++ jg 1f\n"; ++ &poly_add("0*8($itr2)"); ++ &poly_mul_mulx(); $code.=" ++ lea 16($itr2), $itr2 ++ jmp 1b ++1: \n"; ++ &finalize_state_avx2(3); ++ &xor_stream_avx2($A2, $B2, $C2, $D2, 0*32, $T0); ++ &xor_stream_avx2($A1, $B1, $C1, $D1, 4*32, $T0); ++ &finish_stream_avx2($A0, $B0, $C0, $D0, $T0); $code.=" ++ lea 8*32($inp), $inp ++ lea 8*32($oup), $oup ++ sub \$8*32, $inl ++ jmp open_avx2_tail_loop ++3: \n"; ++############################################################################### ++ # 384-512 bytes left ++ &prep_state_avx2(4); $code.=" ++ xor $itr1, $itr1 ++ mov $inp, $itr2 ++1: \n"; ++ &poly_add("0*8($itr2)"); ++ &poly_mul(); $code.=" ++ lea 2*8($itr2), $itr2 ++2: \n"; ++ &emit_body(37); ++ &poly_add("0*8($itr2)"); ++ &poly_mul_mulx(); ++ &emit_body(48); ++ &poly_add("2*8($itr2)"); ++ &poly_mul_mulx(); $code.=" ++ lea 4*8($itr2), $itr2\n"; ++ foreach $l (@loop_body) {$code.=$l."\n";} ++ @loop_body = split /\n/, $chacha_body; $code.=" ++ inc $itr1 ++ cmp \$4, $itr1 ++ jl 1b ++ cmp \$10, $itr1 ++ jne 2b ++ mov $inl, $itr1 ++ sub \$12*32, $itr1 ++ and \$-16, $itr1 ++1: ++ test $itr1, $itr1 ++ je 1f\n"; ++ &poly_add("0*8($itr2)"); ++ &poly_mul_mulx(); $code.=" ++ lea 2*8($itr2), $itr2 ++ sub \$2*8, $itr1 ++ jmp 1b ++1: \n"; ++ &finalize_state_avx2(4); $code.=" ++ vmovdqa $A0, $tmp_store\n"; ++ &xor_stream_avx2($A3, $B3, $C3, $D3, 0*32, $A0); $code.=" ++ vmovdqa $tmp_store, $A0\n"; ++ &xor_stream_avx2($A2, $B2, $C2, $D2, 4*32, $A3); ++ &xor_stream_avx2($A1, $B1, $C1, $D1, 8*32, $A3); ++ &finish_stream_avx2($A0, $B0, $C0, $D0, $A3); $code.=" ++ lea 12*32($inp), $inp ++ lea 12*32($oup), $oup ++ sub \$12*32, $inl ++open_avx2_tail_loop: ++ cmp \$32, $inl ++ jb open_avx2_tail ++ sub \$32, $inl ++ vpxor ($inp), $A0, $A0 ++ vmovdqu $A0, ($oup) ++ lea 1*32($inp), $inp ++ lea 1*32($oup), $oup ++ vmovdqa $B0, $A0 ++ vmovdqa $C0, $B0 ++ vmovdqa $D0, $C0 ++ jmp open_avx2_tail_loop ++open_avx2_tail: ++ cmp \$16, $inl ++ vmovdqa $A0x, $A1x ++ jb 1f ++ sub \$16, $inl ++ #load for decryption ++ vpxor ($inp), $A0x, $A1x ++ vmovdqu $A1x, ($oup) ++ lea 1*16($inp), $inp ++ lea 1*16($oup), $oup ++ vperm2i128 \$0x11, $A0, $A0, $A0 ++ vmovdqa $A0x, $A1x ++1: ++ vzeroupper ++ jmp open_sse_tail_16 ++############################################################################### ++open_avx2_192: ++ vmovdqa $A0, $A1 ++ vmovdqa $A0, $A2 ++ vmovdqa $B0, $B1 ++ vmovdqa $B0, $B2 ++ vmovdqa $C0, $C1 ++ vmovdqa $C0, $C2 ++ vpaddd .avx2_inc(%rip), $D0, $D1 ++ vmovdqa $D0, $T2 ++ vmovdqa $D1, $T3 ++ mov \$10, $acc0 ++1: \n"; ++ &chacha_qr_avx2($A0,$B0,$C0,$D0,$T0,"left"); ++ &chacha_qr_avx2($A1,$B1,$C1,$D1,$T0,"left"); ++ &chacha_qr_avx2($A0,$B0,$C0,$D0,$T0,"right"); ++ &chacha_qr_avx2($A1,$B1,$C1,$D1,$T0,"right"); $code.=" ++ dec $acc0 ++ jne 1b ++ vpaddd $A2, $A0, $A0 ++ vpaddd $A2, $A1, $A1 ++ vpaddd $B2, $B0, $B0 ++ vpaddd $B2, $B1, $B1 ++ vpaddd $C2, $C0, $C0 ++ vpaddd $C2, $C1, $C1 ++ vpaddd $T2, $D0, $D0 ++ vpaddd $T3, $D1, $D1 ++ vperm2i128 \$0x02, $A0, $B0, $T0 ++ # Clamp and store the key ++ vpand .clamp(%rip), $T0, $T0 ++ vmovdqa $T0, $r_store ++ # Stream for up to 192 bytes ++ vperm2i128 \$0x13, $A0, $B0, $A0 ++ vperm2i128 \$0x13, $C0, $D0, $B0 ++ vperm2i128 \$0x02, $A1, $B1, $C0 ++ vperm2i128 \$0x02, $C1, $D1, $D0 ++ vperm2i128 \$0x13, $A1, $B1, $A1 ++ vperm2i128 \$0x13, $C1, $D1, $B1 ++open_avx2_short: ++ mov %r8, $itr2 ++ call poly_hash_ad_internal ++open_avx2_hash_and_xor_loop: ++ cmp \$32, $inl ++ jb open_avx2_short_tail_32 ++ sub \$32, $inl\n"; ++ # Load + hash ++ &poly_add("0*8($inp)"); ++ &poly_mul(); ++ &poly_add("2*8($inp)"); ++ &poly_mul(); $code.=" ++ # Load + decrypt ++ vpxor ($inp), $A0, $A0 ++ vmovdqu $A0, ($oup) ++ lea 1*32($inp), $inp ++ lea 1*32($oup), $oup ++ # Shift stream ++ vmovdqa $B0, $A0 ++ vmovdqa $C0, $B0 ++ vmovdqa $D0, $C0 ++ vmovdqa $A1, $D0 ++ vmovdqa $B1, $A1 ++ vmovdqa $C1, $B1 ++ vmovdqa $D1, $C1 ++ vmovdqa $A2, $D1 ++ vmovdqa $B2, $A2 ++ jmp open_avx2_hash_and_xor_loop ++open_avx2_short_tail_32: ++ cmp \$16, $inl ++ vmovdqa $A0x, $A1x ++ jb 1f ++ sub \$16, $inl\n"; ++ &poly_add("0*8($inp)"); ++ &poly_mul(); $code.=" ++ vpxor ($inp), $A0x, $A3x ++ vmovdqu $A3x, ($oup) ++ lea 1*16($inp), $inp ++ lea 1*16($oup), $oup ++ vextracti128 \$1, $A0, $A1x ++1: ++ vzeroupper ++ jmp open_sse_tail_16 ++############################################################################### ++open_avx2_320: ++ vmovdqa $A0, $A1 ++ vmovdqa $A0, $A2 ++ vmovdqa $B0, $B1 ++ vmovdqa $B0, $B2 ++ vmovdqa $C0, $C1 ++ vmovdqa $C0, $C2 ++ vpaddd .avx2_inc(%rip), $D0, $D1 ++ vpaddd .avx2_inc(%rip), $D1, $D2 ++ vmovdqa $B0, $T1 ++ vmovdqa $C0, $T2 ++ vmovdqa $D0, $ctr0_store ++ vmovdqa $D1, $ctr1_store ++ vmovdqa $D2, $ctr2_store ++ mov \$10, $acc0 ++1: \n"; ++ &chacha_qr_avx2($A0,$B0,$C0,$D0,$T0,"left"); ++ &chacha_qr_avx2($A1,$B1,$C1,$D1,$T0,"left"); ++ &chacha_qr_avx2($A2,$B2,$C2,$D2,$T0,"left"); ++ &chacha_qr_avx2($A0,$B0,$C0,$D0,$T0,"right"); ++ &chacha_qr_avx2($A1,$B1,$C1,$D1,$T0,"right"); ++ &chacha_qr_avx2($A2,$B2,$C2,$D2,$T0,"right"); $code.=" ++ dec $acc0 ++ jne 1b ++ vpaddd .chacha20_consts(%rip), $A0, $A0 ++ vpaddd .chacha20_consts(%rip), $A1, $A1 ++ vpaddd .chacha20_consts(%rip), $A2, $A2 ++ vpaddd $T1, $B0, $B0 ++ vpaddd $T1, $B1, $B1 ++ vpaddd $T1, $B2, $B2 ++ vpaddd $T2, $C0, $C0 ++ vpaddd $T2, $C1, $C1 ++ vpaddd $T2, $C2, $C2 ++ vpaddd $ctr0_store, $D0, $D0 ++ vpaddd $ctr1_store, $D1, $D1 ++ vpaddd $ctr2_store, $D2, $D2 ++ vperm2i128 \$0x02, $A0, $B0, $T0 ++ # Clamp and store the key ++ vpand .clamp(%rip), $T0, $T0 ++ vmovdqa $T0, $r_store ++ # Stream for up to 320 bytes ++ vperm2i128 \$0x13, $A0, $B0, $A0 ++ vperm2i128 \$0x13, $C0, $D0, $B0 ++ vperm2i128 \$0x02, $A1, $B1, $C0 ++ vperm2i128 \$0x02, $C1, $D1, $D0 ++ vperm2i128 \$0x13, $A1, $B1, $A1 ++ vperm2i128 \$0x13, $C1, $D1, $B1 ++ vperm2i128 \$0x02, $A2, $B2, $C1 ++ vperm2i128 \$0x02, $C2, $D2, $D1 ++ vperm2i128 \$0x13, $A2, $B2, $A2 ++ vperm2i128 \$0x13, $C2, $D2, $B2 ++ jmp open_avx2_short ++.size chacha20_poly1305_open_avx2, .-chacha20_poly1305_open_avx2 ++############################################################################### ++############################################################################### ++.type chacha20_poly1305_seal_avx2,\@function,2 ++.align 64 ++chacha20_poly1305_seal_avx2: ++ vzeroupper ++ vmovdqa .chacha20_consts(%rip), $A0 ++ vbroadcasti128 0*16($keyp), $B0 ++ vbroadcasti128 1*16($keyp), $C0 ++ vbroadcasti128 2*16($keyp), $D0 ++ vpaddd .avx2_init(%rip), $D0, $D0 ++ cmp \$6*32, $inl ++ jbe seal_avx2_192 ++ cmp \$10*32, $inl ++ jbe seal_avx2_320 ++ vmovdqa $A0, $A1 ++ vmovdqa $A0, $A2 ++ vmovdqa $A0, $A3 ++ vmovdqa $B0, $B1 ++ vmovdqa $B0, $B2 ++ vmovdqa $B0, $B3 ++ vmovdqa $B0, $state1_store ++ vmovdqa $C0, $C1 ++ vmovdqa $C0, $C2 ++ vmovdqa $C0, $C3 ++ vmovdqa $C0, $state2_store ++ vmovdqa $D0, $D3 ++ vpaddd .avx2_inc(%rip), $D3, $D2 ++ vpaddd .avx2_inc(%rip), $D2, $D1 ++ vpaddd .avx2_inc(%rip), $D1, $D0 ++ vmovdqa $D0, $ctr0_store ++ vmovdqa $D1, $ctr1_store ++ vmovdqa $D2, $ctr2_store ++ vmovdqa $D3, $ctr3_store ++ mov \$10, $acc0 ++1: \n"; ++ foreach $l (@loop_body) {$code.=$l."\n";} ++ @loop_body = split /\n/, $chacha_body; $code.=" ++ dec $acc0 ++ jnz 1b\n"; ++ &finalize_state_avx2(4); $code.=" ++ vperm2i128 \$0x13, $C3, $D3, $C3 ++ vperm2i128 \$0x02, $A3, $B3, $D3 ++ vperm2i128 \$0x13, $A3, $B3, $A3 ++ vpand .clamp(%rip), $D3, $D3 ++ vmovdqa $D3, $r_store ++ mov %r8, $itr2 ++ call poly_hash_ad_internal ++ # Safely store 320 bytes (otherwise would handle with optimized call) ++ vpxor 0*32($inp), $A3, $A3 ++ vpxor 1*32($inp), $C3, $C3 ++ vmovdqu $A3, 0*32($oup) ++ vmovdqu $C3, 1*32($oup)\n"; ++ &xor_stream_avx2($A2,$B2,$C2,$D2,2*32,$T3); ++ &xor_stream_avx2($A1,$B1,$C1,$D1,6*32,$T3); ++ &finish_stream_avx2($A0,$B0,$C0,$D0,$T3); $code.=" ++ lea 10*32($inp), $inp ++ sub \$10*32, $inl ++ mov \$10*32, $itr1 ++ cmp \$4*32, $inl ++ jbe seal_avx2_hash ++ vpxor 0*32($inp), $A0, $A0 ++ vpxor 1*32($inp), $B0, $B0 ++ vpxor 2*32($inp), $C0, $C0 ++ vpxor 3*32($inp), $D0, $D0 ++ vmovdqu $A0, 10*32($oup) ++ vmovdqu $B0, 11*32($oup) ++ vmovdqu $C0, 12*32($oup) ++ vmovdqu $D0, 13*32($oup) ++ lea 4*32($inp), $inp ++ sub \$4*32, $inl ++ mov \$8, $itr1 ++ mov \$2, $itr2 ++ cmp \$4*32, $inl ++ jbe seal_avx2_tail_128 ++ cmp \$8*32, $inl ++ jbe seal_avx2_tail_256 ++ cmp \$12*32, $inl ++ jbe seal_avx2_tail_384 ++ cmp \$16*32, $inl ++ jbe seal_avx2_tail_512\n"; ++ # We have 448 bytes to hash, but main loop hashes 512 bytes at a time - perform some rounds, before the main loop ++ &prep_state_avx2(4); ++ foreach $l (@loop_body) {$code.=$l."\n";} ++ @loop_body = split /\n/, $chacha_body; ++ &emit_body(41); ++ @loop_body = split /\n/, $chacha_body; $code.=" ++ sub \$16, $oup ++ mov \$9, $itr1 ++ jmp 4f ++1: \n"; ++ &prep_state_avx2(4); $code.=" ++ mov \$10, $itr1 ++2: \n"; ++ &poly_add("0*8($oup)"); ++ &emit_body(10); ++ &poly_stage1_mulx(); ++ &emit_body(9); ++ &poly_stage2_mulx(); ++ &emit_body(12); ++ &poly_stage3_mulx(); ++ &emit_body(10); ++ &poly_reduce_stage(); $code.=" ++4: \n"; ++ &emit_body(9); ++ &poly_add("2*8($oup)"); ++ &emit_body(8); ++ &poly_stage1_mulx(); ++ &emit_body(18); ++ &poly_stage2_mulx(); ++ &emit_body(18); ++ &poly_stage3_mulx(); ++ &emit_body(9); ++ &poly_reduce_stage(); ++ &emit_body(8); ++ &poly_add("4*8($oup)"); $code.=" ++ lea 6*8($oup), $oup\n"; ++ &emit_body(18); ++ &poly_stage1_mulx(); ++ &emit_body(8); ++ &poly_stage2_mulx(); ++ &emit_body(8); ++ &poly_stage3_mulx(); ++ &emit_body(18); ++ &poly_reduce_stage(); ++ foreach $l (@loop_body) {$code.=$l."\n";} ++ @loop_body = split /\n/, $chacha_body; $code.=" ++ dec $itr1 ++ jne 2b\n"; ++ &finalize_state_avx2(4); $code.=" ++ lea 4*8($oup), $oup ++ vmovdqa $A0, $tmp_store\n"; ++ &poly_add("-4*8($oup)"); ++ &xor_stream_avx2($A3, $B3, $C3, $D3, 0*32, $A0); $code.=" ++ vmovdqa $tmp_store, $A0\n"; ++ &poly_mul(); ++ &xor_stream_avx2($A2, $B2, $C2, $D2, 4*32, $A3); ++ &poly_add("-2*8($oup)"); ++ &xor_stream_avx2($A1, $B1, $C1, $D1, 8*32, $A3); ++ &poly_mul(); ++ &xor_stream_avx2($A0, $B0, $C0, $D0, 12*32, $A3); $code.=" ++ lea 16*32($inp), $inp ++ sub \$16*32, $inl ++ cmp \$16*32, $inl ++ jg 1b\n"; ++ &poly_add("0*8($oup)"); ++ &poly_mul(); ++ &poly_add("2*8($oup)"); ++ &poly_mul(); $code.=" ++ lea 4*8($oup), $oup ++ mov \$10, $itr1 ++ xor $itr2, $itr2 ++ cmp \$4*32, $inl ++ ja 3f ++############################################################################### ++seal_avx2_tail_128:\n"; ++ &prep_state_avx2(1); $code.=" ++1: \n"; ++ &poly_add("0($oup)"); ++ &poly_mul(); $code.=" ++ lea 2*8($oup), $oup ++2: \n"; ++ &chacha_qr_avx2($A0,$B0,$C0,$D0,$T0,"left"); ++ &poly_add("0*8($oup)"); ++ &poly_mul(); ++ &chacha_qr_avx2($A0,$B0,$C0,$D0,$T0,"right"); ++ &poly_add("2*8($oup)"); ++ &poly_mul(); $code.=" ++ lea 4*8($oup), $oup ++ dec $itr1 ++ jg 1b ++ dec $itr2 ++ jge 2b\n"; ++ &finalize_state_avx2(1); ++ &finish_stream_avx2($A0,$B0,$C0,$D0,$T0); $code.=" ++ jmp seal_avx2_short_loop ++3: ++ cmp \$8*32, $inl ++ ja 3f ++############################################################################### ++seal_avx2_tail_256:\n"; ++ &prep_state_avx2(2); $code.=" ++1: \n"; ++ &poly_add("0($oup)"); ++ &poly_mul(); $code.=" ++ lea 2*8($oup), $oup ++2: \n"; ++ &chacha_qr_avx2($A0,$B0,$C0,$D0,$T0,"left"); ++ &chacha_qr_avx2($A1,$B1,$C1,$D1,$T0,"left"); ++ &poly_add("0*8($oup)"); ++ &poly_mul(); ++ &chacha_qr_avx2($A0,$B0,$C0,$D0,$T0,"right"); ++ &chacha_qr_avx2($A1,$B1,$C1,$D1,$T0,"right"); ++ &poly_add("2*8($oup)"); ++ &poly_mul(); $code.=" ++ lea 4*8($oup), $oup ++ dec $itr1 ++ jg 1b ++ dec $itr2 ++ jge 2b\n"; ++ &finalize_state_avx2(2); ++ &xor_stream_avx2($A1,$B1,$C1,$D1,0*32,$T0); ++ &finish_stream_avx2($A0,$B0,$C0,$D0,$T0); $code.=" ++ mov \$4*32, $itr1 ++ lea 4*32($inp), $inp ++ sub \$4*32, $inl ++ jmp seal_avx2_hash ++3: ++ cmp \$12*32, $inl ++ ja seal_avx2_tail_512 ++############################################################################### ++seal_avx2_tail_384:\n"; ++ &prep_state_avx2(3); $code.=" ++1: \n"; ++ &poly_add("0($oup)"); ++ &poly_mul(); $code.=" ++ lea 2*8($oup), $oup ++2: \n"; ++ &chacha_qr_avx2($A0,$B0,$C0,$D0,$T0,"left"); ++ &chacha_qr_avx2($A1,$B1,$C1,$D1,$T0,"left"); ++ &poly_add("0*8($oup)"); ++ &poly_mul(); ++ &chacha_qr_avx2($A2,$B2,$C2,$D2,$T0,"left"); ++ &chacha_qr_avx2($A0,$B0,$C0,$D0,$T0,"right"); ++ &poly_add("2*8($oup)"); ++ &poly_mul(); ++ &chacha_qr_avx2($A1,$B1,$C1,$D1,$T0,"right"); ++ &chacha_qr_avx2($A2,$B2,$C2,$D2,$T0,"right"); $code.=" ++ lea 4*8($oup), $oup ++ dec $itr1 ++ jg 1b ++ dec $itr2 ++ jge 2b\n"; ++ &finalize_state_avx2(3); ++ &xor_stream_avx2($A2,$B2,$C2,$D2,0*32,$T0); ++ &xor_stream_avx2($A1,$B1,$C1,$D1,4*32,$T0); ++ &finish_stream_avx2($A0,$B0,$C0,$D0,$T0); $code.=" ++ mov \$8*32, $itr1 ++ lea 8*32($inp), $inp ++ sub \$8*32, $inl ++ jmp seal_avx2_hash ++############################################################################### ++seal_avx2_tail_512:\n"; ++ &prep_state_avx2(4); $code.=" ++1: \n"; ++ &poly_add("0($oup)"); ++ &poly_mul_mulx(); $code.=" ++ lea 2*8($oup), $oup ++2: \n"; ++ &emit_body(20); ++ &poly_add("0*8($oup)"); ++ &emit_body(20); ++ &poly_stage1_mulx(); ++ &emit_body(20); ++ &poly_stage2_mulx(); ++ &emit_body(20); ++ &poly_stage3_mulx(); ++ &emit_body(20); ++ &poly_reduce_stage(); ++ &emit_body(20); ++ &poly_add("2*8($oup)"); ++ &emit_body(20); ++ &poly_stage1_mulx(); ++ &emit_body(20); ++ &poly_stage2_mulx(); ++ &emit_body(20); ++ &poly_stage3_mulx(); ++ &emit_body(20); ++ &poly_reduce_stage(); ++ foreach $l (@loop_body) {$code.=$l."\n";} ++ @loop_body = split /\n/, $chacha_body; $code.=" ++ lea 4*8($oup), $oup ++ dec $itr1 ++ jg 1b ++ dec $itr2 ++ jge 2b\n"; ++ &finalize_state_avx2(4); $code.=" ++ vmovdqa $A0, $tmp_store\n"; ++ &xor_stream_avx2($A3, $B3, $C3, $D3, 0*32, $A0); $code.=" ++ vmovdqa $tmp_store, $A0\n"; ++ &xor_stream_avx2($A2, $B2, $C2, $D2, 4*32, $A3); ++ &xor_stream_avx2($A1, $B1, $C1, $D1, 8*32, $A3); ++ &finish_stream_avx2($A0,$B0,$C0,$D0,$T0); $code.=" ++ mov \$12*32, $itr1 ++ lea 12*32($inp), $inp ++ sub \$12*32, $inl ++ jmp seal_avx2_hash ++################################################################################ ++seal_avx2_320: ++ vmovdqa $A0, $A1 ++ vmovdqa $A0, $A2 ++ vmovdqa $B0, $B1 ++ vmovdqa $B0, $B2 ++ vmovdqa $C0, $C1 ++ vmovdqa $C0, $C2 ++ vpaddd .avx2_inc(%rip), $D0, $D1 ++ vpaddd .avx2_inc(%rip), $D1, $D2 ++ vmovdqa $B0, $T1 ++ vmovdqa $C0, $T2 ++ vmovdqa $D0, $ctr0_store ++ vmovdqa $D1, $ctr1_store ++ vmovdqa $D2, $ctr2_store ++ mov \$10, $acc0 ++1: \n"; ++ &chacha_qr_avx2($A0,$B0,$C0,$D0,$T0,"left"); ++ &chacha_qr_avx2($A1,$B1,$C1,$D1,$T0,"left"); ++ &chacha_qr_avx2($A2,$B2,$C2,$D2,$T0,"left"); ++ &chacha_qr_avx2($A0,$B0,$C0,$D0,$T0,"right"); ++ &chacha_qr_avx2($A1,$B1,$C1,$D1,$T0,"right"); ++ &chacha_qr_avx2($A2,$B2,$C2,$D2,$T0,"right"); $code.=" ++ dec $acc0 ++ jne 1b ++ vpaddd .chacha20_consts(%rip), $A0, $A0 ++ vpaddd .chacha20_consts(%rip), $A1, $A1 ++ vpaddd .chacha20_consts(%rip), $A2, $A2 ++ vpaddd $T1, $B0, $B0 ++ vpaddd $T1, $B1, $B1 ++ vpaddd $T1, $B2, $B2 ++ vpaddd $T2, $C0, $C0 ++ vpaddd $T2, $C1, $C1 ++ vpaddd $T2, $C2, $C2 ++ vpaddd $ctr0_store, $D0, $D0 ++ vpaddd $ctr1_store, $D1, $D1 ++ vpaddd $ctr2_store, $D2, $D2 ++ vperm2i128 \$0x02, $A0, $B0, $T0 ++ # Clamp and store the key ++ vpand .clamp(%rip), $T0, $T0 ++ vmovdqa $T0, $r_store ++ # Stream for up to 320 bytes ++ vperm2i128 \$0x13, $A0, $B0, $A0 ++ vperm2i128 \$0x13, $C0, $D0, $B0 ++ vperm2i128 \$0x02, $A1, $B1, $C0 ++ vperm2i128 \$0x02, $C1, $D1, $D0 ++ vperm2i128 \$0x13, $A1, $B1, $A1 ++ vperm2i128 \$0x13, $C1, $D1, $B1 ++ vperm2i128 \$0x02, $A2, $B2, $C1 ++ vperm2i128 \$0x02, $C2, $D2, $D1 ++ vperm2i128 \$0x13, $A2, $B2, $A2 ++ vperm2i128 \$0x13, $C2, $D2, $B2 ++ jmp seal_avx2_short ++################################################################################ ++seal_avx2_192: ++ vmovdqa $A0, $A1 ++ vmovdqa $A0, $A2 ++ vmovdqa $B0, $B1 ++ vmovdqa $B0, $B2 ++ vmovdqa $C0, $C1 ++ vmovdqa $C0, $C2 ++ vpaddd .avx2_inc(%rip), $D0, $D1 ++ vmovdqa $D0, $T2 ++ vmovdqa $D1, $T3 ++ mov \$10, $acc0 ++1: \n"; ++ &chacha_qr_avx2($A0,$B0,$C0,$D0,$T0,"left"); ++ &chacha_qr_avx2($A1,$B1,$C1,$D1,$T0,"left"); ++ &chacha_qr_avx2($A0,$B0,$C0,$D0,$T0,"right"); ++ &chacha_qr_avx2($A1,$B1,$C1,$D1,$T0,"right"); $code.=" ++ dec $acc0 ++ jne 1b ++ vpaddd $A2, $A0, $A0 ++ vpaddd $A2, $A1, $A1 ++ vpaddd $B2, $B0, $B0 ++ vpaddd $B2, $B1, $B1 ++ vpaddd $C2, $C0, $C0 ++ vpaddd $C2, $C1, $C1 ++ vpaddd $T2, $D0, $D0 ++ vpaddd $T3, $D1, $D1 ++ vperm2i128 \$0x02, $A0, $B0, $T0 ++ # Clamp and store the key ++ vpand .clamp(%rip), $T0, $T0 ++ vmovdqa $T0, $r_store ++ # Stream for up to 192 bytes ++ vperm2i128 \$0x13, $A0, $B0, $A0 ++ vperm2i128 \$0x13, $C0, $D0, $B0 ++ vperm2i128 \$0x02, $A1, $B1, $C0 ++ vperm2i128 \$0x02, $C1, $D1, $D0 ++ vperm2i128 \$0x13, $A1, $B1, $A1 ++ vperm2i128 \$0x13, $C1, $D1, $B1 ++seal_avx2_short: ++ mov %r8, $itr2 ++ call poly_hash_ad_internal ++ xor $itr1, $itr1 ++seal_avx2_hash: ++ cmp \$16, $itr1 ++ jb seal_avx2_short_loop\n"; ++ &poly_add("0($oup)"); ++ &poly_mul(); $code.=" ++ sub \$16, $itr1 ++ add \$16, $oup ++ jmp seal_avx2_hash ++seal_avx2_short_loop: ++ cmp \$32, $inl ++ jb seal_avx2_short_tail ++ sub \$32, $inl ++ # Encrypt ++ vpxor ($inp), $A0, $A0 ++ vmovdqu $A0, ($oup) ++ lea 1*32($inp), $inp ++ # Load + hash\n"; ++ &poly_add("0*8($oup)"); ++ &poly_mul(); ++ &poly_add("2*8($oup)"); ++ &poly_mul(); $code.=" ++ lea 1*32($oup), $oup ++ # Shift stream ++ vmovdqa $B0, $A0 ++ vmovdqa $C0, $B0 ++ vmovdqa $D0, $C0 ++ vmovdqa $A1, $D0 ++ vmovdqa $B1, $A1 ++ vmovdqa $C1, $B1 ++ vmovdqa $D1, $C1 ++ vmovdqa $A2, $D1 ++ vmovdqa $B2, $A2 ++ jmp seal_avx2_short_loop ++seal_avx2_short_tail: ++ cmp \$16, $inl ++ jb 1f ++ sub \$16, $inl ++ vpxor ($inp), $A0x, $A3x ++ vmovdqu $A3x, ($oup) ++ lea 1*16($inp), $inp\n"; ++ &poly_add("0*8($oup)"); ++ &poly_mul(); $code.=" ++ lea 1*16($oup), $oup ++ vextracti128 \$1, $A0, $A0x ++1: ++ vzeroupper ++ jmp seal_sse_tail_16 ++"; ++} ++ ++$code =~ s/\`([^\`]*)\`/eval $1/gem; ++print $code; ++close STDOUT; +diff --git a/crypto/chacha20_poly1305/asm/chacha20_x86_64.pl b/crypto/chacha20_poly1305/asm/chacha20_x86_64.pl +new file mode 100644 +index 0000000..538af42 +--- /dev/null ++++ b/crypto/chacha20_poly1305/asm/chacha20_x86_64.pl +@@ -0,0 +1,415 @@ ++#!/usr/bin/env perl ++ ++############################################################################## ++# # ++# Copyright 2014 Intel Corporation # ++# # ++# Licensed under the Apache License, Version 2.0 (the "License"); # ++# you may not use this file except in compliance with the License. # ++# You may obtain a copy of the License at # ++# # ++# http://www.apache.org/licenses/LICENSE-2.0 # ++# # ++# Unless required by applicable law or agreed to in writing, software # ++# distributed under the License is distributed on an "AS IS" BASIS, # ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # ++# See the License for the specific language governing permissions and # ++# limitations under the License. # ++# # ++############################################################################## ++# # ++# Developers and authors: # ++# Shay Gueron (1, 2), and Vlad Krasnov (1) # ++# (1) Intel Corporation, Israel Development Center # ++# (2) University of Haifa # ++# # ++# Related work: # ++# M. Goll, S. Gueron, "Vectorization on ChaCha Stream Cipher", IEEE # ++# Proceedings of 11th International Conference on Information # ++# Technology: New Generations (ITNG 2014), 612-615 (2014). # ++# M. Goll, S. Gueron, "Vectorization on Poly1305 Message Authentication Code"# ++# to be published. # ++# A. Langley, chacha20poly1305 for the AEAD head # ++# https://git.openssl.org/gitweb/?p=openssl.git;a=commit;h=9a8646510b3d0a48e950748f7a2aaa12ed40d5e0 # ++############################################################################## ++ ++ ++$flavour = shift; ++$output = shift; ++if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } ++ ++$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); ++ ++$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; ++( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or ++( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or ++die "can't locate x86_64-xlate.pl"; ++ ++open OUT,"| \"$^X\" $xlate $flavour $output"; ++*STDOUT=*OUT; ++ ++if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` ++ =~ /GNU assembler version ([2-9]\.[0-9]+)/) { ++ $avx = ($1>=2.19) + ($1>=2.22); ++} ++ ++if ($win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && ++ `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { ++ $avx = ($1>=2.09) + ($1>=2.10); ++} ++ ++if ($win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && ++ `ml64 2>&1` =~ /Version ([0-9]+)\./) { ++ $avx = ($1>=10) + ($1>=11); ++} ++ ++if (`$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9])\.([0-9]+)/) { ++ my $ver = $2 + $3/100.0; # 3.1->3.01, 3.10->3.10 ++ $avx = ($ver>=3.0) + ($ver>=3.01); ++} ++ ++{ ++ ++my ($rol8, $rol16, $state_cdef, $tmp, ++ $v0, $v1, $v2, $v3, $v4, $v5, $v6, $v7, ++ $v8, $v9, $v10, $v11)=map("%xmm$_",(0..15)); ++ ++sub chacha_qr { ++ ++my ($a,$b,$c,$d)=@_; ++$code.=<<___; ++ paddd $b, $a # a += b ++ pxor $a, $d # d ^= a ++ pshufb $rol16, $d # d <<<= 16 ++ ++ paddd $d, $c # c += d ++ pxor $c, $b # b ^= c ++ ++ movdqa $b, $tmp ++ pslld \$12, $tmp ++ psrld \$20, $b ++ pxor $tmp, $b # b <<<= 12 ++ ++ paddd $b, $a # a += b ++ pxor $a, $d # d ^= a ++ pshufb $rol8, $d # d <<<= 8 ++ ++ paddd $d, $c # c += d ++ pxor $c, $b # b ^= c ++ ++ movdqa $b, $tmp ++ pslld \$7, $tmp ++ psrld \$25, $b ++ pxor $tmp, $b # b <<<= 7 ++___ ++ ++} ++ ++$code.=<<___; ++.text ++.align 16 ++chacha20_consts: ++.byte 'e','x','p','a','n','d',' ','3','2','-','b','y','t','e',' ','k' ++.rol8: ++.byte 3,0,1,2, 7,4,5,6, 11,8,9,10, 15,12,13,14 ++.rol16: ++.byte 2,3,0,1, 6,7,4,5, 10,11,8,9, 14,15,12,13 ++.avxInc: ++.quad 1,0 ++___ ++ ++{ ++my ($out, $in, $in_len, $key_ptr, $nr) ++ =("%rdi", "%rsi", "%rdx", "%rcx", "%r8"); ++ ++$code.=<<___; ++.globl chacha_20_core_asm ++.type chacha_20_core_asm ,\@function,2 ++.align 64 ++chacha_20_core_asm: ++ ++ # Init state ++ movdqa .rol8(%rip), $rol8 ++ movdqa .rol16(%rip), $rol16 ++ movdqu 2*16($key_ptr), $state_cdef ++ ++2: ++ cmp \$3*64, $in_len ++ jb 2f ++ ++ movdqa chacha20_consts(%rip), $v0 ++ movdqu 0*16($key_ptr), $v1 ++ movdqu 1*16($key_ptr), $v2 ++ movdqa $state_cdef, $v3 ++ movdqa $v0, $v4 ++ movdqa $v0, $v8 ++ movdqa $v1, $v5 ++ movdqa $v1, $v9 ++ movdqa $v2, $v6 ++ movdqa $v2, $v10 ++ movdqa $v3, $v7 ++ paddd .avxInc(%rip), $v7 ++ movdqa $v7, $v11 ++ paddd .avxInc(%rip), $v11 ++ ++ mov \$10, $nr ++ ++ 1: ++___ ++ &chacha_qr( $v0, $v1, $v2, $v3); ++ &chacha_qr( $v4, $v5, $v6, $v7); ++ &chacha_qr( $v8, $v9,$v10,$v11); ++$code.=<<___; ++ palignr \$4, $v1, $v1 ++ palignr \$8, $v2, $v2 ++ palignr \$12, $v3, $v3 ++ palignr \$4, $v5, $v5 ++ palignr \$8, $v6, $v6 ++ palignr \$12, $v7, $v7 ++ palignr \$4, $v9, $v9 ++ palignr \$8, $v10, $v10 ++ palignr \$12, $v11, $v11 ++___ ++ &chacha_qr( $v0, $v1, $v2, $v3); ++ &chacha_qr( $v4, $v5, $v6, $v7); ++ &chacha_qr( $v8, $v9,$v10,$v11); ++$code.=<<___; ++ palignr \$12, $v1, $v1 ++ palignr \$8, $v2, $v2 ++ palignr \$4, $v3, $v3 ++ palignr \$12, $v5, $v5 ++ palignr \$8, $v6, $v6 ++ palignr \$4, $v7, $v7 ++ palignr \$12, $v9, $v9 ++ palignr \$8, $v10, $v10 ++ palignr \$4, $v11, $v11 ++ dec $nr ++ ++ jnz 1b ++ paddd chacha20_consts(%rip), $v0 ++ paddd chacha20_consts(%rip), $v4 ++ paddd chacha20_consts(%rip), $v8 ++ ++ movdqu 16*0($key_ptr), $tmp ++ paddd $tmp, $v1 ++ paddd $tmp, $v5 ++ paddd $tmp, $v9 ++ ++ movdqu 16*1($key_ptr), $tmp ++ paddd $tmp, $v2 ++ paddd $tmp, $v6 ++ paddd $tmp, $v10 ++ ++ paddd $state_cdef, $v3 ++ paddq .avxInc(%rip), $state_cdef ++ paddd $state_cdef, $v7 ++ paddq .avxInc(%rip), $state_cdef ++ paddd $state_cdef, $v11 ++ paddq .avxInc(%rip), $state_cdef ++ ++ movdqu 16*0($in), $tmp ++ pxor $tmp, $v0 ++ movdqu 16*1($in), $tmp ++ pxor $tmp, $v1 ++ movdqu 16*2($in), $tmp ++ pxor $tmp, $v2 ++ movdqu 16*3($in), $tmp ++ pxor $tmp, $v3 ++ ++ movdqu $v0, 16*0($out) ++ movdqu $v1, 16*1($out) ++ movdqu $v2, 16*2($out) ++ movdqu $v3, 16*3($out) ++ ++ movdqu 16*4($in), $tmp ++ pxor $tmp, $v4 ++ movdqu 16*5($in), $tmp ++ pxor $tmp, $v5 ++ movdqu 16*6($in), $tmp ++ pxor $tmp, $v6 ++ movdqu 16*7($in), $tmp ++ pxor $tmp, $v7 ++ ++ movdqu $v4, 16*4($out) ++ movdqu $v5, 16*5($out) ++ movdqu $v6, 16*6($out) ++ movdqu $v7, 16*7($out) ++ ++ movdqu 16*8($in), $tmp ++ pxor $tmp, $v8 ++ movdqu 16*9($in), $tmp ++ pxor $tmp, $v9 ++ movdqu 16*10($in), $tmp ++ pxor $tmp, $v10 ++ movdqu 16*11($in), $tmp ++ pxor $tmp, $v11 ++ ++ movdqu $v8, 16*8($out) ++ movdqu $v9, 16*9($out) ++ movdqu $v10, 16*10($out) ++ movdqu $v11, 16*11($out) ++ ++ lea 16*12($in), $in ++ lea 16*12($out), $out ++ sub \$16*12, $in_len ++ ++ jmp 2b ++ ++2: ++ cmp \$2*64, $in_len ++ jb 2f ++ ++ movdqa chacha20_consts(%rip), $v0 ++ movdqa chacha20_consts(%rip), $v4 ++ movdqu 16*0($key_ptr), $v1 ++ movdqu 16*0($key_ptr), $v5 ++ movdqu 16*1($key_ptr), $v2 ++ movdqu 16*1($key_ptr), $v6 ++ movdqa $state_cdef, $v3 ++ movdqa $v3, $v7 ++ paddd .avxInc(%rip), $v7 ++ ++ mov \$10, $nr ++ 1: ++___ ++ &chacha_qr($v0,$v1,$v2,$v3); ++ &chacha_qr($v4,$v5,$v6,$v7); ++$code.=<<___; ++ palignr \$4, $v1, $v1 ++ palignr \$8, $v2, $v2 ++ palignr \$12, $v3, $v3 ++ palignr \$4, $v5, $v5 ++ palignr \$8, $v6, $v6 ++ palignr \$12, $v7, $v7 ++___ ++ &chacha_qr($v0,$v1,$v2,$v3); ++ &chacha_qr($v4,$v5,$v6,$v7); ++$code.=<<___; ++ palignr \$12, $v1, $v1 ++ palignr \$8, $v2, $v2 ++ palignr \$4, $v3, $v3 ++ palignr \$12, $v5, $v5 ++ palignr \$8, $v6, $v6 ++ palignr \$4, $v7, $v7 ++ dec $nr ++ jnz 1b ++ ++ paddd chacha20_consts(%rip), $v0 ++ paddd chacha20_consts(%rip), $v4 ++ ++ movdqu 16*0($key_ptr), $tmp ++ paddd $tmp, $v1 ++ paddd $tmp, $v5 ++ ++ movdqu 16*1($key_ptr), $tmp ++ paddd $tmp, $v2 ++ paddd $tmp, $v6 ++ ++ paddd $state_cdef, $v3 ++ paddq .avxInc(%rip), $state_cdef ++ paddd $state_cdef, $v7 ++ paddq .avxInc(%rip), $state_cdef ++ ++ movdqu 16*0($in), $tmp ++ pxor $tmp, $v0 ++ movdqu 16*1($in), $tmp ++ pxor $tmp, $v1 ++ movdqu 16*2($in), $tmp ++ pxor $tmp, $v2 ++ movdqu 16*3($in), $tmp ++ pxor $tmp, $v3 ++ ++ movdqu $v0, 16*0($out) ++ movdqu $v1, 16*1($out) ++ movdqu $v2, 16*2($out) ++ movdqu $v3, 16*3($out) ++ ++ movdqu 16*4($in), $tmp ++ pxor $tmp, $v4 ++ movdqu 16*5($in), $tmp ++ pxor $tmp, $v5 ++ movdqu 16*6($in), $tmp ++ pxor $tmp, $v6 ++ movdqu 16*7($in), $tmp ++ pxor $tmp, $v7 ++ ++ movdqu $v4, 16*4($out) ++ movdqu $v5, 16*5($out) ++ movdqu $v6, 16*6($out) ++ movdqu $v7, 16*7($out) ++ ++ lea 16*8($in), $in ++ lea 16*8($out), $out ++ sub \$16*8, $in_len ++ ++ jmp 2b ++2: ++ cmp \$64, $in_len ++ jb 2f ++ ++ movdqa chacha20_consts(%rip), $v0 ++ movdqu 16*0($key_ptr), $v1 ++ movdqu 16*1($key_ptr), $v2 ++ movdqa $state_cdef, $v3 ++ ++ mov \$10, $nr ++ ++ 1: ++___ ++ &chacha_qr($v0,$v1,$v2,$v3); ++$code.=<<___; ++ palignr \$4, $v1, $v1 ++ palignr \$8, $v2, $v2 ++ palignr \$12, $v3, $v3 ++___ ++ &chacha_qr($v0,$v1,$v2,$v3); ++$code.=<<___; ++ palignr \$12, $v1, $v1 ++ palignr \$8, $v2, $v2 ++ palignr \$4, $v3, $v3 ++ dec $nr ++ jnz 1b ++ ++ paddd chacha20_consts(%rip), $v0 ++ ++ movdqu 16*0($key_ptr), $tmp ++ paddd $tmp, $v1 ++ ++ movdqu 16*1($key_ptr), $tmp ++ paddd $tmp, $v2 ++ ++ paddd $state_cdef, $v3 ++ paddq .avxInc(%rip), $state_cdef ++ ++ movdqu 16*0($in), $tmp ++ pxor $tmp, $v0 ++ movdqu 16*1($in), $tmp ++ pxor $tmp, $v1 ++ movdqu 16*2($in), $tmp ++ pxor $tmp, $v2 ++ movdqu 16*3($in), $tmp ++ pxor $tmp, $v3 ++ ++ movdqu $v0, 16*0($out) ++ movdqu $v1, 16*1($out) ++ movdqu $v2, 16*2($out) ++ movdqu $v3, 16*3($out) ++ ++ lea 16*4($in), $in ++ lea 16*4($out), $out ++ sub \$16*4, $in_len ++ jmp 2b ++ ++2: ++ movdqu $state_cdef, 16*2($key_ptr) ++ ret ++.size chacha_20_core_asm,.-chacha_20_core_asm ++___ ++} ++} ++ ++$code =~ s/\`([^\`]*)\`/eval($1)/gem; ++ ++print $code; ++ ++close STDOUT; +diff --git a/crypto/chacha20_poly1305/asm/poly1305_x86_64.pl b/crypto/chacha20_poly1305/asm/poly1305_x86_64.pl +new file mode 100644 +index 0000000..05e4bc5 +--- /dev/null ++++ b/crypto/chacha20_poly1305/asm/poly1305_x86_64.pl +@@ -0,0 +1,280 @@ ++############################################################################## ++# # ++# Copyright 2016 CloudFlare LTD # ++# # ++# Licensed under the Apache License, Version 2.0 (the "License"); # ++# you may not use this file except in compliance with the License. # ++# You may obtain a copy of the License at # ++# # ++# http://www.apache.org/licenses/LICENSE-2.0 # ++# # ++# Unless required by applicable law or agreed to in writing, software # ++# distributed under the License is distributed on an "AS IS" BASIS, # ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # ++# See the License for the specific language governing permissions and # ++# limitations under the License. # ++# # ++############################################################################## ++# # ++# Author: Vlad Krasnov # ++# # ++############################################################################## ++ ++$flavour = shift; ++$output = shift; ++if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } ++ ++$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); ++ ++$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; ++( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or ++( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or ++die "can't locate x86_64-xlate.pl"; ++ ++open OUT,"| \"$^X\" $xlate $flavour $output"; ++*STDOUT=*OUT; ++ ++if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` ++ =~ /GNU assembler version ([2-9]\.[0-9]+)/) { ++ $avx = ($1>=2.19) + ($1>=2.22); ++} ++ ++if ($win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && ++ `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { ++ $avx = ($1>=2.09) + ($1>=2.10); ++} ++ ++if ($win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && ++ `ml64 2>&1` =~ /Version ([0-9]+)\./) { ++ $avx = ($1>=10) + ($1>=11); ++} ++ ++if (`$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9])\.([0-9]+)/) { ++ my $ver = $2 + $3/100.0; # 3.1->3.01, 3.10->3.10 ++ $avx = ($ver>=3.0) + ($ver>=3.01); ++} ++ ++ ++{ ++{ ++ ++my ($state, $key) ++ =("%rdi", "%rsi"); ++ ++$code.=<<___; ++ ++.LrSet: ++.align 16 ++.quad 0x0FFFFFFC0FFFFFFF, 0x0FFFFFFC0FFFFFFC ++############################################################################### ++# void poly1305_init_x64(void *state, uint8_t key[32]) ++ ++.globl poly1305_init_x64 ++.type poly1305_init_x64, \@function, 2 ++.align 64 ++poly1305_init_x64: ++ ++ xor %rax, %rax ++ mov %rax, 8*0($state) ++ mov %rax, 8*1($state) ++ mov %rax, 8*2($state) ++ ++ movdqu 16*0($key), %xmm0 ++ movdqu 16*1($key), %xmm1 ++ pand .LrSet(%rip), %xmm0 ++ ++ movdqu %xmm0, 8*3($state) ++ movdqu %xmm1, 8*3+16($state) ++ movq \$0, 8*7($state) ++ ++ ret ++.size poly1305_init_x64,.-poly1305_init_x64 ++___ ++} ++ ++{ ++ ++my ($state, $inp) ++ =("%rdi", "%rsi"); ++ ++my ($acc0, $acc1, $acc2, $inl, $t0, $t1, $t2, $t3, $r0) ++ =("%rcx", "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15"); ++ ++my ($r1) ++ =("8*4($state)"); ++ ++$code.=<<___; ++############################################################################### ++# void* poly1305_update_x64(void* state, void* in, uint64_t in_len) ++.globl poly1305_update_x64 ++.type poly1305_update_x64, \@function, 2 ++.align 64 ++poly1305_update_x64: ++ ++ push %r11 ++ push %r12 ++ push %r13 ++ push %r14 ++ push %r15 ++ ++ mov %rdx, $inl ++ ++ mov 8*0($state), $acc0 ++ mov 8*1($state), $acc1 ++ mov 8*2($state), $acc2 ++ mov 8*3($state), $r0 ++ ++ cmp \$16, $inl ++ jb 2f ++ jmp 1f ++ ++.align 64 ++1: ++############################ ++ add 8*0($inp), $acc0 ++ adc 8*1($inp), $acc1 ++ lea 16($inp), $inp ++ adc \$1, $acc2 ++ ++5: ++ mov $r0, %rax ++ mulq $acc0 ++ mov %rax, $t0 ++ mov %rdx, $t1 ++ ++ mov $r0, %rax ++ mulq $acc1 ++ add %rax, $t1 ++ adc \$0, %rdx ++ ++ mov $r0, $t2 ++ imul $acc2, $t2 ++ add %rdx, $t2 ++############################ ++ mov $r1, %rax ++ mulq $acc0 ++ add %rax, $t1 ++ adc \$0, %rdx ++ mov %rdx, $acc0 ++ ++ mov $r1, %rax ++ mulq $acc1 ++ add $acc0, $t2 ++ adc \$0, %rdx ++ add %rax, $t2 ++ adc \$0, %rdx ++ ++ mov $r1, $t3 ++ imul $acc2, $t3 ++ add %rdx, $t3 ++############################ ++ ++ mov $t0, $acc0 ++ mov $t1, $acc1 ++ mov $t2, $acc2 ++ and \$3, $acc2 ++ ++ mov $t2, $t0 ++ mov $t3, $t1 ++ ++ and \$-4, $t0 ++ shrd \$2, $t3, $t2 ++ shr \$2, $t3 ++ ++ add $t0, $acc0 ++ adc $t1, $acc1 ++ adc \$0, $acc2 ++ ++ add $t2, $acc0 ++ adc $t3, $acc1 ++ adc \$0, $acc2 ++ ++ sub \$16, $inl ++ cmp \$16, $inl ++ jae 1b ++ ++2: ++ test $inl, $inl ++ jz 3f ++ ++ mov \$1, $t0 ++ xor $t1, $t1 ++ xor $t2, $t2 ++ add $inl, $inp ++ ++4: ++ shld \$8, $t0, $t1 ++ shl \$8, $t0 ++ movzxb -1($inp), $t2 ++ xor $t2, $t0 ++ dec $inp ++ dec $inl ++ jnz 4b ++ ++ add $t0, $acc0 ++ adc $t1, $acc1 ++ adc \$0, $acc2 ++ ++ mov \$16, $inl ++ jmp 5b ++ ++3: ++ ++ mov $acc0, 8*0($state) ++ mov $acc1, 8*1($state) ++ mov $acc2, 8*2($state) ++ ++ pop %r15 ++ pop %r14 ++ pop %r13 ++ pop %r12 ++ pop %r11 ++ ret ++.size poly1305_update_x64, .-poly1305_update_x64 ++___ ++} ++ ++{ ++ ++my ($mac, $state)=("%rsi", "%rdi"); ++ ++my ($acc0, $acc1, $acc2, $t0, $t1, $t2) ++ =("%rcx", "%rax", "%rdx", "%r8", "%r9", "%r10"); ++ ++$code.=<<___; ++############################################################################### ++# void poly1305_finish_x64(void* state, uint64_t mac[2]); ++.type poly1305_finish_x64,\@function, 2 ++.align 64 ++.globl poly1305_finish_x64 ++poly1305_finish_x64: ++ ++ mov 8*0($state), $acc0 ++ mov 8*1($state), $acc1 ++ mov 8*2($state), $acc2 ++ ++ mov $acc0, $t0 ++ mov $acc1, $t1 ++ mov $acc2, $t2 ++ ++ sub \$-5, $acc0 ++ sbb \$-1, $acc1 ++ sbb \$3, $acc2 ++ ++ cmovc $t0, $acc0 ++ cmovc $t1, $acc1 ++ cmovc $t2, $acc2 ++ ++ add 8*5($state), $acc0 ++ adc 8*6($state), $acc1 ++ mov $acc0, ($mac) ++ mov $acc1, 8($mac) ++ ++ ret ++.size poly1305_finish_x64, .-poly1305_finish_x64 ++___ ++} ++} ++$code =~ s/\`([^\`]*)\`/eval($1)/gem; ++print $code; ++close STDOUT; +diff --git a/crypto/chacha20_poly1305/chacha20.c b/crypto/chacha20_poly1305/chacha20.c +new file mode 100644 +index 0000000..b48d857 +--- /dev/null ++++ b/crypto/chacha20_poly1305/chacha20.c +@@ -0,0 +1,142 @@ ++/* Copyright (c) 2014, Google Inc. ++ * ++ * Permission to use, copy, modify, and/or distribute this software for any ++ * purpose with or without fee is hereby granted, provided that the above ++ * copyright notice and this permission notice appear in all copies. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES ++ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF ++ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY ++ * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES ++ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION ++ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN ++ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ ++ ++/* Adapted from the public domain, estream code by D. Bernstein. */ ++ ++#include "chacha20poly1305.h" ++ ++/* sigma contains the ChaCha constants, which happen to be an ASCII string. */ ++static const char sigma[16] = "expand 32-byte k"; ++ ++#define ROTATE(v, n) (((v) << (n)) | ((v) >> (32 - (n)))) ++#define XOR(v, w) ((v) ^ (w)) ++#define PLUS(x, y) ((x) + (y)) ++#define PLUSONE(v) (PLUS((v), 1)) ++ ++#define U32TO8_LITTLE(p, v) \ ++ { \ ++ (p)[0] = (v >> 0) & 0xff; \ ++ (p)[1] = (v >> 8) & 0xff; \ ++ (p)[2] = (v >> 16) & 0xff; \ ++ (p)[3] = (v >> 24) & 0xff; \ ++ } ++ ++#define U8TO32_LITTLE(p) \ ++ (((uint32_t)((p)[0])) | ((uint32_t)((p)[1]) << 8) | \ ++ ((uint32_t)((p)[2]) << 16) | ((uint32_t)((p)[3]) << 24)) ++ ++/* QUARTERROUND updates a, b, c, d with a ChaCha "quarter" round. */ ++#define QUARTERROUND(a,b,c,d) \ ++ x[a] = PLUS(x[a],x[b]); x[d] = ROTATE(XOR(x[d],x[a]),16); \ ++ x[c] = PLUS(x[c],x[d]); x[b] = ROTATE(XOR(x[b],x[c]),12); \ ++ x[a] = PLUS(x[a],x[b]); x[d] = ROTATE(XOR(x[d],x[a]), 8); \ ++ x[c] = PLUS(x[c],x[d]); x[b] = ROTATE(XOR(x[b],x[c]), 7); ++ ++/* chacha_core performs |num_rounds| rounds of ChaCha20 on the input words in ++ * |input| and writes the 64 output bytes to |output|. */ ++static void chacha_core(uint8_t output[64], const uint32_t input[16]) { ++ uint32_t x[16]; ++ int i; ++ ++ memcpy(x, input, sizeof(uint32_t) * 16); ++ for (i = 20; i > 0; i -= 2) { ++ QUARTERROUND(0, 4, 8, 12) ++ QUARTERROUND(1, 5, 9, 13) ++ QUARTERROUND(2, 6, 10, 14) ++ QUARTERROUND(3, 7, 11, 15) ++ QUARTERROUND(0, 5, 10, 15) ++ QUARTERROUND(1, 6, 11, 12) ++ QUARTERROUND(2, 7, 8, 13) ++ QUARTERROUND(3, 4, 9, 14) ++ } ++ ++ for (i = 0; i < 16; ++i) { ++ x[i] = PLUS(x[i], input[i]); ++ } ++ for (i = 0; i < 16; ++i) { ++ U32TO8_LITTLE(output + 4 * i, x[i]); ++ } ++} ++ ++#if CHAPOLY_ASM ++void chacha_20_core_asm(uint8_t *out, const uint8_t *in, size_t in_len, ++ uint8_t nonce[48]); ++#endif ++ ++void CRYPTO_chacha_20(uint8_t *out, const uint8_t *in, size_t in_len, ++ uint8_t nonce[48]) { ++ ++ uint8_t buf[64]; ++ uint32_t input[16]; ++ size_t todo, i; ++ ++#ifdef CHAPOLY_ASM ++ chacha_20_core_asm(out, in, in_len, nonce); ++ todo = in_len & (63); ++ ++ if(todo) { ++ out += in_len - todo; ++ in += in_len - todo; ++ memcpy(buf, in, todo); ++ ++ chacha_20_core_asm(buf, buf, sizeof(buf), nonce); ++ ++ memcpy(out, buf, todo); ++ memset(buf, 0, sizeof(buf)); ++ } ++ return; ++#endif ++ ++ input[0] = U8TO32_LITTLE(sigma + 0); ++ input[1] = U8TO32_LITTLE(sigma + 4); ++ input[2] = U8TO32_LITTLE(sigma + 8); ++ input[3] = U8TO32_LITTLE(sigma + 12); ++ ++ input[4] = U8TO32_LITTLE(nonce + 0); ++ input[5] = U8TO32_LITTLE(nonce + 4); ++ input[6] = U8TO32_LITTLE(nonce + 8); ++ input[7] = U8TO32_LITTLE(nonce + 12); ++ ++ input[8] = U8TO32_LITTLE(nonce + 16); ++ input[9] = U8TO32_LITTLE(nonce + 20); ++ input[10] = U8TO32_LITTLE(nonce + 24); ++ input[11] = U8TO32_LITTLE(nonce + 28); ++ ++ input[12] = U8TO32_LITTLE(nonce + 32); ++ input[13] = U8TO32_LITTLE(nonce + 36); ++ input[14] = U8TO32_LITTLE(nonce + 40); ++ input[15] = U8TO32_LITTLE(nonce + 44); ++ ++ while (in_len > 0) { ++ todo = 64; ++ if (in_len < todo) { ++ todo = in_len; ++ } ++ ++ chacha_core(buf, input); ++ for (i = 0; i < todo; i++) { ++ out[i] = in[i] ^ buf[i]; ++ } ++ ++ out += todo; ++ in += todo; ++ in_len -= todo; ++ ++ ((uint64_t*)input)[6]++; ++ } ++ ++ U32TO8_LITTLE(nonce + 32, input[12]); ++ U32TO8_LITTLE(nonce + 36, input[13]); ++} ++ +diff --git a/crypto/chacha20_poly1305/chacha20poly1305.h b/crypto/chacha20_poly1305/chacha20poly1305.h +new file mode 100644 +index 0000000..3968c40 +--- /dev/null ++++ b/crypto/chacha20_poly1305/chacha20poly1305.h +@@ -0,0 +1,64 @@ ++/* Copyright (c) 2014, Google Inc. ++ * ++ * Permission to use, copy, modify, and/or distribute this software for any ++ * purpose with or without fee is hereby granted, provided that the above ++ * copyright notice and this permission notice appear in all copies. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES ++ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF ++ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY ++ * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES ++ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION ++ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN ++ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ ++ ++#ifndef OPENSSL_HEADER_POLY1305_H ++#define OPENSSL_HEADER_POLY1305_H ++ ++#include ++#include ++#include ++#include "crypto.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++#define POLY1305_MAC_LEN (16) ++#define POLY1305_PAD_LEN (16) ++ ++typedef unsigned char poly1305_state[92]; ++ ++ ++/* CRYPTO_poly1305_init sets up |state| so that it can be used to calculate an ++ * authentication tag with the one-time key |key|. Note that |key| is a ++ * one-time key and therefore there is no `reset' method because that would ++ * enable several messages to be authenticated with the same key. */ ++void CRYPTO_poly1305_init(poly1305_state* state, const uint8_t key[32]); ++ ++/* CRYPTO_poly1305_update processes |in_len| bytes from |in|. It can be called ++ * zero or more times after poly1305_init. */ ++void CRYPTO_poly1305_update(poly1305_state* state, const uint8_t* in, ++ size_t in_len); ++ ++/* CRYPTO_poly1305_finish completes the poly1305 calculation and writes a 16 ++ * byte authentication tag to |mac|. */ ++void CRYPTO_poly1305_finish(poly1305_state* state, ++ uint8_t mac[POLY1305_MAC_LEN]); ++ ++/* CRYPTO_chacha_20 encrypts |in_len| bytes from |in| with the given key and ++ * nonce and writes the result to |out|, which may be equal to |in|. The ++ * initial block counter is specified by |counter|. */ ++void CRYPTO_chacha_20(uint8_t *out, const uint8_t *in, size_t in_len, ++ uint8_t nonce[48]); ++ ++#if CHAPOLY_ASM ++int chacha20_poly1305_open(uint8_t *pt, const uint8_t *ct, size_t len_in, uint8_t *ad, size_t len_ad, uint8_t *key); ++void chacha20_poly1305_seal(uint8_t *ct, const uint8_t *pt, size_t len_in, uint8_t *ad, size_t len_ad, uint8_t *key); ++#endif ++ ++#if defined(__cplusplus) ++} /* extern C */ ++#endif ++ ++#endif /* OPENSSL_HEADER_POLY1305_H */ +diff --git a/crypto/chacha20_poly1305/poly1305.c b/crypto/chacha20_poly1305/poly1305.c +new file mode 100644 +index 0000000..6bd553b +--- /dev/null ++++ b/crypto/chacha20_poly1305/poly1305.c +@@ -0,0 +1,355 @@ ++/* Copyright (c) 2014, Google Inc. ++ * ++ * Permission to use, copy, modify, and/or distribute this software for any ++ * purpose with or without fee is hereby granted, provided that the above ++ * copyright notice and this permission notice appear in all copies. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES ++ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF ++ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY ++ * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES ++ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION ++ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN ++ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ ++ ++/* This implementation of poly1305 is by Andrew Moon ++ * (https://github.com/floodyberry/poly1305-donna) and released as public ++ * domain. */ ++ ++#include "chacha20poly1305.h" ++ ++#include ++#ifndef CHAPOLY_ASM ++ ++#if !defined(B_ENDIAN) ++/* We can assume little-endian. */ ++static uint32_t U8TO32_LE(const uint8_t *m) { ++ uint32_t r; ++ memcpy(&r, m, sizeof(r)); ++ return r; ++} ++ ++static void U32TO8_LE(uint8_t *m, uint32_t v) { memcpy(m, &v, sizeof(v)); } ++#else ++static uint32_t U8TO32_LE(const uint8_t *m) { ++ return (uint32_t)m[0] | (uint32_t)m[1] << 8 | (uint32_t)m[2] << 16 | ++ (uint32_t)m[3] << 24; ++} ++ ++static void U32TO8_LE(uint8_t *m, uint32_t v) { ++ m[0] = v; ++ m[1] = v >> 8; ++ m[2] = v >> 16; ++ m[3] = v >> 24; ++} ++#endif ++ ++static uint64_t mul32x32_64(uint32_t a, uint32_t b) { return (uint64_t)a * b; } ++ ++struct poly1305_state_st { ++ uint32_t r0, r1, r2, r3, r4; ++ uint32_t s1, s2, s3, s4; ++ uint32_t h0, h1, h2, h3, h4; ++ uint8_t buf[16]; ++ unsigned int buf_used; ++ uint8_t key[16]; ++}; ++ ++/* poly1305_blocks updates |state| given some amount of input data. This ++ * function may only be called with a |len| that is not a multiple of 16 at the ++ * end of the data. Otherwise the input must be buffered into 16 byte blocks. */ ++static void poly1305_update(struct poly1305_state_st *state, const uint8_t *in, ++ size_t len) { ++ uint32_t t0, t1, t2, t3; ++ uint64_t t[5]; ++ uint32_t b; ++ uint64_t c; ++ size_t j; ++ uint8_t mp[16]; ++ ++ if (len < 16) { ++ goto poly1305_donna_atmost15bytes; ++ } ++ ++poly1305_donna_16bytes: ++ t0 = U8TO32_LE(in); ++ t1 = U8TO32_LE(in + 4); ++ t2 = U8TO32_LE(in + 8); ++ t3 = U8TO32_LE(in + 12); ++ ++ in += 16; ++ len -= 16; ++ ++ state->h0 += t0 & 0x3ffffff; ++ state->h1 += ((((uint64_t)t1 << 32) | t0) >> 26) & 0x3ffffff; ++ state->h2 += ((((uint64_t)t2 << 32) | t1) >> 20) & 0x3ffffff; ++ state->h3 += ((((uint64_t)t3 << 32) | t2) >> 14) & 0x3ffffff; ++ state->h4 += (t3 >> 8) | (1 << 24); ++ ++poly1305_donna_mul: ++ t[0] = mul32x32_64(state->h0, state->r0) + mul32x32_64(state->h1, state->s4) + ++ mul32x32_64(state->h2, state->s3) + mul32x32_64(state->h3, state->s2) + ++ mul32x32_64(state->h4, state->s1); ++ t[1] = mul32x32_64(state->h0, state->r1) + mul32x32_64(state->h1, state->r0) + ++ mul32x32_64(state->h2, state->s4) + mul32x32_64(state->h3, state->s3) + ++ mul32x32_64(state->h4, state->s2); ++ t[2] = mul32x32_64(state->h0, state->r2) + mul32x32_64(state->h1, state->r1) + ++ mul32x32_64(state->h2, state->r0) + mul32x32_64(state->h3, state->s4) + ++ mul32x32_64(state->h4, state->s3); ++ t[3] = mul32x32_64(state->h0, state->r3) + mul32x32_64(state->h1, state->r2) + ++ mul32x32_64(state->h2, state->r1) + mul32x32_64(state->h3, state->r0) + ++ mul32x32_64(state->h4, state->s4); ++ t[4] = mul32x32_64(state->h0, state->r4) + mul32x32_64(state->h1, state->r3) + ++ mul32x32_64(state->h2, state->r2) + mul32x32_64(state->h3, state->r1) + ++ mul32x32_64(state->h4, state->r0); ++ ++ state->h0 = (uint32_t)t[0] & 0x3ffffff; ++ c = (t[0] >> 26); ++ t[1] += c; ++ state->h1 = (uint32_t)t[1] & 0x3ffffff; ++ b = (uint32_t)(t[1] >> 26); ++ t[2] += b; ++ state->h2 = (uint32_t)t[2] & 0x3ffffff; ++ b = (uint32_t)(t[2] >> 26); ++ t[3] += b; ++ state->h3 = (uint32_t)t[3] & 0x3ffffff; ++ b = (uint32_t)(t[3] >> 26); ++ t[4] += b; ++ state->h4 = (uint32_t)t[4] & 0x3ffffff; ++ b = (uint32_t)(t[4] >> 26); ++ state->h0 += b * 5; ++ ++ if (len >= 16) ++ goto poly1305_donna_16bytes; ++ ++/* final bytes */ ++poly1305_donna_atmost15bytes: ++ if (!len) ++ return; ++ ++ for (j = 0; j < len; j++) ++ mp[j] = in[j]; ++ mp[j++] = 1; ++ for (; j < 16; j++) ++ mp[j] = 0; ++ len = 0; ++ ++ t0 = U8TO32_LE(mp + 0); ++ t1 = U8TO32_LE(mp + 4); ++ t2 = U8TO32_LE(mp + 8); ++ t3 = U8TO32_LE(mp + 12); ++ ++ state->h0 += t0 & 0x3ffffff; ++ state->h1 += ((((uint64_t)t1 << 32) | t0) >> 26) & 0x3ffffff; ++ state->h2 += ((((uint64_t)t2 << 32) | t1) >> 20) & 0x3ffffff; ++ state->h3 += ((((uint64_t)t3 << 32) | t2) >> 14) & 0x3ffffff; ++ state->h4 += (t3 >> 8); ++ ++ goto poly1305_donna_mul; ++} ++ ++void CRYPTO_poly1305_init(poly1305_state *statep, const uint8_t key[32]) { ++ struct poly1305_state_st *state = (struct poly1305_state_st *)statep; ++ uint32_t t0, t1, t2, t3; ++ ++ t0 = U8TO32_LE(key + 0); ++ t1 = U8TO32_LE(key + 4); ++ t2 = U8TO32_LE(key + 8); ++ t3 = U8TO32_LE(key + 12); ++ ++ /* precompute multipliers */ ++ state->r0 = t0 & 0x3ffffff; ++ t0 >>= 26; ++ t0 |= t1 << 6; ++ state->r1 = t0 & 0x3ffff03; ++ t1 >>= 20; ++ t1 |= t2 << 12; ++ state->r2 = t1 & 0x3ffc0ff; ++ t2 >>= 14; ++ t2 |= t3 << 18; ++ state->r3 = t2 & 0x3f03fff; ++ t3 >>= 8; ++ state->r4 = t3 & 0x00fffff; ++ ++ state->s1 = state->r1 * 5; ++ state->s2 = state->r2 * 5; ++ state->s3 = state->r3 * 5; ++ state->s4 = state->r4 * 5; ++ ++ /* init state */ ++ state->h0 = 0; ++ state->h1 = 0; ++ state->h2 = 0; ++ state->h3 = 0; ++ state->h4 = 0; ++ ++ state->buf_used = 0; ++ memcpy(state->key, key + 16, sizeof(state->key)); ++} ++ ++void CRYPTO_poly1305_update(poly1305_state *statep, const uint8_t *in, ++ size_t in_len) { ++ unsigned int i; ++ struct poly1305_state_st *state = (struct poly1305_state_st *)statep; ++ ++ if (state->buf_used) { ++ unsigned int todo = 16 - state->buf_used; ++ if (todo > in_len) ++ todo = in_len; ++ for (i = 0; i < todo; i++) ++ state->buf[state->buf_used + i] = in[i]; ++ state->buf_used += todo; ++ in_len -= todo; ++ in += todo; ++ ++ if (state->buf_used == 16) { ++ poly1305_update(state, state->buf, 16); ++ state->buf_used = 0; ++ } ++ } ++ ++ if (in_len >= 16) { ++ size_t todo = in_len & ~0xf; ++ poly1305_update(state, in, todo); ++ in += todo; ++ in_len &= 0xf; ++ } ++ ++ if (in_len) { ++ for (i = 0; i < in_len; i++) ++ state->buf[i] = in[i]; ++ state->buf_used = in_len; ++ } ++} ++ ++void CRYPTO_poly1305_finish(poly1305_state *statep, uint8_t mac[16]) { ++ struct poly1305_state_st *state = (struct poly1305_state_st *)statep; ++ ++ uint64_t f0, f1, f2, f3; ++ uint32_t g0, g1, g2, g3, g4; ++ uint32_t b, nb; ++ ++ if (state->buf_used) ++ poly1305_update(state, state->buf, state->buf_used); ++ ++ b = state->h0 >> 26; ++ state->h0 = state->h0 & 0x3ffffff; ++ state->h1 += b; ++ b = state->h1 >> 26; ++ state->h1 = state->h1 & 0x3ffffff; ++ state->h2 += b; ++ b = state->h2 >> 26; ++ state->h2 = state->h2 & 0x3ffffff; ++ state->h3 += b; ++ b = state->h3 >> 26; ++ state->h3 = state->h3 & 0x3ffffff; ++ state->h4 += b; ++ b = state->h4 >> 26; ++ state->h4 = state->h4 & 0x3ffffff; ++ state->h0 += b * 5; ++ ++ g0 = state->h0 + 5; ++ b = g0 >> 26; ++ g0 &= 0x3ffffff; ++ g1 = state->h1 + b; ++ b = g1 >> 26; ++ g1 &= 0x3ffffff; ++ g2 = state->h2 + b; ++ b = g2 >> 26; ++ g2 &= 0x3ffffff; ++ g3 = state->h3 + b; ++ b = g3 >> 26; ++ g3 &= 0x3ffffff; ++ g4 = state->h4 + b - (1 << 26); ++ ++ b = (g4 >> 31) - 1; ++ nb = ~b; ++ state->h0 = (state->h0 & nb) | (g0 & b); ++ state->h1 = (state->h1 & nb) | (g1 & b); ++ state->h2 = (state->h2 & nb) | (g2 & b); ++ state->h3 = (state->h3 & nb) | (g3 & b); ++ state->h4 = (state->h4 & nb) | (g4 & b); ++ ++ f0 = ((state->h0) | (state->h1 << 26)) + (uint64_t)U8TO32_LE(&state->key[0]); ++ f1 = ((state->h1 >> 6) | (state->h2 << 20)) + ++ (uint64_t)U8TO32_LE(&state->key[4]); ++ f2 = ((state->h2 >> 12) | (state->h3 << 14)) + ++ (uint64_t)U8TO32_LE(&state->key[8]); ++ f3 = ((state->h3 >> 18) | (state->h4 << 8)) + ++ (uint64_t)U8TO32_LE(&state->key[12]); ++ ++ U32TO8_LE(&mac[0], f0); ++ f1 += (f0 >> 32); ++ U32TO8_LE(&mac[4], f1); ++ f2 += (f1 >> 32); ++ U32TO8_LE(&mac[8], f2); ++ f3 += (f2 >> 32); ++ U32TO8_LE(&mac[12], f3); ++} ++ ++#else ++ ++struct poly1305_state_st { ++ uint8_t opaque[8*8]; ++ uint8_t buf[16]; ++ unsigned int buf_used; ++}; ++ ++void poly1305_init_x64(struct poly1305_state_st* state, const uint8_t key[32]); ++void poly1305_update_x64(struct poly1305_state_st* state, const uint8_t *in, size_t in_len); ++void poly1305_finish_x64(struct poly1305_state_st* state, uint8_t mac[16]); ++ ++#define poly1305_update poly1305_update_x64 ++ ++void CRYPTO_poly1305_init(poly1305_state *statep, const uint8_t key[32]) { ++ struct poly1305_state_st *state = (struct poly1305_state_st *)statep; ++ state->buf_used = 0; ++ return poly1305_init_x64(state, key); ++} ++ ++void CRYPTO_poly1305_update(poly1305_state *statep, const uint8_t *in, ++ size_t in_len) { ++ struct poly1305_state_st *state = (struct poly1305_state_st *)statep; ++ int todo; ++ /* Attempt to fill as many bytes as possible before calling the update ++ function */ ++ if (in_len < 16 || state->buf_used) { ++ todo = 16 - state->buf_used; ++ todo = in_len < todo ? in_len : todo; ++ memcpy(state->buf + state->buf_used, in, todo); ++ state->buf_used += todo; ++ in += todo; ++ in_len -= todo; ++ ++ if (state->buf_used == 16) { ++ poly1305_update_x64(state, state->buf, 16); ++ state->buf_used = 0; ++ } ++ } ++ ++ if (in_len >= 16) { ++ poly1305_update_x64(state, in, in_len & (-16)); ++ in += in_len & (-16); ++ in_len &= (15); ++ } ++ ++ if (in_len) { ++ memcpy(state->buf, in, in_len); ++ state->buf_used = in_len; ++ } ++} ++ ++void CRYPTO_poly1305_finish(poly1305_state *statep, uint8_t mac[16]) { ++ struct poly1305_state_st *state = (struct poly1305_state_st *)statep; ++ ++ if (state->buf_used) { ++ if (state->buf_used % POLY1305_PAD_LEN) { ++ memset(state->buf + state->buf_used, 0, ++ POLY1305_PAD_LEN - (state->buf_used % POLY1305_PAD_LEN)); ++ } ++ poly1305_update_x64(state, state->buf, state->buf_used); ++ } ++ ++ poly1305_finish_x64(state, mac); ++} ++#endif +diff --git a/crypto/evp/Makefile b/crypto/evp/Makefile +index fa138d0..c87896b 100644 +--- a/crypto/evp/Makefile ++++ b/crypto/evp/Makefile +@@ -29,7 +29,8 @@ LIBSRC= encode.c digest.c evp_enc.c evp_key.c evp_acnf.c evp_cnf.c \ + c_all.c c_allc.c c_alld.c evp_lib.c bio_ok.c \ + evp_pkey.c evp_pbe.c p5_crpt.c p5_crpt2.c \ + e_old.c pmeth_lib.c pmeth_fn.c pmeth_gn.c m_sigver.c \ +- e_aes_cbc_hmac_sha1.c e_aes_cbc_hmac_sha256.c e_rc4_hmac_md5.c ++ e_aes_cbc_hmac_sha1.c e_aes_cbc_hmac_sha256.c e_rc4_hmac_md5.c \ ++ e_chacha20_poly1305.c + + LIBOBJ= encode.o digest.o evp_enc.o evp_key.o evp_acnf.o evp_cnf.o \ + e_des.o e_bf.o e_idea.o e_des3.o e_camellia.o\ +@@ -42,7 +43,8 @@ LIBOBJ= encode.o digest.o evp_enc.o evp_key.o evp_acnf.o evp_cnf.o \ + c_all.o c_allc.o c_alld.o evp_lib.o bio_ok.o \ + evp_pkey.o evp_pbe.o p5_crpt.o p5_crpt2.o \ + e_old.o pmeth_lib.o pmeth_fn.o pmeth_gn.o m_sigver.o \ +- e_aes_cbc_hmac_sha1.o e_aes_cbc_hmac_sha256.o e_rc4_hmac_md5.o ++ e_aes_cbc_hmac_sha1.o e_aes_cbc_hmac_sha256.o e_rc4_hmac_md5.o \ ++ e_chacha20_poly1305.o + + SRC= $(LIBSRC) + +@@ -793,3 +795,5 @@ pmeth_lib.o: ../../include/openssl/sha.h ../../include/openssl/stack.h + pmeth_lib.o: ../../include/openssl/symhacks.h ../../include/openssl/x509.h + pmeth_lib.o: ../../include/openssl/x509_vfy.h ../asn1/asn1_locl.h ../cryptlib.h + pmeth_lib.o: evp_locl.h pmeth_lib.c ++e_chacha20_poly1305.o: ../../include/openssl/chacha20poly1305.h ++e_chacha20_poly1305.o: e_chacha20_poly1305.c +diff --git a/crypto/evp/c_allc.c b/crypto/evp/c_allc.c +index 280e584..694f168 100644 +--- a/crypto/evp/c_allc.c ++++ b/crypto/evp/c_allc.c +@@ -238,4 +238,9 @@ void OpenSSL_add_all_ciphers(void) + EVP_add_cipher_alias(SN_camellia_256_cbc, "CAMELLIA256"); + EVP_add_cipher_alias(SN_camellia_256_cbc, "camellia256"); + #endif ++ ++#ifndef OPENSSL_NO_CHACHA_POLY ++ EVP_add_cipher(EVP_chacha20_poly1305()); ++ EVP_add_cipher(EVP_chacha20_poly1305_draft()); ++#endif + } +diff --git a/crypto/evp/e_chacha20_poly1305.c b/crypto/evp/e_chacha20_poly1305.c +new file mode 100644 +index 0000000..1e072ec +--- /dev/null ++++ b/crypto/evp/e_chacha20_poly1305.c +@@ -0,0 +1,362 @@ ++/* ==================================================================== ++ * Copyright (c) 2001-2014 The OpenSSL Project. All rights reserved. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions ++ * are met: ++ * ++ * 1. Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * 2. Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in ++ * the documentation and/or other materials provided with the ++ * distribution. ++ * ++ * 3. All advertising materials mentioning features or use of this ++ * software must display the following acknowledgment: ++ * "This product includes software developed by the OpenSSL Project ++ * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" ++ * ++ * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to ++ * endorse or promote products derived from this software without ++ * prior written permission. For written permission, please contact ++ * openssl-core@openssl.org. ++ * ++ * 5. Products derived from this software may not be called "OpenSSL" ++ * nor may "OpenSSL" appear in their names without prior written ++ * permission of the OpenSSL Project. ++ * ++ * 6. Redistributions of any form whatsoever must retain the following ++ * acknowledgment: ++ * "This product includes software developed by the OpenSSL Project ++ * for use in the OpenSSL Toolkit (http://www.openssl.org/)" ++ * ++ * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY ++ * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE ++ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR ++ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR ++ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT ++ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) ++ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, ++ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ++ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED ++ * OF THE POSSIBILITY OF SUCH DAMAGE. ++ * ==================================================================== ++ * ++ */ ++ ++#include ++#ifndef OPENSSL_NO_CHACHA_POLY ++# include ++# include ++ ++#define FILL_BUFFER ((size_t)128) ++ ++typedef struct { ++ uint8_t iv[12]; ++ uint8_t nonce[48]; ++ size_t aad_l; ++ size_t ct_l; ++ unsigned valid:1; ++ unsigned draft:1; ++ uint8_t poly_buffer[FILL_BUFFER]; ++ uint8_t chacha_buffer[FILL_BUFFER]; ++ uint16_t poly_buffer_used; ++ uint16_t chacha_used; ++ poly1305_state poly_state; ++ #define poly_finish(c,m) CRYPTO_poly1305_finish(&c->poly_state,m) ++} EVP_CHACHA20_POLY1305_CTX; ++ ++static int EVP_chacha20_poly1305_init_draft(EVP_CIPHER_CTX *ctx, ++ const unsigned char *key, ++ const unsigned char *iv, ++ int enc) ++{ ++ EVP_CHACHA20_POLY1305_CTX *aead_ctx = ctx->cipher_data; ++ memcpy(aead_ctx->nonce, key, 32); ++ aead_ctx->valid = 0; ++ aead_ctx->draft = 1; ++ return 1; ++} ++ ++static int EVP_chacha20_poly1305_init(EVP_CIPHER_CTX *ctx, ++ const unsigned char *key, ++ const unsigned char *iv, ++ int enc) ++{ ++ EVP_CHACHA20_POLY1305_CTX *aead_ctx = ctx->cipher_data; ++ memcpy(aead_ctx->nonce, key, 32); ++ memcpy(aead_ctx->iv, iv, 12); ++ aead_ctx->valid = 0; ++ aead_ctx->draft = 0; ++ return 1; ++} ++ ++static int EVP_chacha20_poly1305_cipher(EVP_CIPHER_CTX *ctx, ++ unsigned char *out, ++ const unsigned char *in, ++ size_t inl) ++{ ++ EVP_CHACHA20_POLY1305_CTX *aead_ctx = ctx->cipher_data; ++ uint8_t poly_mac[POLY1305_MAC_LEN]; ++ uint8_t zero[POLY1305_PAD_LEN] = {0}; ++ uint64_t cmp; ++ int i, todo; ++ ++ if (!aead_ctx->valid) ++ return 0; ++ ++ if (inl < POLY1305_MAC_LEN) ++ return -1; ++ ++ /* Fix for MAC */ ++ inl -= POLY1305_MAC_LEN; ++ ++#if (CHAPOLY_ASM) ++ if (!aead_ctx->draft) { ++ aead_ctx->valid = 0; ++ if (ctx->encrypt) { ++ chacha20_poly1305_seal(out, in, inl, ++ aead_ctx->poly_buffer, ++ aead_ctx->poly_buffer_used, ++ aead_ctx->nonce); ++ } else { ++ int cmp = chacha20_poly1305_open(out, in, inl, ++ aead_ctx->poly_buffer, ++ aead_ctx->poly_buffer_used, ++ aead_ctx->nonce); ++ if (!cmp) { ++ OPENSSL_cleanse(out, inl); ++ return -1; ++ } ++ } ++ return inl; ++ } ++#endif ++ ++ if (!ctx->encrypt) { ++ CRYPTO_poly1305_update(&aead_ctx->poly_state, in, inl); ++ } ++ ++ i = 0; ++ if (inl < 256) { ++ /* Consume the buffer we computed during poly initialization */ ++ todo = inl > (FILL_BUFFER - aead_ctx->chacha_used) ? ++ FILL_BUFFER - aead_ctx->chacha_used : ++ inl; ++ ++ for (; i < todo; i++) { ++ out[i] = in[i] ^ aead_ctx->chacha_buffer[i + 64 /*aead_ctx->chacha_used*/]; ++ } ++ ++ } else { ++ /* For long messages don't use precomputed buffer */ ++ ((uint64_t *)(aead_ctx->nonce))[4]--; ++ } ++ ++ todo = inl - i; ++ ++ if (todo) { ++ CRYPTO_chacha_20(&out[i], &in[i], todo, aead_ctx->nonce); ++ } ++ ++ if (ctx->encrypt) { ++ CRYPTO_poly1305_update(&aead_ctx->poly_state, out, inl); ++ } ++ ++ aead_ctx->ct_l += inl; ++ ++ if (!aead_ctx->draft) { ++ /* For RFC padd ciphertext with zeroes, then mac len(aad)||len(ct) */ ++ todo = aead_ctx->ct_l % POLY1305_PAD_LEN ? ++ POLY1305_PAD_LEN - (aead_ctx->ct_l % POLY1305_PAD_LEN) : ++ 0; ++ ++ if (todo) { ++ CRYPTO_poly1305_update(&aead_ctx->poly_state, zero, todo); ++ } ++ ++ CRYPTO_poly1305_update(&aead_ctx->poly_state, (uint8_t*)&aead_ctx->aad_l, 8); ++ CRYPTO_poly1305_update(&aead_ctx->poly_state, (uint8_t*)&aead_ctx->ct_l, 8); ++ ++ } else { ++ /* For the draft don't pad, mac len(ct) */ ++ CRYPTO_poly1305_update(&aead_ctx->poly_state, (uint8_t*)&aead_ctx->ct_l, 8); ++ } ++ aead_ctx->valid = 0; ++ ++ if (ctx->encrypt) { ++ poly_finish(aead_ctx, &out[inl]); ++ return inl + POLY1305_MAC_LEN; ++ ++ } else { /* Decryption */ ++ poly_finish(aead_ctx, poly_mac); ++ /* Constant time comparison */ ++ cmp = (*(uint64_t *)(poly_mac)) ^ (*(uint64_t *)(in + inl)); ++ cmp |= (*(uint64_t *)(poly_mac + 8)) ^ (*(uint64_t *)(in + inl + 8)); ++ ++ if (cmp) { ++ OPENSSL_cleanse(out, inl); ++ return -1; ++ } ++ ++ return inl; ++ } ++} ++ ++ ++static int EVP_chacha20_poly1305_cleanup(EVP_CIPHER_CTX *ctx) ++{ ++ return 1; ++} ++ ++ ++static int EVP_chacha20_poly1305_ctrl(EVP_CIPHER_CTX *ctx, ++ int type, ++ int arg, ++ void *ptr) ++{ ++ EVP_CHACHA20_POLY1305_CTX *aead_ctx = ctx->cipher_data; ++ uint8_t aad[EVP_AEAD_TLS1_AAD_LEN + 8]; ++ uint64_t thirteen = EVP_AEAD_TLS1_AAD_LEN; ++ ++ switch (type) { ++ case EVP_CTRL_AEAD_TLS1_AAD: ++ ++ /* Initialize poly keys */ ++ memset(aead_ctx->chacha_buffer, 0, FILL_BUFFER); ++ ++ if (!aead_ctx->draft) { ++ /* RFC IV = (0 || iv) ^ seq_num */ ++ memset(aead_ctx->nonce + 32, 0, 4); ++ memcpy(aead_ctx->nonce + 36, aead_ctx->iv, 12); ++ *(uint64_t *)(aead_ctx->nonce + 40) ^= *(uint64_t *)(ptr); ++ ++ } else { ++ /* draft IV = 0 || seq_num */ ++ memset(aead_ctx->nonce + 32, 0, 8); ++ memcpy(aead_ctx->nonce + 40, ptr, 8); ++ } ++ ++#if (CHAPOLY_ASM) ++ if (!aead_ctx->draft) { ++ if (arg == EVP_AEAD_TLS1_AAD_LEN) { ++ /* For RFC, use optimized seal/open */ ++ memcpy(aad, ptr, arg); ++ unsigned int len = (aad[arg-2] << 8) | aad[arg-1]; ++ if (!ctx->encrypt) { ++ len -= POLY1305_MAC_LEN; ++ aad[arg-2] = len>>8; ++ aad[arg-1] = len & 0xff; ++ } ++ memcpy(aead_ctx->poly_buffer, aad, arg); ++ } else if (arg <= FILL_BUFFER) { ++ memcpy(aead_ctx->poly_buffer, ptr, arg); ++ } else { ++ aead_ctx->valid = 0; ++ return 0; ++ } ++ aead_ctx->valid = 1; ++ aead_ctx->poly_buffer_used = arg; ++ return POLY1305_MAC_LEN; ++ } ++#endif ++ /* Poly keys = ENC(0) */ ++ CRYPTO_chacha_20(aead_ctx->chacha_buffer, ++ aead_ctx->chacha_buffer, ++ FILL_BUFFER, ++ aead_ctx->nonce); ++ ++ CRYPTO_poly1305_init(&aead_ctx->poly_state, aead_ctx->chacha_buffer); ++ ++ aead_ctx->chacha_used = 64; ++ aead_ctx->poly_buffer_used = 0; ++ aead_ctx->aad_l = arg; ++ aead_ctx->ct_l = 0; ++ ++ /* Absorb AAD */ ++ memcpy(aad, ptr, arg); ++ memset(aad + arg, 0, sizeof(aad) - arg); ++ ++ /* If decrypting fix length for tag */ ++ if (!ctx->encrypt) { ++ unsigned int len = (aad[arg-2] << 8) | aad[arg-1]; ++ len -= POLY1305_MAC_LEN; ++ aad[arg-2] = len>>8; ++ aad[arg-1] = len & 0xff; ++ } ++ ++ if (!aead_ctx->draft) { ++ /* In the RFC, AAD is padded with zeroes */ ++ CRYPTO_poly1305_update(&aead_ctx->poly_state, aad, POLY1305_PAD_LEN); ++ ++ } else { ++ /* In the draft AAD is followed by len(AAD) */ ++ memcpy(&aad[arg], &thirteen, sizeof(thirteen)); ++ CRYPTO_poly1305_update(&aead_ctx->poly_state, aad, arg + sizeof(thirteen)); ++ } ++ ++ aead_ctx->valid = 1; ++ return POLY1305_MAC_LEN; ++ ++ break; ++ ++ default: ++ return 0; ++ break; ++ } ++ ++ return 0; ++} ++ ++ ++#define CUSTOM_FLAGS (\ ++ EVP_CIPH_CUSTOM_IV | EVP_CIPH_FLAG_CUSTOM_CIPHER \ ++ | EVP_CIPH_ALWAYS_CALL_INIT \ ++ | EVP_CIPH_CUSTOM_COPY) ++ ++ ++static const EVP_CIPHER chacha20_poly1305_d = { ++ NID_chacha20_poly1305_draft, ++ 1, /* block size, sorta */ ++ 32, /* key len */ ++ 0, /* iv len */ ++ CUSTOM_FLAGS|EVP_CIPH_FLAG_AEAD_CIPHER, /* flags */ ++ EVP_chacha20_poly1305_init_draft, ++ EVP_chacha20_poly1305_cipher, ++ EVP_chacha20_poly1305_cleanup, ++ sizeof(EVP_CHACHA20_POLY1305_CTX), /* ctx size */ ++ NULL, ++ NULL, ++ EVP_chacha20_poly1305_ctrl, ++ NULL ++ }; ++ ++ ++static const EVP_CIPHER chacha20_poly1305 = { ++ NID_chacha20_poly1305, ++ 1, /* block size, sorta */ ++ 32, /* key len */ ++ 12, /* iv len */ ++ CUSTOM_FLAGS|EVP_CIPH_FLAG_AEAD_CIPHER, /* flags */ ++ EVP_chacha20_poly1305_init, ++ EVP_chacha20_poly1305_cipher, ++ EVP_chacha20_poly1305_cleanup, ++ sizeof(EVP_CHACHA20_POLY1305_CTX), /* ctx size */ ++ NULL, ++ NULL, ++ EVP_chacha20_poly1305_ctrl, ++ NULL ++ }; ++ ++ ++const EVP_CIPHER *EVP_chacha20_poly1305_draft(void) ++{ return &chacha20_poly1305_d; } ++ ++ ++const EVP_CIPHER *EVP_chacha20_poly1305(void) ++{ return &chacha20_poly1305; } ++#endif +diff --git a/crypto/evp/evp.h b/crypto/evp/evp.h +index 39ab793..8feaabc 100644 +--- a/crypto/evp/evp.h ++++ b/crypto/evp/evp.h +@@ -902,6 +902,11 @@ const EVP_CIPHER *EVP_seed_cfb128(void); + const EVP_CIPHER *EVP_seed_ofb(void); + # endif + ++# ifndef OPENSSL_NO_CHACHA_POLY ++const EVP_CIPHER *EVP_chacha20_poly1305(void); ++const EVP_CIPHER *EVP_chacha20_poly1305_draft(void); ++# endif ++ + void OPENSSL_add_all_algorithms_noconf(void); + void OPENSSL_add_all_algorithms_conf(void); + +diff --git a/crypto/objects/obj_dat.h b/crypto/objects/obj_dat.h +index b7e3cf2..26612e2 100644 +--- a/crypto/objects/obj_dat.h ++++ b/crypto/objects/obj_dat.h +@@ -62,9 +62,9 @@ + * [including the GNU Public Licence.] + */ + +-#define NUM_NID 958 +-#define NUM_SN 951 +-#define NUM_LN 951 ++#define NUM_NID 960 ++#define NUM_SN 953 ++#define NUM_LN 953 + #define NUM_OBJ 890 + + static const unsigned char lvalues[6255]={ +@@ -2514,6 +2514,9 @@ static const ASN1_OBJECT nid_objs[NUM_NID]={ + NID_jurisdictionStateOrProvinceName,11,&(lvalues[6232]),0}, + {"jurisdictionC","jurisdictionCountryName", + NID_jurisdictionCountryName,11,&(lvalues[6243]),0}, ++{"CHACHA20-POLY1305","chacha20-poly1305",NID_chacha20_poly1305,0,NULL,0}, ++{"CHACHA20-POLY1305-D","chacha20-poly1305-draft", ++ NID_chacha20_poly1305_draft,0,NULL,0}, + }; + + static const unsigned int sn_objs[NUM_SN]={ +@@ -2574,6 +2577,8 @@ static const unsigned int sn_objs[NUM_SN]={ + 110, /* "CAST5-CFB" */ + 109, /* "CAST5-ECB" */ + 111, /* "CAST5-OFB" */ ++958, /* "CHACHA20-POLY1305" */ ++959, /* "CHACHA20-POLY1305-D" */ + 894, /* "CMAC" */ + 13, /* "CN" */ + 141, /* "CRLReason" */ +@@ -3728,6 +3733,8 @@ static const unsigned int ln_objs[NUM_LN]={ + 677, /* "certicom-arc" */ + 517, /* "certificate extensions" */ + 883, /* "certificateRevocationList" */ ++958, /* "chacha20-poly1305" */ ++959, /* "chacha20-poly1305-draft" */ + 54, /* "challengePassword" */ + 407, /* "characteristic-two-field" */ + 395, /* "clearance" */ +diff --git a/crypto/objects/obj_mac.h b/crypto/objects/obj_mac.h +index 779c309..35a2364 100644 +--- a/crypto/objects/obj_mac.h ++++ b/crypto/objects/obj_mac.h +@@ -4047,6 +4047,14 @@ + #define LN_aes_256_cbc_hmac_sha256 "aes-256-cbc-hmac-sha256" + #define NID_aes_256_cbc_hmac_sha256 950 + ++#define SN_chacha20_poly1305 "CHACHA20-POLY1305" ++#define LN_chacha20_poly1305 "chacha20-poly1305" ++#define NID_chacha20_poly1305 958 ++ ++#define SN_chacha20_poly1305_draft "CHACHA20-POLY1305-D" ++#define LN_chacha20_poly1305_draft "chacha20-poly1305-draft" ++#define NID_chacha20_poly1305_draft 959 ++ + #define SN_dhpublicnumber "dhpublicnumber" + #define LN_dhpublicnumber "X9.42 DH" + #define NID_dhpublicnumber 920 +diff --git a/crypto/objects/obj_mac.num b/crypto/objects/obj_mac.num +index 8e5ea83..a3da329 100644 +--- a/crypto/objects/obj_mac.num ++++ b/crypto/objects/obj_mac.num +@@ -955,3 +955,5 @@ ct_cert_scts 954 + jurisdictionLocalityName 955 + jurisdictionStateOrProvinceName 956 + jurisdictionCountryName 957 ++chacha20_poly1305 958 ++chacha20_poly1305_draft 959 +diff --git a/crypto/objects/objects.txt b/crypto/objects/objects.txt +index b57aabb..6a34a33 100644 +--- a/crypto/objects/objects.txt ++++ b/crypto/objects/objects.txt +@@ -1294,6 +1294,8 @@ kisa 1 6 : SEED-OFB : seed-ofb + : AES-128-CBC-HMAC-SHA256 : aes-128-cbc-hmac-sha256 + : AES-192-CBC-HMAC-SHA256 : aes-192-cbc-hmac-sha256 + : AES-256-CBC-HMAC-SHA256 : aes-256-cbc-hmac-sha256 ++ : CHACHA20-POLY1305 : chacha20-poly1305 ++ : CHACHA20-POLY1305-D : chacha20-poly1305-draft + + ISO-US 10046 2 1 : dhpublicnumber : X9.42 DH + +diff --git a/ssl/s3_lib.c b/ssl/s3_lib.c +index 0385e03..65fdc59 100644 +--- a/ssl/s3_lib.c ++++ b/ssl/s3_lib.c +@@ -2945,6 +2945,110 @@ OPENSSL_GLOBAL SSL_CIPHER ssl3_ciphers[] = { + 256}, + #endif + ++#if !defined(OPENSSL_NO_CHACHA_POLY) ++/* Draft ciphers */ ++ { ++ 1, ++ TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305_D, ++ TLS1_CK_ECDHE_RSA_WITH_CHACHA20_POLY1305_D, ++ SSL_kEECDH, ++ SSL_aRSA, ++ SSL_CHACHA20POLY1305_D, ++ SSL_AEAD, ++ SSL_TLSV1_2, ++ SSL_HIGH, ++ SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, ++ 256, ++ 256, ++ }, ++ ++ { ++ 1, ++ TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_D, ++ TLS1_CK_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_D, ++ SSL_kEECDH, ++ SSL_aECDSA, ++ SSL_CHACHA20POLY1305_D, ++ SSL_AEAD, ++ SSL_TLSV1_2, ++ SSL_HIGH, ++ SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, ++ 256, ++ 256, ++ }, ++ ++ { ++ 1, ++ TLS1_TXT_DHE_RSA_WITH_CHACHA20_POLY1305_D, ++ TLS1_CK_DHE_RSA_WITH_CHACHA20_POLY1305_D, ++ SSL_kEDH, ++ SSL_aRSA, ++ SSL_CHACHA20POLY1305_D, ++ SSL_AEAD, ++ SSL_TLSV1_2, ++ SSL_HIGH, ++ SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, ++ 256, ++ 256, ++ }, ++ /* RFC ciphers */ ++ { ++ 1, ++ TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305, ++ TLS1_CK_ECDHE_RSA_WITH_CHACHA20_POLY1305, ++ SSL_kECDHE, ++ SSL_aRSA, ++ SSL_CHACHA20POLY1305, ++ SSL_AEAD, ++ SSL_TLSV1_2, ++ SSL_HIGH, ++ SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, ++ 256, ++ 256, ++ }, ++ { ++ 1, ++ TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, ++ TLS1_CK_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, ++ SSL_kECDHE, ++ SSL_aECDSA, ++ SSL_CHACHA20POLY1305, ++ SSL_AEAD, ++ SSL_TLSV1_2, ++ SSL_HIGH, ++ SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, ++ 256, ++ 256, ++ }, ++ { ++ 1, ++ TLS1_TXT_DHE_RSA_WITH_CHACHA20_POLY1305, ++ TLS1_CK_DHE_RSA_WITH_CHACHA20_POLY1305, ++ SSL_kDHE, ++ SSL_aRSA, ++ SSL_CHACHA20POLY1305, ++ SSL_AEAD, ++ SSL_TLSV1_2, ++ SSL_HIGH, ++ SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, ++ 256, ++ 256, ++ }, ++ { ++ 1, ++ TLS1_TXT_PSK_WITH_CHACHA20_POLY1305, ++ TLS1_CK_PSK_WITH_CHACHA20_POLY1305, ++ SSL_kPSK, ++ SSL_aPSK, ++ SSL_CHACHA20POLY1305, ++ SSL_AEAD, ++ SSL_TLSV1_2, ++ SSL_HIGH, ++ SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, ++ 256, ++ 256, ++ }, ++#endif + /* end of list */ + }; + +@@ -4090,6 +4194,7 @@ SSL_CIPHER *ssl3_choose_cipher(SSL *s, STACK_OF(SSL_CIPHER) *clnt, + int i, ii, ok; + CERT *cert; + unsigned long alg_k, alg_a, mask_k, mask_a, emask_k, emask_a; ++ int use_chacha = 0; + + /* Let's see which ciphers we can support */ + cert = s->cert; +@@ -4119,13 +4224,21 @@ SSL_CIPHER *ssl3_choose_cipher(SSL *s, STACK_OF(SSL_CIPHER) *clnt, + fprintf(stderr, "%p:%s\n", (void *)c, c->name); + } + #endif +- ++retry: + if (s->options & SSL_OP_CIPHER_SERVER_PREFERENCE || tls1_suiteb(s)) { + prio = srvr; + allow = clnt; ++ /* Use ChaCha20+Poly1305 iff it's client's most preferred cipher suite */ ++ if (sk_SSL_CIPHER_num(clnt) > 0) { ++ c = sk_SSL_CIPHER_value(clnt, 0); ++ if (c->algorithm_enc == SSL_CHACHA20POLY1305 || ++ c->algorithm_enc == SSL_CHACHA20POLY1305_D) ++ use_chacha = 1; ++ } + } else { + prio = clnt; + allow = srvr; ++ use_chacha = 1; + } + + tls1_set_cert_validity(s); +@@ -4137,6 +4250,11 @@ SSL_CIPHER *ssl3_choose_cipher(SSL *s, STACK_OF(SSL_CIPHER) *clnt, + if ((c->algorithm_ssl & SSL_TLSV1_2) && !SSL_USE_TLS1_2_CIPHERS(s)) + continue; + ++ /* Skip ChaCha unless top client priority */ ++ if ((c->algorithm_enc == SSL_CHACHA20POLY1305 || ++ c->algorithm_enc == SSL_CHACHA20POLY1305_D) && !use_chacha) ++ continue; ++ + ssl_set_cert_masks(cert, c); + mask_k = cert->mask_k; + mask_a = cert->mask_a; +@@ -4216,6 +4334,14 @@ SSL_CIPHER *ssl3_choose_cipher(SSL *s, STACK_OF(SSL_CIPHER) *clnt, + break; + } + } ++ ++ if (ret == NULL && !use_chacha) { ++ /* If no shared cipher was found due to some unusual preferences, try ++ * again with CHACHA enabled even if not top priority */ ++ use_chacha = 1; ++ goto retry; ++ } ++ + return (ret); + } + +diff --git a/ssl/ssl.h b/ssl/ssl.h +index 90aeb0c..f783baa 100644 +--- a/ssl/ssl.h ++++ b/ssl/ssl.h +@@ -297,6 +297,8 @@ extern "C" { + # define SSL_TXT_CAMELLIA128 "CAMELLIA128" + # define SSL_TXT_CAMELLIA256 "CAMELLIA256" + # define SSL_TXT_CAMELLIA "CAMELLIA" ++# define SSL_TXT_CHACHA20_D "CHACHA20-draft" ++# define SSL_TXT_CHACHA20 "CHACHA20" + + # define SSL_TXT_MD5 "MD5" + # define SSL_TXT_SHA1 "SHA1" +diff --git a/ssl/ssl_ciph.c b/ssl/ssl_ciph.c +index 2ad8f43..23c1c68 100644 +--- a/ssl/ssl_ciph.c ++++ b/ssl/ssl_ciph.c +@@ -164,11 +164,13 @@ + #define SSL_ENC_SEED_IDX 11 + #define SSL_ENC_AES128GCM_IDX 12 + #define SSL_ENC_AES256GCM_IDX 13 +-#define SSL_ENC_NUM_IDX 14 ++#define SSL_ENC_CHACHA20POLY1305_DRAFT_IDX 14 ++#define SSL_ENC_CHACHA20POLY1305_IDX 15 ++#define SSL_ENC_NUM_IDX 16 + + static const EVP_CIPHER *ssl_cipher_methods[SSL_ENC_NUM_IDX] = { + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, +- NULL, NULL ++ NULL, NULL, NULL, NULL + }; + + #define SSL_COMP_NULL_IDX 0 +@@ -315,6 +317,8 @@ static const SSL_CIPHER cipher_aliases[] = { + {0, SSL_TXT_CAMELLIA256, 0, 0, 0, SSL_CAMELLIA256, 0, 0, 0, 0, 0, 0}, + {0, SSL_TXT_CAMELLIA, 0, 0, 0, SSL_CAMELLIA128 | SSL_CAMELLIA256, 0, 0, 0, + 0, 0, 0}, ++ {0, SSL_TXT_CHACHA20_D, 0, 0, 0, SSL_CHACHA20POLY1305_D, 0, 0, 0, 0, 0, 0}, ++ {0, SSL_TXT_CHACHA20, 0, 0, 0, SSL_CHACHA20POLY1305, 0, 0, 0, 0, 0, 0}, + + /* MAC aliases */ + {0, SSL_TXT_MD5, 0, 0, 0, 0, SSL_MD5, 0, 0, 0, 0, 0}, +@@ -431,6 +435,11 @@ void ssl_load_ciphers(void) + ssl_cipher_methods[SSL_ENC_AES256GCM_IDX] = + EVP_get_cipherbyname(SN_aes_256_gcm); + ++ ssl_cipher_methods[SSL_ENC_CHACHA20POLY1305_DRAFT_IDX] = ++ EVP_chacha20_poly1305_draft(); ++ ssl_cipher_methods[SSL_ENC_CHACHA20POLY1305_IDX] = ++ EVP_chacha20_poly1305(); ++ + ssl_digest_methods[SSL_MD_MD5_IDX] = EVP_get_digestbyname(SN_md5); + ssl_mac_secret_size[SSL_MD_MD5_IDX] = + EVP_MD_size(ssl_digest_methods[SSL_MD_MD5_IDX]); +@@ -581,6 +590,12 @@ int ssl_cipher_get_evp(const SSL_SESSION *s, const EVP_CIPHER **enc, + case SSL_AES256GCM: + i = SSL_ENC_AES256GCM_IDX; + break; ++ case SSL_CHACHA20POLY1305_D: ++ i = SSL_ENC_CHACHA20POLY1305_DRAFT_IDX; ++ break; ++ case SSL_CHACHA20POLY1305: ++ i = SSL_ENC_CHACHA20POLY1305_IDX; ++ break; + default: + i = -1; + break; +@@ -805,6 +820,12 @@ static void ssl_cipher_get_disabled(unsigned long *mkey, unsigned long *auth, + (ssl_cipher_methods[SSL_ENC_GOST89_IDX] == + NULL) ? SSL_eGOST2814789CNT : 0; + *enc |= (ssl_cipher_methods[SSL_ENC_SEED_IDX] == NULL) ? SSL_SEED : 0; ++ *enc |= ++ (ssl_cipher_methods[SSL_ENC_CHACHA20POLY1305_DRAFT_IDX] == ++ NULL) ? SSL_CHACHA20POLY1305_D : 0; ++ *enc |= ++ (ssl_cipher_methods[SSL_ENC_CHACHA20POLY1305_IDX] == ++ NULL) ? SSL_CHACHA20POLY1305 : 0; + + *mac |= (ssl_digest_methods[SSL_MD_MD5_IDX] == NULL) ? SSL_MD5 : 0; + *mac |= (ssl_digest_methods[SSL_MD_SHA1_IDX] == NULL) ? SSL_SHA1 : 0; +@@ -1824,6 +1845,12 @@ char *SSL_CIPHER_description(const SSL_CIPHER *cipher, char *buf, int len) + case SSL_eGOST2814789CNT: + enc = "GOST89(256)"; + break; ++ case SSL_CHACHA20POLY1305_D: ++ enc = "ChaCha20-Poly1305-draft"; ++ break; ++ case SSL_CHACHA20POLY1305: ++ enc = "ChaCha20-Poly1305"; ++ break; + default: + enc = "unknown"; + break; +diff --git a/ssl/ssl_locl.h b/ssl/ssl_locl.h +index 6df725f..dbe68f2 100644 +--- a/ssl/ssl_locl.h ++++ b/ssl/ssl_locl.h +@@ -354,6 +354,8 @@ + # define SSL_SEED 0x00000800L + # define SSL_AES128GCM 0x00001000L + # define SSL_AES256GCM 0x00002000L ++# define SSL_CHACHA20POLY1305_D 0x00040000L ++# define SSL_CHACHA20POLY1305 0x00080000L /* Value from openssl */ + + # define SSL_AES (SSL_AES128|SSL_AES256|SSL_AES128GCM|SSL_AES256GCM) + # define SSL_CAMELLIA (SSL_CAMELLIA128|SSL_CAMELLIA256) +diff --git a/ssl/tls1.h b/ssl/tls1.h +index 7e237d0..ff2e259 100644 +--- a/ssl/tls1.h ++++ b/ssl/tls1.h +@@ -563,6 +563,19 @@ SSL_CTX_callback_ctrl(ssl,SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB,(void (*)(void))cb) + # define TLS1_CK_ECDH_RSA_WITH_AES_128_GCM_SHA256 0x0300C031 + # define TLS1_CK_ECDH_RSA_WITH_AES_256_GCM_SHA384 0x0300C032 + ++/* ChaCha20-Poly1305 ciphersuites draft-agl-tls-chacha20poly1305-01 */ ++# define TLS1_CK_ECDHE_RSA_WITH_CHACHA20_POLY1305_D 0x0300CC13 ++# define TLS1_CK_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_D 0x0300CC14 ++# define TLS1_CK_DHE_RSA_WITH_CHACHA20_POLY1305_D 0x0300CC15 ++/* ChaCha20-Poly1305 ciphersuites from RFC */ ++# define TLS1_CK_ECDHE_RSA_WITH_CHACHA20_POLY1305 0x0300CCA8 ++# define TLS1_CK_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 0x0300CCA9 ++# define TLS1_CK_DHE_RSA_WITH_CHACHA20_POLY1305 0x0300CCAA ++# define TLS1_CK_PSK_WITH_CHACHA20_POLY1305 0x0300CCAB ++# define TLS1_CK_ECDHE_PSK_WITH_CHACHA20_POLY1305 0x0300CCAC ++# define TLS1_CK_DHE_PSK_WITH_CHACHA20_POLY1305 0x0300CCAD ++# define TLS1_CK_RSA_PSK_WITH_CHACHA20_POLY1305 0x0300CCAE ++ + /* + * XXX * Backward compatibility alert: + * Older versions of OpenSSL gave + * some DHE ciphers names with "EDH" + * instead of "DHE". Going forward, we +@@ -713,6 +726,19 @@ SSL_CTX_callback_ctrl(ssl,SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB,(void (*)(void))cb) + # define TLS1_TXT_ECDH_RSA_WITH_AES_128_GCM_SHA256 "ECDH-RSA-AES128-GCM-SHA256" + # define TLS1_TXT_ECDH_RSA_WITH_AES_256_GCM_SHA384 "ECDH-RSA-AES256-GCM-SHA384" + ++/* ChaCha20-Poly1305 ciphersuites draft-agl-tls-chacha20poly1305-01 */ ++# define TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305_D "ECDHE-RSA-CHACHA20-POLY1305-D" ++# define TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_D "ECDHE-ECDSA-CHACHA20-POLY1305-D" ++# define TLS1_TXT_DHE_RSA_WITH_CHACHA20_POLY1305_D "DHE-RSA-CHACHA20-POLY1305-D" ++/* Chacha20-Poly1305 ciphersuites from RFC */ ++# define TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305 "ECDHE-RSA-CHACHA20-POLY1305" ++# define TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 "ECDHE-ECDSA-CHACHA20-POLY1305" ++# define TLS1_TXT_DHE_RSA_WITH_CHACHA20_POLY1305 "DHE-RSA-CHACHA20-POLY1305" ++# define TLS1_TXT_PSK_WITH_CHACHA20_POLY1305 "PSK-CHACHA20-POLY1305" ++# define TLS1_TXT_ECDHE_PSK_WITH_CHACHA20_POLY1305 "ECDHE-PSK-CHACHA20-POLY1305" ++# define TLS1_TXT_DHE_PSK_WITH_CHACHA20_POLY1305 "DHE-PSK-CHACHA20-POLY1305" ++# define TLS1_TXT_RSA_PSK_WITH_CHACHA20_POLY1305 "RSA-PSK-CHACHA20-POLY1305" ++ + # define TLS_CT_RSA_SIGN 1 + # define TLS_CT_DSS_SIGN 2 + # define TLS_CT_RSA_FIXED_DH 3 +-- +2.10.1 + From a0a893e339b5c368cae5a6bdfbfa743e4aa17670 Mon Sep 17 00:00:00 2001 From: Justin Date: Wed, 22 Mar 2017 11:44:30 -0400 Subject: [PATCH 06/16] New patches for nginx 1.11.11 and OpenSSL 1.1.0 --- patches/nginx__1.11.11_http2_spdy.patch | 6305 +++++++++++++++++ .../openssl__1.1.0_chacha20_poly1305.patch | 60 + 2 files changed, 6365 insertions(+) create mode 100644 patches/nginx__1.11.11_http2_spdy.patch create mode 100644 patches/openssl__1.1.0_chacha20_poly1305.patch diff --git a/patches/nginx__1.11.11_http2_spdy.patch b/patches/nginx__1.11.11_http2_spdy.patch new file mode 100644 index 0000000..fe6f248 --- /dev/null +++ b/patches/nginx__1.11.11_http2_spdy.patch @@ -0,0 +1,6305 @@ +diff --git a/auto/modules b/auto/modules +index be3561e..5a86643 100644 +--- a/auto/modules ++++ b/auto/modules +@@ -134,6 +134,7 @@ if [ $HTTP = YES ]; then + # ngx_http_header_filter + # ngx_http_chunked_filter + # ngx_http_v2_filter ++ # ngx_http_spdy_filter + # ngx_http_range_header_filter + # ngx_http_gzip_filter + # ngx_http_postpone_filter +@@ -166,6 +167,7 @@ if [ $HTTP = YES ]; then + ngx_http_header_filter_module \ + ngx_http_chunked_filter_module \ + ngx_http_v2_filter_module \ ++ ngx_http_spdy_filter_module \ + ngx_http_range_header_filter_module \ + ngx_http_gzip_filter_module \ + ngx_http_postpone_filter_module \ +@@ -227,6 +229,19 @@ if [ $HTTP = YES ]; then + . auto/module + fi + ++ if [ $HTTP_SPDY = YES ]; then ++ have=NGX_HTTP_SPDY . auto/have ++ USE_ZLIB=YES ++ ngx_module_name=ngx_http_spdy_filter_module ++ ngx_module_incs= ++ ngx_module_deps= ++ ngx_module_srcs=src/http/ngx_http_spdy_filter_module.c ++ ngx_module_libs= ++ ngx_module_link=$HTTP_SPDY ++ ++ . auto/module ++ fi ++ + if :; then + ngx_module_name=ngx_http_range_header_filter_module + ngx_module_incs= +@@ -436,6 +451,19 @@ if [ $HTTP = YES ]; then + . auto/module + fi + ++ if [ $HTTP_SPDY = YES ]; then ++ have=NGX_HTTP_SPDY . auto/have ++ ngx_module_name=ngx_http_spdy_module ++ ngx_module_incs=src/http ++ ngx_module_deps="src/http/ngx_http_spdy.h src/http/ngx_http_spdy_module.h" ++ ngx_module_srcs="src/http/ngx_http_spdy.c \ ++ src/http/ngx_http_spdy_module.c" ++ ngx_module_libs= ++ ngx_module_link=$HTTP_SPDY ++ ++ . auto/module ++ fi ++ + if :; then + ngx_module_name=ngx_http_static_module + ngx_module_incs= +diff --git a/auto/options b/auto/options +index 66b822a..dbcca32 100644 +--- a/auto/options ++++ b/auto/options +@@ -59,6 +59,7 @@ HTTP_CHARSET=YES + HTTP_GZIP=YES + HTTP_SSL=NO + HTTP_V2=NO ++HTTP_SPDY=NO + HTTP_SSI=YES + HTTP_POSTPONE=NO + HTTP_REALIP=NO +@@ -221,6 +222,7 @@ $0: warning: the \"--with-ipv6\" option is deprecated" + + --with-http_ssl_module) HTTP_SSL=YES ;; + --with-http_v2_module) HTTP_V2=YES ;; ++ --with-http_spdy_module) HTTP_SPDY=YES ;; + --with-http_realip_module) HTTP_REALIP=YES ;; + --with-http_addition_module) HTTP_ADDITION=YES ;; + --with-http_xslt_module) HTTP_XSLT=YES ;; +@@ -430,6 +432,7 @@ cat << END + + --with-http_ssl_module enable ngx_http_ssl_module + --with-http_v2_module enable ngx_http_v2_module ++ --with-http_spdy_module enable ngx_http_spdy_module + --with-http_realip_module enable ngx_http_realip_module + --with-http_addition_module enable ngx_http_addition_module + --with-http_xslt_module enable ngx_http_xslt_module +diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h +index 1d3e3a3..41f9882 100644 +--- a/src/core/ngx_connection.h ++++ b/src/core/ngx_connection.h +@@ -116,6 +116,7 @@ typedef enum { + #define NGX_LOWLEVEL_BUFFERED 0x0f + #define NGX_SSL_BUFFERED 0x01 + #define NGX_HTTP_V2_BUFFERED 0x02 ++#define NGX_SPDY_BUFFERED 0x04 + + + struct ngx_connection_s { +diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c +index 2771ac1..d189993 100644 +--- a/src/http/modules/ngx_http_ssl_module.c ++++ b/src/http/modules/ngx_http_ssl_module.c +@@ -347,10 +347,10 @@ ngx_http_ssl_alpn_select(ngx_ssl_conn_t *ssl_conn, const unsigned char **out, + #if (NGX_DEBUG) + unsigned int i; + #endif +-#if (NGX_HTTP_V2) ++#if (NGX_HTTP_V2 || NGX_HTTP_SPDY) + ngx_http_connection_t *hc; + #endif +-#if (NGX_HTTP_V2 || NGX_DEBUG) ++#if (NGX_HTTP_V2 || NGX_HTTP_SPDY || NGX_DEBUG) + ngx_connection_t *c; + + c = ngx_ssl_get_connection(ssl_conn); +@@ -364,9 +364,20 @@ ngx_http_ssl_alpn_select(ngx_ssl_conn_t *ssl_conn, const unsigned char **out, + } + #endif + +-#if (NGX_HTTP_V2) ++#if (NGX_HTTP_V2 || NGX_HTTP_SPDY) + hc = c->data; ++#endif + ++#if (NGX_HTTP_V2 && NGX_HTTP_SPDY) ++ if (hc->addr_conf->http2 && hc->addr_conf->spdy) { ++ srv = (unsigned char *) NGX_HTTP_V2_ALPN_ADVERTISE ++ NGX_SPDY_NPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE; ++ srvlen = sizeof(NGX_HTTP_V2_ALPN_ADVERTISE NGX_SPDY_NPN_ADVERTISE ++ NGX_HTTP_NPN_ADVERTISE) - 1; ++ ++ } else ++#endif ++#if (NGX_HTTP_V2) + if (hc->addr_conf->http2) { + srv = + (unsigned char *) NGX_HTTP_V2_ALPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE; +@@ -374,6 +385,13 @@ ngx_http_ssl_alpn_select(ngx_ssl_conn_t *ssl_conn, const unsigned char **out, + + } else + #endif ++#if (NGX_HTTP_SPDY) ++ if (hc->addr_conf->spdy) { ++ srv = (unsigned char *) NGX_SPDY_NPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE; ++ srvlen = sizeof(NGX_SPDY_NPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE) - 1; ++ ++ } else ++#endif + { + srv = (unsigned char *) NGX_HTTP_NPN_ADVERTISE; + srvlen = sizeof(NGX_HTTP_NPN_ADVERTISE) - 1; +@@ -401,19 +419,32 @@ static int + ngx_http_ssl_npn_advertised(ngx_ssl_conn_t *ssl_conn, + const unsigned char **out, unsigned int *outlen, void *arg) + { +-#if (NGX_HTTP_V2 || NGX_DEBUG) ++#if (NGX_HTTP_V2 || NGX_HTTP_SPDY || NGX_DEBUG) + ngx_connection_t *c; + + c = ngx_ssl_get_connection(ssl_conn); + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "SSL NPN advertised"); + #endif + +-#if (NGX_HTTP_V2) ++#if (NGX_HTTP_V2 || NGX_HTTP_SPDY) + { + ngx_http_connection_t *hc; + + hc = c->data; ++#endif + ++#if (NGX_HTTP_V2 && NGX_HTTP_SPDY) ++ if (hc->addr_conf->http2 && hc->addr_conf->spdy) { ++ *out = (unsigned char *) NGX_HTTP_V2_NPN_ADVERTISE ++ NGX_SPDY_NPN_ADVERTISE ++ NGX_HTTP_NPN_ADVERTISE; ++ *outlen = sizeof(NGX_HTTP_V2_NPN_ADVERTISE NGX_SPDY_NPN_ADVERTISE ++ NGX_HTTP_NPN_ADVERTISE) - 1; ++ ++ return SSL_TLSEXT_ERR_OK; ++ } else ++#endif ++#if (NGX_HTTP_V2) + if (hc->addr_conf->http2) { + *out = + (unsigned char *) NGX_HTTP_V2_NPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE; +@@ -421,6 +452,20 @@ ngx_http_ssl_npn_advertised(ngx_ssl_conn_t *ssl_conn, + + return SSL_TLSEXT_ERR_OK; + } ++#endif ++#if (NGX_HTTP_V2 && NGX_HTTP_SPDY) ++ else ++#endif ++#if (NGX_HTTP_SPDY) ++ if (hc->addr_conf->spdy) { ++ *out = (unsigned char *) NGX_SPDY_NPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE; ++ *outlen = sizeof(NGX_SPDY_NPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE) - 1; ++ ++ return SSL_TLSEXT_ERR_OK; ++ } ++#endif ++ ++#if (NGX_HTTP_V2 || NGX_HTTP_SPDY) + } + #endif + +diff --git a/src/http/ngx_http.c b/src/http/ngx_http.c +index c036389..78b671c 100644 +--- a/src/http/ngx_http.c ++++ b/src/http/ngx_http.c +@@ -1202,6 +1202,9 @@ ngx_http_add_addresses(ngx_conf_t *cf, ngx_http_core_srv_conf_t *cscf, + #if (NGX_HTTP_V2) + ngx_uint_t http2; + #endif ++#if (NGX_HTTP_SPDY) ++ ngx_uint_t spdy; ++#endif + + /* + * we cannot compare whole sockaddr struct's as kernel +@@ -1237,6 +1240,9 @@ ngx_http_add_addresses(ngx_conf_t *cf, ngx_http_core_srv_conf_t *cscf, + #if (NGX_HTTP_V2) + http2 = lsopt->http2 || addr[i].opt.http2; + #endif ++#if (NGX_HTTP_SPDY) ++ spdy = lsopt->spdy || addr[i].opt.spdy; ++#endif + + if (lsopt->set) { + +@@ -1271,6 +1277,9 @@ ngx_http_add_addresses(ngx_conf_t *cf, ngx_http_core_srv_conf_t *cscf, + #if (NGX_HTTP_V2) + addr[i].opt.http2 = http2; + #endif ++#if (NGX_HTTP_SPDY) ++ addr[i].opt.spdy = spdy; ++#endif + + return NGX_OK; + } +@@ -1314,6 +1323,18 @@ ngx_http_add_address(ngx_conf_t *cf, ngx_http_core_srv_conf_t *cscf, + + #endif + ++#if (NGX_HTTP_SPDY && NGX_HTTP_SSL \ ++ && !defined TLSEXT_TYPE_application_layer_protocol_negotiation \ ++ && !defined TLSEXT_TYPE_next_proto_neg) ++ if (lsopt->spdy && lsopt->ssl) { ++ ngx_conf_log_error(NGX_LOG_WARN, cf, 0, ++ "nginx was built with OpenSSL that lacks ALPN " ++ "and NPN support, SPDY is not enabled for %s", ++ lsopt->addr); ++ } ++ ++#endif ++ + addr = ngx_array_push(&port->addrs); + if (addr == NULL) { + return NGX_ERROR; +@@ -1807,6 +1828,9 @@ ngx_http_add_addrs(ngx_conf_t *cf, ngx_http_port_t *hport, + #if (NGX_HTTP_V2) + addrs[i].conf.http2 = addr[i].opt.http2; + #endif ++#if (NGX_HTTP_SPDY) ++ addrs[i].conf.spdy = addr[i].opt.spdy; ++#endif + addrs[i].conf.proxy_protocol = addr[i].opt.proxy_protocol; + + if (addr[i].hash.buckets == NULL +@@ -1872,6 +1896,9 @@ ngx_http_add_addrs6(ngx_conf_t *cf, ngx_http_port_t *hport, + #if (NGX_HTTP_V2) + addrs6[i].conf.http2 = addr[i].opt.http2; + #endif ++#if (NGX_HTTP_SPDY) ++ addrs6[i].conf.spdy = addr[i].opt.spdy; ++#endif + addrs6[i].conf.proxy_protocol = addr[i].opt.proxy_protocol; + + if (addr[i].hash.buckets == NULL +diff --git a/src/http/ngx_http.h b/src/http/ngx_http.h +index afab4f6..7b70426 100644 +--- a/src/http/ngx_http.h ++++ b/src/http/ngx_http.h +@@ -19,6 +19,9 @@ typedef struct ngx_http_cache_s ngx_http_cache_t; + typedef struct ngx_http_file_cache_s ngx_http_file_cache_t; + typedef struct ngx_http_log_ctx_s ngx_http_log_ctx_t; + typedef struct ngx_http_chunked_s ngx_http_chunked_t; ++#if (NGX_HTTP_SPDY) ++typedef struct ngx_http_spdy_stream_s ngx_http_spdy_stream_t; ++#endif + typedef struct ngx_http_v2_stream_s ngx_http_v2_stream_t; + + typedef ngx_int_t (*ngx_http_header_handler_pt)(ngx_http_request_t *r, +@@ -35,9 +38,13 @@ typedef u_char *(*ngx_http_log_handler_pt)(ngx_http_request_t *r, + #include + #include + ++ + #if (NGX_HTTP_V2) + #include + #endif ++#if (NGX_HTTP_SPDY) ++#include ++#endif + #if (NGX_HTTP_CACHE) + #include + #endif +diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c +index 6e31a2a..afe7aa1 100644 +--- a/src/http/ngx_http_core_module.c ++++ b/src/http/ngx_http_core_module.c +@@ -2151,6 +2151,13 @@ ngx_http_gzip_ok(ngx_http_request_t *r) + return NGX_DECLINED; + } + ++#if (NGX_HTTP_SPDY) ++ if (r->spdy_stream) { ++ r->gzip_ok = 1; ++ return NGX_OK; ++ } ++#endif ++ + ae = r->headers_in.accept_encoding; + if (ae == NULL) { + return NGX_DECLINED; +@@ -2495,6 +2502,9 @@ ngx_http_subrequest(ngx_http_request_t *r, + #if (NGX_HTTP_V2) + sr->stream = r->stream; + #endif ++#if (NGX_HTTP_SPDY) ++ sr->spdy_stream = r->spdy_stream; ++#endif + + sr->method = NGX_HTTP_GET; + sr->http_version = r->http_version; +@@ -4173,11 +4183,15 @@ ngx_http_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) + } + + if (ngx_strcmp(value[n].data, "spdy") == 0) { +- ngx_conf_log_error(NGX_LOG_WARN, cf, 0, +- "invalid parameter \"spdy\": " +- "ngx_http_spdy_module was superseded " +- "by ngx_http_v2_module"); ++#if (NGX_HTTP_SPDY) ++ lsopt.spdy = 1; + continue; ++#else ++ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, ++ "the \"spdy\" parameter requires " ++ "ngx_http_spdy_module"); ++ return NGX_CONF_ERROR; ++#endif + } + + if (ngx_strncmp(value[n].data, "so_keepalive=", 13) == 0) { +diff --git a/src/http/ngx_http_core_module.h b/src/http/ngx_http_core_module.h +index 5018da0..718bc35 100644 +--- a/src/http/ngx_http_core_module.h ++++ b/src/http/ngx_http_core_module.h +@@ -74,6 +74,9 @@ typedef struct { + unsigned wildcard:1; + unsigned ssl:1; + unsigned http2:1; ++#if (NGX_HTTP_SPDY) ++ unsigned spdy:1; ++#endif + #if (NGX_HAVE_INET6) + unsigned ipv6only:1; + #endif +@@ -236,6 +239,9 @@ struct ngx_http_addr_conf_s { + + unsigned ssl:1; + unsigned http2:1; ++#if (NGX_HTTP_SPDY) ++ unsigned spdy:1; ++#endif + unsigned proxy_protocol:1; + }; + +diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c +index 1788bf3..1b88009 100644 +--- a/src/http/ngx_http_request.c ++++ b/src/http/ngx_http_request.c +@@ -316,6 +316,11 @@ ngx_http_init_connection(ngx_connection_t *c) + rev->handler = ngx_http_wait_request_handler; + c->write->handler = ngx_http_empty_handler; + ++#if (NGX_HTTP_SPDY) ++ if (hc->addr_conf->spdy) { ++ rev->handler = ngx_http_spdy_init; ++ } ++#endif + #if (NGX_HTTP_V2) + if (hc->addr_conf->http2) { + rev->handler = ngx_http_v2_init; +@@ -801,6 +806,34 @@ ngx_http_ssl_handshake_handler(ngx_connection_t *c) + } + #endif + ++#if (NGX_HTTP_SPDY \ ++ && (defined TLSEXT_TYPE_application_layer_protocol_negotiation \ ++ || defined TLSEXT_TYPE_next_proto_neg)) ++ { ++ unsigned int len; ++ const unsigned char *data; ++ static const ngx_str_t spdy = ngx_string(NGX_SPDY_NPN_NEGOTIATED); ++ ++#ifdef TLSEXT_TYPE_application_layer_protocol_negotiation ++ SSL_get0_alpn_selected(c->ssl->connection, &data, &len); ++ ++#ifdef TLSEXT_TYPE_next_proto_neg ++ if (len == 0) { ++ SSL_get0_next_proto_negotiated(c->ssl->connection, &data, &len); ++ } ++#endif ++ ++#else /* TLSEXT_TYPE_next_proto_neg */ ++ SSL_get0_next_proto_negotiated(c->ssl->connection, &data, &len); ++#endif ++ ++ if (len == spdy.len && ngx_strncmp(data, spdy.data, spdy.len) == 0) { ++ ngx_http_spdy_init(c->read); ++ return; ++ } ++ } ++#endif ++ + c->log->action = "waiting for request"; + + c->read->handler = ngx_http_wait_request_handler; +@@ -2524,6 +2557,12 @@ ngx_http_finalize_connection(ngx_http_request_t *r) + return; + } + #endif ++#if (NGX_HTTP_SPDY) ++ if (r->spdy_stream) { ++ ngx_http_close_request(r, 0); ++ return; ++ } ++#endif + + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); + +@@ -2743,6 +2782,18 @@ ngx_http_test_reading(ngx_http_request_t *r) + } + + #endif ++#if (NGX_HTTP_SPDY) ++ ++ if (r->spdy_stream) { ++ if (c->error) { ++ err = 0; ++ goto closed; ++ } ++ ++ return; ++ } ++ ++#endif + + #if (NGX_HAVE_KQUEUE) + +@@ -3426,6 +3477,12 @@ ngx_http_close_request(ngx_http_request_t *r, ngx_int_t rc) + return; + } + #endif ++#if (NGX_HTTP_SPDY) ++ if (r->spdy_stream) { ++ ngx_http_spdy_close_stream(r->spdy_stream, rc); ++ return; ++ } ++#endif + + ngx_http_free_request(r, rc); + ngx_http_close_connection(c); +diff --git a/src/http/ngx_http_request.h b/src/http/ngx_http_request.h +index 780a99f..50a0eff 100644 +--- a/src/http/ngx_http_request.h ++++ b/src/http/ngx_http_request.h +@@ -423,6 +423,9 @@ struct ngx_http_request_s { + int *captures; + u_char *captures_data; + #endif ++#if (NGX_HTTP_SPDY) ++ ngx_http_spdy_stream_t *spdy_stream; ++#endif + + size_t limit_rate; + size_t limit_rate_after; +diff --git a/src/http/ngx_http_request_body.c b/src/http/ngx_http_request_body.c +index 2f66484..c6006a0 100644 +--- a/src/http/ngx_http_request_body.c ++++ b/src/http/ngx_http_request_body.c +@@ -52,6 +52,12 @@ ngx_http_read_client_request_body(ngx_http_request_t *r, + goto done; + } + #endif ++#if (NGX_HTTP_SPDY) ++ if (r->spdy_stream) { ++ rc = ngx_http_spdy_read_request_body(r, post_handler); ++ goto done; ++ } ++#endif + + if (ngx_http_test_expect(r) != NGX_OK) { + rc = NGX_HTTP_INTERNAL_SERVER_ERROR; +@@ -524,6 +530,12 @@ ngx_http_discard_request_body(ngx_http_request_t *r) + return NGX_OK; + } + #endif ++#if (NGX_HTTP_SPDY) ++ if (r->spdy_stream) { ++ r->spdy_stream->skip_data = 1; ++ return NGX_OK; ++ } ++#endif + + if (ngx_http_test_expect(r) != NGX_OK) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; +diff --git a/b/src/http/ngx_http_spdy.c b/src/http/ngx_http_spdy.c +new file mode 100644 +index 0000000..21c5217 +--- /dev/null ++++ b/src/http/ngx_http_spdy.c +@@ -0,0 +1,3701 @@ ++ ++/* ++ * Copyright (C) Nginx, Inc. ++ * Copyright (C) Valentin V. Bartenev ++ */ ++ ++ ++#include ++#include ++#include ++#include ++ ++#include ++ ++ ++#if (NGX_HAVE_LITTLE_ENDIAN && NGX_HAVE_NONALIGNED) ++ ++#define ngx_str5cmp(m, c0, c1, c2, c3, c4) \ ++ *(uint32_t *) m == (c3 << 24 | c2 << 16 | c1 << 8 | c0) \ ++ && m[4] == c4 ++ ++#else ++ ++#define ngx_str5cmp(m, c0, c1, c2, c3, c4) \ ++ m[0] == c0 && m[1] == c1 && m[2] == c2 && m[3] == c3 && m[4] == c4 ++ ++#endif ++ ++ ++#if (NGX_HAVE_NONALIGNED) ++ ++#define ngx_spdy_frame_parse_uint16(p) ntohs(*(uint16_t *) (p)) ++#define ngx_spdy_frame_parse_uint32(p) ntohl(*(uint32_t *) (p)) ++ ++#else ++ ++#define ngx_spdy_frame_parse_uint16(p) ((p)[0] << 8 | (p)[1]) ++#define ngx_spdy_frame_parse_uint32(p) \ ++ ((p)[0] << 24 | (p)[1] << 16 | (p)[2] << 8 | (p)[3]) ++ ++#endif ++ ++#define ngx_spdy_frame_parse_sid(p) \ ++ (ngx_spdy_frame_parse_uint32(p) & 0x7fffffff) ++#define ngx_spdy_frame_parse_delta(p) \ ++ (ngx_spdy_frame_parse_uint32(p) & 0x7fffffff) ++ ++ ++#define ngx_spdy_ctl_frame_check(h) \ ++ (((h) & 0xffff0000) == ngx_spdy_ctl_frame_head(0)) ++#define ngx_spdy_data_frame_check(h) \ ++ (!((h) & (uint32_t) NGX_SPDY_CTL_BIT << 31)) ++ ++#define ngx_spdy_ctl_frame_type(h) ((h) & 0x0000ffff) ++#define ngx_spdy_frame_flags(p) ((p) >> 24) ++#define ngx_spdy_frame_length(p) ((p) & 0x00ffffff) ++#define ngx_spdy_frame_id(p) ((p) & 0x00ffffff) ++ ++ ++#define NGX_SPDY_SKIP_HEADERS_BUFFER_SIZE 4096 ++#define NGX_SPDY_CTL_FRAME_BUFFER_SIZE 16 ++ ++#define NGX_SPDY_PROTOCOL_ERROR 1 ++#define NGX_SPDY_INVALID_STREAM 2 ++#define NGX_SPDY_REFUSED_STREAM 3 ++#define NGX_SPDY_UNSUPPORTED_VERSION 4 ++#define NGX_SPDY_CANCEL 5 ++#define NGX_SPDY_INTERNAL_ERROR 6 ++#define NGX_SPDY_FLOW_CONTROL_ERROR 7 ++#define NGX_SPDY_STREAM_IN_USE 8 ++#define NGX_SPDY_STREAM_ALREADY_CLOSED 9 ++/* deprecated 10 */ ++#define NGX_SPDY_FRAME_TOO_LARGE 11 ++ ++#define NGX_SPDY_SETTINGS_MAX_STREAMS 4 ++#define NGX_SPDY_SETTINGS_INIT_WINDOW 7 ++ ++#define NGX_SPDY_SETTINGS_FLAG_PERSIST 0x01 ++#define NGX_SPDY_SETTINGS_FLAG_PERSISTED 0x02 ++ ++#define NGX_SPDY_MAX_WINDOW NGX_MAX_INT32_VALUE ++#define NGX_SPDY_CONNECTION_WINDOW 65536 ++#define NGX_SPDY_INIT_STREAM_WINDOW 65536 ++#define NGX_SPDY_STREAM_WINDOW NGX_SPDY_MAX_WINDOW ++ ++typedef struct { ++ ngx_uint_t hash; ++ u_char len; ++ u_char header[7]; ++ ngx_int_t (*handler)(ngx_http_request_t *r); ++} ngx_http_spdy_request_header_t; ++ ++ ++static void ngx_http_spdy_read_handler(ngx_event_t *rev); ++static void ngx_http_spdy_write_handler(ngx_event_t *wev); ++static void ngx_http_spdy_handle_connection(ngx_http_spdy_connection_t *sc); ++ ++static u_char *ngx_http_spdy_proxy_protocol(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_head(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_syn_stream(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_headers(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_headers_skip(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_headers_error(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_window_update(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_data(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_read_data(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_rst_stream(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_ping(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_skip(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_settings(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_complete(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_save(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end, ngx_http_spdy_handler_pt handler); ++ ++static u_char *ngx_http_spdy_state_inflate_error( ++ ngx_http_spdy_connection_t *sc, int rc); ++static u_char *ngx_http_spdy_state_protocol_error( ++ ngx_http_spdy_connection_t *sc); ++static u_char *ngx_http_spdy_state_internal_error( ++ ngx_http_spdy_connection_t *sc); ++ ++static ngx_int_t ngx_http_spdy_send_window_update( ++ ngx_http_spdy_connection_t *sc, ngx_uint_t sid, ngx_uint_t delta); ++static ngx_int_t ngx_http_spdy_send_rst_stream(ngx_http_spdy_connection_t *sc, ++ ngx_uint_t sid, ngx_uint_t status, ngx_uint_t priority); ++static ngx_int_t ngx_http_spdy_send_settings(ngx_http_spdy_connection_t *sc); ++static ngx_int_t ngx_http_spdy_settings_frame_handler( ++ ngx_http_spdy_connection_t *sc, ngx_http_spdy_out_frame_t *frame); ++static ngx_http_spdy_out_frame_t *ngx_http_spdy_get_ctl_frame( ++ ngx_http_spdy_connection_t *sc, size_t size, ngx_uint_t priority); ++static ngx_int_t ngx_http_spdy_ctl_frame_handler( ++ ngx_http_spdy_connection_t *sc, ngx_http_spdy_out_frame_t *frame); ++ ++static ngx_http_spdy_stream_t *ngx_http_spdy_create_stream( ++ ngx_http_spdy_connection_t *sc, ngx_uint_t id, ngx_uint_t priority); ++static ngx_http_spdy_stream_t *ngx_http_spdy_get_stream_by_id( ++ ngx_http_spdy_connection_t *sc, ngx_uint_t sid); ++#define ngx_http_spdy_streams_index_size(sscf) (sscf->streams_index_mask + 1) ++#define ngx_http_spdy_stream_index(sscf, sid) \ ++ ((sid >> 1) & sscf->streams_index_mask) ++ ++static ngx_int_t ngx_http_spdy_parse_header(ngx_http_request_t *r); ++static ngx_int_t ngx_http_spdy_alloc_large_header_buffer(ngx_http_request_t *r); ++ ++static ngx_int_t ngx_http_spdy_handle_request_header(ngx_http_request_t *r); ++static ngx_int_t ngx_http_spdy_parse_method(ngx_http_request_t *r); ++static ngx_int_t ngx_http_spdy_parse_scheme(ngx_http_request_t *r); ++static ngx_int_t ngx_http_spdy_parse_host(ngx_http_request_t *r); ++static ngx_int_t ngx_http_spdy_parse_path(ngx_http_request_t *r); ++static ngx_int_t ngx_http_spdy_parse_version(ngx_http_request_t *r); ++ ++static ngx_int_t ngx_http_spdy_construct_request_line(ngx_http_request_t *r); ++static void ngx_http_spdy_run_request(ngx_http_request_t *r); ++static ngx_int_t ngx_http_spdy_init_request_body(ngx_http_request_t *r); ++ ++static ngx_int_t ngx_http_spdy_terminate_stream(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_stream_t *stream, ngx_uint_t status); ++ ++static void ngx_http_spdy_close_stream_handler(ngx_event_t *ev); ++ ++static void ngx_http_spdy_handle_connection_handler(ngx_event_t *rev); ++static void ngx_http_spdy_keepalive_handler(ngx_event_t *rev); ++static void ngx_http_spdy_finalize_connection(ngx_http_spdy_connection_t *sc, ++ ngx_int_t rc); ++ ++static ngx_int_t ngx_http_spdy_adjust_windows(ngx_http_spdy_connection_t *sc, ++ ssize_t delta); ++ ++static void ngx_http_spdy_pool_cleanup(void *data); ++ ++static void *ngx_http_spdy_zalloc(void *opaque, u_int items, u_int size); ++static void ngx_http_spdy_zfree(void *opaque, void *address); ++ ++ ++static const u_char ngx_http_spdy_dict[] = { ++ 0x00, 0x00, 0x00, 0x07, 0x6f, 0x70, 0x74, 0x69, /* - - - - o p t i */ ++ 0x6f, 0x6e, 0x73, 0x00, 0x00, 0x00, 0x04, 0x68, /* o n s - - - - h */ ++ 0x65, 0x61, 0x64, 0x00, 0x00, 0x00, 0x04, 0x70, /* e a d - - - - p */ ++ 0x6f, 0x73, 0x74, 0x00, 0x00, 0x00, 0x03, 0x70, /* o s t - - - - p */ ++ 0x75, 0x74, 0x00, 0x00, 0x00, 0x06, 0x64, 0x65, /* u t - - - - d e */ ++ 0x6c, 0x65, 0x74, 0x65, 0x00, 0x00, 0x00, 0x05, /* l e t e - - - - */ ++ 0x74, 0x72, 0x61, 0x63, 0x65, 0x00, 0x00, 0x00, /* t r a c e - - - */ ++ 0x06, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x00, /* - a c c e p t - */ ++ 0x00, 0x00, 0x0e, 0x61, 0x63, 0x63, 0x65, 0x70, /* - - - a c c e p */ ++ 0x74, 0x2d, 0x63, 0x68, 0x61, 0x72, 0x73, 0x65, /* t - c h a r s e */ ++ 0x74, 0x00, 0x00, 0x00, 0x0f, 0x61, 0x63, 0x63, /* t - - - - a c c */ ++ 0x65, 0x70, 0x74, 0x2d, 0x65, 0x6e, 0x63, 0x6f, /* e p t - e n c o */ ++ 0x64, 0x69, 0x6e, 0x67, 0x00, 0x00, 0x00, 0x0f, /* d i n g - - - - */ ++ 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x2d, 0x6c, /* a c c e p t - l */ ++ 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, 0x00, /* a n g u a g e - */ ++ 0x00, 0x00, 0x0d, 0x61, 0x63, 0x63, 0x65, 0x70, /* - - - a c c e p */ ++ 0x74, 0x2d, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x73, /* t - r a n g e s */ ++ 0x00, 0x00, 0x00, 0x03, 0x61, 0x67, 0x65, 0x00, /* - - - - a g e - */ ++ 0x00, 0x00, 0x05, 0x61, 0x6c, 0x6c, 0x6f, 0x77, /* - - - a l l o w */ ++ 0x00, 0x00, 0x00, 0x0d, 0x61, 0x75, 0x74, 0x68, /* - - - - a u t h */ ++ 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, /* o r i z a t i o */ ++ 0x6e, 0x00, 0x00, 0x00, 0x0d, 0x63, 0x61, 0x63, /* n - - - - c a c */ ++ 0x68, 0x65, 0x2d, 0x63, 0x6f, 0x6e, 0x74, 0x72, /* h e - c o n t r */ ++ 0x6f, 0x6c, 0x00, 0x00, 0x00, 0x0a, 0x63, 0x6f, /* o l - - - - c o */ ++ 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, /* n n e c t i o n */ ++ 0x00, 0x00, 0x00, 0x0c, 0x63, 0x6f, 0x6e, 0x74, /* - - - - c o n t */ ++ 0x65, 0x6e, 0x74, 0x2d, 0x62, 0x61, 0x73, 0x65, /* e n t - b a s e */ ++ 0x00, 0x00, 0x00, 0x10, 0x63, 0x6f, 0x6e, 0x74, /* - - - - c o n t */ ++ 0x65, 0x6e, 0x74, 0x2d, 0x65, 0x6e, 0x63, 0x6f, /* e n t - e n c o */ ++ 0x64, 0x69, 0x6e, 0x67, 0x00, 0x00, 0x00, 0x10, /* d i n g - - - - */ ++ 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, /* c o n t e n t - */ ++ 0x6c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, /* l a n g u a g e */ ++ 0x00, 0x00, 0x00, 0x0e, 0x63, 0x6f, 0x6e, 0x74, /* - - - - c o n t */ ++ 0x65, 0x6e, 0x74, 0x2d, 0x6c, 0x65, 0x6e, 0x67, /* e n t - l e n g */ ++ 0x74, 0x68, 0x00, 0x00, 0x00, 0x10, 0x63, 0x6f, /* t h - - - - c o */ ++ 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x6c, 0x6f, /* n t e n t - l o */ ++ 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x00, 0x00, /* c a t i o n - - */ ++ 0x00, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, /* - - c o n t e n */ ++ 0x74, 0x2d, 0x6d, 0x64, 0x35, 0x00, 0x00, 0x00, /* t - m d 5 - - - */ ++ 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, /* - c o n t e n t */ ++ 0x2d, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x00, 0x00, /* - r a n g e - - */ ++ 0x00, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, /* - - c o n t e n */ ++ 0x74, 0x2d, 0x74, 0x79, 0x70, 0x65, 0x00, 0x00, /* t - t y p e - - */ ++ 0x00, 0x04, 0x64, 0x61, 0x74, 0x65, 0x00, 0x00, /* - - d a t e - - */ ++ 0x00, 0x04, 0x65, 0x74, 0x61, 0x67, 0x00, 0x00, /* - - e t a g - - */ ++ 0x00, 0x06, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, /* - - e x p e c t */ ++ 0x00, 0x00, 0x00, 0x07, 0x65, 0x78, 0x70, 0x69, /* - - - - e x p i */ ++ 0x72, 0x65, 0x73, 0x00, 0x00, 0x00, 0x04, 0x66, /* r e s - - - - f */ ++ 0x72, 0x6f, 0x6d, 0x00, 0x00, 0x00, 0x04, 0x68, /* r o m - - - - h */ ++ 0x6f, 0x73, 0x74, 0x00, 0x00, 0x00, 0x08, 0x69, /* o s t - - - - i */ ++ 0x66, 0x2d, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x00, /* f - m a t c h - */ ++ 0x00, 0x00, 0x11, 0x69, 0x66, 0x2d, 0x6d, 0x6f, /* - - - i f - m o */ ++ 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x2d, 0x73, /* d i f i e d - s */ ++ 0x69, 0x6e, 0x63, 0x65, 0x00, 0x00, 0x00, 0x0d, /* i n c e - - - - */ ++ 0x69, 0x66, 0x2d, 0x6e, 0x6f, 0x6e, 0x65, 0x2d, /* i f - n o n e - */ ++ 0x6d, 0x61, 0x74, 0x63, 0x68, 0x00, 0x00, 0x00, /* m a t c h - - - */ ++ 0x08, 0x69, 0x66, 0x2d, 0x72, 0x61, 0x6e, 0x67, /* - i f - r a n g */ ++ 0x65, 0x00, 0x00, 0x00, 0x13, 0x69, 0x66, 0x2d, /* e - - - - i f - */ ++ 0x75, 0x6e, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, /* u n m o d i f i */ ++ 0x65, 0x64, 0x2d, 0x73, 0x69, 0x6e, 0x63, 0x65, /* e d - s i n c e */ ++ 0x00, 0x00, 0x00, 0x0d, 0x6c, 0x61, 0x73, 0x74, /* - - - - l a s t */ ++ 0x2d, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, /* - m o d i f i e */ ++ 0x64, 0x00, 0x00, 0x00, 0x08, 0x6c, 0x6f, 0x63, /* d - - - - l o c */ ++ 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x00, 0x00, 0x00, /* a t i o n - - - */ ++ 0x0c, 0x6d, 0x61, 0x78, 0x2d, 0x66, 0x6f, 0x72, /* - m a x - f o r */ ++ 0x77, 0x61, 0x72, 0x64, 0x73, 0x00, 0x00, 0x00, /* w a r d s - - - */ ++ 0x06, 0x70, 0x72, 0x61, 0x67, 0x6d, 0x61, 0x00, /* - p r a g m a - */ ++ 0x00, 0x00, 0x12, 0x70, 0x72, 0x6f, 0x78, 0x79, /* - - - p r o x y */ ++ 0x2d, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, /* - a u t h e n t */ ++ 0x69, 0x63, 0x61, 0x74, 0x65, 0x00, 0x00, 0x00, /* i c a t e - - - */ ++ 0x13, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x2d, 0x61, /* - p r o x y - a */ ++ 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, /* u t h o r i z a */ ++ 0x74, 0x69, 0x6f, 0x6e, 0x00, 0x00, 0x00, 0x05, /* t i o n - - - - */ ++ 0x72, 0x61, 0x6e, 0x67, 0x65, 0x00, 0x00, 0x00, /* r a n g e - - - */ ++ 0x07, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x72, /* - r e f e r e r */ ++ 0x00, 0x00, 0x00, 0x0b, 0x72, 0x65, 0x74, 0x72, /* - - - - r e t r */ ++ 0x79, 0x2d, 0x61, 0x66, 0x74, 0x65, 0x72, 0x00, /* y - a f t e r - */ ++ 0x00, 0x00, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, /* - - - s e r v e */ ++ 0x72, 0x00, 0x00, 0x00, 0x02, 0x74, 0x65, 0x00, /* r - - - - t e - */ ++ 0x00, 0x00, 0x07, 0x74, 0x72, 0x61, 0x69, 0x6c, /* - - - t r a i l */ ++ 0x65, 0x72, 0x00, 0x00, 0x00, 0x11, 0x74, 0x72, /* e r - - - - t r */ ++ 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x2d, 0x65, /* a n s f e r - e */ ++ 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x00, /* n c o d i n g - */ ++ 0x00, 0x00, 0x07, 0x75, 0x70, 0x67, 0x72, 0x61, /* - - - u p g r a */ ++ 0x64, 0x65, 0x00, 0x00, 0x00, 0x0a, 0x75, 0x73, /* d e - - - - u s */ ++ 0x65, 0x72, 0x2d, 0x61, 0x67, 0x65, 0x6e, 0x74, /* e r - a g e n t */ ++ 0x00, 0x00, 0x00, 0x04, 0x76, 0x61, 0x72, 0x79, /* - - - - v a r y */ ++ 0x00, 0x00, 0x00, 0x03, 0x76, 0x69, 0x61, 0x00, /* - - - - v i a - */ ++ 0x00, 0x00, 0x07, 0x77, 0x61, 0x72, 0x6e, 0x69, /* - - - w a r n i */ ++ 0x6e, 0x67, 0x00, 0x00, 0x00, 0x10, 0x77, 0x77, /* n g - - - - w w */ ++ 0x77, 0x2d, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, /* w - a u t h e n */ ++ 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x00, 0x00, /* t i c a t e - - */ ++ 0x00, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, /* - - m e t h o d */ ++ 0x00, 0x00, 0x00, 0x03, 0x67, 0x65, 0x74, 0x00, /* - - - - g e t - */ ++ 0x00, 0x00, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, /* - - - s t a t u */ ++ 0x73, 0x00, 0x00, 0x00, 0x06, 0x32, 0x30, 0x30, /* s - - - - 2 0 0 */ ++ 0x20, 0x4f, 0x4b, 0x00, 0x00, 0x00, 0x07, 0x76, /* - O K - - - - v */ ++ 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x00, 0x00, /* e r s i o n - - */ ++ 0x00, 0x08, 0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, /* - - H T T P - 1 */ ++ 0x2e, 0x31, 0x00, 0x00, 0x00, 0x03, 0x75, 0x72, /* - 1 - - - - u r */ ++ 0x6c, 0x00, 0x00, 0x00, 0x06, 0x70, 0x75, 0x62, /* l - - - - p u b */ ++ 0x6c, 0x69, 0x63, 0x00, 0x00, 0x00, 0x0a, 0x73, /* l i c - - - - s */ ++ 0x65, 0x74, 0x2d, 0x63, 0x6f, 0x6f, 0x6b, 0x69, /* e t - c o o k i */ ++ 0x65, 0x00, 0x00, 0x00, 0x0a, 0x6b, 0x65, 0x65, /* e - - - - k e e */ ++ 0x70, 0x2d, 0x61, 0x6c, 0x69, 0x76, 0x65, 0x00, /* p - a l i v e - */ ++ 0x00, 0x00, 0x06, 0x6f, 0x72, 0x69, 0x67, 0x69, /* - - - o r i g i */ ++ 0x6e, 0x31, 0x30, 0x30, 0x31, 0x30, 0x31, 0x32, /* n 1 0 0 1 0 1 2 */ ++ 0x30, 0x31, 0x32, 0x30, 0x32, 0x32, 0x30, 0x35, /* 0 1 2 0 2 2 0 5 */ ++ 0x32, 0x30, 0x36, 0x33, 0x30, 0x30, 0x33, 0x30, /* 2 0 6 3 0 0 3 0 */ ++ 0x32, 0x33, 0x30, 0x33, 0x33, 0x30, 0x34, 0x33, /* 2 3 0 3 3 0 4 3 */ ++ 0x30, 0x35, 0x33, 0x30, 0x36, 0x33, 0x30, 0x37, /* 0 5 3 0 6 3 0 7 */ ++ 0x34, 0x30, 0x32, 0x34, 0x30, 0x35, 0x34, 0x30, /* 4 0 2 4 0 5 4 0 */ ++ 0x36, 0x34, 0x30, 0x37, 0x34, 0x30, 0x38, 0x34, /* 6 4 0 7 4 0 8 4 */ ++ 0x30, 0x39, 0x34, 0x31, 0x30, 0x34, 0x31, 0x31, /* 0 9 4 1 0 4 1 1 */ ++ 0x34, 0x31, 0x32, 0x34, 0x31, 0x33, 0x34, 0x31, /* 4 1 2 4 1 3 4 1 */ ++ 0x34, 0x34, 0x31, 0x35, 0x34, 0x31, 0x36, 0x34, /* 4 4 1 5 4 1 6 4 */ ++ 0x31, 0x37, 0x35, 0x30, 0x32, 0x35, 0x30, 0x34, /* 1 7 5 0 2 5 0 4 */ ++ 0x35, 0x30, 0x35, 0x32, 0x30, 0x33, 0x20, 0x4e, /* 5 0 5 2 0 3 - N */ ++ 0x6f, 0x6e, 0x2d, 0x41, 0x75, 0x74, 0x68, 0x6f, /* o n - A u t h o */ ++ 0x72, 0x69, 0x74, 0x61, 0x74, 0x69, 0x76, 0x65, /* r i t a t i v e */ ++ 0x20, 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, /* - I n f o r m a */ ++ 0x74, 0x69, 0x6f, 0x6e, 0x32, 0x30, 0x34, 0x20, /* t i o n 2 0 4 - */ ++ 0x4e, 0x6f, 0x20, 0x43, 0x6f, 0x6e, 0x74, 0x65, /* N o - C o n t e */ ++ 0x6e, 0x74, 0x33, 0x30, 0x31, 0x20, 0x4d, 0x6f, /* n t 3 0 1 - M o */ ++ 0x76, 0x65, 0x64, 0x20, 0x50, 0x65, 0x72, 0x6d, /* v e d - P e r m */ ++ 0x61, 0x6e, 0x65, 0x6e, 0x74, 0x6c, 0x79, 0x34, /* a n e n t l y 4 */ ++ 0x30, 0x30, 0x20, 0x42, 0x61, 0x64, 0x20, 0x52, /* 0 0 - B a d - R */ ++ 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x34, 0x30, /* e q u e s t 4 0 */ ++ 0x31, 0x20, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, /* 1 - U n a u t h */ ++ 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x34, 0x30, /* o r i z e d 4 0 */ ++ 0x33, 0x20, 0x46, 0x6f, 0x72, 0x62, 0x69, 0x64, /* 3 - F o r b i d */ ++ 0x64, 0x65, 0x6e, 0x34, 0x30, 0x34, 0x20, 0x4e, /* d e n 4 0 4 - N */ ++ 0x6f, 0x74, 0x20, 0x46, 0x6f, 0x75, 0x6e, 0x64, /* o t - F o u n d */ ++ 0x35, 0x30, 0x30, 0x20, 0x49, 0x6e, 0x74, 0x65, /* 5 0 0 - I n t e */ ++ 0x72, 0x6e, 0x61, 0x6c, 0x20, 0x53, 0x65, 0x72, /* r n a l - S e r */ ++ 0x76, 0x65, 0x72, 0x20, 0x45, 0x72, 0x72, 0x6f, /* v e r - E r r o */ ++ 0x72, 0x35, 0x30, 0x31, 0x20, 0x4e, 0x6f, 0x74, /* r 5 0 1 - N o t */ ++ 0x20, 0x49, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, /* - I m p l e m e */ ++ 0x6e, 0x74, 0x65, 0x64, 0x35, 0x30, 0x33, 0x20, /* n t e d 5 0 3 - */ ++ 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x20, /* S e r v i c e - */ ++ 0x55, 0x6e, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, /* U n a v a i l a */ ++ 0x62, 0x6c, 0x65, 0x4a, 0x61, 0x6e, 0x20, 0x46, /* b l e J a n - F */ ++ 0x65, 0x62, 0x20, 0x4d, 0x61, 0x72, 0x20, 0x41, /* e b - M a r - A */ ++ 0x70, 0x72, 0x20, 0x4d, 0x61, 0x79, 0x20, 0x4a, /* p r - M a y - J */ ++ 0x75, 0x6e, 0x20, 0x4a, 0x75, 0x6c, 0x20, 0x41, /* u n - J u l - A */ ++ 0x75, 0x67, 0x20, 0x53, 0x65, 0x70, 0x74, 0x20, /* u g - S e p t - */ ++ 0x4f, 0x63, 0x74, 0x20, 0x4e, 0x6f, 0x76, 0x20, /* O c t - N o v - */ ++ 0x44, 0x65, 0x63, 0x20, 0x30, 0x30, 0x3a, 0x30, /* D e c - 0 0 - 0 */ ++ 0x30, 0x3a, 0x30, 0x30, 0x20, 0x4d, 0x6f, 0x6e, /* 0 - 0 0 - M o n */ ++ 0x2c, 0x20, 0x54, 0x75, 0x65, 0x2c, 0x20, 0x57, /* - - T u e - - W */ ++ 0x65, 0x64, 0x2c, 0x20, 0x54, 0x68, 0x75, 0x2c, /* e d - - T h u - */ ++ 0x20, 0x46, 0x72, 0x69, 0x2c, 0x20, 0x53, 0x61, /* - F r i - - S a */ ++ 0x74, 0x2c, 0x20, 0x53, 0x75, 0x6e, 0x2c, 0x20, /* t - - S u n - - */ ++ 0x47, 0x4d, 0x54, 0x63, 0x68, 0x75, 0x6e, 0x6b, /* G M T c h u n k */ ++ 0x65, 0x64, 0x2c, 0x74, 0x65, 0x78, 0x74, 0x2f, /* e d - t e x t - */ ++ 0x68, 0x74, 0x6d, 0x6c, 0x2c, 0x69, 0x6d, 0x61, /* h t m l - i m a */ ++ 0x67, 0x65, 0x2f, 0x70, 0x6e, 0x67, 0x2c, 0x69, /* g e - p n g - i */ ++ 0x6d, 0x61, 0x67, 0x65, 0x2f, 0x6a, 0x70, 0x67, /* m a g e - j p g */ ++ 0x2c, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x2f, 0x67, /* - i m a g e - g */ ++ 0x69, 0x66, 0x2c, 0x61, 0x70, 0x70, 0x6c, 0x69, /* i f - a p p l i */ ++ 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x78, /* c a t i o n - x */ ++ 0x6d, 0x6c, 0x2c, 0x61, 0x70, 0x70, 0x6c, 0x69, /* m l - a p p l i */ ++ 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x78, /* c a t i o n - x */ ++ 0x68, 0x74, 0x6d, 0x6c, 0x2b, 0x78, 0x6d, 0x6c, /* h t m l - x m l */ ++ 0x2c, 0x74, 0x65, 0x78, 0x74, 0x2f, 0x70, 0x6c, /* - t e x t - p l */ ++ 0x61, 0x69, 0x6e, 0x2c, 0x74, 0x65, 0x78, 0x74, /* a i n - t e x t */ ++ 0x2f, 0x6a, 0x61, 0x76, 0x61, 0x73, 0x63, 0x72, /* - j a v a s c r */ ++ 0x69, 0x70, 0x74, 0x2c, 0x70, 0x75, 0x62, 0x6c, /* i p t - p u b l */ ++ 0x69, 0x63, 0x70, 0x72, 0x69, 0x76, 0x61, 0x74, /* i c p r i v a t */ ++ 0x65, 0x6d, 0x61, 0x78, 0x2d, 0x61, 0x67, 0x65, /* e m a x - a g e */ ++ 0x3d, 0x67, 0x7a, 0x69, 0x70, 0x2c, 0x64, 0x65, /* - g z i p - d e */ ++ 0x66, 0x6c, 0x61, 0x74, 0x65, 0x2c, 0x73, 0x64, /* f l a t e - s d */ ++ 0x63, 0x68, 0x63, 0x68, 0x61, 0x72, 0x73, 0x65, /* c h c h a r s e */ ++ 0x74, 0x3d, 0x75, 0x74, 0x66, 0x2d, 0x38, 0x63, /* t - u t f - 8 c */ ++ 0x68, 0x61, 0x72, 0x73, 0x65, 0x74, 0x3d, 0x69, /* h a r s e t - i */ ++ 0x73, 0x6f, 0x2d, 0x38, 0x38, 0x35, 0x39, 0x2d, /* s o - 8 8 5 9 - */ ++ 0x31, 0x2c, 0x75, 0x74, 0x66, 0x2d, 0x2c, 0x2a, /* 1 - u t f - - - */ ++ 0x2c, 0x65, 0x6e, 0x71, 0x3d, 0x30, 0x2e /* - e n q - 0 - */ ++}; ++ ++ ++static ngx_http_spdy_request_header_t ngx_http_spdy_request_headers[] = { ++ { 0, 6, "method", ngx_http_spdy_parse_method }, ++ { 0, 6, "scheme", ngx_http_spdy_parse_scheme }, ++ { 0, 4, "host", ngx_http_spdy_parse_host }, ++ { 0, 4, "path", ngx_http_spdy_parse_path }, ++ { 0, 7, "version", ngx_http_spdy_parse_version }, ++}; ++ ++#define NGX_SPDY_REQUEST_HEADERS \ ++ (sizeof(ngx_http_spdy_request_headers) \ ++ / sizeof(ngx_http_spdy_request_header_t)) ++ ++ ++void ++ngx_http_spdy_init(ngx_event_t *rev) ++{ ++ int rc; ++ ngx_connection_t *c; ++ ngx_pool_cleanup_t *cln; ++ ngx_http_connection_t *hc; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ngx_http_spdy_main_conf_t *smcf; ++ ngx_http_spdy_connection_t *sc; ++ ++ c = rev->data; ++ hc = c->data; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "init spdy request"); ++ ++ c->log->action = "processing SPDY"; ++ ++ smcf = ngx_http_get_module_main_conf(hc->conf_ctx, ngx_http_spdy_module); ++ ++ if (smcf->recv_buffer == NULL) { ++ smcf->recv_buffer = ngx_palloc(ngx_cycle->pool, smcf->recv_buffer_size); ++ if (smcf->recv_buffer == NULL) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ } ++ ++ sc = ngx_pcalloc(c->pool, sizeof(ngx_http_spdy_connection_t)); ++ if (sc == NULL) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ sc->connection = c; ++ sc->http_connection = hc; ++ ++ sc->send_window = NGX_SPDY_CONNECTION_WINDOW; ++ sc->recv_window = NGX_SPDY_CONNECTION_WINDOW; ++ ++ sc->init_window = NGX_SPDY_INIT_STREAM_WINDOW; ++ ++ sc->handler = hc->proxy_protocol ? ngx_http_spdy_proxy_protocol ++ : ngx_http_spdy_state_head; ++ ++ sc->zstream_in.zalloc = ngx_http_spdy_zalloc; ++ sc->zstream_in.zfree = ngx_http_spdy_zfree; ++ sc->zstream_in.opaque = sc; ++ ++ rc = inflateInit(&sc->zstream_in); ++ if (rc != Z_OK) { ++ ngx_log_error(NGX_LOG_ALERT, c->log, 0, ++ "inflateInit() failed: %d", rc); ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ sc->zstream_out.zalloc = ngx_http_spdy_zalloc; ++ sc->zstream_out.zfree = ngx_http_spdy_zfree; ++ sc->zstream_out.opaque = sc; ++ ++ sscf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_spdy_module); ++ ++ rc = deflateInit2(&sc->zstream_out, (int) sscf->headers_comp, ++ Z_DEFLATED, 11, 4, Z_DEFAULT_STRATEGY); ++ ++ if (rc != Z_OK) { ++ ngx_log_error(NGX_LOG_ALERT, c->log, 0, ++ "deflateInit2() failed: %d", rc); ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ rc = deflateSetDictionary(&sc->zstream_out, ngx_http_spdy_dict, ++ sizeof(ngx_http_spdy_dict)); ++ if (rc != Z_OK) { ++ ngx_log_error(NGX_LOG_ALERT, c->log, 0, ++ "deflateSetDictionary() failed: %d", rc); ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ sc->pool = ngx_create_pool(sscf->pool_size, sc->connection->log); ++ if (sc->pool == NULL) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ cln = ngx_pool_cleanup_add(c->pool, sizeof(ngx_pool_cleanup_file_t)); ++ if (cln == NULL) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ cln->handler = ngx_http_spdy_pool_cleanup; ++ cln->data = sc; ++ ++ sc->streams_index = ngx_pcalloc(sc->pool, ++ ngx_http_spdy_streams_index_size(sscf) ++ * sizeof(ngx_http_spdy_stream_t *)); ++ if (sc->streams_index == NULL) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ if (ngx_http_spdy_send_settings(sc) == NGX_ERROR) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ if (ngx_http_spdy_send_window_update(sc, 0, NGX_SPDY_MAX_WINDOW ++ - sc->recv_window) ++ == NGX_ERROR) ++ { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ sc->recv_window = NGX_SPDY_MAX_WINDOW; ++ ++ ngx_queue_init(&sc->waiting); ++ ngx_queue_init(&sc->posted); ++ ++ c->data = sc; ++ ++ rev->handler = ngx_http_spdy_read_handler; ++ c->write->handler = ngx_http_spdy_write_handler; ++ ++ ngx_http_spdy_read_handler(rev); ++} ++ ++ ++static void ++ngx_http_spdy_read_handler(ngx_event_t *rev) ++{ ++ u_char *p, *end; ++ size_t available; ++ ssize_t n; ++ ngx_connection_t *c; ++ ngx_http_spdy_main_conf_t *smcf; ++ ngx_http_spdy_connection_t *sc; ++ ++ c = rev->data; ++ sc = c->data; ++ ++ if (rev->timedout) { ++ ngx_log_error(NGX_LOG_INFO, c->log, NGX_ETIMEDOUT, "client timed out"); ++ ngx_http_spdy_finalize_connection(sc, NGX_HTTP_REQUEST_TIME_OUT); ++ return; ++ } ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "spdy read handler"); ++ ++ sc->blocked = 1; ++ ++ smcf = ngx_http_get_module_main_conf(sc->http_connection->conf_ctx, ++ ngx_http_spdy_module); ++ ++ available = smcf->recv_buffer_size - 2 * NGX_SPDY_STATE_BUFFER_SIZE; ++ ++ do { ++ p = smcf->recv_buffer; ++ ++ ngx_memcpy(p, sc->buffer, NGX_SPDY_STATE_BUFFER_SIZE); ++ end = p + sc->buffer_used; ++ ++ n = c->recv(c, end, available); ++ ++ if (n == NGX_AGAIN) { ++ break; ++ } ++ ++ if (n == 0 && (sc->incomplete || sc->processing)) { ++ ngx_log_error(NGX_LOG_INFO, c->log, 0, ++ "client prematurely closed connection"); ++ } ++ ++ if (n == 0 || n == NGX_ERROR) { ++ ngx_http_spdy_finalize_connection(sc, ++ NGX_HTTP_CLIENT_CLOSED_REQUEST); ++ return; ++ } ++ ++ end += n; ++ ++ sc->buffer_used = 0; ++ sc->incomplete = 0; ++ ++ do { ++ p = sc->handler(sc, p, end); ++ ++ if (p == NULL) { ++ return; ++ } ++ ++ } while (p != end); ++ ++ } while (rev->ready); ++ ++ if (ngx_handle_read_event(rev, 0) != NGX_OK) { ++ ngx_http_spdy_finalize_connection(sc, NGX_HTTP_INTERNAL_SERVER_ERROR); ++ return; ++ } ++ ++ if (sc->last_out && ngx_http_spdy_send_output_queue(sc) == NGX_ERROR) { ++ ngx_http_spdy_finalize_connection(sc, NGX_HTTP_CLIENT_CLOSED_REQUEST); ++ return; ++ } ++ ++ sc->blocked = 0; ++ ++ if (sc->processing) { ++ if (rev->timer_set) { ++ ngx_del_timer(rev); ++ } ++ return; ++ } ++ ++ ngx_http_spdy_handle_connection(sc); ++} ++ ++ ++static void ++ngx_http_spdy_write_handler(ngx_event_t *wev) ++{ ++ ngx_int_t rc; ++ ngx_queue_t *q; ++ ngx_connection_t *c; ++ ngx_http_spdy_stream_t *stream; ++ ngx_http_spdy_connection_t *sc; ++ ++ c = wev->data; ++ sc = c->data; ++ ++ if (wev->timedout) { ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, ++ "spdy write event timed out"); ++ ngx_http_spdy_finalize_connection(sc, NGX_HTTP_CLIENT_CLOSED_REQUEST); ++ return; ++ } ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "spdy write handler"); ++ ++ sc->blocked = 1; ++ ++ rc = ngx_http_spdy_send_output_queue(sc); ++ ++ if (rc == NGX_ERROR) { ++ ngx_http_spdy_finalize_connection(sc, NGX_HTTP_CLIENT_CLOSED_REQUEST); ++ return; ++ } ++ ++ while (!ngx_queue_empty(&sc->posted)) { ++ q = ngx_queue_head(&sc->posted); ++ ++ ngx_queue_remove(q); ++ ++ stream = ngx_queue_data(q, ngx_http_spdy_stream_t, queue); ++ ++ stream->handled = 0; ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, ++ "run spdy stream %ui", stream->id); ++ ++ wev = stream->request->connection->write; ++ wev->handler(wev); ++ } ++ ++ sc->blocked = 0; ++ ++ if (rc == NGX_AGAIN) { ++ return; ++ } ++ ++ ngx_http_spdy_handle_connection(sc); ++} ++ ++ ++ngx_int_t ++ngx_http_spdy_send_output_queue(ngx_http_spdy_connection_t *sc) ++{ ++ int tcp_nodelay; ++ ngx_chain_t *cl; ++ ngx_event_t *wev; ++ ngx_connection_t *c; ++ ngx_http_core_loc_conf_t *clcf; ++ ngx_http_spdy_out_frame_t *out, *frame, *fn; ++ ++ c = sc->connection; ++ ++ if (c->error) { ++ return NGX_ERROR; ++ } ++ ++ wev = c->write; ++ ++ if (!wev->ready) { ++ return NGX_OK; ++ } ++ ++ cl = NULL; ++ out = NULL; ++ ++ for (frame = sc->last_out; frame; frame = fn) { ++ frame->last->next = cl; ++ cl = frame->first; ++ ++ fn = frame->next; ++ frame->next = out; ++ out = frame; ++ ++ ngx_log_debug5(NGX_LOG_DEBUG_HTTP, c->log, 0, ++ "spdy frame out: %p sid:%ui prio:%ui bl:%d len:%uz", ++ out, out->stream ? out->stream->id : 0, out->priority, ++ out->blocked, out->length); ++ } ++ ++ cl = c->send_chain(c, cl, 0); ++ ++ if (cl == NGX_CHAIN_ERROR) { ++ goto error; ++ } ++ ++ clcf = ngx_http_get_module_loc_conf(sc->http_connection->conf_ctx, ++ ngx_http_core_module); ++ ++ if (ngx_handle_write_event(wev, clcf->send_lowat) != NGX_OK) { ++ goto error; ++ } ++ ++ if (c->tcp_nopush == NGX_TCP_NOPUSH_SET) { ++ if (ngx_tcp_push(c->fd) == -1) { ++ ngx_connection_error(c, ngx_socket_errno, ngx_tcp_push_n " failed"); ++ goto error; ++ } ++ ++ c->tcp_nopush = NGX_TCP_NOPUSH_UNSET; ++ tcp_nodelay = ngx_tcp_nodelay_and_tcp_nopush ? 1 : 0; ++ ++ } else { ++ tcp_nodelay = 1; ++ } ++ ++ if (tcp_nodelay ++ && clcf->tcp_nodelay ++ && c->tcp_nodelay == NGX_TCP_NODELAY_UNSET) ++ { ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "tcp_nodelay"); ++ ++ if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY, ++ (const void *) &tcp_nodelay, sizeof(int)) ++ == -1) ++ { ++#if (NGX_SOLARIS) ++ /* Solaris returns EINVAL if a socket has been shut down */ ++ c->log_error = NGX_ERROR_IGNORE_EINVAL; ++#endif ++ ++ ngx_connection_error(c, ngx_socket_errno, ++ "setsockopt(TCP_NODELAY) failed"); ++ ++ c->log_error = NGX_ERROR_INFO; ++ goto error; ++ } ++ ++ c->tcp_nodelay = NGX_TCP_NODELAY_SET; ++ } ++ ++ if (cl) { ++ ngx_add_timer(wev, clcf->send_timeout); ++ ++ } else { ++ if (wev->timer_set) { ++ ngx_del_timer(wev); ++ } ++ } ++ ++ for ( /* void */ ; out; out = fn) { ++ fn = out->next; ++ ++ if (out->handler(sc, out) != NGX_OK) { ++ out->blocked = 1; ++ out->priority = NGX_SPDY_HIGHEST_PRIORITY; ++ break; ++ } ++ ++ ngx_log_debug4(NGX_LOG_DEBUG_HTTP, c->log, 0, ++ "spdy frame sent: %p sid:%ui bl:%d len:%uz", ++ out, out->stream ? out->stream->id : 0, ++ out->blocked, out->length); ++ } ++ ++ frame = NULL; ++ ++ for ( /* void */ ; out; out = fn) { ++ fn = out->next; ++ out->next = frame; ++ frame = out; ++ } ++ ++ sc->last_out = frame; ++ ++ return NGX_OK; ++ ++error: ++ ++ c->error = 1; ++ ++ if (!sc->blocked) { ++ ngx_post_event(wev, &ngx_posted_events); ++ } ++ ++ return NGX_ERROR; ++} ++ ++ ++static void ++ngx_http_spdy_handle_connection(ngx_http_spdy_connection_t *sc) ++{ ++ ngx_connection_t *c; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ++ if (sc->last_out || sc->processing) { ++ return; ++ } ++ ++ c = sc->connection; ++ ++ if (c->error) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ if (c->buffered) { ++ return; ++ } ++ ++ sscf = ngx_http_get_module_srv_conf(sc->http_connection->conf_ctx, ++ ngx_http_spdy_module); ++ if (sc->incomplete) { ++ ngx_add_timer(c->read, sscf->recv_timeout); ++ return; ++ } ++ ++ if (ngx_terminate || ngx_exiting) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ ngx_destroy_pool(sc->pool); ++ ++ sc->pool = NULL; ++ sc->free_ctl_frames = NULL; ++ sc->free_fake_connections = NULL; ++ ++#if (NGX_HTTP_SSL) ++ if (c->ssl) { ++ ngx_ssl_free_buffer(c); ++ } ++#endif ++ ++ c->destroyed = 1; ++ c->idle = 1; ++ ngx_reusable_connection(c, 1); ++ ++ c->write->handler = ngx_http_empty_handler; ++ c->read->handler = ngx_http_spdy_keepalive_handler; ++ ++ if (c->write->timer_set) { ++ ngx_del_timer(c->write); ++ } ++ ++ ngx_add_timer(c->read, sscf->keepalive_timeout); ++} ++ ++ ++static u_char * ++ngx_http_spdy_proxy_protocol(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ ngx_log_t *log; ++ ++ log = sc->connection->log; ++ log->action = "reading PROXY protocol"; ++ ++ pos = ngx_proxy_protocol_read(sc->connection, pos, end); ++ ++ log->action = "processing SPDY"; ++ ++ if (pos == NULL) { ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_head(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ uint32_t head, flen; ++ ngx_uint_t type; ++ ++ if (end - pos < NGX_SPDY_FRAME_HEADER_SIZE) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_head); ++ } ++ ++ head = ngx_spdy_frame_parse_uint32(pos); ++ ++ pos += sizeof(uint32_t); ++ ++ flen = ngx_spdy_frame_parse_uint32(pos); ++ ++ sc->flags = ngx_spdy_frame_flags(flen); ++ sc->length = ngx_spdy_frame_length(flen); ++ ++ pos += sizeof(uint32_t); ++ ++ ngx_log_debug3(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "process spdy frame head:%08XD f:%Xd l:%uz", ++ head, sc->flags, sc->length); ++ ++ if (ngx_spdy_ctl_frame_check(head)) { ++ type = ngx_spdy_ctl_frame_type(head); ++ ++ switch (type) { ++ ++ case NGX_SPDY_SYN_STREAM: ++ return ngx_http_spdy_state_syn_stream(sc, pos, end); ++ ++ case NGX_SPDY_SYN_REPLY: ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent unexpected SYN_REPLY frame"); ++ return ngx_http_spdy_state_protocol_error(sc); ++ ++ case NGX_SPDY_RST_STREAM: ++ return ngx_http_spdy_state_rst_stream(sc, pos, end); ++ ++ case NGX_SPDY_SETTINGS: ++ return ngx_http_spdy_state_settings(sc, pos, end); ++ ++ case NGX_SPDY_PING: ++ return ngx_http_spdy_state_ping(sc, pos, end); ++ ++ case NGX_SPDY_GOAWAY: ++ return ngx_http_spdy_state_skip(sc, pos, end); /* TODO */ ++ ++ case NGX_SPDY_HEADERS: ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent unexpected HEADERS frame"); ++ return ngx_http_spdy_state_protocol_error(sc); ++ ++ case NGX_SPDY_WINDOW_UPDATE: ++ return ngx_http_spdy_state_window_update(sc, pos, end); ++ ++ default: ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy control frame with unknown type %ui", type); ++ return ngx_http_spdy_state_skip(sc, pos, end); ++ } ++ } ++ ++ if (ngx_spdy_data_frame_check(head)) { ++ sc->stream = ngx_http_spdy_get_stream_by_id(sc, head); ++ return ngx_http_spdy_state_data(sc, pos, end); ++ } ++ ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent invalid frame"); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_syn_stream(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ ngx_uint_t sid, prio; ++ ngx_http_spdy_stream_t *stream; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ++ if (end - pos < NGX_SPDY_SYN_STREAM_SIZE) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_syn_stream); ++ } ++ ++ if (sc->length <= NGX_SPDY_SYN_STREAM_SIZE) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent SYN_STREAM frame with incorrect length %uz", ++ sc->length); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ sc->length -= NGX_SPDY_SYN_STREAM_SIZE; ++ ++ sid = ngx_spdy_frame_parse_sid(pos); ++ prio = pos[8] >> 5; ++ ++ pos += NGX_SPDY_SYN_STREAM_SIZE; ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy SYN_STREAM frame sid:%ui prio:%ui", sid, prio); ++ ++ if (sid % 2 == 0 || sid <= sc->last_sid) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent SYN_STREAM frame " ++ "with invalid Stream-ID %ui", sid); ++ ++ stream = ngx_http_spdy_get_stream_by_id(sc, sid); ++ ++ if (stream) { ++ if (ngx_http_spdy_terminate_stream(sc, stream, ++ NGX_SPDY_PROTOCOL_ERROR) ++ != NGX_OK) ++ { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ } ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ sc->last_sid = sid; ++ ++ sscf = ngx_http_get_module_srv_conf(sc->http_connection->conf_ctx, ++ ngx_http_spdy_module); ++ ++ if (sc->processing >= sscf->concurrent_streams) { ++ ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "concurrent streams exceeded %ui", sc->processing); ++ ++ if (ngx_http_spdy_send_rst_stream(sc, sid, NGX_SPDY_REFUSED_STREAM, ++ prio) ++ != NGX_OK) ++ { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ return ngx_http_spdy_state_headers_skip(sc, pos, end); ++ } ++ ++ stream = ngx_http_spdy_create_stream(sc, sid, prio); ++ if (stream == NULL) { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ stream->in_closed = (sc->flags & NGX_SPDY_FLAG_FIN) ? 1 : 0; ++ ++ stream->request->request_length = NGX_SPDY_FRAME_HEADER_SIZE ++ + NGX_SPDY_SYN_STREAM_SIZE ++ + sc->length; ++ ++ sc->stream = stream; ++ ++ return ngx_http_spdy_state_headers(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_headers(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ int z; ++ size_t size; ++ ngx_buf_t *buf; ++ ngx_int_t rc; ++ ngx_http_request_t *r; ++ ++ size = end - pos; ++ ++ if (size == 0) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_headers); ++ } ++ ++ if (size > sc->length) { ++ size = sc->length; ++ } ++ ++ r = sc->stream->request; ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "process spdy header block %uz of %uz", size, sc->length); ++ ++ buf = r->header_in; ++ ++ sc->zstream_in.next_in = pos; ++ sc->zstream_in.avail_in = size; ++ sc->zstream_in.next_out = buf->last; ++ ++ /* one byte is reserved for null-termination of the last header value */ ++ sc->zstream_in.avail_out = buf->end - buf->last - 1; ++ ++ z = inflate(&sc->zstream_in, Z_NO_FLUSH); ++ ++ if (z == Z_NEED_DICT) { ++ z = inflateSetDictionary(&sc->zstream_in, ngx_http_spdy_dict, ++ sizeof(ngx_http_spdy_dict)); ++ ++ if (z != Z_OK) { ++ if (z == Z_DATA_ERROR) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent SYN_STREAM frame with header " ++ "block encoded using wrong dictionary: %ul", ++ (u_long) sc->zstream_in.adler); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, ++ "inflateSetDictionary() failed: %d", z); ++ ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy inflateSetDictionary(): %d", z); ++ ++ z = sc->zstream_in.avail_in ? inflate(&sc->zstream_in, Z_NO_FLUSH) ++ : Z_OK; ++ } ++ ++ ngx_log_debug5(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy inflate out: ni:%p no:%p ai:%ud ao:%ud rc:%d", ++ sc->zstream_in.next_in, sc->zstream_in.next_out, ++ sc->zstream_in.avail_in, sc->zstream_in.avail_out, ++ z); ++ ++ if (z != Z_OK) { ++ return ngx_http_spdy_state_inflate_error(sc, z); ++ } ++ ++ sc->length -= sc->zstream_in.next_in - pos; ++ pos = sc->zstream_in.next_in; ++ ++ buf->last = sc->zstream_in.next_out; ++ ++ if (r->headers_in.headers.part.elts == NULL) { ++ ++ if (buf->last - buf->pos < NGX_SPDY_NV_NUM_SIZE) { ++ ++ if (sc->length == 0) { ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "premature end of spdy header block"); ++ ++ return ngx_http_spdy_state_headers_error(sc, pos, end); ++ } ++ ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_headers); ++ } ++ ++ sc->entries = ngx_spdy_frame_parse_uint32(buf->pos); ++ ++ buf->pos += NGX_SPDY_NV_NUM_SIZE; ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy header block has %ui entries", ++ sc->entries); ++ ++ if (ngx_list_init(&r->headers_in.headers, r->pool, 20, ++ sizeof(ngx_table_elt_t)) ++ != NGX_OK) ++ { ++ ngx_http_spdy_close_stream(sc->stream, ++ NGX_HTTP_INTERNAL_SERVER_ERROR); ++ return ngx_http_spdy_state_headers_skip(sc, pos, end); ++ } ++ ++ if (ngx_array_init(&r->headers_in.cookies, r->pool, 2, ++ sizeof(ngx_table_elt_t *)) ++ != NGX_OK) ++ { ++ ngx_http_spdy_close_stream(sc->stream, ++ NGX_HTTP_INTERNAL_SERVER_ERROR); ++ return ngx_http_spdy_state_headers_skip(sc, pos, end); ++ } ++ } ++ ++ while (sc->entries) { ++ ++ rc = ngx_http_spdy_parse_header(r); ++ ++ switch (rc) { ++ ++ case NGX_DONE: ++ sc->entries--; ++ ++ case NGX_OK: ++ break; ++ ++ case NGX_AGAIN: ++ ++ if (sc->zstream_in.avail_in) { ++ ++ rc = ngx_http_spdy_alloc_large_header_buffer(r); ++ ++ if (rc == NGX_DECLINED) { ++ ngx_http_finalize_request(r, ++ NGX_HTTP_REQUEST_HEADER_TOO_LARGE); ++ return ngx_http_spdy_state_headers_skip(sc, pos, end); ++ } ++ ++ if (rc != NGX_OK) { ++ ngx_http_spdy_close_stream(sc->stream, ++ NGX_HTTP_INTERNAL_SERVER_ERROR); ++ return ngx_http_spdy_state_headers_skip(sc, pos, end); ++ } ++ ++ /* null-terminate the last processed header name or value */ ++ *buf->pos = '\0'; ++ ++ buf = r->header_in; ++ ++ sc->zstream_in.next_out = buf->last; ++ ++ /* one byte is reserved for null-termination */ ++ sc->zstream_in.avail_out = buf->end - buf->last - 1; ++ ++ z = inflate(&sc->zstream_in, Z_NO_FLUSH); ++ ++ ngx_log_debug5(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy inflate out: ni:%p no:%p ai:%ud ao:%ud rc:%d", ++ sc->zstream_in.next_in, sc->zstream_in.next_out, ++ sc->zstream_in.avail_in, sc->zstream_in.avail_out, ++ z); ++ ++ if (z != Z_OK) { ++ return ngx_http_spdy_state_inflate_error(sc, z); ++ } ++ ++ sc->length -= sc->zstream_in.next_in - pos; ++ pos = sc->zstream_in.next_in; ++ ++ buf->last = sc->zstream_in.next_out; ++ ++ continue; ++ } ++ ++ if (sc->length == 0) { ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "premature end of spdy header block"); ++ ++ return ngx_http_spdy_state_headers_error(sc, pos, end); ++ } ++ ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_headers); ++ ++ case NGX_HTTP_PARSE_INVALID_HEADER: ++ ngx_http_finalize_request(r, NGX_HTTP_BAD_REQUEST); ++ return ngx_http_spdy_state_headers_skip(sc, pos, end); ++ ++ default: /* NGX_ERROR */ ++ return ngx_http_spdy_state_headers_error(sc, pos, end); ++ } ++ ++ /* a header line has been parsed successfully */ ++ ++ rc = ngx_http_spdy_handle_request_header(r); ++ ++ if (rc != NGX_OK) { ++ if (rc == NGX_HTTP_PARSE_INVALID_HEADER) { ++ ngx_http_finalize_request(r, NGX_HTTP_BAD_REQUEST); ++ return ngx_http_spdy_state_headers_skip(sc, pos, end); ++ } ++ ++ if (rc != NGX_ABORT) { ++ ngx_http_spdy_close_stream(sc->stream, ++ NGX_HTTP_INTERNAL_SERVER_ERROR); ++ } ++ ++ return ngx_http_spdy_state_headers_skip(sc, pos, end); ++ } ++ } ++ ++ if (buf->pos != buf->last || sc->zstream_in.avail_in) { ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "incorrect number of spdy header block entries"); ++ ++ return ngx_http_spdy_state_headers_error(sc, pos, end); ++ } ++ ++ if (sc->length) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_headers); ++ } ++ ++ /* null-terminate the last header value */ ++ *buf->pos = '\0'; ++ ++ ngx_http_spdy_run_request(r); ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_headers_skip(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ int n; ++ size_t size; ++ u_char buffer[NGX_SPDY_SKIP_HEADERS_BUFFER_SIZE]; ++ ++ if (sc->length == 0) { ++ return ngx_http_spdy_state_complete(sc, pos, end); ++ } ++ ++ size = end - pos; ++ ++ if (size == 0) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_headers_skip); ++ } ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy header block skip %uz of %uz", size, sc->length); ++ ++ sc->zstream_in.next_in = pos; ++ sc->zstream_in.avail_in = (size < sc->length) ? size : sc->length; ++ ++ while (sc->zstream_in.avail_in) { ++ sc->zstream_in.next_out = buffer; ++ sc->zstream_in.avail_out = NGX_SPDY_SKIP_HEADERS_BUFFER_SIZE; ++ ++ n = inflate(&sc->zstream_in, Z_NO_FLUSH); ++ ++ ngx_log_debug5(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy inflate out: ni:%p no:%p ai:%ud ao:%ud rc:%d", ++ sc->zstream_in.next_in, sc->zstream_in.next_out, ++ sc->zstream_in.avail_in, sc->zstream_in.avail_out, ++ n); ++ ++ if (n != Z_OK) { ++ return ngx_http_spdy_state_inflate_error(sc, n); ++ } ++ } ++ ++ pos = sc->zstream_in.next_in; ++ ++ if (size < sc->length) { ++ sc->length -= size; ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_headers_skip); ++ } ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_headers_error(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ ngx_http_spdy_stream_t *stream; ++ ++ stream = sc->stream; ++ ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent SYN_STREAM frame for stream %ui " ++ "with invalid header block", stream->id); ++ ++ if (ngx_http_spdy_send_rst_stream(sc, stream->id, NGX_SPDY_PROTOCOL_ERROR, ++ stream->priority) ++ != NGX_OK) ++ { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ stream->out_closed = 1; ++ ++ ngx_http_spdy_close_stream(stream, NGX_HTTP_BAD_REQUEST); ++ ++ return ngx_http_spdy_state_headers_skip(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_window_update(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ size_t delta; ++ ngx_uint_t sid; ++ ngx_event_t *wev; ++ ngx_queue_t *q; ++ ngx_http_spdy_stream_t *stream; ++ ++ if (end - pos < NGX_SPDY_WINDOW_UPDATE_SIZE) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_window_update); ++ } ++ ++ if (sc->length != NGX_SPDY_WINDOW_UPDATE_SIZE) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent WINDOW_UPDATE frame " ++ "with incorrect length %uz", sc->length); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ sid = ngx_spdy_frame_parse_sid(pos); ++ ++ pos += NGX_SPDY_SID_SIZE; ++ ++ delta = ngx_spdy_frame_parse_delta(pos); ++ ++ pos += NGX_SPDY_DELTA_SIZE; ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy WINDOW_UPDATE sid:%ui delta:%uz", sid, delta); ++ ++ if (sid) { ++ stream = ngx_http_spdy_get_stream_by_id(sc, sid); ++ ++ if (stream == NULL) { ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "unknown spdy stream"); ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++ } ++ ++ if (stream->send_window > (ssize_t) (NGX_SPDY_MAX_WINDOW - delta)) { ++ ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client violated flow control for stream %ui: " ++ "received WINDOW_UPDATE frame with delta %uz " ++ "not allowed for window %z", ++ sid, delta, stream->send_window); ++ ++ if (ngx_http_spdy_terminate_stream(sc, stream, ++ NGX_SPDY_FLOW_CONTROL_ERROR) ++ == NGX_ERROR) ++ { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++ } ++ ++ stream->send_window += delta; ++ ++ if (stream->exhausted) { ++ stream->exhausted = 0; ++ ++ wev = stream->request->connection->write; ++ ++ if (!wev->timer_set) { ++ wev->delayed = 0; ++ wev->handler(wev); ++ } ++ } ++ ++ } else { ++ sc->send_window += delta; ++ ++ if (sc->send_window > NGX_SPDY_MAX_WINDOW) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client violated connection flow control: " ++ "received WINDOW_UPDATE frame with delta %uz " ++ "not allowed for window %uz", ++ delta, sc->send_window); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ while (!ngx_queue_empty(&sc->waiting)) { ++ q = ngx_queue_head(&sc->waiting); ++ ++ ngx_queue_remove(q); ++ ++ stream = ngx_queue_data(q, ngx_http_spdy_stream_t, queue); ++ ++ stream->handled = 0; ++ ++ wev = stream->request->connection->write; ++ ++ if (!wev->timer_set) { ++ wev->delayed = 0; ++ wev->handler(wev); ++ ++ if (sc->send_window == 0) { ++ break; ++ } ++ } ++ } ++ } ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_data(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ ngx_http_spdy_stream_t *stream; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy DATA frame"); ++ ++ if (sc->length > sc->recv_window) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client violated connection flow control: " ++ "received DATA frame length %uz, available window %uz", ++ sc->length, sc->recv_window); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ sc->recv_window -= sc->length; ++ ++ if (sc->recv_window < NGX_SPDY_MAX_WINDOW / 4) { ++ ++ if (ngx_http_spdy_send_window_update(sc, 0, ++ NGX_SPDY_MAX_WINDOW ++ - sc->recv_window) ++ == NGX_ERROR) ++ { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ sc->recv_window = NGX_SPDY_MAX_WINDOW; ++ } ++ ++ stream = sc->stream; ++ ++ if (stream == NULL) { ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "unknown spdy stream"); ++ ++ return ngx_http_spdy_state_skip(sc, pos, end); ++ } ++ ++ if (sc->length > stream->recv_window) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client violated flow control for stream %ui: " ++ "received DATA frame length %uz, available window %uz", ++ stream->id, sc->length, stream->recv_window); ++ ++ if (ngx_http_spdy_terminate_stream(sc, stream, ++ NGX_SPDY_FLOW_CONTROL_ERROR) ++ == NGX_ERROR) ++ { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ return ngx_http_spdy_state_skip(sc, pos, end); ++ } ++ ++ stream->recv_window -= sc->length; ++ ++ if (stream->recv_window < NGX_SPDY_STREAM_WINDOW / 4) { ++ ++ if (ngx_http_spdy_send_window_update(sc, stream->id, ++ NGX_SPDY_STREAM_WINDOW ++ - stream->recv_window) ++ == NGX_ERROR) ++ { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ stream->recv_window = NGX_SPDY_STREAM_WINDOW; ++ } ++ ++ if (stream->in_closed) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent DATA frame for half-closed stream %ui", ++ stream->id); ++ ++ if (ngx_http_spdy_terminate_stream(sc, stream, ++ NGX_SPDY_STREAM_ALREADY_CLOSED) ++ == NGX_ERROR) ++ { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ return ngx_http_spdy_state_skip(sc, pos, end); ++ } ++ ++ return ngx_http_spdy_state_read_data(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_read_data(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ size_t size; ++ ssize_t n; ++ ngx_buf_t *buf; ++ ngx_int_t rc; ++ ngx_temp_file_t *tf; ++ ngx_http_request_t *r; ++ ngx_http_spdy_stream_t *stream; ++ ngx_http_request_body_t *rb; ++ ngx_http_core_loc_conf_t *clcf; ++ ++ stream = sc->stream; ++ ++ if (stream == NULL) { ++ return ngx_http_spdy_state_skip(sc, pos, end); ++ } ++ ++ if (stream->skip_data) { ++ ++ if (sc->flags & NGX_SPDY_FLAG_FIN) { ++ stream->in_closed = 1; ++ } ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "skipping spdy DATA frame, reason: %d", ++ stream->skip_data); ++ ++ return ngx_http_spdy_state_skip(sc, pos, end); ++ } ++ ++ size = end - pos; ++ ++ if (size > sc->length) { ++ size = sc->length; ++ } ++ ++ r = stream->request; ++ ++ if (r->request_body == NULL ++ && ngx_http_spdy_init_request_body(r) != NGX_OK) ++ { ++ stream->skip_data = NGX_SPDY_DATA_INTERNAL_ERROR; ++ return ngx_http_spdy_state_skip(sc, pos, end); ++ } ++ ++ rb = r->request_body; ++ tf = rb->temp_file; ++ buf = rb->buf; ++ ++ if (size) { ++ rb->rest += size; ++ ++ if (r->headers_in.content_length_n != -1 ++ && r->headers_in.content_length_n < rb->rest) ++ { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client intended to send body data " ++ "larger than declared"); ++ ++ stream->skip_data = NGX_SPDY_DATA_ERROR; ++ goto error; ++ ++ } else { ++ clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); ++ ++ if (clcf->client_max_body_size ++ && clcf->client_max_body_size < rb->rest) ++ { ++ ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, ++ "client intended to send " ++ "too large chunked body: %O bytes", rb->rest); ++ ++ stream->skip_data = NGX_SPDY_DATA_ERROR; ++ goto error; ++ } ++ } ++ ++ sc->length -= size; ++ ++ if (tf) { ++ buf->start = pos; ++ buf->pos = pos; ++ ++ pos += size; ++ ++ buf->end = pos; ++ buf->last = pos; ++ ++ n = ngx_write_chain_to_temp_file(tf, rb->bufs); ++ ++ /* TODO: n == 0 or not complete and level event */ ++ ++ if (n == NGX_ERROR) { ++ stream->skip_data = NGX_SPDY_DATA_INTERNAL_ERROR; ++ goto error; ++ } ++ ++ tf->offset += n; ++ ++ } else { ++ buf->last = ngx_cpymem(buf->last, pos, size); ++ pos += size; ++ } ++ ++ r->request_length += size; ++ } ++ ++ if (sc->length) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_read_data); ++ } ++ ++ if (sc->flags & NGX_SPDY_FLAG_FIN) { ++ ++ stream->in_closed = 1; ++ ++ if (r->headers_in.content_length_n < 0) { ++ r->headers_in.content_length_n = rb->rest; ++ ++ } else if (r->headers_in.content_length_n != rb->rest) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client prematurely closed stream: " ++ "only %O out of %O bytes of request body received", ++ rb->rest, r->headers_in.content_length_n); ++ ++ stream->skip_data = NGX_SPDY_DATA_ERROR; ++ goto error; ++ } ++ ++ if (tf) { ++ ngx_memzero(buf, sizeof(ngx_buf_t)); ++ ++ buf->in_file = 1; ++ buf->file_last = tf->file.offset; ++ buf->file = &tf->file; ++ ++ rb->buf = NULL; ++ } ++ ++ if (rb->post_handler) { ++ r->read_event_handler = ngx_http_block_reading; ++ rb->post_handler(r); ++ } ++ } ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++ ++error: ++ ++ if (rb->post_handler) { ++ ++ if (stream->skip_data == NGX_SPDY_DATA_ERROR) { ++ rc = (r->headers_in.content_length_n == -1) ++ ? NGX_HTTP_REQUEST_ENTITY_TOO_LARGE ++ : NGX_HTTP_BAD_REQUEST; ++ ++ } else { ++ rc = NGX_HTTP_INTERNAL_SERVER_ERROR; ++ } ++ ++ ngx_http_finalize_request(r, rc); ++ } ++ ++ return ngx_http_spdy_state_skip(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_rst_stream(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ ngx_uint_t sid, status; ++ ngx_event_t *ev; ++ ngx_connection_t *fc; ++ ngx_http_spdy_stream_t *stream; ++ ++ if (end - pos < NGX_SPDY_RST_STREAM_SIZE) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_rst_stream); ++ } ++ ++ if (sc->length != NGX_SPDY_RST_STREAM_SIZE) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent RST_STREAM frame with incorrect length %uz", ++ sc->length); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ sid = ngx_spdy_frame_parse_sid(pos); ++ ++ pos += NGX_SPDY_SID_SIZE; ++ ++ status = ngx_spdy_frame_parse_uint32(pos); ++ ++ pos += sizeof(uint32_t); ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy RST_STREAM sid:%ui st:%ui", sid, status); ++ ++ stream = ngx_http_spdy_get_stream_by_id(sc, sid); ++ ++ if (stream == NULL) { ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "unknown spdy stream"); ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++ } ++ ++ stream->in_closed = 1; ++ stream->out_closed = 1; ++ ++ fc = stream->request->connection; ++ fc->error = 1; ++ ++ switch (status) { ++ ++ case NGX_SPDY_CANCEL: ++ ngx_log_error(NGX_LOG_INFO, fc->log, 0, ++ "client canceled stream %ui", sid); ++ break; ++ ++ case NGX_SPDY_INTERNAL_ERROR: ++ ngx_log_error(NGX_LOG_INFO, fc->log, 0, ++ "client terminated stream %ui due to internal error", ++ sid); ++ break; ++ ++ default: ++ ngx_log_error(NGX_LOG_INFO, fc->log, 0, ++ "client terminated stream %ui with status %ui", ++ sid, status); ++ break; ++ } ++ ++ ev = fc->read; ++ ev->handler(ev); ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_ping(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ u_char *p; ++ ngx_buf_t *buf; ++ ngx_http_spdy_out_frame_t *frame; ++ ++ if (end - pos < NGX_SPDY_PING_SIZE) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_ping); ++ } ++ ++ if (sc->length != NGX_SPDY_PING_SIZE) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent PING frame with incorrect length %uz", ++ sc->length); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy PING frame"); ++ ++ frame = ngx_http_spdy_get_ctl_frame(sc, NGX_SPDY_PING_SIZE, ++ NGX_SPDY_HIGHEST_PRIORITY); ++ if (frame == NULL) { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ buf = frame->first->buf; ++ ++ p = buf->pos; ++ ++ p = ngx_spdy_frame_write_head(p, NGX_SPDY_PING); ++ p = ngx_spdy_frame_write_flags_and_len(p, 0, NGX_SPDY_PING_SIZE); ++ ++ p = ngx_cpymem(p, pos, NGX_SPDY_PING_SIZE); ++ ++ buf->last = p; ++ ++ ngx_http_spdy_queue_frame(sc, frame); ++ ++ pos += NGX_SPDY_PING_SIZE; ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_skip(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ size_t size; ++ ++ size = end - pos; ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy frame skip %uz of %uz", size, sc->length); ++ ++ if (size < sc->length) { ++ sc->length -= size; ++ return ngx_http_spdy_state_save(sc, end, end, ++ ngx_http_spdy_state_skip); ++ } ++ ++ return ngx_http_spdy_state_complete(sc, pos + sc->length, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_settings(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ ngx_uint_t fid, val; ++ ++ if (sc->entries == 0) { ++ ++ if (end - pos < NGX_SPDY_SETTINGS_NUM_SIZE) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_settings); ++ } ++ ++ sc->entries = ngx_spdy_frame_parse_uint32(pos); ++ ++ pos += NGX_SPDY_SETTINGS_NUM_SIZE; ++ sc->length -= NGX_SPDY_SETTINGS_NUM_SIZE; ++ ++ if (sc->length < sc->entries * NGX_SPDY_SETTINGS_PAIR_SIZE) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent SETTINGS frame with incorrect " ++ "length %uz or number of entries %ui", ++ sc->length, sc->entries); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy SETTINGS frame has %ui entries", sc->entries); ++ } ++ ++ while (sc->entries) { ++ if (end - pos < NGX_SPDY_SETTINGS_PAIR_SIZE) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_settings); ++ } ++ ++ sc->entries--; ++ sc->length -= NGX_SPDY_SETTINGS_PAIR_SIZE; ++ ++ fid = ngx_spdy_frame_parse_uint32(pos); ++ ++ pos += NGX_SPDY_SETTINGS_FID_SIZE; ++ ++ val = ngx_spdy_frame_parse_uint32(pos); ++ ++ pos += NGX_SPDY_SETTINGS_VAL_SIZE; ++ ++ ngx_log_debug3(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy SETTINGS entry fl:%ui id:%ui val:%ui", ++ ngx_spdy_frame_flags(fid), ngx_spdy_frame_id(fid), val); ++ ++ if (ngx_spdy_frame_flags(fid) == NGX_SPDY_SETTINGS_FLAG_PERSISTED) { ++ continue; ++ } ++ ++ switch (ngx_spdy_frame_id(fid)) { ++ ++ case NGX_SPDY_SETTINGS_INIT_WINDOW: ++ ++ if (val > NGX_SPDY_MAX_WINDOW) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent SETTINGS frame with " ++ "incorrect INIT_WINDOW value: %ui", val); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ if (ngx_http_spdy_adjust_windows(sc, val - sc->init_window) ++ != NGX_OK) ++ { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ sc->init_window = val; ++ ++ continue; ++ } ++ } ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_complete(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy frame complete pos:%p end:%p", pos, end); ++ ++ if (pos > end) { ++ ngx_log_error(NGX_LOG_ALERT, sc->connection->log, 0, ++ "receive buffer overrun"); ++ ++ ngx_debug_point(); ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ sc->handler = ngx_http_spdy_state_head; ++ sc->stream = NULL; ++ ++ return pos; ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_save(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end, ngx_http_spdy_handler_pt handler) ++{ ++ size_t size; ++ ++ ngx_log_debug3(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy frame state save pos:%p end:%p handler:%p", ++ pos, end, handler); ++ ++ size = end - pos; ++ ++ if (size > NGX_SPDY_STATE_BUFFER_SIZE) { ++ ngx_log_error(NGX_LOG_ALERT, sc->connection->log, 0, ++ "state buffer overflow: %uz bytes required", size); ++ ++ ngx_debug_point(); ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ ngx_memcpy(sc->buffer, pos, NGX_SPDY_STATE_BUFFER_SIZE); ++ ++ sc->buffer_used = size; ++ sc->handler = handler; ++ sc->incomplete = 1; ++ ++ return end; ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_inflate_error(ngx_http_spdy_connection_t *sc, int rc) ++{ ++ if (rc == Z_DATA_ERROR || rc == Z_STREAM_END) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent SYN_STREAM frame with " ++ "corrupted header block, inflate() failed: %d", rc); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ ngx_log_error(NGX_LOG_ERR, sc->connection->log, 0, ++ "inflate() failed: %d", rc); ++ ++ return ngx_http_spdy_state_internal_error(sc); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_protocol_error(ngx_http_spdy_connection_t *sc) ++{ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy state protocol error"); ++ ++ if (sc->stream) { ++ sc->stream->out_closed = 1; ++ ngx_http_spdy_close_stream(sc->stream, NGX_HTTP_BAD_REQUEST); ++ } ++ ++ ngx_http_spdy_finalize_connection(sc, NGX_HTTP_CLIENT_CLOSED_REQUEST); ++ ++ return NULL; ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_internal_error(ngx_http_spdy_connection_t *sc) ++{ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy state internal error"); ++ ++ if (sc->stream) { ++ sc->stream->out_closed = 1; ++ ngx_http_spdy_close_stream(sc->stream, NGX_HTTP_INTERNAL_SERVER_ERROR); ++ } ++ ++ ngx_http_spdy_finalize_connection(sc, NGX_HTTP_INTERNAL_SERVER_ERROR); ++ ++ return NULL; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_send_window_update(ngx_http_spdy_connection_t *sc, ngx_uint_t sid, ++ ngx_uint_t delta) ++{ ++ u_char *p; ++ ngx_buf_t *buf; ++ ngx_http_spdy_out_frame_t *frame; ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy send WINDOW_UPDATE sid:%ui delta:%ui", sid, delta); ++ ++ frame = ngx_http_spdy_get_ctl_frame(sc, NGX_SPDY_WINDOW_UPDATE_SIZE, ++ NGX_SPDY_HIGHEST_PRIORITY); ++ if (frame == NULL) { ++ return NGX_ERROR; ++ } ++ ++ buf = frame->first->buf; ++ ++ p = buf->pos; ++ ++ p = ngx_spdy_frame_write_head(p, NGX_SPDY_WINDOW_UPDATE); ++ p = ngx_spdy_frame_write_flags_and_len(p, 0, NGX_SPDY_WINDOW_UPDATE_SIZE); ++ ++ p = ngx_spdy_frame_write_sid(p, sid); ++ p = ngx_spdy_frame_aligned_write_uint32(p, delta); ++ ++ buf->last = p; ++ ++ ngx_http_spdy_queue_frame(sc, frame); ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_send_rst_stream(ngx_http_spdy_connection_t *sc, ngx_uint_t sid, ++ ngx_uint_t status, ngx_uint_t priority) ++{ ++ u_char *p; ++ ngx_buf_t *buf; ++ ngx_http_spdy_out_frame_t *frame; ++ ++ if (sc->connection->error) { ++ return NGX_OK; ++ } ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy send RST_STREAM sid:%ui st:%ui", sid, status); ++ ++ frame = ngx_http_spdy_get_ctl_frame(sc, NGX_SPDY_RST_STREAM_SIZE, ++ priority); ++ if (frame == NULL) { ++ return NGX_ERROR; ++ } ++ ++ buf = frame->first->buf; ++ ++ p = buf->pos; ++ ++ p = ngx_spdy_frame_write_head(p, NGX_SPDY_RST_STREAM); ++ p = ngx_spdy_frame_write_flags_and_len(p, 0, NGX_SPDY_RST_STREAM_SIZE); ++ ++ p = ngx_spdy_frame_write_sid(p, sid); ++ p = ngx_spdy_frame_aligned_write_uint32(p, status); ++ ++ buf->last = p; ++ ++ ngx_http_spdy_queue_frame(sc, frame); ++ ++ return NGX_OK; ++} ++ ++ ++#if 0 ++static ngx_int_t ++ngx_http_spdy_send_goaway(ngx_http_spdy_connection_t *sc) ++{ ++ u_char *p; ++ ngx_buf_t *buf; ++ ngx_http_spdy_out_frame_t *frame; ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy send GOAWAY sid:%ui", sc->last_sid); ++ ++ frame = ngx_http_spdy_get_ctl_frame(sc, NGX_SPDY_GOAWAY_SIZE, ++ NGX_SPDY_HIGHEST_PRIORITY); ++ if (frame == NULL) { ++ return NGX_ERROR; ++ } ++ ++ buf = frame->first->buf; ++ ++ p = buf->pos; ++ ++ p = ngx_spdy_frame_write_head(p, NGX_SPDY_GOAWAY); ++ p = ngx_spdy_frame_write_flags_and_len(p, 0, NGX_SPDY_GOAWAY_SIZE); ++ ++ p = ngx_spdy_frame_write_sid(p, sc->last_sid); ++ ++ buf->last = p; ++ ++ ngx_http_spdy_queue_frame(sc, frame); ++ ++ return NGX_OK; ++} ++#endif ++ ++ ++static ngx_int_t ++ngx_http_spdy_send_settings(ngx_http_spdy_connection_t *sc) ++{ ++ u_char *p; ++ ngx_buf_t *buf; ++ ngx_chain_t *cl; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ngx_http_spdy_out_frame_t *frame; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy send SETTINGS frame"); ++ ++ frame = ngx_palloc(sc->pool, sizeof(ngx_http_spdy_out_frame_t)); ++ if (frame == NULL) { ++ return NGX_ERROR; ++ } ++ ++ cl = ngx_alloc_chain_link(sc->pool); ++ if (cl == NULL) { ++ return NGX_ERROR; ++ } ++ ++ buf = ngx_create_temp_buf(sc->pool, NGX_SPDY_FRAME_HEADER_SIZE ++ + NGX_SPDY_SETTINGS_NUM_SIZE ++ + 2 * NGX_SPDY_SETTINGS_PAIR_SIZE); ++ if (buf == NULL) { ++ return NGX_ERROR; ++ } ++ ++ buf->last_buf = 1; ++ ++ cl->buf = buf; ++ cl->next = NULL; ++ ++ frame->first = cl; ++ frame->last = cl; ++ frame->handler = ngx_http_spdy_settings_frame_handler; ++ frame->stream = NULL; ++#if (NGX_DEBUG) ++ frame->length = NGX_SPDY_SETTINGS_NUM_SIZE ++ + 2 * NGX_SPDY_SETTINGS_PAIR_SIZE; ++#endif ++ frame->priority = NGX_SPDY_HIGHEST_PRIORITY; ++ frame->blocked = 0; ++ ++ p = buf->pos; ++ ++ p = ngx_spdy_frame_write_head(p, NGX_SPDY_SETTINGS); ++ p = ngx_spdy_frame_write_flags_and_len(p, NGX_SPDY_FLAG_CLEAR_SETTINGS, ++ NGX_SPDY_SETTINGS_NUM_SIZE ++ + 2 * NGX_SPDY_SETTINGS_PAIR_SIZE); ++ ++ p = ngx_spdy_frame_aligned_write_uint32(p, 2); ++ ++ sscf = ngx_http_get_module_srv_conf(sc->http_connection->conf_ctx, ++ ngx_http_spdy_module); ++ ++ p = ngx_spdy_frame_write_flags_and_id(p, 0, NGX_SPDY_SETTINGS_MAX_STREAMS); ++ p = ngx_spdy_frame_aligned_write_uint32(p, sscf->concurrent_streams); ++ ++ p = ngx_spdy_frame_write_flags_and_id(p, 0, NGX_SPDY_SETTINGS_INIT_WINDOW); ++ p = ngx_spdy_frame_aligned_write_uint32(p, NGX_SPDY_STREAM_WINDOW); ++ ++ buf->last = p; ++ ++ ngx_http_spdy_queue_frame(sc, frame); ++ ++ return NGX_OK; ++} ++ ++ ++ngx_int_t ++ngx_http_spdy_settings_frame_handler(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_out_frame_t *frame) ++{ ++ ngx_buf_t *buf; ++ ++ buf = frame->first->buf; ++ ++ if (buf->pos != buf->last) { ++ return NGX_AGAIN; ++ } ++ ++ ngx_free_chain(sc->pool, frame->first); ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_http_spdy_out_frame_t * ++ngx_http_spdy_get_ctl_frame(ngx_http_spdy_connection_t *sc, size_t length, ++ ngx_uint_t priority) ++{ ++ ngx_chain_t *cl; ++ ngx_http_spdy_out_frame_t *frame; ++ ++ frame = sc->free_ctl_frames; ++ ++ if (frame) { ++ sc->free_ctl_frames = frame->next; ++ ++ cl = frame->first; ++ cl->buf->pos = cl->buf->start; ++ ++ } else { ++ frame = ngx_palloc(sc->pool, sizeof(ngx_http_spdy_out_frame_t)); ++ if (frame == NULL) { ++ return NULL; ++ } ++ ++ cl = ngx_alloc_chain_link(sc->pool); ++ if (cl == NULL) { ++ return NULL; ++ } ++ ++ cl->buf = ngx_create_temp_buf(sc->pool, ++ NGX_SPDY_CTL_FRAME_BUFFER_SIZE); ++ if (cl->buf == NULL) { ++ return NULL; ++ } ++ ++ cl->buf->last_buf = 1; ++ ++ frame->first = cl; ++ frame->last = cl; ++ frame->handler = ngx_http_spdy_ctl_frame_handler; ++ frame->stream = NULL; ++ } ++ ++#if (NGX_DEBUG) ++ if (length > NGX_SPDY_CTL_FRAME_BUFFER_SIZE - NGX_SPDY_FRAME_HEADER_SIZE) { ++ ngx_log_error(NGX_LOG_ALERT, sc->pool->log, 0, ++ "requested control frame is too large: %uz", length); ++ return NULL; ++ } ++ ++ frame->length = length; ++#endif ++ ++ frame->priority = priority; ++ frame->blocked = 0; ++ ++ return frame; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_ctl_frame_handler(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_out_frame_t *frame) ++{ ++ ngx_buf_t *buf; ++ ++ buf = frame->first->buf; ++ ++ if (buf->pos != buf->last) { ++ return NGX_AGAIN; ++ } ++ ++ frame->next = sc->free_ctl_frames; ++ sc->free_ctl_frames = frame; ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_http_spdy_stream_t * ++ngx_http_spdy_create_stream(ngx_http_spdy_connection_t *sc, ngx_uint_t id, ++ ngx_uint_t priority) ++{ ++ ngx_log_t *log; ++ ngx_uint_t index; ++ ngx_event_t *rev, *wev; ++ ngx_connection_t *fc; ++ ngx_http_log_ctx_t *ctx; ++ ngx_http_request_t *r; ++ ngx_http_spdy_stream_t *stream; ++ ngx_http_core_srv_conf_t *cscf; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ++ fc = sc->free_fake_connections; ++ ++ if (fc) { ++ sc->free_fake_connections = fc->data; ++ ++ rev = fc->read; ++ wev = fc->write; ++ log = fc->log; ++ ctx = log->data; ++ ++ } else { ++ fc = ngx_palloc(sc->pool, sizeof(ngx_connection_t)); ++ if (fc == NULL) { ++ return NULL; ++ } ++ ++ rev = ngx_palloc(sc->pool, sizeof(ngx_event_t)); ++ if (rev == NULL) { ++ return NULL; ++ } ++ ++ wev = ngx_palloc(sc->pool, sizeof(ngx_event_t)); ++ if (wev == NULL) { ++ return NULL; ++ } ++ ++ log = ngx_palloc(sc->pool, sizeof(ngx_log_t)); ++ if (log == NULL) { ++ return NULL; ++ } ++ ++ ctx = ngx_palloc(sc->pool, sizeof(ngx_http_log_ctx_t)); ++ if (ctx == NULL) { ++ return NULL; ++ } ++ ++ ctx->connection = fc; ++ ctx->request = NULL; ++ } ++ ++ ngx_memcpy(log, sc->connection->log, sizeof(ngx_log_t)); ++ ++ log->data = ctx; ++ ++ ngx_memzero(rev, sizeof(ngx_event_t)); ++ ++ rev->data = fc; ++ rev->ready = 1; ++ rev->handler = ngx_http_spdy_close_stream_handler; ++ rev->log = log; ++ ++ ngx_memcpy(wev, rev, sizeof(ngx_event_t)); ++ ++ wev->write = 1; ++ ++ ngx_memcpy(fc, sc->connection, sizeof(ngx_connection_t)); ++ ++ fc->data = sc->http_connection; ++ fc->read = rev; ++ fc->write = wev; ++ fc->sent = 0; ++ fc->log = log; ++ fc->buffered = 0; ++ fc->sndlowat = 1; ++ fc->tcp_nodelay = NGX_TCP_NODELAY_DISABLED; ++ ++ r = ngx_http_create_request(fc); ++ if (r == NULL) { ++ return NULL; ++ } ++ ++ r->valid_location = 1; ++ ++ fc->data = r; ++ sc->connection->requests++; ++ ++ cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module); ++ ++ r->header_in = ngx_create_temp_buf(r->pool, ++ cscf->client_header_buffer_size); ++ if (r->header_in == NULL) { ++ ngx_http_free_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR); ++ return NULL; ++ } ++ ++ r->headers_in.connection_type = NGX_HTTP_CONNECTION_CLOSE; ++ ++ stream = ngx_pcalloc(r->pool, sizeof(ngx_http_spdy_stream_t)); ++ if (stream == NULL) { ++ ngx_http_free_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR); ++ return NULL; ++ } ++ ++ r->spdy_stream = stream; ++ ++ stream->id = id; ++ stream->request = r; ++ stream->connection = sc; ++ ++ stream->send_window = sc->init_window; ++ stream->recv_window = NGX_SPDY_STREAM_WINDOW; ++ ++ stream->priority = priority; ++ ++ sscf = ngx_http_get_module_srv_conf(r, ngx_http_spdy_module); ++ ++ index = ngx_http_spdy_stream_index(sscf, id); ++ ++ stream->index = sc->streams_index[index]; ++ sc->streams_index[index] = stream; ++ ++ sc->processing++; ++ ++ return stream; ++} ++ ++ ++static ngx_http_spdy_stream_t * ++ngx_http_spdy_get_stream_by_id(ngx_http_spdy_connection_t *sc, ++ ngx_uint_t sid) ++{ ++ ngx_http_spdy_stream_t *stream; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ++ sscf = ngx_http_get_module_srv_conf(sc->http_connection->conf_ctx, ++ ngx_http_spdy_module); ++ ++ stream = sc->streams_index[ngx_http_spdy_stream_index(sscf, sid)]; ++ ++ while (stream) { ++ if (stream->id == sid) { ++ return stream; ++ } ++ ++ stream = stream->index; ++ } ++ ++ return NULL; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_parse_header(ngx_http_request_t *r) ++{ ++ u_char *p, *end, ch; ++ ngx_uint_t hash; ++ ngx_http_core_srv_conf_t *cscf; ++ ++ enum { ++ sw_name_len = 0, ++ sw_name, ++ sw_value_len, ++ sw_value ++ } state; ++ ++ state = r->state; ++ ++ p = r->header_in->pos; ++ end = r->header_in->last; ++ ++ switch (state) { ++ ++ case sw_name_len: ++ ++ if (end - p < NGX_SPDY_NV_NLEN_SIZE) { ++ return NGX_AGAIN; ++ } ++ ++ r->lowcase_index = ngx_spdy_frame_parse_uint32(p); ++ ++ if (r->lowcase_index == 0) { ++ return NGX_ERROR; ++ } ++ ++ /* null-terminate the previous header value */ ++ *p = '\0'; ++ ++ p += NGX_SPDY_NV_NLEN_SIZE; ++ ++ r->invalid_header = 0; ++ ++ state = sw_name; ++ ++ /* fall through */ ++ ++ case sw_name: ++ ++ if ((ngx_uint_t) (end - p) < r->lowcase_index) { ++ break; ++ } ++ ++ cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module); ++ ++ r->header_name_start = p; ++ r->header_name_end = p + r->lowcase_index; ++ ++ if (p[0] == ':') { ++ p++; ++ } ++ ++ hash = 0; ++ ++ for ( /* void */ ; p != r->header_name_end; p++) { ++ ++ ch = *p; ++ ++ hash = ngx_hash(hash, ch); ++ ++ if ((ch >= 'a' && ch <= 'z') ++ || (ch == '-') ++ || (ch >= '0' && ch <= '9') ++ || (ch == '_' && cscf->underscores_in_headers)) ++ { ++ continue; ++ } ++ ++ switch (ch) { ++ case '\0': ++ case LF: ++ case CR: ++ case ':': ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent invalid header name: \"%*s\"", ++ r->lowcase_index, r->header_name_start); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ if (ch >= 'A' && ch <= 'Z') { ++ return NGX_ERROR; ++ } ++ ++ r->invalid_header = 1; ++ } ++ ++ r->header_hash = hash; ++ ++ state = sw_value_len; ++ ++ /* fall through */ ++ ++ case sw_value_len: ++ ++ if (end - p < NGX_SPDY_NV_VLEN_SIZE) { ++ break; ++ } ++ ++ r->lowcase_index = ngx_spdy_frame_parse_uint32(p); ++ ++ /* null-terminate header name */ ++ *p = '\0'; ++ ++ p += NGX_SPDY_NV_VLEN_SIZE; ++ ++ state = sw_value; ++ ++ /* fall through */ ++ ++ case sw_value: ++ ++ if ((ngx_uint_t) (end - p) < r->lowcase_index) { ++ break; ++ } ++ ++ r->header_start = p; ++ ++ while (r->lowcase_index--) { ++ ch = *p; ++ ++ if (ch == '\0') { ++ ++ if (p == r->header_start) { ++ return NGX_ERROR; ++ } ++ ++ r->header_end = p; ++ r->header_in->pos = p + 1; ++ ++ r->state = sw_value; ++ ++ return NGX_OK; ++ } ++ ++ if (ch == CR || ch == LF) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent header \"%*s\" with " ++ "invalid value: \"%*s\\%c...\"", ++ r->header_name_end - r->header_name_start, ++ r->header_name_start, ++ p - r->header_start, ++ r->header_start, ++ ch == CR ? 'r' : 'n'); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ p++; ++ } ++ ++ r->header_end = p; ++ r->header_in->pos = p; ++ ++ r->state = 0; ++ ++ return NGX_DONE; ++ } ++ ++ r->header_in->pos = p; ++ r->state = state; ++ ++ return NGX_AGAIN; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_alloc_large_header_buffer(ngx_http_request_t *r) ++{ ++ u_char *old, *new, *p; ++ size_t rest; ++ ngx_buf_t *buf; ++ ngx_http_spdy_stream_t *stream; ++ ngx_http_core_srv_conf_t *cscf; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy alloc large header buffer"); ++ ++ stream = r->spdy_stream; ++ ++ cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module); ++ ++ if (stream->header_buffers ++ == (ngx_uint_t) cscf->large_client_header_buffers.num) ++ { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent too large request"); ++ ++ return NGX_DECLINED; ++ } ++ ++ rest = r->header_in->last - r->header_in->pos; ++ ++ /* ++ * One more byte is needed for null-termination ++ * and another one for further progress. ++ */ ++ if (rest > cscf->large_client_header_buffers.size - 2) { ++ p = r->header_in->pos; ++ ++ if (rest > NGX_MAX_ERROR_STR - 300) { ++ rest = NGX_MAX_ERROR_STR - 300; ++ } ++ ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent too long header name or value: \"%*s...\"", ++ rest, p); ++ ++ return NGX_DECLINED; ++ } ++ ++ buf = ngx_create_temp_buf(r->pool, cscf->large_client_header_buffers.size); ++ if (buf == NULL) { ++ return NGX_ERROR; ++ } ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy large header alloc: %p %uz", ++ buf->pos, buf->end - buf->last); ++ ++ old = r->header_in->pos; ++ new = buf->pos; ++ ++ if (rest) { ++ buf->last = ngx_cpymem(new, old, rest); ++ } ++ ++ r->header_in = buf; ++ ++ stream->header_buffers++; ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_handle_request_header(ngx_http_request_t *r) ++{ ++ ngx_uint_t i; ++ ngx_table_elt_t *h; ++ ngx_http_core_srv_conf_t *cscf; ++ ngx_http_spdy_request_header_t *sh; ++ ++ if (r->invalid_header) { ++ cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module); ++ ++ if (cscf->ignore_invalid_headers) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent invalid header: \"%*s\"", ++ r->header_end - r->header_name_start, ++ r->header_name_start); ++ return NGX_OK; ++ } ++ ++ } ++ ++ if (r->header_name_start[0] == ':') { ++ r->header_name_start++; ++ ++ for (i = 0; i < NGX_SPDY_REQUEST_HEADERS; i++) { ++ sh = &ngx_http_spdy_request_headers[i]; ++ ++ if (sh->hash != r->header_hash ++ || sh->len != r->header_name_end - r->header_name_start ++ || ngx_strncmp(sh->header, r->header_name_start, sh->len) != 0) ++ { ++ continue; ++ } ++ ++ return sh->handler(r); ++ } ++ ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent invalid header name: \":%*s\"", ++ r->header_end - r->header_name_start, ++ r->header_name_start); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ h = ngx_list_push(&r->headers_in.headers); ++ if (h == NULL) { ++ return NGX_ERROR; ++ } ++ ++ h->hash = r->header_hash; ++ ++ h->key.len = r->header_name_end - r->header_name_start; ++ h->key.data = r->header_name_start; ++ ++ h->value.len = r->header_end - r->header_start; ++ h->value.data = r->header_start; ++ ++ h->lowcase_key = h->key.data; ++ ++ return NGX_OK; ++} ++ ++ ++void ++ngx_http_spdy_request_headers_init(void) ++{ ++ ngx_uint_t i; ++ ngx_http_spdy_request_header_t *h; ++ ++ for (i = 0; i < NGX_SPDY_REQUEST_HEADERS; i++) { ++ h = &ngx_http_spdy_request_headers[i]; ++ h->hash = ngx_hash_key(h->header, h->len); ++ } ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_parse_method(ngx_http_request_t *r) ++{ ++ size_t k, len; ++ ngx_uint_t n; ++ const u_char *p, *m; ++ ++ /* ++ * This array takes less than 256 sequential bytes, ++ * and if typical CPU cache line size is 64 bytes, ++ * it is prefetched for 4 load operations. ++ */ ++ static const struct { ++ u_char len; ++ const u_char method[11]; ++ uint32_t value; ++ } tests[] = { ++ { 3, "GET", NGX_HTTP_GET }, ++ { 4, "POST", NGX_HTTP_POST }, ++ { 4, "HEAD", NGX_HTTP_HEAD }, ++ { 7, "OPTIONS", NGX_HTTP_OPTIONS }, ++ { 8, "PROPFIND", NGX_HTTP_PROPFIND }, ++ { 3, "PUT", NGX_HTTP_PUT }, ++ { 5, "MKCOL", NGX_HTTP_MKCOL }, ++ { 6, "DELETE", NGX_HTTP_DELETE }, ++ { 4, "COPY", NGX_HTTP_COPY }, ++ { 4, "MOVE", NGX_HTTP_MOVE }, ++ { 9, "PROPPATCH", NGX_HTTP_PROPPATCH }, ++ { 4, "LOCK", NGX_HTTP_LOCK }, ++ { 6, "UNLOCK", NGX_HTTP_UNLOCK }, ++ { 5, "PATCH", NGX_HTTP_PATCH }, ++ { 5, "TRACE", NGX_HTTP_TRACE } ++ }, *test; ++ ++ if (r->method_name.len) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent duplicate :method header"); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ len = r->header_end - r->header_start; ++ ++ r->method_name.len = len; ++ r->method_name.data = r->header_start; ++ ++ test = tests; ++ n = sizeof(tests) / sizeof(tests[0]); ++ ++ do { ++ if (len == test->len) { ++ p = r->method_name.data; ++ m = test->method; ++ k = len; ++ ++ do { ++ if (*p++ != *m++) { ++ goto next; ++ } ++ } while (--k); ++ ++ r->method = test->value; ++ return NGX_OK; ++ } ++ ++ next: ++ test++; ++ ++ } while (--n); ++ ++ p = r->method_name.data; ++ ++ do { ++ if ((*p < 'A' || *p > 'Z') && *p != '_') { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent invalid method: \"%V\"", ++ &r->method_name); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ p++; ++ ++ } while (--len); ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_parse_scheme(ngx_http_request_t *r) ++{ ++ if (r->schema_start) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent duplicate :schema header"); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ r->schema_start = r->header_start; ++ r->schema_end = r->header_end; ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_parse_host(ngx_http_request_t *r) ++{ ++ ngx_table_elt_t *h; ++ ++ if (r->headers_in.host) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent duplicate :host header"); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ h = ngx_list_push(&r->headers_in.headers); ++ if (h == NULL) { ++ return NGX_ERROR; ++ } ++ ++ r->headers_in.host = h; ++ ++ h->hash = r->header_hash; ++ ++ h->key.len = r->header_name_end - r->header_name_start; ++ h->key.data = r->header_name_start; ++ ++ h->value.len = r->header_end - r->header_start; ++ h->value.data = r->header_start; ++ ++ h->lowcase_key = h->key.data; ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_parse_path(ngx_http_request_t *r) ++{ ++ if (r->unparsed_uri.len) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent duplicate :path header"); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ r->uri_start = r->header_start; ++ r->uri_end = r->header_end; ++ ++ if (ngx_http_parse_uri(r) != NGX_OK) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent invalid URI: \"%*s\"", ++ r->uri_end - r->uri_start, r->uri_start); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ if (ngx_http_process_request_uri(r) != NGX_OK) { ++ /* ++ * request has been finalized already ++ * in ngx_http_process_request_uri() ++ */ ++ return NGX_ABORT; ++ } ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_parse_version(ngx_http_request_t *r) ++{ ++ u_char *p, ch; ++ ++ if (r->http_protocol.len) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent duplicate :version header"); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ p = r->header_start; ++ ++ if (r->header_end - p < 8 || !(ngx_str5cmp(p, 'H', 'T', 'T', 'P', '/'))) { ++ goto invalid; ++ } ++ ++ ch = *(p + 5); ++ ++ if (ch < '1' || ch > '9') { ++ goto invalid; ++ } ++ ++ r->http_major = ch - '0'; ++ ++ for (p += 6; p != r->header_end - 2; p++) { ++ ++ ch = *p; ++ ++ if (ch == '.') { ++ break; ++ } ++ ++ if (ch < '0' || ch > '9') { ++ goto invalid; ++ } ++ ++ r->http_major = r->http_major * 10 + ch - '0'; ++ } ++ ++ if (*p != '.') { ++ goto invalid; ++ } ++ ++ ch = *(p + 1); ++ ++ if (ch < '0' || ch > '9') { ++ goto invalid; ++ } ++ ++ r->http_minor = ch - '0'; ++ ++ for (p += 2; p != r->header_end; p++) { ++ ++ ch = *p; ++ ++ if (ch < '0' || ch > '9') { ++ goto invalid; ++ } ++ ++ r->http_minor = r->http_minor * 10 + ch - '0'; ++ } ++ ++ r->http_protocol.len = r->header_end - r->header_start; ++ r->http_protocol.data = r->header_start; ++ r->http_version = r->http_major * 1000 + r->http_minor; ++ ++ return NGX_OK; ++ ++invalid: ++ ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent invalid http version: \"%*s\"", ++ r->header_end - r->header_start, r->header_start); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_construct_request_line(ngx_http_request_t *r) ++{ ++ u_char *p; ++ ++ if (r->method_name.len == 0 ++ || r->unparsed_uri.len == 0 ++ || r->http_protocol.len == 0) ++ { ++ ngx_http_finalize_request(r, NGX_HTTP_BAD_REQUEST); ++ return NGX_ERROR; ++ } ++ ++ r->request_line.len = r->method_name.len + 1 ++ + r->unparsed_uri.len + 1 ++ + r->http_protocol.len; ++ ++ p = ngx_pnalloc(r->pool, r->request_line.len + 1); ++ if (p == NULL) { ++ ngx_http_spdy_close_stream(r->spdy_stream, ++ NGX_HTTP_INTERNAL_SERVER_ERROR); ++ return NGX_ERROR; ++ } ++ ++ r->request_line.data = p; ++ ++ p = ngx_cpymem(p, r->method_name.data, r->method_name.len); ++ ++ *p++ = ' '; ++ ++ p = ngx_cpymem(p, r->unparsed_uri.data, r->unparsed_uri.len); ++ ++ *p++ = ' '; ++ ++ ngx_memcpy(p, r->http_protocol.data, r->http_protocol.len + 1); ++ ++ /* some modules expect the space character after method name */ ++ r->method_name.data = r->request_line.data; ++ ++ return NGX_OK; ++} ++ ++ ++static void ++ngx_http_spdy_run_request(ngx_http_request_t *r) ++{ ++ ngx_uint_t i; ++ ngx_list_part_t *part; ++ ngx_table_elt_t *h; ++ ngx_http_header_t *hh; ++ ngx_http_core_main_conf_t *cmcf; ++ ++ if (ngx_http_spdy_construct_request_line(r) != NGX_OK) { ++ return; ++ } ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy http request line: \"%V\"", &r->request_line); ++ ++ cmcf = ngx_http_get_module_main_conf(r, ngx_http_core_module); ++ ++ part = &r->headers_in.headers.part; ++ h = part->elts; ++ ++ for (i = 0 ;; i++) { ++ ++ if (i >= part->nelts) { ++ if (part->next == NULL) { ++ break; ++ } ++ ++ part = part->next; ++ h = part->elts; ++ i = 0; ++ } ++ ++ hh = ngx_hash_find(&cmcf->headers_in_hash, h[i].hash, ++ h[i].lowcase_key, h[i].key.len); ++ ++ if (hh && hh->handler(r, &h[i], hh->offset) != NGX_OK) { ++ return; ++ } ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy http header: \"%V: %V\"", &h[i].key, &h[i].value); ++ } ++ ++ r->http_state = NGX_HTTP_PROCESS_REQUEST_STATE; ++ ++ if (ngx_http_process_request_header(r) != NGX_OK) { ++ return; ++ } ++ ++ if (r->headers_in.content_length_n > 0 && r->spdy_stream->in_closed) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client prematurely closed stream"); ++ ++ r->spdy_stream->skip_data = NGX_SPDY_DATA_ERROR; ++ ++ ngx_http_finalize_request(r, NGX_HTTP_BAD_REQUEST); ++ return; ++ } ++ ++ ngx_http_process_request(r); ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_init_request_body(ngx_http_request_t *r) ++{ ++ ngx_buf_t *buf; ++ ngx_temp_file_t *tf; ++ ngx_http_request_body_t *rb; ++ ngx_http_core_loc_conf_t *clcf; ++ ++ rb = ngx_pcalloc(r->pool, sizeof(ngx_http_request_body_t)); ++ if (rb == NULL) { ++ return NGX_ERROR; ++ } ++ ++ r->request_body = rb; ++ ++ if (r->spdy_stream->in_closed) { ++ return NGX_OK; ++ } ++ ++ rb->rest = r->headers_in.content_length_n; ++ ++ clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); ++ ++ if (r->request_body_in_file_only ++ || rb->rest > (off_t) clcf->client_body_buffer_size ++ || rb->rest < 0) ++ { ++ tf = ngx_pcalloc(r->pool, sizeof(ngx_temp_file_t)); ++ if (tf == NULL) { ++ return NGX_ERROR; ++ } ++ ++ tf->file.fd = NGX_INVALID_FILE; ++ tf->file.log = r->connection->log; ++ tf->path = clcf->client_body_temp_path; ++ tf->pool = r->pool; ++ tf->warn = "a client request body is buffered to a temporary file"; ++ tf->log_level = r->request_body_file_log_level; ++ tf->persistent = r->request_body_in_persistent_file; ++ tf->clean = r->request_body_in_clean_file; ++ ++ if (r->request_body_file_group_access) { ++ tf->access = 0660; ++ } ++ ++ rb->temp_file = tf; ++ ++ if (r->spdy_stream->in_closed ++ && ngx_create_temp_file(&tf->file, tf->path, tf->pool, ++ tf->persistent, tf->clean, tf->access) ++ != NGX_OK) ++ { ++ return NGX_ERROR; ++ } ++ ++ buf = ngx_calloc_buf(r->pool); ++ if (buf == NULL) { ++ return NGX_ERROR; ++ } ++ ++ } else { ++ ++ if (rb->rest == 0) { ++ return NGX_OK; ++ } ++ ++ buf = ngx_create_temp_buf(r->pool, (size_t) rb->rest); ++ if (buf == NULL) { ++ return NGX_ERROR; ++ } ++ } ++ ++ rb->buf = buf; ++ ++ rb->bufs = ngx_alloc_chain_link(r->pool); ++ if (rb->bufs == NULL) { ++ return NGX_ERROR; ++ } ++ ++ rb->bufs->buf = buf; ++ rb->bufs->next = NULL; ++ ++ rb->rest = 0; ++ ++ return NGX_OK; ++} ++ ++ ++ngx_int_t ++ngx_http_spdy_read_request_body(ngx_http_request_t *r, ++ ngx_http_client_body_handler_pt post_handler) ++{ ++ ngx_http_spdy_stream_t *stream; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy read request body"); ++ ++ stream = r->spdy_stream; ++ ++ switch (stream->skip_data) { ++ ++ case NGX_SPDY_DATA_DISCARD: ++ post_handler(r); ++ return NGX_OK; ++ ++ case NGX_SPDY_DATA_ERROR: ++ if (r->headers_in.content_length_n == -1) { ++ return NGX_HTTP_REQUEST_ENTITY_TOO_LARGE; ++ } else { ++ return NGX_HTTP_BAD_REQUEST; ++ } ++ ++ case NGX_SPDY_DATA_INTERNAL_ERROR: ++ return NGX_HTTP_INTERNAL_SERVER_ERROR; ++ } ++ ++ if (!r->request_body && ngx_http_spdy_init_request_body(r) != NGX_OK) { ++ stream->skip_data = NGX_SPDY_DATA_INTERNAL_ERROR; ++ return NGX_HTTP_INTERNAL_SERVER_ERROR; ++ } ++ ++ if (stream->in_closed) { ++ post_handler(r); ++ return NGX_OK; ++ } ++ ++ r->request_body->post_handler = post_handler; ++ ++ r->read_event_handler = ngx_http_test_reading; ++ r->write_event_handler = ngx_http_request_empty_handler; ++ ++ return NGX_AGAIN; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_terminate_stream(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_stream_t *stream, ngx_uint_t status) ++{ ++ ngx_event_t *rev; ++ ngx_connection_t *fc; ++ ++ if (ngx_http_spdy_send_rst_stream(sc, stream->id, status, ++ NGX_SPDY_HIGHEST_PRIORITY) ++ == NGX_ERROR) ++ { ++ return NGX_ERROR; ++ } ++ ++ stream->out_closed = 1; ++ ++ fc = stream->request->connection; ++ fc->error = 1; ++ ++ rev = fc->read; ++ rev->handler(rev); ++ ++ return NGX_OK; ++} ++ ++ ++static void ++ngx_http_spdy_close_stream_handler(ngx_event_t *ev) ++{ ++ ngx_connection_t *fc; ++ ngx_http_request_t *r; ++ ++ fc = ev->data; ++ r = fc->data; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy close stream handler"); ++ ++ ngx_http_spdy_close_stream(r->spdy_stream, 0); ++} ++ ++ ++void ++ngx_http_spdy_close_stream(ngx_http_spdy_stream_t *stream, ngx_int_t rc) ++{ ++ ngx_event_t *ev; ++ ngx_connection_t *fc; ++ ngx_http_spdy_stream_t **index, *s; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ngx_http_spdy_connection_t *sc; ++ ++ sc = stream->connection; ++ ++ ngx_log_debug3(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy close stream %ui, queued %ui, processing %ui", ++ stream->id, stream->queued, sc->processing); ++ ++ fc = stream->request->connection; ++ ++ if (stream->queued) { ++ fc->write->handler = ngx_http_spdy_close_stream_handler; ++ return; ++ } ++ ++ if (!stream->out_closed) { ++ if (ngx_http_spdy_send_rst_stream(sc, stream->id, ++ NGX_SPDY_INTERNAL_ERROR, ++ stream->priority) ++ != NGX_OK) ++ { ++ sc->connection->error = 1; ++ } ++ } ++ ++ if (sc->stream == stream) { ++ sc->stream = NULL; ++ } ++ ++ sscf = ngx_http_get_module_srv_conf(sc->http_connection->conf_ctx, ++ ngx_http_spdy_module); ++ ++ index = sc->streams_index + ngx_http_spdy_stream_index(sscf, stream->id); ++ ++ for ( ;; ) { ++ s = *index; ++ ++ if (s == NULL) { ++ break; ++ } ++ ++ if (s == stream) { ++ *index = s->index; ++ break; ++ } ++ ++ index = &s->index; ++ } ++ ++ ngx_http_free_request(stream->request, rc); ++ ++ ev = fc->read; ++ ++ if (ev->active || ev->disabled) { ++ ngx_log_error(NGX_LOG_ALERT, sc->connection->log, 0, ++ "fake read event was activated"); ++ } ++ ++ if (ev->timer_set) { ++ ngx_del_timer(ev); ++ } ++ ++ if (ev->posted) { ++ ngx_delete_posted_event(ev); ++ } ++ ++ ev = fc->write; ++ ++ if (ev->active || ev->disabled) { ++ ngx_log_error(NGX_LOG_ALERT, sc->connection->log, 0, ++ "fake write event was activated"); ++ } ++ ++ if (ev->timer_set) { ++ ngx_del_timer(ev); ++ } ++ ++ if (ev->posted) { ++ ngx_delete_posted_event(ev); ++ } ++ ++ fc->data = sc->free_fake_connections; ++ sc->free_fake_connections = fc; ++ ++ sc->processing--; ++ ++ if (sc->processing || sc->blocked) { ++ return; ++ } ++ ++ ev = sc->connection->read; ++ ++ ev->handler = ngx_http_spdy_handle_connection_handler; ++ ngx_post_event(ev, &ngx_posted_events); ++} ++ ++ ++static void ++ngx_http_spdy_handle_connection_handler(ngx_event_t *rev) ++{ ++ ngx_connection_t *c; ++ ++ rev->handler = ngx_http_spdy_read_handler; ++ ++ if (rev->ready) { ++ ngx_http_spdy_read_handler(rev); ++ return; ++ } ++ ++ c = rev->data; ++ ++ ngx_http_spdy_handle_connection(c->data); ++} ++ ++ ++static void ++ngx_http_spdy_keepalive_handler(ngx_event_t *rev) ++{ ++ ngx_connection_t *c; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ngx_http_spdy_connection_t *sc; ++ ++ c = rev->data; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "spdy keepalive handler"); ++ ++ if (rev->timedout || c->close) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++#if (NGX_HAVE_KQUEUE) ++ ++ if (ngx_event_flags & NGX_USE_KQUEUE_EVENT) { ++ if (rev->pending_eof) { ++ c->log->handler = NULL; ++ ngx_log_error(NGX_LOG_INFO, c->log, rev->kq_errno, ++ "kevent() reported that client %V closed " ++ "keepalive connection", &c->addr_text); ++#if (NGX_HTTP_SSL) ++ if (c->ssl) { ++ c->ssl->no_send_shutdown = 1; ++ } ++#endif ++ ngx_http_close_connection(c); ++ return; ++ } ++ } ++ ++#endif ++ ++ c->destroyed = 0; ++ c->idle = 0; ++ ngx_reusable_connection(c, 0); ++ ++ sc = c->data; ++ ++ sscf = ngx_http_get_module_srv_conf(sc->http_connection->conf_ctx, ++ ngx_http_spdy_module); ++ ++ sc->pool = ngx_create_pool(sscf->pool_size, sc->connection->log); ++ if (sc->pool == NULL) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ sc->streams_index = ngx_pcalloc(sc->pool, ++ ngx_http_spdy_streams_index_size(sscf) ++ * sizeof(ngx_http_spdy_stream_t *)); ++ if (sc->streams_index == NULL) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ c->write->handler = ngx_http_spdy_write_handler; ++ ++ rev->handler = ngx_http_spdy_read_handler; ++ ngx_http_spdy_read_handler(rev); ++} ++ ++ ++static void ++ngx_http_spdy_finalize_connection(ngx_http_spdy_connection_t *sc, ++ ngx_int_t rc) ++{ ++ ngx_uint_t i, size; ++ ngx_event_t *ev; ++ ngx_connection_t *c, *fc; ++ ngx_http_request_t *r; ++ ngx_http_spdy_stream_t *stream; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ++ c = sc->connection; ++ ++ if (!sc->processing) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ c->error = 1; ++ c->read->handler = ngx_http_empty_handler; ++ c->write->handler = ngx_http_empty_handler; ++ ++ sc->last_out = NULL; ++ ++ sc->blocked = 1; ++ ++ sscf = ngx_http_get_module_srv_conf(sc->http_connection->conf_ctx, ++ ngx_http_spdy_module); ++ ++ size = ngx_http_spdy_streams_index_size(sscf); ++ ++ for (i = 0; i < size; i++) { ++ stream = sc->streams_index[i]; ++ ++ while (stream) { ++ stream->handled = 0; ++ ++ r = stream->request; ++ fc = r->connection; ++ ++ fc->error = 1; ++ ++ if (stream->queued) { ++ stream->queued = 0; ++ ++ ev = fc->write; ++ ev->delayed = 0; ++ ++ } else { ++ ev = fc->read; ++ } ++ ++ stream = stream->index; ++ ++ ev->eof = 1; ++ ev->handler(ev); ++ } ++ } ++ ++ sc->blocked = 0; ++ ++ if (sc->processing) { ++ return; ++ } ++ ++ ngx_http_close_connection(c); ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_adjust_windows(ngx_http_spdy_connection_t *sc, ssize_t delta) ++{ ++ ngx_uint_t i, size; ++ ngx_event_t *wev; ++ ngx_http_spdy_stream_t *stream, *sn; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ++ sscf = ngx_http_get_module_srv_conf(sc->http_connection->conf_ctx, ++ ngx_http_spdy_module); ++ ++ size = ngx_http_spdy_streams_index_size(sscf); ++ ++ for (i = 0; i < size; i++) { ++ ++ for (stream = sc->streams_index[i]; stream; stream = sn) { ++ sn = stream->index; ++ ++ if (delta > 0 ++ && stream->send_window ++ > (ssize_t) (NGX_SPDY_MAX_WINDOW - delta)) ++ { ++ if (ngx_http_spdy_terminate_stream(sc, stream, ++ NGX_SPDY_FLOW_CONTROL_ERROR) ++ == NGX_ERROR) ++ { ++ return NGX_ERROR; ++ } ++ ++ continue; ++ } ++ ++ stream->send_window += delta; ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy:%ui adjust window:%z", ++ stream->id, stream->send_window); ++ ++ if (stream->send_window > 0 && stream->exhausted) { ++ stream->exhausted = 0; ++ ++ wev = stream->request->connection->write; ++ ++ if (!wev->timer_set) { ++ wev->delayed = 0; ++ wev->handler(wev); ++ } ++ } ++ } ++ } ++ ++ return NGX_OK; ++} ++ ++ ++static void ++ngx_http_spdy_pool_cleanup(void *data) ++{ ++ ngx_http_spdy_connection_t *sc = data; ++ ++ if (sc->pool) { ++ ngx_destroy_pool(sc->pool); ++ } ++} ++ ++ ++static void * ++ngx_http_spdy_zalloc(void *opaque, u_int items, u_int size) ++{ ++ ngx_http_spdy_connection_t *sc = opaque; ++ ++ return ngx_palloc(sc->connection->pool, items * size); ++} ++ ++ ++static void ++ngx_http_spdy_zfree(void *opaque, void *address) ++{ ++#if 0 ++ ngx_http_spdy_connection_t *sc = opaque; ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy zfree: %p", address); ++#endif ++} +diff --git a/b/src/http/ngx_http_spdy.h b/src/http/ngx_http_spdy.h +new file mode 100644 +index 0000000..df24495 +--- /dev/null ++++ b/src/http/ngx_http_spdy.h +@@ -0,0 +1,261 @@ ++/* ++ * Copyright (C) Nginx, Inc. ++ * Copyright (C) Valentin V. Bartenev ++ */ ++ ++ ++#ifndef _NGX_HTTP_SPDY_H_INCLUDED_ ++#define _NGX_HTTP_SPDY_H_INCLUDED_ ++ ++ ++#include ++#include ++#include ++ ++#include ++ ++ ++#define NGX_SPDY_VERSION 3 ++ ++#define NGX_SPDY_NPN_ADVERTISE "\x08spdy/3.1" ++#define NGX_SPDY_NPN_NEGOTIATED "spdy/3.1" ++ ++#define NGX_SPDY_STATE_BUFFER_SIZE 16 ++ ++#define NGX_SPDY_CTL_BIT 1 ++ ++#define NGX_SPDY_SYN_STREAM 1 ++#define NGX_SPDY_SYN_REPLY 2 ++#define NGX_SPDY_RST_STREAM 3 ++#define NGX_SPDY_SETTINGS 4 ++#define NGX_SPDY_PING 6 ++#define NGX_SPDY_GOAWAY 7 ++#define NGX_SPDY_HEADERS 8 ++#define NGX_SPDY_WINDOW_UPDATE 9 ++ ++#define NGX_SPDY_FRAME_HEADER_SIZE 8 ++ ++#define NGX_SPDY_SID_SIZE 4 ++#define NGX_SPDY_DELTA_SIZE 4 ++ ++#define NGX_SPDY_SYN_STREAM_SIZE 10 ++#define NGX_SPDY_SYN_REPLY_SIZE 4 ++#define NGX_SPDY_RST_STREAM_SIZE 8 ++#define NGX_SPDY_PING_SIZE 4 ++#define NGX_SPDY_GOAWAY_SIZE 8 ++#define NGX_SPDY_WINDOW_UPDATE_SIZE 8 ++#define NGX_SPDY_NV_NUM_SIZE 4 ++#define NGX_SPDY_NV_NLEN_SIZE 4 ++#define NGX_SPDY_NV_VLEN_SIZE 4 ++#define NGX_SPDY_SETTINGS_NUM_SIZE 4 ++#define NGX_SPDY_SETTINGS_FID_SIZE 4 ++#define NGX_SPDY_SETTINGS_VAL_SIZE 4 ++ ++#define NGX_SPDY_SETTINGS_PAIR_SIZE \ ++ (NGX_SPDY_SETTINGS_FID_SIZE + NGX_SPDY_SETTINGS_VAL_SIZE) ++ ++#define NGX_SPDY_HIGHEST_PRIORITY 0 ++#define NGX_SPDY_LOWEST_PRIORITY 7 ++ ++#define NGX_SPDY_FLAG_FIN 0x01 ++#define NGX_SPDY_FLAG_UNIDIRECTIONAL 0x02 ++#define NGX_SPDY_FLAG_CLEAR_SETTINGS 0x01 ++ ++#define NGX_SPDY_MAX_FRAME_SIZE ((1 << 24) - 1) ++ ++#define NGX_SPDY_DATA_DISCARD 1 ++#define NGX_SPDY_DATA_ERROR 2 ++#define NGX_SPDY_DATA_INTERNAL_ERROR 3 ++ ++ ++typedef struct ngx_http_spdy_connection_s ngx_http_spdy_connection_t; ++typedef struct ngx_http_spdy_out_frame_s ngx_http_spdy_out_frame_t; ++ ++ ++typedef u_char *(*ngx_http_spdy_handler_pt) (ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++ ++struct ngx_http_spdy_connection_s { ++ ngx_connection_t *connection; ++ ngx_http_connection_t *http_connection; ++ ++ ngx_uint_t processing; ++ ++ size_t send_window; ++ size_t recv_window; ++ size_t init_window; ++ ++ ngx_queue_t waiting; ++ ++ u_char buffer[NGX_SPDY_STATE_BUFFER_SIZE]; ++ size_t buffer_used; ++ ngx_http_spdy_handler_pt handler; ++ ++ z_stream zstream_in; ++ z_stream zstream_out; ++ ++ ngx_pool_t *pool; ++ ++ ngx_http_spdy_out_frame_t *free_ctl_frames; ++ ngx_connection_t *free_fake_connections; ++ ++ ngx_http_spdy_stream_t **streams_index; ++ ++ ngx_http_spdy_out_frame_t *last_out; ++ ++ ngx_queue_t posted; ++ ++ ngx_http_spdy_stream_t *stream; ++ ++ ngx_uint_t entries; ++ size_t length; ++ u_char flags; ++ ++ ngx_uint_t last_sid; ++ ++ unsigned blocked:1; ++ unsigned incomplete:1; ++}; ++ ++ ++struct ngx_http_spdy_stream_s { ++ ngx_uint_t id; ++ ngx_http_request_t *request; ++ ngx_http_spdy_connection_t *connection; ++ ngx_http_spdy_stream_t *index; ++ ++ ngx_uint_t header_buffers; ++ ngx_uint_t queued; ++ ++ /* ++ * A change to SETTINGS_INITIAL_WINDOW_SIZE could cause the ++ * send_window to become negative, hence it's signed. ++ */ ++ ssize_t send_window; ++ size_t recv_window; ++ ++ ngx_http_spdy_out_frame_t *free_frames; ++ ngx_chain_t *free_data_headers; ++ ngx_chain_t *free_bufs; ++ ++ ngx_queue_t queue; ++ ++ unsigned priority:3; ++ unsigned handled:1; ++ unsigned blocked:1; ++ unsigned exhausted:1; ++ unsigned in_closed:1; ++ unsigned out_closed:1; ++ unsigned skip_data:2; ++}; ++ ++ ++struct ngx_http_spdy_out_frame_s { ++ ngx_http_spdy_out_frame_t *next; ++ ngx_chain_t *first; ++ ngx_chain_t *last; ++ ngx_int_t (*handler)(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_out_frame_t *frame); ++ ++ ngx_http_spdy_stream_t *stream; ++ size_t length; ++ ++ ngx_uint_t priority; ++ unsigned blocked:1; ++ unsigned fin:1; ++}; ++ ++ ++static ngx_inline void ++ngx_http_spdy_queue_frame(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_out_frame_t *frame) ++{ ++ ngx_http_spdy_out_frame_t **out; ++ ++ for (out = &sc->last_out; *out; out = &(*out)->next) ++ { ++ /* ++ * NB: higher values represent lower priorities. ++ */ ++ if (frame->priority >= (*out)->priority) { ++ break; ++ } ++ } ++ ++ frame->next = *out; ++ *out = frame; ++} ++ ++ ++static ngx_inline void ++ngx_http_spdy_queue_blocked_frame(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_out_frame_t *frame) ++{ ++ ngx_http_spdy_out_frame_t **out; ++ ++ for (out = &sc->last_out; *out; out = &(*out)->next) ++ { ++ if ((*out)->blocked) { ++ break; ++ } ++ } ++ ++ frame->next = *out; ++ *out = frame; ++} ++ ++ ++void ngx_http_spdy_init(ngx_event_t *rev); ++void ngx_http_spdy_request_headers_init(void); ++ ++ngx_int_t ngx_http_spdy_read_request_body(ngx_http_request_t *r, ++ ngx_http_client_body_handler_pt post_handler); ++ ++void ngx_http_spdy_close_stream(ngx_http_spdy_stream_t *stream, ngx_int_t rc); ++ ++ngx_int_t ngx_http_spdy_send_output_queue(ngx_http_spdy_connection_t *sc); ++ ++ ++#define ngx_spdy_frame_aligned_write_uint16(p, s) \ ++ (*(uint16_t *) (p) = htons((uint16_t) (s)), (p) + sizeof(uint16_t)) ++ ++#define ngx_spdy_frame_aligned_write_uint32(p, s) \ ++ (*(uint32_t *) (p) = htonl((uint32_t) (s)), (p) + sizeof(uint32_t)) ++ ++#if (NGX_HAVE_NONALIGNED) ++ ++#define ngx_spdy_frame_write_uint16 ngx_spdy_frame_aligned_write_uint16 ++#define ngx_spdy_frame_write_uint32 ngx_spdy_frame_aligned_write_uint32 ++ ++#else ++ ++#define ngx_spdy_frame_write_uint16(p, s) \ ++ ((p)[0] = (u_char) ((s) >> 8), \ ++ (p)[1] = (u_char) (s), \ ++ (p) + sizeof(uint16_t)) ++ ++#define ngx_spdy_frame_write_uint32(p, s) \ ++ ((p)[0] = (u_char) ((s) >> 24), \ ++ (p)[1] = (u_char) ((s) >> 16), \ ++ (p)[2] = (u_char) ((s) >> 8), \ ++ (p)[3] = (u_char) (s), \ ++ (p) + sizeof(uint32_t)) ++ ++#endif ++ ++ ++#define ngx_spdy_ctl_frame_head(t) \ ++ ((uint32_t) NGX_SPDY_CTL_BIT << 31 | NGX_SPDY_VERSION << 16 | (t)) ++ ++#define ngx_spdy_frame_write_head(p, t) \ ++ ngx_spdy_frame_aligned_write_uint32(p, ngx_spdy_ctl_frame_head(t)) ++ ++#define ngx_spdy_frame_write_flags_and_len(p, f, l) \ ++ ngx_spdy_frame_aligned_write_uint32(p, (f) << 24 | (l)) ++#define ngx_spdy_frame_write_flags_and_id(p, f, i) \ ++ ngx_spdy_frame_aligned_write_uint32(p, (f) << 24 | (i)) ++ ++#define ngx_spdy_frame_write_sid ngx_spdy_frame_aligned_write_uint32 ++#define ngx_spdy_frame_write_window ngx_spdy_frame_aligned_write_uint32 ++ ++#endif /* _NGX_HTTP_SPDY_H_INCLUDED_ */ +diff --git a/b/src/http/ngx_http_spdy_filter_module.c b/src/http/ngx_http_spdy_filter_module.c +new file mode 100644 +index 0000000..377e935 +--- /dev/null ++++ b/src/http/ngx_http_spdy_filter_module.c +@@ -0,0 +1,1222 @@ ++ ++/* ++ * Copyright (C) Nginx, Inc. ++ * Copyright (C) Valentin V. Bartenev ++ */ ++ ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++ ++#define ngx_http_spdy_nv_nsize(h) (NGX_SPDY_NV_NLEN_SIZE + sizeof(h) - 1) ++#define ngx_http_spdy_nv_vsize(h) (NGX_SPDY_NV_VLEN_SIZE + sizeof(h) - 1) ++ ++#define ngx_http_spdy_nv_write_num ngx_spdy_frame_write_uint32 ++#define ngx_http_spdy_nv_write_nlen ngx_spdy_frame_write_uint32 ++#define ngx_http_spdy_nv_write_vlen ngx_spdy_frame_write_uint32 ++ ++#define ngx_http_spdy_nv_write_name(p, h) \ ++ ngx_cpymem(ngx_http_spdy_nv_write_nlen(p, sizeof(h) - 1), h, sizeof(h) - 1) ++ ++#define ngx_http_spdy_nv_write_val(p, h) \ ++ ngx_cpymem(ngx_http_spdy_nv_write_vlen(p, sizeof(h) - 1), h, sizeof(h) - 1) ++ ++ ++static ngx_chain_t *ngx_http_spdy_send_chain(ngx_connection_t *fc, ++ ngx_chain_t *in, off_t limit); ++ ++static ngx_inline ngx_int_t ngx_http_spdy_filter_send( ++ ngx_connection_t *fc, ngx_http_spdy_stream_t *stream); ++static ngx_inline ngx_int_t ngx_http_spdy_flow_control( ++ ngx_http_spdy_connection_t *sc, ngx_http_spdy_stream_t *stream); ++static void ngx_http_spdy_waiting_queue(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_stream_t *stream); ++ ++static ngx_chain_t *ngx_http_spdy_filter_get_shadow( ++ ngx_http_spdy_stream_t *stream, ngx_buf_t *buf, off_t offset, off_t size); ++static ngx_http_spdy_out_frame_t *ngx_http_spdy_filter_get_data_frame( ++ ngx_http_spdy_stream_t *stream, size_t len, ngx_chain_t *first, ++ ngx_chain_t *last); ++ ++static ngx_int_t ngx_http_spdy_syn_frame_handler( ++ ngx_http_spdy_connection_t *sc, ngx_http_spdy_out_frame_t *frame); ++static ngx_int_t ngx_http_spdy_data_frame_handler( ++ ngx_http_spdy_connection_t *sc, ngx_http_spdy_out_frame_t *frame); ++static ngx_inline void ngx_http_spdy_handle_frame( ++ ngx_http_spdy_stream_t *stream, ngx_http_spdy_out_frame_t *frame); ++static ngx_inline void ngx_http_spdy_handle_stream( ++ ngx_http_spdy_connection_t *sc, ngx_http_spdy_stream_t *stream); ++ ++static void ngx_http_spdy_filter_cleanup(void *data); ++ ++static ngx_int_t ngx_http_spdy_filter_init(ngx_conf_t *cf); ++ ++ ++static ngx_http_module_t ngx_http_spdy_filter_module_ctx = { ++ NULL, /* preconfiguration */ ++ ngx_http_spdy_filter_init, /* postconfiguration */ ++ ++ NULL, /* create main configuration */ ++ NULL, /* init main configuration */ ++ ++ NULL, /* create server configuration */ ++ NULL, /* merge server configuration */ ++ ++ NULL, /* create location configuration */ ++ NULL /* merge location configuration */ ++}; ++ ++ ++ngx_module_t ngx_http_spdy_filter_module = { ++ NGX_MODULE_V1, ++ &ngx_http_spdy_filter_module_ctx, /* module context */ ++ NULL, /* module directives */ ++ NGX_HTTP_MODULE, /* module type */ ++ NULL, /* init master */ ++ NULL, /* init module */ ++ NULL, /* init process */ ++ NULL, /* init thread */ ++ NULL, /* exit thread */ ++ NULL, /* exit process */ ++ NULL, /* exit master */ ++ NGX_MODULE_V1_PADDING ++}; ++ ++ ++static ngx_http_output_header_filter_pt ngx_http_next_header_filter; ++ ++ ++static ngx_int_t ++ngx_http_spdy_header_filter(ngx_http_request_t *r) ++{ ++ int rc; ++ size_t len; ++ u_char *p, *buf, *last; ++ ngx_buf_t *b; ++ ngx_str_t host; ++ ngx_uint_t i, j, count, port; ++ ngx_chain_t *cl; ++ ngx_list_part_t *part, *pt; ++ ngx_table_elt_t *header, *h; ++ ngx_connection_t *c; ++ ngx_http_cleanup_t *cln; ++ ngx_http_core_loc_conf_t *clcf; ++ ngx_http_core_srv_conf_t *cscf; ++ ngx_http_spdy_stream_t *stream; ++ ngx_http_spdy_out_frame_t *frame; ++ ngx_http_spdy_connection_t *sc; ++ struct sockaddr_in *sin; ++#if (NGX_HAVE_INET6) ++ struct sockaddr_in6 *sin6; ++#endif ++ u_char addr[NGX_SOCKADDR_STRLEN]; ++ ++ if (!r->spdy_stream) { ++ return ngx_http_next_header_filter(r); ++ } ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy header filter"); ++ ++ if (r->header_sent) { ++ return NGX_OK; ++ } ++ ++ r->header_sent = 1; ++ ++ if (r != r->main) { ++ return NGX_OK; ++ } ++ ++ c = r->connection; ++ ++ if (r->method == NGX_HTTP_HEAD) { ++ r->header_only = 1; ++ } ++ ++ switch (r->headers_out.status) { ++ ++ case NGX_HTTP_OK: ++ case NGX_HTTP_PARTIAL_CONTENT: ++ break; ++ ++ case NGX_HTTP_NOT_MODIFIED: ++ r->header_only = 1; ++ break; ++ ++ case NGX_HTTP_NO_CONTENT: ++ r->header_only = 1; ++ ++ ngx_str_null(&r->headers_out.content_type); ++ ++ r->headers_out.content_length = NULL; ++ r->headers_out.content_length_n = -1; ++ ++ /* fall through */ ++ ++ default: ++ r->headers_out.last_modified_time = -1; ++ r->headers_out.last_modified = NULL; ++ } ++ ++ len = NGX_SPDY_NV_NUM_SIZE ++ + ngx_http_spdy_nv_nsize(":version") ++ + ngx_http_spdy_nv_vsize("HTTP/1.1") ++ + ngx_http_spdy_nv_nsize(":status") ++ + (r->headers_out.status_line.len ++ ? NGX_SPDY_NV_VLEN_SIZE + r->headers_out.status_line.len ++ : ngx_http_spdy_nv_vsize("418")); ++ ++ clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); ++ ++ if (r->headers_out.server == NULL) { ++ len += ngx_http_spdy_nv_nsize("server"); ++ len += clcf->server_tokens ? ngx_http_spdy_nv_vsize(NGINX_VER) ++ : ngx_http_spdy_nv_vsize("nginx"); ++ } ++ ++ if (r->headers_out.date == NULL) { ++ len += ngx_http_spdy_nv_nsize("date") ++ + ngx_http_spdy_nv_vsize("Wed, 31 Dec 1986 10:00:00 GMT"); ++ } ++ ++ if (r->headers_out.content_type.len) { ++ len += ngx_http_spdy_nv_nsize("content-type") ++ + NGX_SPDY_NV_VLEN_SIZE + r->headers_out.content_type.len; ++ ++ if (r->headers_out.content_type_len == r->headers_out.content_type.len ++ && r->headers_out.charset.len) ++ { ++ len += sizeof("; charset=") - 1 + r->headers_out.charset.len; ++ } ++ } ++ ++ if (r->headers_out.content_length == NULL ++ && r->headers_out.content_length_n >= 0) ++ { ++ len += ngx_http_spdy_nv_nsize("content-length") ++ + NGX_SPDY_NV_VLEN_SIZE + NGX_OFF_T_LEN; ++ } ++ ++ if (r->headers_out.last_modified == NULL ++ && r->headers_out.last_modified_time != -1) ++ { ++ len += ngx_http_spdy_nv_nsize("last-modified") ++ + ngx_http_spdy_nv_vsize("Wed, 31 Dec 1986 10:00:00 GMT"); ++ } ++ ++ if (r->headers_out.location ++ && r->headers_out.location->value.len ++ && r->headers_out.location->value.data[0] == '/') ++ { ++ r->headers_out.location->hash = 0; ++ ++ if (clcf->server_name_in_redirect) { ++ cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module); ++ host = cscf->server_name; ++ ++ } else if (r->headers_in.server.len) { ++ host = r->headers_in.server; ++ ++ } else { ++ host.len = NGX_SOCKADDR_STRLEN; ++ host.data = addr; ++ ++ if (ngx_connection_local_sockaddr(c, &host, 0) != NGX_OK) { ++ return NGX_ERROR; ++ } ++ } ++ ++ switch (c->local_sockaddr->sa_family) { ++ ++#if (NGX_HAVE_INET6) ++ case AF_INET6: ++ sin6 = (struct sockaddr_in6 *) c->local_sockaddr; ++ port = ntohs(sin6->sin6_port); ++ break; ++#endif ++#if (NGX_HAVE_UNIX_DOMAIN) ++ case AF_UNIX: ++ port = 0; ++ break; ++#endif ++ default: /* AF_INET */ ++ sin = (struct sockaddr_in *) c->local_sockaddr; ++ port = ntohs(sin->sin_port); ++ break; ++ } ++ ++ len += ngx_http_spdy_nv_nsize("location") ++ + ngx_http_spdy_nv_vsize("https://") ++ + host.len ++ + r->headers_out.location->value.len; ++ ++ if (clcf->port_in_redirect) { ++ ++#if (NGX_HTTP_SSL) ++ if (c->ssl) ++ port = (port == 443) ? 0 : port; ++ else ++#endif ++ port = (port == 80) ? 0 : port; ++ ++ } else { ++ port = 0; ++ } ++ ++ if (port) { ++ len += sizeof(":65535") - 1; ++ } ++ ++ } else { ++ ngx_str_null(&host); ++ port = 0; ++ } ++ ++ part = &r->headers_out.headers.part; ++ header = part->elts; ++ ++ for (i = 0; /* void */; i++) { ++ ++ if (i >= part->nelts) { ++ if (part->next == NULL) { ++ break; ++ } ++ ++ part = part->next; ++ header = part->elts; ++ i = 0; ++ } ++ ++ if (header[i].hash == 0) { ++ continue; ++ } ++ ++ len += NGX_SPDY_NV_NLEN_SIZE + header[i].key.len ++ + NGX_SPDY_NV_VLEN_SIZE + header[i].value.len; ++ } ++ ++ buf = ngx_alloc(len, r->pool->log); ++ if (buf == NULL) { ++ return NGX_ERROR; ++ } ++ ++ last = buf + NGX_SPDY_NV_NUM_SIZE; ++ ++ last = ngx_http_spdy_nv_write_name(last, ":version"); ++ last = ngx_http_spdy_nv_write_val(last, "HTTP/1.1"); ++ ++ last = ngx_http_spdy_nv_write_name(last, ":status"); ++ ++ if (r->headers_out.status_line.len) { ++ last = ngx_http_spdy_nv_write_vlen(last, ++ r->headers_out.status_line.len); ++ last = ngx_cpymem(last, r->headers_out.status_line.data, ++ r->headers_out.status_line.len); ++ } else { ++ last = ngx_http_spdy_nv_write_vlen(last, 3); ++ last = ngx_sprintf(last, "%03ui", r->headers_out.status); ++ } ++ ++ count = 2; ++ ++ if (r->headers_out.server == NULL) { ++ last = ngx_http_spdy_nv_write_name(last, "server"); ++ last = clcf->server_tokens ++ ? ngx_http_spdy_nv_write_val(last, NGINX_VER) ++ : ngx_http_spdy_nv_write_val(last, "nginx"); ++ ++ count++; ++ } ++ ++ if (r->headers_out.date == NULL) { ++ last = ngx_http_spdy_nv_write_name(last, "date"); ++ ++ last = ngx_http_spdy_nv_write_vlen(last, ngx_cached_http_time.len); ++ ++ last = ngx_cpymem(last, ngx_cached_http_time.data, ++ ngx_cached_http_time.len); ++ ++ count++; ++ } ++ ++ if (r->headers_out.content_type.len) { ++ ++ last = ngx_http_spdy_nv_write_name(last, "content-type"); ++ ++ p = last + NGX_SPDY_NV_VLEN_SIZE; ++ ++ last = ngx_cpymem(p, r->headers_out.content_type.data, ++ r->headers_out.content_type.len); ++ ++ if (r->headers_out.content_type_len == r->headers_out.content_type.len ++ && r->headers_out.charset.len) ++ { ++ last = ngx_cpymem(last, "; charset=", sizeof("; charset=") - 1); ++ ++ last = ngx_cpymem(last, r->headers_out.charset.data, ++ r->headers_out.charset.len); ++ ++ /* update r->headers_out.content_type for possible logging */ ++ ++ r->headers_out.content_type.len = last - p; ++ r->headers_out.content_type.data = p; ++ } ++ ++ (void) ngx_http_spdy_nv_write_vlen(p - NGX_SPDY_NV_VLEN_SIZE, ++ r->headers_out.content_type.len); ++ ++ count++; ++ } ++ ++ if (r->headers_out.content_length == NULL ++ && r->headers_out.content_length_n >= 0) ++ { ++ last = ngx_http_spdy_nv_write_name(last, "content-length"); ++ ++ p = last + NGX_SPDY_NV_VLEN_SIZE; ++ ++ last = ngx_sprintf(p, "%O", r->headers_out.content_length_n); ++ ++ (void) ngx_http_spdy_nv_write_vlen(p - NGX_SPDY_NV_VLEN_SIZE, ++ last - p); ++ ++ count++; ++ } ++ ++ if (r->headers_out.last_modified == NULL ++ && r->headers_out.last_modified_time != -1) ++ { ++ last = ngx_http_spdy_nv_write_name(last, "last-modified"); ++ ++ p = last + NGX_SPDY_NV_VLEN_SIZE; ++ ++ last = ngx_http_time(p, r->headers_out.last_modified_time); ++ ++ (void) ngx_http_spdy_nv_write_vlen(p - NGX_SPDY_NV_VLEN_SIZE, ++ last - p); ++ ++ count++; ++ } ++ ++ if (host.data) { ++ ++ last = ngx_http_spdy_nv_write_name(last, "location"); ++ ++ p = last + NGX_SPDY_NV_VLEN_SIZE; ++ ++ last = ngx_cpymem(p, "http", sizeof("http") - 1); ++ ++#if (NGX_HTTP_SSL) ++ if (c->ssl) { ++ *last++ ='s'; ++ } ++#endif ++ ++ *last++ = ':'; *last++ = '/'; *last++ = '/'; ++ ++ last = ngx_cpymem(last, host.data, host.len); ++ ++ if (port) { ++ last = ngx_sprintf(last, ":%ui", port); ++ } ++ ++ last = ngx_cpymem(last, r->headers_out.location->value.data, ++ r->headers_out.location->value.len); ++ ++ /* update r->headers_out.location->value for possible logging */ ++ ++ r->headers_out.location->value.len = last - p; ++ r->headers_out.location->value.data = p; ++ ngx_str_set(&r->headers_out.location->key, "location"); ++ ++ (void) ngx_http_spdy_nv_write_vlen(p - NGX_SPDY_NV_VLEN_SIZE, ++ r->headers_out.location->value.len); ++ ++ count++; ++ } ++ ++ part = &r->headers_out.headers.part; ++ header = part->elts; ++ ++ for (i = 0; /* void */; i++) { ++ ++ if (i >= part->nelts) { ++ if (part->next == NULL) { ++ break; ++ } ++ ++ part = part->next; ++ header = part->elts; ++ i = 0; ++ } ++ ++ if (header[i].hash == 0 || header[i].hash == 2) { ++ continue; ++ } ++ ++ last = ngx_http_spdy_nv_write_nlen(last, header[i].key.len); ++ ++ ngx_strlow(last, header[i].key.data, header[i].key.len); ++ last += header[i].key.len; ++ ++ p = last + NGX_SPDY_NV_VLEN_SIZE; ++ ++ last = ngx_cpymem(p, header[i].value.data, header[i].value.len); ++ ++ pt = part; ++ h = header; ++ ++ for (j = i + 1; /* void */; j++) { ++ ++ if (j >= pt->nelts) { ++ if (pt->next == NULL) { ++ break; ++ } ++ ++ pt = pt->next; ++ h = pt->elts; ++ j = 0; ++ } ++ ++ if (h[j].hash == 0 || h[j].hash == 2 ++ || h[j].key.len != header[i].key.len ++ || ngx_strncasecmp(header[i].key.data, h[j].key.data, ++ header[i].key.len)) ++ { ++ continue; ++ } ++ ++ if (h[j].value.len) { ++ if (last != p) { ++ *last++ = '\0'; ++ } ++ ++ last = ngx_cpymem(last, h[j].value.data, h[j].value.len); ++ } ++ ++ h[j].hash = 2; ++ } ++ ++ (void) ngx_http_spdy_nv_write_vlen(p - NGX_SPDY_NV_VLEN_SIZE, ++ last - p); ++ ++ count++; ++ } ++ ++ (void) ngx_http_spdy_nv_write_num(buf, count); ++ ++ stream = r->spdy_stream; ++ sc = stream->connection; ++ ++ len = last - buf; ++ ++ b = ngx_create_temp_buf(r->pool, NGX_SPDY_FRAME_HEADER_SIZE ++ + NGX_SPDY_SYN_REPLY_SIZE ++ + deflateBound(&sc->zstream_out, len)); ++ if (b == NULL) { ++ ngx_free(buf); ++ return NGX_ERROR; ++ } ++ ++ b->last += NGX_SPDY_FRAME_HEADER_SIZE + NGX_SPDY_SYN_REPLY_SIZE; ++ ++ sc->zstream_out.next_in = buf; ++ sc->zstream_out.avail_in = len; ++ sc->zstream_out.next_out = b->last; ++ sc->zstream_out.avail_out = b->end - b->last; ++ ++ rc = deflate(&sc->zstream_out, Z_SYNC_FLUSH); ++ ++ ngx_free(buf); ++ ++ if (rc != Z_OK) { ++ ngx_log_error(NGX_LOG_ALERT, c->log, 0, "deflate() failed: %d", rc); ++ return NGX_ERROR; ++ } ++ ++ ngx_log_debug5(NGX_LOG_DEBUG_HTTP, c->log, 0, ++ "spdy deflate out: ni:%p no:%p ai:%ud ao:%ud rc:%d", ++ sc->zstream_out.next_in, sc->zstream_out.next_out, ++ sc->zstream_out.avail_in, sc->zstream_out.avail_out, ++ rc); ++ ++ b->last = sc->zstream_out.next_out; ++ ++ p = b->pos; ++ p = ngx_spdy_frame_write_head(p, NGX_SPDY_SYN_REPLY); ++ ++ len = b->last - b->pos; ++ ++ r->header_size = len; ++ ++ len -= NGX_SPDY_FRAME_HEADER_SIZE; ++ ++ if (r->header_only) { ++ b->last_buf = 1; ++ p = ngx_spdy_frame_write_flags_and_len(p, NGX_SPDY_FLAG_FIN, len); ++ ++ } else { ++ p = ngx_spdy_frame_write_flags_and_len(p, 0, len); ++ } ++ ++ (void) ngx_spdy_frame_write_sid(p, stream->id); ++ ++ cl = ngx_alloc_chain_link(r->pool); ++ if (cl == NULL) { ++ return NGX_ERROR; ++ } ++ ++ cl->buf = b; ++ cl->next = NULL; ++ ++ frame = ngx_palloc(r->pool, sizeof(ngx_http_spdy_out_frame_t)); ++ if (frame == NULL) { ++ return NGX_ERROR; ++ } ++ ++ frame->first = cl; ++ frame->last = cl; ++ frame->handler = ngx_http_spdy_syn_frame_handler; ++ frame->stream = stream; ++ frame->length = len; ++ frame->priority = stream->priority; ++ frame->blocked = 1; ++ frame->fin = r->header_only; ++ ++ ngx_log_debug3(NGX_LOG_DEBUG_HTTP, stream->request->connection->log, 0, ++ "spdy:%ui create SYN_REPLY frame %p: len:%uz", ++ stream->id, frame, frame->length); ++ ++ ngx_http_spdy_queue_blocked_frame(sc, frame); ++ ++ cln = ngx_http_cleanup_add(r, 0); ++ if (cln == NULL) { ++ return NGX_ERROR; ++ } ++ ++ cln->handler = ngx_http_spdy_filter_cleanup; ++ cln->data = stream; ++ ++ stream->queued = 1; ++ ++ c->send_chain = ngx_http_spdy_send_chain; ++ c->need_last_buf = 1; ++ ++ return ngx_http_spdy_filter_send(c, stream); ++} ++ ++ ++static ngx_chain_t * ++ngx_http_spdy_send_chain(ngx_connection_t *fc, ngx_chain_t *in, off_t limit) ++{ ++ off_t size, offset; ++ size_t rest, frame_size; ++ ngx_chain_t *cl, *out, **ln; ++ ngx_http_request_t *r; ++ ngx_http_spdy_stream_t *stream; ++ ngx_http_spdy_loc_conf_t *slcf; ++ ngx_http_spdy_out_frame_t *frame; ++ ngx_http_spdy_connection_t *sc; ++ ++ r = fc->data; ++ stream = r->spdy_stream; ++ ++#if (NGX_SUPPRESS_WARN) ++ size = 0; ++#endif ++ ++ while (in) { ++ size = ngx_buf_size(in->buf); ++ ++ if (size || in->buf->last_buf) { ++ break; ++ } ++ ++ in = in->next; ++ } ++ ++ if (in == NULL) { ++ ++ if (stream->queued) { ++ fc->write->delayed = 1; ++ } else { ++ fc->buffered &= ~NGX_SPDY_BUFFERED; ++ } ++ ++ return NULL; ++ } ++ ++ sc = stream->connection; ++ ++ if (size && ngx_http_spdy_flow_control(sc, stream) == NGX_DECLINED) { ++ fc->write->delayed = 1; ++ return in; ++ } ++ ++ if (limit == 0 || limit > (off_t) sc->send_window) { ++ limit = sc->send_window; ++ } ++ ++ if (limit > stream->send_window) { ++ limit = (stream->send_window > 0) ? stream->send_window : 0; ++ } ++ ++ if (in->buf->tag == (ngx_buf_tag_t) &ngx_http_spdy_filter_get_shadow) { ++ cl = ngx_alloc_chain_link(r->pool); ++ if (cl == NULL) { ++ return NGX_CHAIN_ERROR; ++ } ++ ++ cl->buf = in->buf; ++ in->buf = cl->buf->shadow; ++ ++ offset = ngx_buf_in_memory(in->buf) ++ ? (cl->buf->pos - in->buf->pos) ++ : (cl->buf->file_pos - in->buf->file_pos); ++ ++ cl->next = stream->free_bufs; ++ stream->free_bufs = cl; ++ ++ } else { ++ offset = 0; ++ } ++ ++#if (NGX_SUPPRESS_WARN) ++ cl = NULL; ++#endif ++ ++ slcf = ngx_http_get_module_loc_conf(r, ngx_http_spdy_module); ++ ++ frame_size = (limit <= (off_t) slcf->chunk_size) ? (size_t) limit ++ : slcf->chunk_size; ++ ++ for ( ;; ) { ++ ln = &out; ++ rest = frame_size; ++ ++ while ((off_t) rest >= size) { ++ ++ if (offset) { ++ cl = ngx_http_spdy_filter_get_shadow(stream, in->buf, ++ offset, size); ++ if (cl == NULL) { ++ return NGX_CHAIN_ERROR; ++ } ++ ++ offset = 0; ++ ++ } else { ++ cl = ngx_alloc_chain_link(r->pool); ++ if (cl == NULL) { ++ return NGX_CHAIN_ERROR; ++ } ++ ++ cl->buf = in->buf; ++ } ++ ++ *ln = cl; ++ ln = &cl->next; ++ ++ rest -= (size_t) size; ++ in = in->next; ++ ++ if (in == NULL) { ++ frame_size -= rest; ++ rest = 0; ++ break; ++ } ++ ++ size = ngx_buf_size(in->buf); ++ } ++ ++ if (rest) { ++ cl = ngx_http_spdy_filter_get_shadow(stream, in->buf, ++ offset, rest); ++ if (cl == NULL) { ++ return NGX_CHAIN_ERROR; ++ } ++ ++ cl->buf->flush = 0; ++ cl->buf->last_buf = 0; ++ ++ *ln = cl; ++ ++ offset += rest; ++ size -= rest; ++ } ++ ++ frame = ngx_http_spdy_filter_get_data_frame(stream, frame_size, ++ out, cl); ++ if (frame == NULL) { ++ return NGX_CHAIN_ERROR; ++ } ++ ++ ngx_http_spdy_queue_frame(sc, frame); ++ ++ sc->send_window -= frame_size; ++ ++ stream->send_window -= frame_size; ++ stream->queued++; ++ ++ if (in == NULL) { ++ break; ++ } ++ ++ limit -= frame_size; ++ ++ if (limit == 0) { ++ break; ++ } ++ ++ if (limit < (off_t) slcf->chunk_size) { ++ frame_size = (size_t) limit; ++ } ++ } ++ ++ if (offset) { ++ cl = ngx_http_spdy_filter_get_shadow(stream, in->buf, offset, size); ++ if (cl == NULL) { ++ return NGX_CHAIN_ERROR; ++ } ++ ++ in->buf = cl->buf; ++ ngx_free_chain(r->pool, cl); ++ } ++ ++ if (ngx_http_spdy_filter_send(fc, stream) == NGX_ERROR) { ++ return NGX_CHAIN_ERROR; ++ } ++ ++ if (in && ngx_http_spdy_flow_control(sc, stream) == NGX_DECLINED) { ++ fc->write->delayed = 1; ++ } ++ ++ return in; ++} ++ ++ ++static ngx_chain_t * ++ngx_http_spdy_filter_get_shadow(ngx_http_spdy_stream_t *stream, ngx_buf_t *buf, ++ off_t offset, off_t size) ++{ ++ ngx_buf_t *chunk; ++ ngx_chain_t *cl; ++ ++ cl = ngx_chain_get_free_buf(stream->request->pool, &stream->free_bufs); ++ if (cl == NULL) { ++ return NULL; ++ } ++ ++ chunk = cl->buf; ++ ++ ngx_memcpy(chunk, buf, sizeof(ngx_buf_t)); ++ ++ chunk->tag = (ngx_buf_tag_t) &ngx_http_spdy_filter_get_shadow; ++ chunk->shadow = buf; ++ ++ if (ngx_buf_in_memory(chunk)) { ++ chunk->pos += offset; ++ chunk->last = chunk->pos + size; ++ } ++ ++ if (chunk->in_file) { ++ chunk->file_pos += offset; ++ chunk->file_last = chunk->file_pos + size; ++ } ++ ++ return cl; ++} ++ ++ ++static ngx_http_spdy_out_frame_t * ++ngx_http_spdy_filter_get_data_frame(ngx_http_spdy_stream_t *stream, ++ size_t len, ngx_chain_t *first, ngx_chain_t *last) ++{ ++ u_char *p; ++ ngx_buf_t *buf; ++ ngx_uint_t flags; ++ ngx_chain_t *cl; ++ ngx_http_spdy_out_frame_t *frame; ++ ++ ++ frame = stream->free_frames; ++ ++ if (frame) { ++ stream->free_frames = frame->next; ++ ++ } else { ++ frame = ngx_palloc(stream->request->pool, ++ sizeof(ngx_http_spdy_out_frame_t)); ++ if (frame == NULL) { ++ return NULL; ++ } ++ } ++ ++ flags = last->buf->last_buf ? NGX_SPDY_FLAG_FIN : 0; ++ ++ ngx_log_debug4(NGX_LOG_DEBUG_HTTP, stream->request->connection->log, 0, ++ "spdy:%ui create DATA frame %p: len:%uz flags:%ui", ++ stream->id, frame, len, flags); ++ ++ cl = ngx_chain_get_free_buf(stream->request->pool, ++ &stream->free_data_headers); ++ if (cl == NULL) { ++ return NULL; ++ } ++ ++ buf = cl->buf; ++ ++ if (buf->start) { ++ p = buf->start; ++ buf->pos = p; ++ ++ p += NGX_SPDY_SID_SIZE; ++ ++ (void) ngx_spdy_frame_write_flags_and_len(p, flags, len); ++ ++ } else { ++ p = ngx_palloc(stream->request->pool, NGX_SPDY_FRAME_HEADER_SIZE); ++ if (p == NULL) { ++ return NULL; ++ } ++ ++ buf->pos = p; ++ buf->start = p; ++ ++ p = ngx_spdy_frame_write_sid(p, stream->id); ++ p = ngx_spdy_frame_write_flags_and_len(p, flags, len); ++ ++ buf->last = p; ++ buf->end = p; ++ ++ buf->tag = (ngx_buf_tag_t) &ngx_http_spdy_filter_get_data_frame; ++ buf->memory = 1; ++ } ++ ++ cl->next = first; ++ first = cl; ++ ++ last->buf->flush = 1; ++ ++ frame->first = first; ++ frame->last = last; ++ frame->handler = ngx_http_spdy_data_frame_handler; ++ frame->stream = stream; ++ frame->length = len; ++ frame->priority = stream->priority; ++ frame->blocked = 0; ++ frame->fin = last->buf->last_buf; ++ ++ return frame; ++} ++ ++ ++static ngx_inline ngx_int_t ++ngx_http_spdy_filter_send(ngx_connection_t *fc, ngx_http_spdy_stream_t *stream) ++{ ++ stream->blocked = 1; ++ ++ if (ngx_http_spdy_send_output_queue(stream->connection) == NGX_ERROR) { ++ fc->error = 1; ++ return NGX_ERROR; ++ } ++ ++ stream->blocked = 0; ++ ++ if (stream->queued) { ++ fc->buffered |= NGX_SPDY_BUFFERED; ++ fc->write->delayed = 1; ++ return NGX_AGAIN; ++ } ++ ++ fc->buffered &= ~NGX_SPDY_BUFFERED; ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_inline ngx_int_t ++ngx_http_spdy_flow_control(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_stream_t *stream) ++{ ++ if (stream->send_window <= 0) { ++ stream->exhausted = 1; ++ return NGX_DECLINED; ++ } ++ ++ if (sc->send_window == 0) { ++ ngx_http_spdy_waiting_queue(sc, stream); ++ return NGX_DECLINED; ++ } ++ ++ return NGX_OK; ++} ++ ++ ++static void ++ngx_http_spdy_waiting_queue(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_stream_t *stream) ++{ ++ ngx_queue_t *q; ++ ngx_http_spdy_stream_t *s; ++ ++ if (stream->handled) { ++ return; ++ } ++ ++ stream->handled = 1; ++ ++ for (q = ngx_queue_last(&sc->waiting); ++ q != ngx_queue_sentinel(&sc->waiting); ++ q = ngx_queue_prev(q)) ++ { ++ s = ngx_queue_data(q, ngx_http_spdy_stream_t, queue); ++ ++ /* ++ * NB: higher values represent lower priorities. ++ */ ++ if (stream->priority >= s->priority) { ++ break; ++ } ++ } ++ ++ ngx_queue_insert_after(q, &stream->queue); ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_syn_frame_handler(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_out_frame_t *frame) ++{ ++ ngx_buf_t *buf; ++ ngx_http_spdy_stream_t *stream; ++ ++ buf = frame->first->buf; ++ ++ if (buf->pos != buf->last) { ++ return NGX_AGAIN; ++ } ++ ++ stream = frame->stream; ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy:%ui SYN_REPLY frame %p was sent", stream->id, frame); ++ ++ ngx_free_chain(stream->request->pool, frame->first); ++ ++ ngx_http_spdy_handle_frame(stream, frame); ++ ++ ngx_http_spdy_handle_stream(sc, stream); ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_data_frame_handler(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_out_frame_t *frame) ++{ ++ ngx_buf_t *buf; ++ ngx_chain_t *cl, *ln; ++ ngx_http_spdy_stream_t *stream; ++ ++ stream = frame->stream; ++ ++ cl = frame->first; ++ ++ if (cl->buf->tag == (ngx_buf_tag_t) &ngx_http_spdy_filter_get_data_frame) { ++ ++ if (cl->buf->pos != cl->buf->last) { ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy:%ui DATA frame %p was sent partially", ++ stream->id, frame); ++ ++ return NGX_AGAIN; ++ } ++ ++ ln = cl->next; ++ ++ cl->next = stream->free_data_headers; ++ stream->free_data_headers = cl; ++ ++ if (cl == frame->last) { ++ goto done; ++ } ++ ++ cl = ln; ++ } ++ ++ for ( ;; ) { ++ if (cl->buf->tag == (ngx_buf_tag_t) &ngx_http_spdy_filter_get_shadow) { ++ buf = cl->buf->shadow; ++ ++ if (ngx_buf_in_memory(buf)) { ++ buf->pos = cl->buf->pos; ++ } ++ ++ if (buf->in_file) { ++ buf->file_pos = cl->buf->file_pos; ++ } ++ } ++ ++ if (ngx_buf_size(cl->buf) != 0) { ++ ++ if (cl != frame->first) { ++ frame->first = cl; ++ ngx_http_spdy_handle_stream(sc, stream); ++ } ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy:%ui DATA frame %p was sent partially", ++ stream->id, frame); ++ ++ return NGX_AGAIN; ++ } ++ ++ ln = cl->next; ++ ++ if (cl->buf->tag == (ngx_buf_tag_t) &ngx_http_spdy_filter_get_shadow) { ++ cl->next = stream->free_bufs; ++ stream->free_bufs = cl; ++ ++ } else { ++ ngx_free_chain(stream->request->pool, cl); ++ } ++ ++ if (cl == frame->last) { ++ goto done; ++ } ++ ++ cl = ln; ++ } ++ ++done: ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy:%ui DATA frame %p was sent", stream->id, frame); ++ ++ stream->request->header_size += NGX_SPDY_FRAME_HEADER_SIZE; ++ ++ ngx_http_spdy_handle_frame(stream, frame); ++ ++ ngx_http_spdy_handle_stream(sc, stream); ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_inline void ++ngx_http_spdy_handle_frame(ngx_http_spdy_stream_t *stream, ++ ngx_http_spdy_out_frame_t *frame) ++{ ++ ngx_http_request_t *r; ++ ++ r = stream->request; ++ ++ r->connection->sent += NGX_SPDY_FRAME_HEADER_SIZE + frame->length; ++ ++ if (frame->fin) { ++ stream->out_closed = 1; ++ } ++ ++ frame->next = stream->free_frames; ++ stream->free_frames = frame; ++ ++ stream->queued--; ++} ++ ++ ++static ngx_inline void ++ngx_http_spdy_handle_stream(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_stream_t *stream) ++{ ++ ngx_event_t *wev; ++ ++ if (stream->handled || stream->blocked || stream->exhausted) { ++ return; ++ } ++ ++ wev = stream->request->connection->write; ++ ++ /* ++ * This timer can only be set if the stream was delayed because of rate ++ * limit. In that case the event should be triggered by the timer. ++ */ ++ ++ if (!wev->timer_set) { ++ wev->delayed = 0; ++ ++ stream->handled = 1; ++ ngx_queue_insert_tail(&sc->posted, &stream->queue); ++ } ++} ++ ++ ++static void ++ngx_http_spdy_filter_cleanup(void *data) ++{ ++ ngx_http_spdy_stream_t *stream = data; ++ ++ size_t delta; ++ ngx_http_spdy_out_frame_t *frame, **fn; ++ ngx_http_spdy_connection_t *sc; ++ ++ if (stream->handled) { ++ stream->handled = 0; ++ ngx_queue_remove(&stream->queue); ++ } ++ ++ if (stream->queued == 0) { ++ return; ++ } ++ ++ delta = 0; ++ sc = stream->connection; ++ fn = &sc->last_out; ++ ++ for ( ;; ) { ++ frame = *fn; ++ ++ if (frame == NULL) { ++ break; ++ } ++ ++ if (frame->stream == stream && !frame->blocked) { ++ *fn = frame->next; ++ ++ delta += frame->length; ++ ++ if (--stream->queued == 0) { ++ break; ++ } ++ ++ continue; ++ } ++ ++ fn = &frame->next; ++ } ++ ++ if (sc->send_window == 0 && delta && !ngx_queue_empty(&sc->waiting)) { ++ ngx_queue_add(&sc->posted, &sc->waiting); ++ ngx_queue_init(&sc->waiting); ++ } ++ ++ sc->send_window += delta; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_filter_init(ngx_conf_t *cf) ++{ ++ ngx_http_next_header_filter = ngx_http_top_header_filter; ++ ngx_http_top_header_filter = ngx_http_spdy_header_filter; ++ ++ return NGX_OK; ++} +diff --git a/b/src/http/ngx_http_spdy_module.c b/src/http/ngx_http_spdy_module.c +new file mode 100644 +index 0000000..5178a36 +--- /dev/null ++++ b/src/http/ngx_http_spdy_module.c +@@ -0,0 +1,408 @@ ++ ++/* ++ * Copyright (C) Nginx, Inc. ++ * Copyright (C) Valentin V. Bartenev ++ */ ++ ++ ++#include ++#include ++#include ++#include ++ ++ ++static ngx_int_t ngx_http_spdy_add_variables(ngx_conf_t *cf); ++ ++static ngx_int_t ngx_http_spdy_variable(ngx_http_request_t *r, ++ ngx_http_variable_value_t *v, uintptr_t data); ++static ngx_int_t ngx_http_spdy_request_priority_variable(ngx_http_request_t *r, ++ ngx_http_variable_value_t *v, uintptr_t data); ++ ++static ngx_int_t ngx_http_spdy_module_init(ngx_cycle_t *cycle); ++ ++static void *ngx_http_spdy_create_main_conf(ngx_conf_t *cf); ++static char *ngx_http_spdy_init_main_conf(ngx_conf_t *cf, void *conf); ++static void *ngx_http_spdy_create_srv_conf(ngx_conf_t *cf); ++static char *ngx_http_spdy_merge_srv_conf(ngx_conf_t *cf, void *parent, ++ void *child); ++static void *ngx_http_spdy_create_loc_conf(ngx_conf_t *cf); ++static char *ngx_http_spdy_merge_loc_conf(ngx_conf_t *cf, void *parent, ++ void *child); ++ ++static char *ngx_http_spdy_recv_buffer_size(ngx_conf_t *cf, void *post, ++ void *data); ++static char *ngx_http_spdy_pool_size(ngx_conf_t *cf, void *post, void *data); ++static char *ngx_http_spdy_streams_index_mask(ngx_conf_t *cf, void *post, ++ void *data); ++static char *ngx_http_spdy_chunk_size(ngx_conf_t *cf, void *post, void *data); ++ ++ ++static ngx_conf_num_bounds_t ngx_http_spdy_headers_comp_bounds = { ++ ngx_conf_check_num_bounds, 0, 9 ++}; ++ ++static ngx_conf_post_t ngx_http_spdy_recv_buffer_size_post = ++ { ngx_http_spdy_recv_buffer_size }; ++static ngx_conf_post_t ngx_http_spdy_pool_size_post = ++ { ngx_http_spdy_pool_size }; ++static ngx_conf_post_t ngx_http_spdy_streams_index_mask_post = ++ { ngx_http_spdy_streams_index_mask }; ++static ngx_conf_post_t ngx_http_spdy_chunk_size_post = ++ { ngx_http_spdy_chunk_size }; ++ ++ ++static ngx_command_t ngx_http_spdy_commands[] = { ++ ++ { ngx_string("spdy_recv_buffer_size"), ++ NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1, ++ ngx_conf_set_size_slot, ++ NGX_HTTP_MAIN_CONF_OFFSET, ++ offsetof(ngx_http_spdy_main_conf_t, recv_buffer_size), ++ &ngx_http_spdy_recv_buffer_size_post }, ++ ++ { ngx_string("spdy_pool_size"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, ++ ngx_conf_set_size_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_spdy_srv_conf_t, pool_size), ++ &ngx_http_spdy_pool_size_post }, ++ ++ { ngx_string("spdy_max_concurrent_streams"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, ++ ngx_conf_set_num_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_spdy_srv_conf_t, concurrent_streams), ++ NULL }, ++ ++ { ngx_string("spdy_streams_index_size"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, ++ ngx_conf_set_num_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_spdy_srv_conf_t, streams_index_mask), ++ &ngx_http_spdy_streams_index_mask_post }, ++ ++ { ngx_string("spdy_recv_timeout"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, ++ ngx_conf_set_msec_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_spdy_srv_conf_t, recv_timeout), ++ NULL }, ++ ++ { ngx_string("spdy_keepalive_timeout"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, ++ ngx_conf_set_msec_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_spdy_srv_conf_t, keepalive_timeout), ++ NULL }, ++ ++ { ngx_string("spdy_headers_comp"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, ++ ngx_conf_set_num_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_spdy_srv_conf_t, headers_comp), ++ &ngx_http_spdy_headers_comp_bounds }, ++ ++ { ngx_string("spdy_chunk_size"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, ++ ngx_conf_set_size_slot, ++ NGX_HTTP_LOC_CONF_OFFSET, ++ offsetof(ngx_http_spdy_loc_conf_t, chunk_size), ++ &ngx_http_spdy_chunk_size_post }, ++ ++ ngx_null_command ++}; ++ ++ ++static ngx_http_module_t ngx_http_spdy_module_ctx = { ++ ngx_http_spdy_add_variables, /* preconfiguration */ ++ NULL, /* postconfiguration */ ++ ++ ngx_http_spdy_create_main_conf, /* create main configuration */ ++ ngx_http_spdy_init_main_conf, /* init main configuration */ ++ ++ ngx_http_spdy_create_srv_conf, /* create server configuration */ ++ ngx_http_spdy_merge_srv_conf, /* merge server configuration */ ++ ++ ngx_http_spdy_create_loc_conf, /* create location configuration */ ++ ngx_http_spdy_merge_loc_conf /* merge location configuration */ ++}; ++ ++ ++ngx_module_t ngx_http_spdy_module = { ++ NGX_MODULE_V1, ++ &ngx_http_spdy_module_ctx, /* module context */ ++ ngx_http_spdy_commands, /* module directives */ ++ NGX_HTTP_MODULE, /* module type */ ++ NULL, /* init master */ ++ ngx_http_spdy_module_init, /* init module */ ++ NULL, /* init process */ ++ NULL, /* init thread */ ++ NULL, /* exit thread */ ++ NULL, /* exit process */ ++ NULL, /* exit master */ ++ NGX_MODULE_V1_PADDING ++}; ++ ++ ++static ngx_http_variable_t ngx_http_spdy_vars[] = { ++ ++ { ngx_string("spdy"), NULL, ++ ngx_http_spdy_variable, 0, 0, 0 }, ++ ++ { ngx_string("spdy_request_priority"), NULL, ++ ngx_http_spdy_request_priority_variable, 0, 0, 0 }, ++ ++ { ngx_null_string, NULL, NULL, 0, 0, 0 } ++}; ++ ++ ++static ngx_int_t ++ngx_http_spdy_add_variables(ngx_conf_t *cf) ++{ ++ ngx_http_variable_t *var, *v; ++ ++ for (v = ngx_http_spdy_vars; v->name.len; v++) { ++ var = ngx_http_add_variable(cf, &v->name, v->flags); ++ if (var == NULL) { ++ return NGX_ERROR; ++ } ++ ++ var->get_handler = v->get_handler; ++ var->data = v->data; ++ } ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_variable(ngx_http_request_t *r, ++ ngx_http_variable_value_t *v, uintptr_t data) ++{ ++ if (r->spdy_stream) { ++ v->len = sizeof("3.1") - 1; ++ v->valid = 1; ++ v->no_cacheable = 0; ++ v->not_found = 0; ++ v->data = (u_char *) "3.1"; ++ ++ return NGX_OK; ++ } ++ ++ *v = ngx_http_variable_null_value; ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_request_priority_variable(ngx_http_request_t *r, ++ ngx_http_variable_value_t *v, uintptr_t data) ++{ ++ if (r->spdy_stream) { ++ v->len = 1; ++ v->valid = 1; ++ v->no_cacheable = 0; ++ v->not_found = 0; ++ ++ v->data = ngx_pnalloc(r->pool, 1); ++ if (v->data == NULL) { ++ return NGX_ERROR; ++ } ++ ++ v->data[0] = '0' + (u_char) r->spdy_stream->priority; ++ ++ return NGX_OK; ++ } ++ ++ *v = ngx_http_variable_null_value; ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_module_init(ngx_cycle_t *cycle) ++{ ++ ngx_http_spdy_request_headers_init(); ++ ++ return NGX_OK; ++} ++ ++ ++static void * ++ngx_http_spdy_create_main_conf(ngx_conf_t *cf) ++{ ++ ngx_http_spdy_main_conf_t *smcf; ++ ++ smcf = ngx_pcalloc(cf->pool, sizeof(ngx_http_spdy_main_conf_t)); ++ if (smcf == NULL) { ++ return NULL; ++ } ++ ++ smcf->recv_buffer_size = NGX_CONF_UNSET_SIZE; ++ ++ return smcf; ++} ++ ++ ++static char * ++ngx_http_spdy_init_main_conf(ngx_conf_t *cf, void *conf) ++{ ++ ngx_http_spdy_main_conf_t *smcf = conf; ++ ++ ngx_conf_init_size_value(smcf->recv_buffer_size, 256 * 1024); ++ ++ return NGX_CONF_OK; ++} ++ ++ ++static void * ++ngx_http_spdy_create_srv_conf(ngx_conf_t *cf) ++{ ++ ngx_http_spdy_srv_conf_t *sscf; ++ ++ sscf = ngx_pcalloc(cf->pool, sizeof(ngx_http_spdy_srv_conf_t)); ++ if (sscf == NULL) { ++ return NULL; ++ } ++ ++ sscf->pool_size = NGX_CONF_UNSET_SIZE; ++ ++ sscf->concurrent_streams = NGX_CONF_UNSET_UINT; ++ sscf->streams_index_mask = NGX_CONF_UNSET_UINT; ++ ++ sscf->recv_timeout = NGX_CONF_UNSET_MSEC; ++ sscf->keepalive_timeout = NGX_CONF_UNSET_MSEC; ++ ++ sscf->headers_comp = NGX_CONF_UNSET; ++ ++ return sscf; ++} ++ ++ ++static char * ++ngx_http_spdy_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) ++{ ++ ngx_http_spdy_srv_conf_t *prev = parent; ++ ngx_http_spdy_srv_conf_t *conf = child; ++ ++ ngx_conf_merge_size_value(conf->pool_size, prev->pool_size, 4096); ++ ++ ngx_conf_merge_uint_value(conf->concurrent_streams, ++ prev->concurrent_streams, 100); ++ ++ ngx_conf_merge_uint_value(conf->streams_index_mask, ++ prev->streams_index_mask, 32 - 1); ++ ++ ngx_conf_merge_msec_value(conf->recv_timeout, ++ prev->recv_timeout, 30000); ++ ngx_conf_merge_msec_value(conf->keepalive_timeout, ++ prev->keepalive_timeout, 180000); ++ ++ ngx_conf_merge_value(conf->headers_comp, prev->headers_comp, 0); ++ ++ return NGX_CONF_OK; ++} ++ ++ ++static void * ++ngx_http_spdy_create_loc_conf(ngx_conf_t *cf) ++{ ++ ngx_http_spdy_loc_conf_t *slcf; ++ ++ slcf = ngx_pcalloc(cf->pool, sizeof(ngx_http_spdy_loc_conf_t)); ++ if (slcf == NULL) { ++ return NULL; ++ } ++ ++ slcf->chunk_size = NGX_CONF_UNSET_SIZE; ++ ++ return slcf; ++} ++ ++ ++static char * ++ngx_http_spdy_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) ++{ ++ ngx_http_spdy_loc_conf_t *prev = parent; ++ ngx_http_spdy_loc_conf_t *conf = child; ++ ++ ngx_conf_merge_size_value(conf->chunk_size, prev->chunk_size, 8 * 1024); ++ ++ return NGX_CONF_OK; ++} ++ ++ ++static char * ++ngx_http_spdy_recv_buffer_size(ngx_conf_t *cf, void *post, void *data) ++{ ++ size_t *sp = data; ++ ++ if (*sp <= 2 * NGX_SPDY_STATE_BUFFER_SIZE) { ++ return "value is too small"; ++ } ++ ++ return NGX_CONF_OK; ++} ++ ++ ++static char * ++ngx_http_spdy_pool_size(ngx_conf_t *cf, void *post, void *data) ++{ ++ size_t *sp = data; ++ ++ if (*sp < NGX_MIN_POOL_SIZE) { ++ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, ++ "the pool size must be no less than %uz", ++ NGX_MIN_POOL_SIZE); ++ return NGX_CONF_ERROR; ++ } ++ ++ if (*sp % NGX_POOL_ALIGNMENT) { ++ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, ++ "the pool size must be a multiple of %uz", ++ NGX_POOL_ALIGNMENT); ++ return NGX_CONF_ERROR; ++ } ++ ++ return NGX_CONF_OK; ++} ++ ++ ++static char * ++ngx_http_spdy_streams_index_mask(ngx_conf_t *cf, void *post, void *data) ++{ ++ ngx_uint_t *np = data; ++ ++ ngx_uint_t mask; ++ ++ mask = *np - 1; ++ ++ if (*np == 0 || (*np & mask)) { ++ return "must be a power of two"; ++ } ++ ++ *np = mask; ++ ++ return NGX_CONF_OK; ++} ++ ++ ++static char * ++ngx_http_spdy_chunk_size(ngx_conf_t *cf, void *post, void *data) ++{ ++ size_t *sp = data; ++ ++ if (*sp == 0) { ++ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, ++ "the spdy chunk size cannot be zero"); ++ return NGX_CONF_ERROR; ++ } ++ ++ if (*sp > NGX_SPDY_MAX_FRAME_SIZE) { ++ *sp = NGX_SPDY_MAX_FRAME_SIZE; ++ } ++ ++ return NGX_CONF_OK; ++} +diff --git a/b/src/http/ngx_http_spdy_module.h b/src/http/ngx_http_spdy_module.h +new file mode 100644 +index 0000000..5242322 +--- /dev/null ++++ b/src/http/ngx_http_spdy_module.h +@@ -0,0 +1,41 @@ ++ ++/* ++ * Copyright (C) Nginx, Inc. ++ * Copyright (C) Valentin V. Bartenev ++ */ ++ ++ ++#ifndef _NGX_HTTP_SPDY_MODULE_H_INCLUDED_ ++#define _NGX_HTTP_SPDY_MODULE_H_INCLUDED_ ++ ++ ++#include ++#include ++#include ++ ++ ++typedef struct { ++ size_t recv_buffer_size; ++ u_char *recv_buffer; ++} ngx_http_spdy_main_conf_t; ++ ++ ++typedef struct { ++ size_t pool_size; ++ ngx_uint_t concurrent_streams; ++ ngx_uint_t streams_index_mask; ++ ngx_msec_t recv_timeout; ++ ngx_msec_t keepalive_timeout; ++ ngx_int_t headers_comp; ++} ngx_http_spdy_srv_conf_t; ++ ++ ++typedef struct { ++ size_t chunk_size; ++} ngx_http_spdy_loc_conf_t; ++ ++ ++extern ngx_module_t ngx_http_spdy_module; ++ ++ ++#endif /* _NGX_HTTP_SPDY_MODULE_H_INCLUDED_ */ +diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c +index 1404693..6973ecf 100644 +--- a/src/http/ngx_http_upstream.c ++++ b/src/http/ngx_http_upstream.c +@@ -511,6 +511,12 @@ ngx_http_upstream_init(ngx_http_request_t *r) + return; + } + #endif ++#if (NGX_HTTP_SPDY) ++ if (r->spdy_stream) { ++ ngx_http_upstream_init_request(r); ++ return; ++ } ++#endif + + if (c->read->timer_set) { + ngx_del_timer(c->read); +@@ -1300,6 +1306,11 @@ ngx_http_upstream_check_broken_connection(ngx_http_request_t *r, + return; + } + #endif ++#if (NGX_HTTP_SPDY) ++ if (r->spdy_stream) { ++ return; ++ } ++#endif + + #if (NGX_HAVE_KQUEUE) + +diff --git a/src/http/v2/ngx_http_v2_module.c b/src/http/v2/ngx_http_v2_module.c +index 032abcb..a9cb728 100644 +--- a/src/http/v2/ngx_http_v2_module.c ++++ b/src/http/v2/ngx_http_v2_module.c +@@ -34,8 +34,6 @@ static char *ngx_http_v2_preread_size(ngx_conf_t *cf, void *post, void *data); + static char *ngx_http_v2_streams_index_mask(ngx_conf_t *cf, void *post, + void *data); + static char *ngx_http_v2_chunk_size(ngx_conf_t *cf, void *post, void *data); +-static char *ngx_http_v2_spdy_deprecated(ngx_conf_t *cf, ngx_command_t *cmd, +- void *conf); + + + static ngx_conf_post_t ngx_http_v2_recv_buffer_size_post = +@@ -129,62 +127,6 @@ static ngx_command_t ngx_http_v2_commands[] = { + offsetof(ngx_http_v2_loc_conf_t, chunk_size), + &ngx_http_v2_chunk_size_post }, + +- { ngx_string("spdy_recv_buffer_size"), +- NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1, +- ngx_http_v2_spdy_deprecated, +- NGX_HTTP_MAIN_CONF_OFFSET, +- 0, +- NULL }, +- +- { ngx_string("spdy_pool_size"), +- NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, +- ngx_http_v2_spdy_deprecated, +- NGX_HTTP_SRV_CONF_OFFSET, +- 0, +- NULL }, +- +- { ngx_string("spdy_max_concurrent_streams"), +- NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, +- ngx_http_v2_spdy_deprecated, +- NGX_HTTP_SRV_CONF_OFFSET, +- 0, +- NULL }, +- +- { ngx_string("spdy_streams_index_size"), +- NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, +- ngx_http_v2_spdy_deprecated, +- NGX_HTTP_SRV_CONF_OFFSET, +- 0, +- NULL }, +- +- { ngx_string("spdy_recv_timeout"), +- NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, +- ngx_http_v2_spdy_deprecated, +- NGX_HTTP_SRV_CONF_OFFSET, +- 0, +- NULL }, +- +- { ngx_string("spdy_keepalive_timeout"), +- NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, +- ngx_http_v2_spdy_deprecated, +- NGX_HTTP_SRV_CONF_OFFSET, +- 0, +- NULL }, +- +- { ngx_string("spdy_headers_comp"), +- NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, +- ngx_http_v2_spdy_deprecated, +- NGX_HTTP_SRV_CONF_OFFSET, +- 0, +- NULL }, +- +- { ngx_string("spdy_chunk_size"), +- NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, +- ngx_http_v2_spdy_deprecated, +- NGX_HTTP_LOC_CONF_OFFSET, +- 0, +- NULL }, +- + ngx_null_command + }; + +@@ -496,14 +438,3 @@ ngx_http_v2_chunk_size(ngx_conf_t *cf, void *post, void *data) + + return NGX_CONF_OK; + } +- +- +-static char * +-ngx_http_v2_spdy_deprecated(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +-{ +- ngx_conf_log_error(NGX_LOG_WARN, cf, 0, +- "invalid directive \"%V\": ngx_http_spdy_module " +- "was superseded by ngx_http_v2_module", &cmd->name); +- +- return NGX_CONF_OK; +-} diff --git a/patches/openssl__1.1.0_chacha20_poly1305.patch b/patches/openssl__1.1.0_chacha20_poly1305.patch new file mode 100644 index 0000000..34da57b --- /dev/null +++ b/patches/openssl__1.1.0_chacha20_poly1305.patch @@ -0,0 +1,60 @@ +diff --git a/ssl/s3_lib.c b/ssl/s3_lib.c +index e94ee83..3cd7e3a 100644 +--- a/ssl/s3_lib.c ++++ b/ssl/s3_lib.c +@@ -3582,6 +3582,7 @@ const SSL_CIPHER *ssl3_choose_cipher(SSL *s, STACK_OF(SSL_CIPHER) *clnt, + STACK_OF(SSL_CIPHER) *prio, *allow; + int i, ii, ok; + unsigned long alg_k, alg_a, mask_k, mask_a; ++ int use_chacha = 0; + + /* Let's see which ciphers we can support */ + +@@ -3610,13 +3611,20 @@ const SSL_CIPHER *ssl3_choose_cipher(SSL *s, STACK_OF(SSL_CIPHER) *clnt, + fprintf(stderr, "%p:%s\n", (void *)c, c->name); + } + #endif +- ++retry: + if (s->options & SSL_OP_CIPHER_SERVER_PREFERENCE || tls1_suiteb(s)) { + prio = srvr; + allow = clnt; ++ /* Use ChaCha20+Poly1305 if it's client's most preferred cipher suite */ ++ if (sk_SSL_CIPHER_num(clnt) > 0) { ++ c = sk_SSL_CIPHER_value(clnt, 0); ++ if (c->algorithm_enc == SSL_CHACHA20POLY1305) ++ use_chacha = 1; ++ } + } else { + prio = clnt; + allow = srvr; ++ use_chacha = 1; + } + + tls1_set_cert_validity(s); +@@ -3634,6 +3642,10 @@ const SSL_CIPHER *ssl3_choose_cipher(SSL *s, STACK_OF(SSL_CIPHER) *clnt, + DTLS_VERSION_GT(s->version, c->max_dtls))) + continue; + ++ /* Skip ChaCha unless top client priority */ ++ if (c->algorithm_enc == SSL_CHACHA20POLY1305 && !use_chacha) ++ continue; ++ + mask_k = s->s3->tmp.mask_k; + mask_a = s->s3->tmp.mask_a; + #ifndef OPENSSL_NO_SRP +@@ -3687,6 +3699,14 @@ const SSL_CIPHER *ssl3_choose_cipher(SSL *s, STACK_OF(SSL_CIPHER) *clnt, + break; + } + } ++ ++ if (ret == NULL && !use_chacha) { ++ /* If no shared cipher was found due to some unusual preferences, try ++ * again with CHACHA enabled even if not top priority */ ++ use_chacha = 1; ++ goto retry; ++ } ++ + return (ret); + } + From 7d93da5a827f7291ee8cf59f45b8c6f4a5d7e412 Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 25 Apr 2017 22:34:31 -0400 Subject: [PATCH 07/16] Fix patch --- patches/nginx__1.11.11_http2_spdy.patch | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/patches/nginx__1.11.11_http2_spdy.patch b/patches/nginx__1.11.11_http2_spdy.patch index fe6f248..ba106d5 100644 --- a/patches/nginx__1.11.11_http2_spdy.patch +++ b/patches/nginx__1.11.11_http2_spdy.patch @@ -520,7 +520,7 @@ index 2f66484..c6006a0 100644 if (ngx_http_test_expect(r) != NGX_OK) { return NGX_HTTP_INTERNAL_SERVER_ERROR; -diff --git a/b/src/http/ngx_http_spdy.c b/src/http/ngx_http_spdy.c +diff --git a/src/http/ngx_http_spdy.c b/src/http/ngx_http_spdy.c new file mode 100644 index 0000000..21c5217 --- /dev/null @@ -4227,7 +4227,7 @@ index 0000000..21c5217 + "spdy zfree: %p", address); +#endif +} -diff --git a/b/src/http/ngx_http_spdy.h b/src/http/ngx_http_spdy.h +diff --git a/src/http/ngx_http_spdy.h b/src/http/ngx_http_spdy.h new file mode 100644 index 0000000..df24495 --- /dev/null @@ -4494,7 +4494,7 @@ index 0000000..df24495 +#define ngx_spdy_frame_write_window ngx_spdy_frame_aligned_write_uint32 + +#endif /* _NGX_HTTP_SPDY_H_INCLUDED_ */ -diff --git a/b/src/http/ngx_http_spdy_filter_module.c b/src/http/ngx_http_spdy_filter_module.c +diff --git a/src/http/ngx_http_spdy_filter_module.c b/src/http/ngx_http_spdy_filter_module.c new file mode 100644 index 0000000..377e935 --- /dev/null @@ -5722,7 +5722,7 @@ index 0000000..377e935 + + return NGX_OK; +} -diff --git a/b/src/http/ngx_http_spdy_module.c b/src/http/ngx_http_spdy_module.c +diff --git a/src/http/ngx_http_spdy_module.c b/src/http/ngx_http_spdy_module.c new file mode 100644 index 0000000..5178a36 --- /dev/null @@ -6136,7 +6136,7 @@ index 0000000..5178a36 + + return NGX_CONF_OK; +} -diff --git a/b/src/http/ngx_http_spdy_module.h b/src/http/ngx_http_spdy_module.h +diff --git a/src/http/ngx_http_spdy_module.h b/src/http/ngx_http_spdy_module.h new file mode 100644 index 0000000..5242322 --- /dev/null From 5930c628d475023de6d224d0733a3a2a08b829fe Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 25 Apr 2017 22:34:47 -0400 Subject: [PATCH 08/16] Update patch for nginx 1.13.0 --- patches/nginx__1.13.0_http2_spdy.patch | 6305 ++++++++++++++++++++++++ 1 file changed, 6305 insertions(+) create mode 100644 patches/nginx__1.13.0_http2_spdy.patch diff --git a/patches/nginx__1.13.0_http2_spdy.patch b/patches/nginx__1.13.0_http2_spdy.patch new file mode 100644 index 0000000..10d762c --- /dev/null +++ b/patches/nginx__1.13.0_http2_spdy.patch @@ -0,0 +1,6305 @@ +diff --git a/auto/modules b/auto/modules +index be3561e..5a86643 100644 +--- a/auto/modules ++++ b/auto/modules +@@ -134,6 +134,7 @@ if [ $HTTP = YES ]; then + # ngx_http_header_filter + # ngx_http_chunked_filter + # ngx_http_v2_filter ++ # ngx_http_spdy_filter + # ngx_http_range_header_filter + # ngx_http_gzip_filter + # ngx_http_postpone_filter +@@ -166,6 +167,7 @@ if [ $HTTP = YES ]; then + ngx_http_header_filter_module \ + ngx_http_chunked_filter_module \ + ngx_http_v2_filter_module \ ++ ngx_http_spdy_filter_module \ + ngx_http_range_header_filter_module \ + ngx_http_gzip_filter_module \ + ngx_http_postpone_filter_module \ +@@ -227,6 +229,19 @@ if [ $HTTP = YES ]; then + . auto/module + fi + ++ if [ $HTTP_SPDY = YES ]; then ++ have=NGX_HTTP_SPDY . auto/have ++ USE_ZLIB=YES ++ ngx_module_name=ngx_http_spdy_filter_module ++ ngx_module_incs= ++ ngx_module_deps= ++ ngx_module_srcs=src/http/ngx_http_spdy_filter_module.c ++ ngx_module_libs= ++ ngx_module_link=$HTTP_SPDY ++ ++ . auto/module ++ fi ++ + if :; then + ngx_module_name=ngx_http_range_header_filter_module + ngx_module_incs= +@@ -436,6 +451,19 @@ if [ $HTTP = YES ]; then + . auto/module + fi + ++ if [ $HTTP_SPDY = YES ]; then ++ have=NGX_HTTP_SPDY . auto/have ++ ngx_module_name=ngx_http_spdy_module ++ ngx_module_incs=src/http ++ ngx_module_deps="src/http/ngx_http_spdy.h src/http/ngx_http_spdy_module.h" ++ ngx_module_srcs="src/http/ngx_http_spdy.c \ ++ src/http/ngx_http_spdy_module.c" ++ ngx_module_libs= ++ ngx_module_link=$HTTP_SPDY ++ ++ . auto/module ++ fi ++ + if :; then + ngx_module_name=ngx_http_static_module + ngx_module_incs= +diff --git a/auto/options b/auto/options +index 66b822a..dbcca32 100644 +--- a/auto/options ++++ b/auto/options +@@ -59,6 +59,7 @@ HTTP_CHARSET=YES + HTTP_GZIP=YES + HTTP_SSL=NO + HTTP_V2=NO ++HTTP_SPDY=NO + HTTP_SSI=YES + HTTP_POSTPONE=NO + HTTP_REALIP=NO +@@ -221,6 +222,7 @@ $0: warning: the \"--with-ipv6\" option is deprecated" + + --with-http_ssl_module) HTTP_SSL=YES ;; + --with-http_v2_module) HTTP_V2=YES ;; ++ --with-http_spdy_module) HTTP_SPDY=YES ;; + --with-http_realip_module) HTTP_REALIP=YES ;; + --with-http_addition_module) HTTP_ADDITION=YES ;; + --with-http_xslt_module) HTTP_XSLT=YES ;; +@@ -430,6 +432,7 @@ cat << END + + --with-http_ssl_module enable ngx_http_ssl_module + --with-http_v2_module enable ngx_http_v2_module ++ --with-http_spdy_module enable ngx_http_spdy_module + --with-http_realip_module enable ngx_http_realip_module + --with-http_addition_module enable ngx_http_addition_module + --with-http_xslt_module enable ngx_http_xslt_module +diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h +index 1d3e3a3..41f9882 100644 +--- a/src/core/ngx_connection.h ++++ b/src/core/ngx_connection.h +@@ -116,6 +116,7 @@ typedef enum { + #define NGX_LOWLEVEL_BUFFERED 0x0f + #define NGX_SSL_BUFFERED 0x01 + #define NGX_HTTP_V2_BUFFERED 0x02 ++#define NGX_SPDY_BUFFERED 0x04 + + + struct ngx_connection_s { +diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c +index b466e5d..247039a 100644 +--- a/src/http/modules/ngx_http_ssl_module.c ++++ b/src/http/modules/ngx_http_ssl_module.c +@@ -348,10 +348,10 @@ ngx_http_ssl_alpn_select(ngx_ssl_conn_t *ssl_conn, const unsigned char **out, + #if (NGX_DEBUG) + unsigned int i; + #endif +-#if (NGX_HTTP_V2) ++#if (NGX_HTTP_V2 || NGX_HTTP_SPDY) + ngx_http_connection_t *hc; + #endif +-#if (NGX_HTTP_V2 || NGX_DEBUG) ++#if (NGX_HTTP_V2 || NGX_HTTP_SPDY || NGX_DEBUG) + ngx_connection_t *c; + + c = ngx_ssl_get_connection(ssl_conn); +@@ -365,9 +365,20 @@ ngx_http_ssl_alpn_select(ngx_ssl_conn_t *ssl_conn, const unsigned char **out, + } + #endif + +-#if (NGX_HTTP_V2) ++#if (NGX_HTTP_V2 || NGX_HTTP_SPDY) + hc = c->data; ++#endif + ++#if (NGX_HTTP_V2 && NGX_HTTP_SPDY) ++ if (hc->addr_conf->http2 && hc->addr_conf->spdy) { ++ srv = (unsigned char *) NGX_HTTP_V2_ALPN_ADVERTISE ++ NGX_SPDY_NPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE; ++ srvlen = sizeof(NGX_HTTP_V2_ALPN_ADVERTISE NGX_SPDY_NPN_ADVERTISE ++ NGX_HTTP_NPN_ADVERTISE) - 1; ++ ++ } else ++#endif ++#if (NGX_HTTP_V2) + if (hc->addr_conf->http2) { + srv = + (unsigned char *) NGX_HTTP_V2_ALPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE; +@@ -375,6 +386,13 @@ ngx_http_ssl_alpn_select(ngx_ssl_conn_t *ssl_conn, const unsigned char **out, + + } else + #endif ++#if (NGX_HTTP_SPDY) ++ if (hc->addr_conf->spdy) { ++ srv = (unsigned char *) NGX_SPDY_NPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE; ++ srvlen = sizeof(NGX_SPDY_NPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE) - 1; ++ ++ } else ++#endif + { + srv = (unsigned char *) NGX_HTTP_NPN_ADVERTISE; + srvlen = sizeof(NGX_HTTP_NPN_ADVERTISE) - 1; +@@ -402,19 +420,32 @@ static int + ngx_http_ssl_npn_advertised(ngx_ssl_conn_t *ssl_conn, + const unsigned char **out, unsigned int *outlen, void *arg) + { +-#if (NGX_HTTP_V2 || NGX_DEBUG) ++#if (NGX_HTTP_V2 || NGX_HTTP_SPDY || NGX_DEBUG) + ngx_connection_t *c; + + c = ngx_ssl_get_connection(ssl_conn); + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "SSL NPN advertised"); + #endif + +-#if (NGX_HTTP_V2) ++#if (NGX_HTTP_V2 || NGX_HTTP_SPDY) + { + ngx_http_connection_t *hc; + + hc = c->data; ++#endif + ++#if (NGX_HTTP_V2 && NGX_HTTP_SPDY) ++ if (hc->addr_conf->http2 && hc->addr_conf->spdy) { ++ *out = (unsigned char *) NGX_HTTP_V2_NPN_ADVERTISE ++ NGX_SPDY_NPN_ADVERTISE ++ NGX_HTTP_NPN_ADVERTISE; ++ *outlen = sizeof(NGX_HTTP_V2_NPN_ADVERTISE NGX_SPDY_NPN_ADVERTISE ++ NGX_HTTP_NPN_ADVERTISE) - 1; ++ ++ return SSL_TLSEXT_ERR_OK; ++ } else ++#endif ++#if (NGX_HTTP_V2) + if (hc->addr_conf->http2) { + *out = + (unsigned char *) NGX_HTTP_V2_NPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE; +@@ -422,6 +453,20 @@ ngx_http_ssl_npn_advertised(ngx_ssl_conn_t *ssl_conn, + + return SSL_TLSEXT_ERR_OK; + } ++#endif ++#if (NGX_HTTP_V2 && NGX_HTTP_SPDY) ++ else ++#endif ++#if (NGX_HTTP_SPDY) ++ if (hc->addr_conf->spdy) { ++ *out = (unsigned char *) NGX_SPDY_NPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE; ++ *outlen = sizeof(NGX_SPDY_NPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE) - 1; ++ ++ return SSL_TLSEXT_ERR_OK; ++ } ++#endif ++ ++#if (NGX_HTTP_V2 || NGX_HTTP_SPDY) + } + #endif + +diff --git a/src/http/ngx_http.c b/src/http/ngx_http.c +index c036389..78b671c 100644 +--- a/src/http/ngx_http.c ++++ b/src/http/ngx_http.c +@@ -1202,6 +1202,9 @@ ngx_http_add_addresses(ngx_conf_t *cf, ngx_http_core_srv_conf_t *cscf, + #if (NGX_HTTP_V2) + ngx_uint_t http2; + #endif ++#if (NGX_HTTP_SPDY) ++ ngx_uint_t spdy; ++#endif + + /* + * we cannot compare whole sockaddr struct's as kernel +@@ -1237,6 +1240,9 @@ ngx_http_add_addresses(ngx_conf_t *cf, ngx_http_core_srv_conf_t *cscf, + #if (NGX_HTTP_V2) + http2 = lsopt->http2 || addr[i].opt.http2; + #endif ++#if (NGX_HTTP_SPDY) ++ spdy = lsopt->spdy || addr[i].opt.spdy; ++#endif + + if (lsopt->set) { + +@@ -1271,6 +1277,9 @@ ngx_http_add_addresses(ngx_conf_t *cf, ngx_http_core_srv_conf_t *cscf, + #if (NGX_HTTP_V2) + addr[i].opt.http2 = http2; + #endif ++#if (NGX_HTTP_SPDY) ++ addr[i].opt.spdy = spdy; ++#endif + + return NGX_OK; + } +@@ -1314,6 +1323,18 @@ ngx_http_add_address(ngx_conf_t *cf, ngx_http_core_srv_conf_t *cscf, + + #endif + ++#if (NGX_HTTP_SPDY && NGX_HTTP_SSL \ ++ && !defined TLSEXT_TYPE_application_layer_protocol_negotiation \ ++ && !defined TLSEXT_TYPE_next_proto_neg) ++ if (lsopt->spdy && lsopt->ssl) { ++ ngx_conf_log_error(NGX_LOG_WARN, cf, 0, ++ "nginx was built with OpenSSL that lacks ALPN " ++ "and NPN support, SPDY is not enabled for %s", ++ lsopt->addr); ++ } ++ ++#endif ++ + addr = ngx_array_push(&port->addrs); + if (addr == NULL) { + return NGX_ERROR; +@@ -1807,6 +1828,9 @@ ngx_http_add_addrs(ngx_conf_t *cf, ngx_http_port_t *hport, + #if (NGX_HTTP_V2) + addrs[i].conf.http2 = addr[i].opt.http2; + #endif ++#if (NGX_HTTP_SPDY) ++ addrs[i].conf.spdy = addr[i].opt.spdy; ++#endif + addrs[i].conf.proxy_protocol = addr[i].opt.proxy_protocol; + + if (addr[i].hash.buckets == NULL +@@ -1872,6 +1896,9 @@ ngx_http_add_addrs6(ngx_conf_t *cf, ngx_http_port_t *hport, + #if (NGX_HTTP_V2) + addrs6[i].conf.http2 = addr[i].opt.http2; + #endif ++#if (NGX_HTTP_SPDY) ++ addrs6[i].conf.spdy = addr[i].opt.spdy; ++#endif + addrs6[i].conf.proxy_protocol = addr[i].opt.proxy_protocol; + + if (addr[i].hash.buckets == NULL +diff --git a/src/http/ngx_http.h b/src/http/ngx_http.h +index afab4f6..7b70426 100644 +--- a/src/http/ngx_http.h ++++ b/src/http/ngx_http.h +@@ -19,6 +19,9 @@ typedef struct ngx_http_cache_s ngx_http_cache_t; + typedef struct ngx_http_file_cache_s ngx_http_file_cache_t; + typedef struct ngx_http_log_ctx_s ngx_http_log_ctx_t; + typedef struct ngx_http_chunked_s ngx_http_chunked_t; ++#if (NGX_HTTP_SPDY) ++typedef struct ngx_http_spdy_stream_s ngx_http_spdy_stream_t; ++#endif + typedef struct ngx_http_v2_stream_s ngx_http_v2_stream_t; + + typedef ngx_int_t (*ngx_http_header_handler_pt)(ngx_http_request_t *r, +@@ -35,9 +38,13 @@ typedef u_char *(*ngx_http_log_handler_pt)(ngx_http_request_t *r, + #include + #include + ++ + #if (NGX_HTTP_V2) + #include + #endif ++#if (NGX_HTTP_SPDY) ++#include ++#endif + #if (NGX_HTTP_CACHE) + #include + #endif +diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c +index af67b7f..cb5ea27 100644 +--- a/src/http/ngx_http_core_module.c ++++ b/src/http/ngx_http_core_module.c +@@ -2158,6 +2158,13 @@ ngx_http_gzip_ok(ngx_http_request_t *r) + return NGX_DECLINED; + } + ++#if (NGX_HTTP_SPDY) ++ if (r->spdy_stream) { ++ r->gzip_ok = 1; ++ return NGX_OK; ++ } ++#endif ++ + ae = r->headers_in.accept_encoding; + if (ae == NULL) { + return NGX_DECLINED; +@@ -2502,6 +2509,9 @@ ngx_http_subrequest(ngx_http_request_t *r, + #if (NGX_HTTP_V2) + sr->stream = r->stream; + #endif ++#if (NGX_HTTP_SPDY) ++ sr->spdy_stream = r->spdy_stream; ++#endif + + sr->method = NGX_HTTP_GET; + sr->http_version = r->http_version; +@@ -4180,11 +4190,15 @@ ngx_http_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) + } + + if (ngx_strcmp(value[n].data, "spdy") == 0) { +- ngx_conf_log_error(NGX_LOG_WARN, cf, 0, +- "invalid parameter \"spdy\": " +- "ngx_http_spdy_module was superseded " +- "by ngx_http_v2_module"); ++#if (NGX_HTTP_SPDY) ++ lsopt.spdy = 1; + continue; ++#else ++ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, ++ "the \"spdy\" parameter requires " ++ "ngx_http_spdy_module"); ++ return NGX_CONF_ERROR; ++#endif + } + + if (ngx_strncmp(value[n].data, "so_keepalive=", 13) == 0) { +diff --git a/src/http/ngx_http_core_module.h b/src/http/ngx_http_core_module.h +index 5018da0..718bc35 100644 +--- a/src/http/ngx_http_core_module.h ++++ b/src/http/ngx_http_core_module.h +@@ -74,6 +74,9 @@ typedef struct { + unsigned wildcard:1; + unsigned ssl:1; + unsigned http2:1; ++#if (NGX_HTTP_SPDY) ++ unsigned spdy:1; ++#endif + #if (NGX_HAVE_INET6) + unsigned ipv6only:1; + #endif +@@ -236,6 +239,9 @@ struct ngx_http_addr_conf_s { + + unsigned ssl:1; + unsigned http2:1; ++#if (NGX_HTTP_SPDY) ++ unsigned spdy:1; ++#endif + unsigned proxy_protocol:1; + }; + +diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c +index 476f039..8964b36 100644 +--- a/src/http/ngx_http_request.c ++++ b/src/http/ngx_http_request.c +@@ -316,6 +316,11 @@ ngx_http_init_connection(ngx_connection_t *c) + rev->handler = ngx_http_wait_request_handler; + c->write->handler = ngx_http_empty_handler; + ++#if (NGX_HTTP_SPDY) ++ if (hc->addr_conf->spdy) { ++ rev->handler = ngx_http_spdy_init; ++ } ++#endif + #if (NGX_HTTP_V2) + if (hc->addr_conf->http2) { + rev->handler = ngx_http_v2_init; +@@ -801,6 +806,34 @@ ngx_http_ssl_handshake_handler(ngx_connection_t *c) + } + #endif + ++#if (NGX_HTTP_SPDY \ ++ && (defined TLSEXT_TYPE_application_layer_protocol_negotiation \ ++ || defined TLSEXT_TYPE_next_proto_neg)) ++ { ++ unsigned int len; ++ const unsigned char *data; ++ static const ngx_str_t spdy = ngx_string(NGX_SPDY_NPN_NEGOTIATED); ++ ++#ifdef TLSEXT_TYPE_application_layer_protocol_negotiation ++ SSL_get0_alpn_selected(c->ssl->connection, &data, &len); ++ ++#ifdef TLSEXT_TYPE_next_proto_neg ++ if (len == 0) { ++ SSL_get0_next_proto_negotiated(c->ssl->connection, &data, &len); ++ } ++#endif ++ ++#else /* TLSEXT_TYPE_next_proto_neg */ ++ SSL_get0_next_proto_negotiated(c->ssl->connection, &data, &len); ++#endif ++ ++ if (len == spdy.len && ngx_strncmp(data, spdy.data, spdy.len) == 0) { ++ ngx_http_spdy_init(c->read); ++ return; ++ } ++ } ++#endif ++ + c->log->action = "waiting for request"; + + c->read->handler = ngx_http_wait_request_handler; +@@ -2529,6 +2562,12 @@ ngx_http_finalize_connection(ngx_http_request_t *r) + return; + } + #endif ++#if (NGX_HTTP_SPDY) ++ if (r->spdy_stream) { ++ ngx_http_close_request(r, 0); ++ return; ++ } ++#endif + + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); + +@@ -2736,6 +2775,18 @@ ngx_http_test_reading(ngx_http_request_t *r) + } + + #endif ++#if (NGX_HTTP_SPDY) ++ ++ if (r->spdy_stream) { ++ if (c->error) { ++ err = 0; ++ goto closed; ++ } ++ ++ return; ++ } ++ ++#endif + + #if (NGX_HAVE_KQUEUE) + +@@ -3420,6 +3471,12 @@ ngx_http_close_request(ngx_http_request_t *r, ngx_int_t rc) + return; + } + #endif ++#if (NGX_HTTP_SPDY) ++ if (r->spdy_stream) { ++ ngx_http_spdy_close_stream(r->spdy_stream, rc); ++ return; ++ } ++#endif + + ngx_http_free_request(r, rc); + ngx_http_close_connection(c); +diff --git a/src/http/ngx_http_request.h b/src/http/ngx_http_request.h +index 2c77f93..cf54e81 100644 +--- a/src/http/ngx_http_request.h ++++ b/src/http/ngx_http_request.h +@@ -425,6 +425,9 @@ struct ngx_http_request_s { + int *captures; + u_char *captures_data; + #endif ++#if (NGX_HTTP_SPDY) ++ ngx_http_spdy_stream_t *spdy_stream; ++#endif + + size_t limit_rate; + size_t limit_rate_after; +diff --git a/src/http/ngx_http_request_body.c b/src/http/ngx_http_request_body.c +index c4f092e..2a2970f 100644 +--- a/src/http/ngx_http_request_body.c ++++ b/src/http/ngx_http_request_body.c +@@ -84,6 +84,12 @@ ngx_http_read_client_request_body(ngx_http_request_t *r, + goto done; + } + #endif ++#if (NGX_HTTP_SPDY) ++ if (r->spdy_stream) { ++ rc = ngx_http_spdy_read_request_body(r, post_handler); ++ goto done; ++ } ++#endif + + preread = r->header_in->last - r->header_in->pos; + +@@ -524,6 +530,12 @@ ngx_http_discard_request_body(ngx_http_request_t *r) + return NGX_OK; + } + #endif ++#if (NGX_HTTP_SPDY) ++ if (r->spdy_stream) { ++ r->spdy_stream->skip_data = 1; ++ return NGX_OK; ++ } ++#endif + + if (ngx_http_test_expect(r) != NGX_OK) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; +diff --git a/src/http/ngx_http_spdy.c b/src/http/ngx_http_spdy.c +new file mode 100644 +index 0000000..21c5217 +--- /dev/null ++++ b/src/http/ngx_http_spdy.c +@@ -0,0 +1,3701 @@ ++ ++/* ++ * Copyright (C) Nginx, Inc. ++ * Copyright (C) Valentin V. Bartenev ++ */ ++ ++ ++#include ++#include ++#include ++#include ++ ++#include ++ ++ ++#if (NGX_HAVE_LITTLE_ENDIAN && NGX_HAVE_NONALIGNED) ++ ++#define ngx_str5cmp(m, c0, c1, c2, c3, c4) \ ++ *(uint32_t *) m == (c3 << 24 | c2 << 16 | c1 << 8 | c0) \ ++ && m[4] == c4 ++ ++#else ++ ++#define ngx_str5cmp(m, c0, c1, c2, c3, c4) \ ++ m[0] == c0 && m[1] == c1 && m[2] == c2 && m[3] == c3 && m[4] == c4 ++ ++#endif ++ ++ ++#if (NGX_HAVE_NONALIGNED) ++ ++#define ngx_spdy_frame_parse_uint16(p) ntohs(*(uint16_t *) (p)) ++#define ngx_spdy_frame_parse_uint32(p) ntohl(*(uint32_t *) (p)) ++ ++#else ++ ++#define ngx_spdy_frame_parse_uint16(p) ((p)[0] << 8 | (p)[1]) ++#define ngx_spdy_frame_parse_uint32(p) \ ++ ((p)[0] << 24 | (p)[1] << 16 | (p)[2] << 8 | (p)[3]) ++ ++#endif ++ ++#define ngx_spdy_frame_parse_sid(p) \ ++ (ngx_spdy_frame_parse_uint32(p) & 0x7fffffff) ++#define ngx_spdy_frame_parse_delta(p) \ ++ (ngx_spdy_frame_parse_uint32(p) & 0x7fffffff) ++ ++ ++#define ngx_spdy_ctl_frame_check(h) \ ++ (((h) & 0xffff0000) == ngx_spdy_ctl_frame_head(0)) ++#define ngx_spdy_data_frame_check(h) \ ++ (!((h) & (uint32_t) NGX_SPDY_CTL_BIT << 31)) ++ ++#define ngx_spdy_ctl_frame_type(h) ((h) & 0x0000ffff) ++#define ngx_spdy_frame_flags(p) ((p) >> 24) ++#define ngx_spdy_frame_length(p) ((p) & 0x00ffffff) ++#define ngx_spdy_frame_id(p) ((p) & 0x00ffffff) ++ ++ ++#define NGX_SPDY_SKIP_HEADERS_BUFFER_SIZE 4096 ++#define NGX_SPDY_CTL_FRAME_BUFFER_SIZE 16 ++ ++#define NGX_SPDY_PROTOCOL_ERROR 1 ++#define NGX_SPDY_INVALID_STREAM 2 ++#define NGX_SPDY_REFUSED_STREAM 3 ++#define NGX_SPDY_UNSUPPORTED_VERSION 4 ++#define NGX_SPDY_CANCEL 5 ++#define NGX_SPDY_INTERNAL_ERROR 6 ++#define NGX_SPDY_FLOW_CONTROL_ERROR 7 ++#define NGX_SPDY_STREAM_IN_USE 8 ++#define NGX_SPDY_STREAM_ALREADY_CLOSED 9 ++/* deprecated 10 */ ++#define NGX_SPDY_FRAME_TOO_LARGE 11 ++ ++#define NGX_SPDY_SETTINGS_MAX_STREAMS 4 ++#define NGX_SPDY_SETTINGS_INIT_WINDOW 7 ++ ++#define NGX_SPDY_SETTINGS_FLAG_PERSIST 0x01 ++#define NGX_SPDY_SETTINGS_FLAG_PERSISTED 0x02 ++ ++#define NGX_SPDY_MAX_WINDOW NGX_MAX_INT32_VALUE ++#define NGX_SPDY_CONNECTION_WINDOW 65536 ++#define NGX_SPDY_INIT_STREAM_WINDOW 65536 ++#define NGX_SPDY_STREAM_WINDOW NGX_SPDY_MAX_WINDOW ++ ++typedef struct { ++ ngx_uint_t hash; ++ u_char len; ++ u_char header[7]; ++ ngx_int_t (*handler)(ngx_http_request_t *r); ++} ngx_http_spdy_request_header_t; ++ ++ ++static void ngx_http_spdy_read_handler(ngx_event_t *rev); ++static void ngx_http_spdy_write_handler(ngx_event_t *wev); ++static void ngx_http_spdy_handle_connection(ngx_http_spdy_connection_t *sc); ++ ++static u_char *ngx_http_spdy_proxy_protocol(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_head(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_syn_stream(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_headers(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_headers_skip(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_headers_error(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_window_update(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_data(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_read_data(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_rst_stream(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_ping(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_skip(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_settings(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_complete(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++static u_char *ngx_http_spdy_state_save(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end, ngx_http_spdy_handler_pt handler); ++ ++static u_char *ngx_http_spdy_state_inflate_error( ++ ngx_http_spdy_connection_t *sc, int rc); ++static u_char *ngx_http_spdy_state_protocol_error( ++ ngx_http_spdy_connection_t *sc); ++static u_char *ngx_http_spdy_state_internal_error( ++ ngx_http_spdy_connection_t *sc); ++ ++static ngx_int_t ngx_http_spdy_send_window_update( ++ ngx_http_spdy_connection_t *sc, ngx_uint_t sid, ngx_uint_t delta); ++static ngx_int_t ngx_http_spdy_send_rst_stream(ngx_http_spdy_connection_t *sc, ++ ngx_uint_t sid, ngx_uint_t status, ngx_uint_t priority); ++static ngx_int_t ngx_http_spdy_send_settings(ngx_http_spdy_connection_t *sc); ++static ngx_int_t ngx_http_spdy_settings_frame_handler( ++ ngx_http_spdy_connection_t *sc, ngx_http_spdy_out_frame_t *frame); ++static ngx_http_spdy_out_frame_t *ngx_http_spdy_get_ctl_frame( ++ ngx_http_spdy_connection_t *sc, size_t size, ngx_uint_t priority); ++static ngx_int_t ngx_http_spdy_ctl_frame_handler( ++ ngx_http_spdy_connection_t *sc, ngx_http_spdy_out_frame_t *frame); ++ ++static ngx_http_spdy_stream_t *ngx_http_spdy_create_stream( ++ ngx_http_spdy_connection_t *sc, ngx_uint_t id, ngx_uint_t priority); ++static ngx_http_spdy_stream_t *ngx_http_spdy_get_stream_by_id( ++ ngx_http_spdy_connection_t *sc, ngx_uint_t sid); ++#define ngx_http_spdy_streams_index_size(sscf) (sscf->streams_index_mask + 1) ++#define ngx_http_spdy_stream_index(sscf, sid) \ ++ ((sid >> 1) & sscf->streams_index_mask) ++ ++static ngx_int_t ngx_http_spdy_parse_header(ngx_http_request_t *r); ++static ngx_int_t ngx_http_spdy_alloc_large_header_buffer(ngx_http_request_t *r); ++ ++static ngx_int_t ngx_http_spdy_handle_request_header(ngx_http_request_t *r); ++static ngx_int_t ngx_http_spdy_parse_method(ngx_http_request_t *r); ++static ngx_int_t ngx_http_spdy_parse_scheme(ngx_http_request_t *r); ++static ngx_int_t ngx_http_spdy_parse_host(ngx_http_request_t *r); ++static ngx_int_t ngx_http_spdy_parse_path(ngx_http_request_t *r); ++static ngx_int_t ngx_http_spdy_parse_version(ngx_http_request_t *r); ++ ++static ngx_int_t ngx_http_spdy_construct_request_line(ngx_http_request_t *r); ++static void ngx_http_spdy_run_request(ngx_http_request_t *r); ++static ngx_int_t ngx_http_spdy_init_request_body(ngx_http_request_t *r); ++ ++static ngx_int_t ngx_http_spdy_terminate_stream(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_stream_t *stream, ngx_uint_t status); ++ ++static void ngx_http_spdy_close_stream_handler(ngx_event_t *ev); ++ ++static void ngx_http_spdy_handle_connection_handler(ngx_event_t *rev); ++static void ngx_http_spdy_keepalive_handler(ngx_event_t *rev); ++static void ngx_http_spdy_finalize_connection(ngx_http_spdy_connection_t *sc, ++ ngx_int_t rc); ++ ++static ngx_int_t ngx_http_spdy_adjust_windows(ngx_http_spdy_connection_t *sc, ++ ssize_t delta); ++ ++static void ngx_http_spdy_pool_cleanup(void *data); ++ ++static void *ngx_http_spdy_zalloc(void *opaque, u_int items, u_int size); ++static void ngx_http_spdy_zfree(void *opaque, void *address); ++ ++ ++static const u_char ngx_http_spdy_dict[] = { ++ 0x00, 0x00, 0x00, 0x07, 0x6f, 0x70, 0x74, 0x69, /* - - - - o p t i */ ++ 0x6f, 0x6e, 0x73, 0x00, 0x00, 0x00, 0x04, 0x68, /* o n s - - - - h */ ++ 0x65, 0x61, 0x64, 0x00, 0x00, 0x00, 0x04, 0x70, /* e a d - - - - p */ ++ 0x6f, 0x73, 0x74, 0x00, 0x00, 0x00, 0x03, 0x70, /* o s t - - - - p */ ++ 0x75, 0x74, 0x00, 0x00, 0x00, 0x06, 0x64, 0x65, /* u t - - - - d e */ ++ 0x6c, 0x65, 0x74, 0x65, 0x00, 0x00, 0x00, 0x05, /* l e t e - - - - */ ++ 0x74, 0x72, 0x61, 0x63, 0x65, 0x00, 0x00, 0x00, /* t r a c e - - - */ ++ 0x06, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x00, /* - a c c e p t - */ ++ 0x00, 0x00, 0x0e, 0x61, 0x63, 0x63, 0x65, 0x70, /* - - - a c c e p */ ++ 0x74, 0x2d, 0x63, 0x68, 0x61, 0x72, 0x73, 0x65, /* t - c h a r s e */ ++ 0x74, 0x00, 0x00, 0x00, 0x0f, 0x61, 0x63, 0x63, /* t - - - - a c c */ ++ 0x65, 0x70, 0x74, 0x2d, 0x65, 0x6e, 0x63, 0x6f, /* e p t - e n c o */ ++ 0x64, 0x69, 0x6e, 0x67, 0x00, 0x00, 0x00, 0x0f, /* d i n g - - - - */ ++ 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x2d, 0x6c, /* a c c e p t - l */ ++ 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, 0x00, /* a n g u a g e - */ ++ 0x00, 0x00, 0x0d, 0x61, 0x63, 0x63, 0x65, 0x70, /* - - - a c c e p */ ++ 0x74, 0x2d, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x73, /* t - r a n g e s */ ++ 0x00, 0x00, 0x00, 0x03, 0x61, 0x67, 0x65, 0x00, /* - - - - a g e - */ ++ 0x00, 0x00, 0x05, 0x61, 0x6c, 0x6c, 0x6f, 0x77, /* - - - a l l o w */ ++ 0x00, 0x00, 0x00, 0x0d, 0x61, 0x75, 0x74, 0x68, /* - - - - a u t h */ ++ 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, /* o r i z a t i o */ ++ 0x6e, 0x00, 0x00, 0x00, 0x0d, 0x63, 0x61, 0x63, /* n - - - - c a c */ ++ 0x68, 0x65, 0x2d, 0x63, 0x6f, 0x6e, 0x74, 0x72, /* h e - c o n t r */ ++ 0x6f, 0x6c, 0x00, 0x00, 0x00, 0x0a, 0x63, 0x6f, /* o l - - - - c o */ ++ 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, /* n n e c t i o n */ ++ 0x00, 0x00, 0x00, 0x0c, 0x63, 0x6f, 0x6e, 0x74, /* - - - - c o n t */ ++ 0x65, 0x6e, 0x74, 0x2d, 0x62, 0x61, 0x73, 0x65, /* e n t - b a s e */ ++ 0x00, 0x00, 0x00, 0x10, 0x63, 0x6f, 0x6e, 0x74, /* - - - - c o n t */ ++ 0x65, 0x6e, 0x74, 0x2d, 0x65, 0x6e, 0x63, 0x6f, /* e n t - e n c o */ ++ 0x64, 0x69, 0x6e, 0x67, 0x00, 0x00, 0x00, 0x10, /* d i n g - - - - */ ++ 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, /* c o n t e n t - */ ++ 0x6c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, /* l a n g u a g e */ ++ 0x00, 0x00, 0x00, 0x0e, 0x63, 0x6f, 0x6e, 0x74, /* - - - - c o n t */ ++ 0x65, 0x6e, 0x74, 0x2d, 0x6c, 0x65, 0x6e, 0x67, /* e n t - l e n g */ ++ 0x74, 0x68, 0x00, 0x00, 0x00, 0x10, 0x63, 0x6f, /* t h - - - - c o */ ++ 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x6c, 0x6f, /* n t e n t - l o */ ++ 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x00, 0x00, /* c a t i o n - - */ ++ 0x00, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, /* - - c o n t e n */ ++ 0x74, 0x2d, 0x6d, 0x64, 0x35, 0x00, 0x00, 0x00, /* t - m d 5 - - - */ ++ 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, /* - c o n t e n t */ ++ 0x2d, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x00, 0x00, /* - r a n g e - - */ ++ 0x00, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, /* - - c o n t e n */ ++ 0x74, 0x2d, 0x74, 0x79, 0x70, 0x65, 0x00, 0x00, /* t - t y p e - - */ ++ 0x00, 0x04, 0x64, 0x61, 0x74, 0x65, 0x00, 0x00, /* - - d a t e - - */ ++ 0x00, 0x04, 0x65, 0x74, 0x61, 0x67, 0x00, 0x00, /* - - e t a g - - */ ++ 0x00, 0x06, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, /* - - e x p e c t */ ++ 0x00, 0x00, 0x00, 0x07, 0x65, 0x78, 0x70, 0x69, /* - - - - e x p i */ ++ 0x72, 0x65, 0x73, 0x00, 0x00, 0x00, 0x04, 0x66, /* r e s - - - - f */ ++ 0x72, 0x6f, 0x6d, 0x00, 0x00, 0x00, 0x04, 0x68, /* r o m - - - - h */ ++ 0x6f, 0x73, 0x74, 0x00, 0x00, 0x00, 0x08, 0x69, /* o s t - - - - i */ ++ 0x66, 0x2d, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x00, /* f - m a t c h - */ ++ 0x00, 0x00, 0x11, 0x69, 0x66, 0x2d, 0x6d, 0x6f, /* - - - i f - m o */ ++ 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x2d, 0x73, /* d i f i e d - s */ ++ 0x69, 0x6e, 0x63, 0x65, 0x00, 0x00, 0x00, 0x0d, /* i n c e - - - - */ ++ 0x69, 0x66, 0x2d, 0x6e, 0x6f, 0x6e, 0x65, 0x2d, /* i f - n o n e - */ ++ 0x6d, 0x61, 0x74, 0x63, 0x68, 0x00, 0x00, 0x00, /* m a t c h - - - */ ++ 0x08, 0x69, 0x66, 0x2d, 0x72, 0x61, 0x6e, 0x67, /* - i f - r a n g */ ++ 0x65, 0x00, 0x00, 0x00, 0x13, 0x69, 0x66, 0x2d, /* e - - - - i f - */ ++ 0x75, 0x6e, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, /* u n m o d i f i */ ++ 0x65, 0x64, 0x2d, 0x73, 0x69, 0x6e, 0x63, 0x65, /* e d - s i n c e */ ++ 0x00, 0x00, 0x00, 0x0d, 0x6c, 0x61, 0x73, 0x74, /* - - - - l a s t */ ++ 0x2d, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, /* - m o d i f i e */ ++ 0x64, 0x00, 0x00, 0x00, 0x08, 0x6c, 0x6f, 0x63, /* d - - - - l o c */ ++ 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x00, 0x00, 0x00, /* a t i o n - - - */ ++ 0x0c, 0x6d, 0x61, 0x78, 0x2d, 0x66, 0x6f, 0x72, /* - m a x - f o r */ ++ 0x77, 0x61, 0x72, 0x64, 0x73, 0x00, 0x00, 0x00, /* w a r d s - - - */ ++ 0x06, 0x70, 0x72, 0x61, 0x67, 0x6d, 0x61, 0x00, /* - p r a g m a - */ ++ 0x00, 0x00, 0x12, 0x70, 0x72, 0x6f, 0x78, 0x79, /* - - - p r o x y */ ++ 0x2d, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, /* - a u t h e n t */ ++ 0x69, 0x63, 0x61, 0x74, 0x65, 0x00, 0x00, 0x00, /* i c a t e - - - */ ++ 0x13, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x2d, 0x61, /* - p r o x y - a */ ++ 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, /* u t h o r i z a */ ++ 0x74, 0x69, 0x6f, 0x6e, 0x00, 0x00, 0x00, 0x05, /* t i o n - - - - */ ++ 0x72, 0x61, 0x6e, 0x67, 0x65, 0x00, 0x00, 0x00, /* r a n g e - - - */ ++ 0x07, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x72, /* - r e f e r e r */ ++ 0x00, 0x00, 0x00, 0x0b, 0x72, 0x65, 0x74, 0x72, /* - - - - r e t r */ ++ 0x79, 0x2d, 0x61, 0x66, 0x74, 0x65, 0x72, 0x00, /* y - a f t e r - */ ++ 0x00, 0x00, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, /* - - - s e r v e */ ++ 0x72, 0x00, 0x00, 0x00, 0x02, 0x74, 0x65, 0x00, /* r - - - - t e - */ ++ 0x00, 0x00, 0x07, 0x74, 0x72, 0x61, 0x69, 0x6c, /* - - - t r a i l */ ++ 0x65, 0x72, 0x00, 0x00, 0x00, 0x11, 0x74, 0x72, /* e r - - - - t r */ ++ 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x2d, 0x65, /* a n s f e r - e */ ++ 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x00, /* n c o d i n g - */ ++ 0x00, 0x00, 0x07, 0x75, 0x70, 0x67, 0x72, 0x61, /* - - - u p g r a */ ++ 0x64, 0x65, 0x00, 0x00, 0x00, 0x0a, 0x75, 0x73, /* d e - - - - u s */ ++ 0x65, 0x72, 0x2d, 0x61, 0x67, 0x65, 0x6e, 0x74, /* e r - a g e n t */ ++ 0x00, 0x00, 0x00, 0x04, 0x76, 0x61, 0x72, 0x79, /* - - - - v a r y */ ++ 0x00, 0x00, 0x00, 0x03, 0x76, 0x69, 0x61, 0x00, /* - - - - v i a - */ ++ 0x00, 0x00, 0x07, 0x77, 0x61, 0x72, 0x6e, 0x69, /* - - - w a r n i */ ++ 0x6e, 0x67, 0x00, 0x00, 0x00, 0x10, 0x77, 0x77, /* n g - - - - w w */ ++ 0x77, 0x2d, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, /* w - a u t h e n */ ++ 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x00, 0x00, /* t i c a t e - - */ ++ 0x00, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, /* - - m e t h o d */ ++ 0x00, 0x00, 0x00, 0x03, 0x67, 0x65, 0x74, 0x00, /* - - - - g e t - */ ++ 0x00, 0x00, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, /* - - - s t a t u */ ++ 0x73, 0x00, 0x00, 0x00, 0x06, 0x32, 0x30, 0x30, /* s - - - - 2 0 0 */ ++ 0x20, 0x4f, 0x4b, 0x00, 0x00, 0x00, 0x07, 0x76, /* - O K - - - - v */ ++ 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x00, 0x00, /* e r s i o n - - */ ++ 0x00, 0x08, 0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, /* - - H T T P - 1 */ ++ 0x2e, 0x31, 0x00, 0x00, 0x00, 0x03, 0x75, 0x72, /* - 1 - - - - u r */ ++ 0x6c, 0x00, 0x00, 0x00, 0x06, 0x70, 0x75, 0x62, /* l - - - - p u b */ ++ 0x6c, 0x69, 0x63, 0x00, 0x00, 0x00, 0x0a, 0x73, /* l i c - - - - s */ ++ 0x65, 0x74, 0x2d, 0x63, 0x6f, 0x6f, 0x6b, 0x69, /* e t - c o o k i */ ++ 0x65, 0x00, 0x00, 0x00, 0x0a, 0x6b, 0x65, 0x65, /* e - - - - k e e */ ++ 0x70, 0x2d, 0x61, 0x6c, 0x69, 0x76, 0x65, 0x00, /* p - a l i v e - */ ++ 0x00, 0x00, 0x06, 0x6f, 0x72, 0x69, 0x67, 0x69, /* - - - o r i g i */ ++ 0x6e, 0x31, 0x30, 0x30, 0x31, 0x30, 0x31, 0x32, /* n 1 0 0 1 0 1 2 */ ++ 0x30, 0x31, 0x32, 0x30, 0x32, 0x32, 0x30, 0x35, /* 0 1 2 0 2 2 0 5 */ ++ 0x32, 0x30, 0x36, 0x33, 0x30, 0x30, 0x33, 0x30, /* 2 0 6 3 0 0 3 0 */ ++ 0x32, 0x33, 0x30, 0x33, 0x33, 0x30, 0x34, 0x33, /* 2 3 0 3 3 0 4 3 */ ++ 0x30, 0x35, 0x33, 0x30, 0x36, 0x33, 0x30, 0x37, /* 0 5 3 0 6 3 0 7 */ ++ 0x34, 0x30, 0x32, 0x34, 0x30, 0x35, 0x34, 0x30, /* 4 0 2 4 0 5 4 0 */ ++ 0x36, 0x34, 0x30, 0x37, 0x34, 0x30, 0x38, 0x34, /* 6 4 0 7 4 0 8 4 */ ++ 0x30, 0x39, 0x34, 0x31, 0x30, 0x34, 0x31, 0x31, /* 0 9 4 1 0 4 1 1 */ ++ 0x34, 0x31, 0x32, 0x34, 0x31, 0x33, 0x34, 0x31, /* 4 1 2 4 1 3 4 1 */ ++ 0x34, 0x34, 0x31, 0x35, 0x34, 0x31, 0x36, 0x34, /* 4 4 1 5 4 1 6 4 */ ++ 0x31, 0x37, 0x35, 0x30, 0x32, 0x35, 0x30, 0x34, /* 1 7 5 0 2 5 0 4 */ ++ 0x35, 0x30, 0x35, 0x32, 0x30, 0x33, 0x20, 0x4e, /* 5 0 5 2 0 3 - N */ ++ 0x6f, 0x6e, 0x2d, 0x41, 0x75, 0x74, 0x68, 0x6f, /* o n - A u t h o */ ++ 0x72, 0x69, 0x74, 0x61, 0x74, 0x69, 0x76, 0x65, /* r i t a t i v e */ ++ 0x20, 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, /* - I n f o r m a */ ++ 0x74, 0x69, 0x6f, 0x6e, 0x32, 0x30, 0x34, 0x20, /* t i o n 2 0 4 - */ ++ 0x4e, 0x6f, 0x20, 0x43, 0x6f, 0x6e, 0x74, 0x65, /* N o - C o n t e */ ++ 0x6e, 0x74, 0x33, 0x30, 0x31, 0x20, 0x4d, 0x6f, /* n t 3 0 1 - M o */ ++ 0x76, 0x65, 0x64, 0x20, 0x50, 0x65, 0x72, 0x6d, /* v e d - P e r m */ ++ 0x61, 0x6e, 0x65, 0x6e, 0x74, 0x6c, 0x79, 0x34, /* a n e n t l y 4 */ ++ 0x30, 0x30, 0x20, 0x42, 0x61, 0x64, 0x20, 0x52, /* 0 0 - B a d - R */ ++ 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x34, 0x30, /* e q u e s t 4 0 */ ++ 0x31, 0x20, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, /* 1 - U n a u t h */ ++ 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x34, 0x30, /* o r i z e d 4 0 */ ++ 0x33, 0x20, 0x46, 0x6f, 0x72, 0x62, 0x69, 0x64, /* 3 - F o r b i d */ ++ 0x64, 0x65, 0x6e, 0x34, 0x30, 0x34, 0x20, 0x4e, /* d e n 4 0 4 - N */ ++ 0x6f, 0x74, 0x20, 0x46, 0x6f, 0x75, 0x6e, 0x64, /* o t - F o u n d */ ++ 0x35, 0x30, 0x30, 0x20, 0x49, 0x6e, 0x74, 0x65, /* 5 0 0 - I n t e */ ++ 0x72, 0x6e, 0x61, 0x6c, 0x20, 0x53, 0x65, 0x72, /* r n a l - S e r */ ++ 0x76, 0x65, 0x72, 0x20, 0x45, 0x72, 0x72, 0x6f, /* v e r - E r r o */ ++ 0x72, 0x35, 0x30, 0x31, 0x20, 0x4e, 0x6f, 0x74, /* r 5 0 1 - N o t */ ++ 0x20, 0x49, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, /* - I m p l e m e */ ++ 0x6e, 0x74, 0x65, 0x64, 0x35, 0x30, 0x33, 0x20, /* n t e d 5 0 3 - */ ++ 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x20, /* S e r v i c e - */ ++ 0x55, 0x6e, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, /* U n a v a i l a */ ++ 0x62, 0x6c, 0x65, 0x4a, 0x61, 0x6e, 0x20, 0x46, /* b l e J a n - F */ ++ 0x65, 0x62, 0x20, 0x4d, 0x61, 0x72, 0x20, 0x41, /* e b - M a r - A */ ++ 0x70, 0x72, 0x20, 0x4d, 0x61, 0x79, 0x20, 0x4a, /* p r - M a y - J */ ++ 0x75, 0x6e, 0x20, 0x4a, 0x75, 0x6c, 0x20, 0x41, /* u n - J u l - A */ ++ 0x75, 0x67, 0x20, 0x53, 0x65, 0x70, 0x74, 0x20, /* u g - S e p t - */ ++ 0x4f, 0x63, 0x74, 0x20, 0x4e, 0x6f, 0x76, 0x20, /* O c t - N o v - */ ++ 0x44, 0x65, 0x63, 0x20, 0x30, 0x30, 0x3a, 0x30, /* D e c - 0 0 - 0 */ ++ 0x30, 0x3a, 0x30, 0x30, 0x20, 0x4d, 0x6f, 0x6e, /* 0 - 0 0 - M o n */ ++ 0x2c, 0x20, 0x54, 0x75, 0x65, 0x2c, 0x20, 0x57, /* - - T u e - - W */ ++ 0x65, 0x64, 0x2c, 0x20, 0x54, 0x68, 0x75, 0x2c, /* e d - - T h u - */ ++ 0x20, 0x46, 0x72, 0x69, 0x2c, 0x20, 0x53, 0x61, /* - F r i - - S a */ ++ 0x74, 0x2c, 0x20, 0x53, 0x75, 0x6e, 0x2c, 0x20, /* t - - S u n - - */ ++ 0x47, 0x4d, 0x54, 0x63, 0x68, 0x75, 0x6e, 0x6b, /* G M T c h u n k */ ++ 0x65, 0x64, 0x2c, 0x74, 0x65, 0x78, 0x74, 0x2f, /* e d - t e x t - */ ++ 0x68, 0x74, 0x6d, 0x6c, 0x2c, 0x69, 0x6d, 0x61, /* h t m l - i m a */ ++ 0x67, 0x65, 0x2f, 0x70, 0x6e, 0x67, 0x2c, 0x69, /* g e - p n g - i */ ++ 0x6d, 0x61, 0x67, 0x65, 0x2f, 0x6a, 0x70, 0x67, /* m a g e - j p g */ ++ 0x2c, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x2f, 0x67, /* - i m a g e - g */ ++ 0x69, 0x66, 0x2c, 0x61, 0x70, 0x70, 0x6c, 0x69, /* i f - a p p l i */ ++ 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x78, /* c a t i o n - x */ ++ 0x6d, 0x6c, 0x2c, 0x61, 0x70, 0x70, 0x6c, 0x69, /* m l - a p p l i */ ++ 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x78, /* c a t i o n - x */ ++ 0x68, 0x74, 0x6d, 0x6c, 0x2b, 0x78, 0x6d, 0x6c, /* h t m l - x m l */ ++ 0x2c, 0x74, 0x65, 0x78, 0x74, 0x2f, 0x70, 0x6c, /* - t e x t - p l */ ++ 0x61, 0x69, 0x6e, 0x2c, 0x74, 0x65, 0x78, 0x74, /* a i n - t e x t */ ++ 0x2f, 0x6a, 0x61, 0x76, 0x61, 0x73, 0x63, 0x72, /* - j a v a s c r */ ++ 0x69, 0x70, 0x74, 0x2c, 0x70, 0x75, 0x62, 0x6c, /* i p t - p u b l */ ++ 0x69, 0x63, 0x70, 0x72, 0x69, 0x76, 0x61, 0x74, /* i c p r i v a t */ ++ 0x65, 0x6d, 0x61, 0x78, 0x2d, 0x61, 0x67, 0x65, /* e m a x - a g e */ ++ 0x3d, 0x67, 0x7a, 0x69, 0x70, 0x2c, 0x64, 0x65, /* - g z i p - d e */ ++ 0x66, 0x6c, 0x61, 0x74, 0x65, 0x2c, 0x73, 0x64, /* f l a t e - s d */ ++ 0x63, 0x68, 0x63, 0x68, 0x61, 0x72, 0x73, 0x65, /* c h c h a r s e */ ++ 0x74, 0x3d, 0x75, 0x74, 0x66, 0x2d, 0x38, 0x63, /* t - u t f - 8 c */ ++ 0x68, 0x61, 0x72, 0x73, 0x65, 0x74, 0x3d, 0x69, /* h a r s e t - i */ ++ 0x73, 0x6f, 0x2d, 0x38, 0x38, 0x35, 0x39, 0x2d, /* s o - 8 8 5 9 - */ ++ 0x31, 0x2c, 0x75, 0x74, 0x66, 0x2d, 0x2c, 0x2a, /* 1 - u t f - - - */ ++ 0x2c, 0x65, 0x6e, 0x71, 0x3d, 0x30, 0x2e /* - e n q - 0 - */ ++}; ++ ++ ++static ngx_http_spdy_request_header_t ngx_http_spdy_request_headers[] = { ++ { 0, 6, "method", ngx_http_spdy_parse_method }, ++ { 0, 6, "scheme", ngx_http_spdy_parse_scheme }, ++ { 0, 4, "host", ngx_http_spdy_parse_host }, ++ { 0, 4, "path", ngx_http_spdy_parse_path }, ++ { 0, 7, "version", ngx_http_spdy_parse_version }, ++}; ++ ++#define NGX_SPDY_REQUEST_HEADERS \ ++ (sizeof(ngx_http_spdy_request_headers) \ ++ / sizeof(ngx_http_spdy_request_header_t)) ++ ++ ++void ++ngx_http_spdy_init(ngx_event_t *rev) ++{ ++ int rc; ++ ngx_connection_t *c; ++ ngx_pool_cleanup_t *cln; ++ ngx_http_connection_t *hc; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ngx_http_spdy_main_conf_t *smcf; ++ ngx_http_spdy_connection_t *sc; ++ ++ c = rev->data; ++ hc = c->data; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "init spdy request"); ++ ++ c->log->action = "processing SPDY"; ++ ++ smcf = ngx_http_get_module_main_conf(hc->conf_ctx, ngx_http_spdy_module); ++ ++ if (smcf->recv_buffer == NULL) { ++ smcf->recv_buffer = ngx_palloc(ngx_cycle->pool, smcf->recv_buffer_size); ++ if (smcf->recv_buffer == NULL) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ } ++ ++ sc = ngx_pcalloc(c->pool, sizeof(ngx_http_spdy_connection_t)); ++ if (sc == NULL) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ sc->connection = c; ++ sc->http_connection = hc; ++ ++ sc->send_window = NGX_SPDY_CONNECTION_WINDOW; ++ sc->recv_window = NGX_SPDY_CONNECTION_WINDOW; ++ ++ sc->init_window = NGX_SPDY_INIT_STREAM_WINDOW; ++ ++ sc->handler = hc->proxy_protocol ? ngx_http_spdy_proxy_protocol ++ : ngx_http_spdy_state_head; ++ ++ sc->zstream_in.zalloc = ngx_http_spdy_zalloc; ++ sc->zstream_in.zfree = ngx_http_spdy_zfree; ++ sc->zstream_in.opaque = sc; ++ ++ rc = inflateInit(&sc->zstream_in); ++ if (rc != Z_OK) { ++ ngx_log_error(NGX_LOG_ALERT, c->log, 0, ++ "inflateInit() failed: %d", rc); ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ sc->zstream_out.zalloc = ngx_http_spdy_zalloc; ++ sc->zstream_out.zfree = ngx_http_spdy_zfree; ++ sc->zstream_out.opaque = sc; ++ ++ sscf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_spdy_module); ++ ++ rc = deflateInit2(&sc->zstream_out, (int) sscf->headers_comp, ++ Z_DEFLATED, 11, 4, Z_DEFAULT_STRATEGY); ++ ++ if (rc != Z_OK) { ++ ngx_log_error(NGX_LOG_ALERT, c->log, 0, ++ "deflateInit2() failed: %d", rc); ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ rc = deflateSetDictionary(&sc->zstream_out, ngx_http_spdy_dict, ++ sizeof(ngx_http_spdy_dict)); ++ if (rc != Z_OK) { ++ ngx_log_error(NGX_LOG_ALERT, c->log, 0, ++ "deflateSetDictionary() failed: %d", rc); ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ sc->pool = ngx_create_pool(sscf->pool_size, sc->connection->log); ++ if (sc->pool == NULL) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ cln = ngx_pool_cleanup_add(c->pool, sizeof(ngx_pool_cleanup_file_t)); ++ if (cln == NULL) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ cln->handler = ngx_http_spdy_pool_cleanup; ++ cln->data = sc; ++ ++ sc->streams_index = ngx_pcalloc(sc->pool, ++ ngx_http_spdy_streams_index_size(sscf) ++ * sizeof(ngx_http_spdy_stream_t *)); ++ if (sc->streams_index == NULL) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ if (ngx_http_spdy_send_settings(sc) == NGX_ERROR) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ if (ngx_http_spdy_send_window_update(sc, 0, NGX_SPDY_MAX_WINDOW ++ - sc->recv_window) ++ == NGX_ERROR) ++ { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ sc->recv_window = NGX_SPDY_MAX_WINDOW; ++ ++ ngx_queue_init(&sc->waiting); ++ ngx_queue_init(&sc->posted); ++ ++ c->data = sc; ++ ++ rev->handler = ngx_http_spdy_read_handler; ++ c->write->handler = ngx_http_spdy_write_handler; ++ ++ ngx_http_spdy_read_handler(rev); ++} ++ ++ ++static void ++ngx_http_spdy_read_handler(ngx_event_t *rev) ++{ ++ u_char *p, *end; ++ size_t available; ++ ssize_t n; ++ ngx_connection_t *c; ++ ngx_http_spdy_main_conf_t *smcf; ++ ngx_http_spdy_connection_t *sc; ++ ++ c = rev->data; ++ sc = c->data; ++ ++ if (rev->timedout) { ++ ngx_log_error(NGX_LOG_INFO, c->log, NGX_ETIMEDOUT, "client timed out"); ++ ngx_http_spdy_finalize_connection(sc, NGX_HTTP_REQUEST_TIME_OUT); ++ return; ++ } ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "spdy read handler"); ++ ++ sc->blocked = 1; ++ ++ smcf = ngx_http_get_module_main_conf(sc->http_connection->conf_ctx, ++ ngx_http_spdy_module); ++ ++ available = smcf->recv_buffer_size - 2 * NGX_SPDY_STATE_BUFFER_SIZE; ++ ++ do { ++ p = smcf->recv_buffer; ++ ++ ngx_memcpy(p, sc->buffer, NGX_SPDY_STATE_BUFFER_SIZE); ++ end = p + sc->buffer_used; ++ ++ n = c->recv(c, end, available); ++ ++ if (n == NGX_AGAIN) { ++ break; ++ } ++ ++ if (n == 0 && (sc->incomplete || sc->processing)) { ++ ngx_log_error(NGX_LOG_INFO, c->log, 0, ++ "client prematurely closed connection"); ++ } ++ ++ if (n == 0 || n == NGX_ERROR) { ++ ngx_http_spdy_finalize_connection(sc, ++ NGX_HTTP_CLIENT_CLOSED_REQUEST); ++ return; ++ } ++ ++ end += n; ++ ++ sc->buffer_used = 0; ++ sc->incomplete = 0; ++ ++ do { ++ p = sc->handler(sc, p, end); ++ ++ if (p == NULL) { ++ return; ++ } ++ ++ } while (p != end); ++ ++ } while (rev->ready); ++ ++ if (ngx_handle_read_event(rev, 0) != NGX_OK) { ++ ngx_http_spdy_finalize_connection(sc, NGX_HTTP_INTERNAL_SERVER_ERROR); ++ return; ++ } ++ ++ if (sc->last_out && ngx_http_spdy_send_output_queue(sc) == NGX_ERROR) { ++ ngx_http_spdy_finalize_connection(sc, NGX_HTTP_CLIENT_CLOSED_REQUEST); ++ return; ++ } ++ ++ sc->blocked = 0; ++ ++ if (sc->processing) { ++ if (rev->timer_set) { ++ ngx_del_timer(rev); ++ } ++ return; ++ } ++ ++ ngx_http_spdy_handle_connection(sc); ++} ++ ++ ++static void ++ngx_http_spdy_write_handler(ngx_event_t *wev) ++{ ++ ngx_int_t rc; ++ ngx_queue_t *q; ++ ngx_connection_t *c; ++ ngx_http_spdy_stream_t *stream; ++ ngx_http_spdy_connection_t *sc; ++ ++ c = wev->data; ++ sc = c->data; ++ ++ if (wev->timedout) { ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, ++ "spdy write event timed out"); ++ ngx_http_spdy_finalize_connection(sc, NGX_HTTP_CLIENT_CLOSED_REQUEST); ++ return; ++ } ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "spdy write handler"); ++ ++ sc->blocked = 1; ++ ++ rc = ngx_http_spdy_send_output_queue(sc); ++ ++ if (rc == NGX_ERROR) { ++ ngx_http_spdy_finalize_connection(sc, NGX_HTTP_CLIENT_CLOSED_REQUEST); ++ return; ++ } ++ ++ while (!ngx_queue_empty(&sc->posted)) { ++ q = ngx_queue_head(&sc->posted); ++ ++ ngx_queue_remove(q); ++ ++ stream = ngx_queue_data(q, ngx_http_spdy_stream_t, queue); ++ ++ stream->handled = 0; ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, ++ "run spdy stream %ui", stream->id); ++ ++ wev = stream->request->connection->write; ++ wev->handler(wev); ++ } ++ ++ sc->blocked = 0; ++ ++ if (rc == NGX_AGAIN) { ++ return; ++ } ++ ++ ngx_http_spdy_handle_connection(sc); ++} ++ ++ ++ngx_int_t ++ngx_http_spdy_send_output_queue(ngx_http_spdy_connection_t *sc) ++{ ++ int tcp_nodelay; ++ ngx_chain_t *cl; ++ ngx_event_t *wev; ++ ngx_connection_t *c; ++ ngx_http_core_loc_conf_t *clcf; ++ ngx_http_spdy_out_frame_t *out, *frame, *fn; ++ ++ c = sc->connection; ++ ++ if (c->error) { ++ return NGX_ERROR; ++ } ++ ++ wev = c->write; ++ ++ if (!wev->ready) { ++ return NGX_OK; ++ } ++ ++ cl = NULL; ++ out = NULL; ++ ++ for (frame = sc->last_out; frame; frame = fn) { ++ frame->last->next = cl; ++ cl = frame->first; ++ ++ fn = frame->next; ++ frame->next = out; ++ out = frame; ++ ++ ngx_log_debug5(NGX_LOG_DEBUG_HTTP, c->log, 0, ++ "spdy frame out: %p sid:%ui prio:%ui bl:%d len:%uz", ++ out, out->stream ? out->stream->id : 0, out->priority, ++ out->blocked, out->length); ++ } ++ ++ cl = c->send_chain(c, cl, 0); ++ ++ if (cl == NGX_CHAIN_ERROR) { ++ goto error; ++ } ++ ++ clcf = ngx_http_get_module_loc_conf(sc->http_connection->conf_ctx, ++ ngx_http_core_module); ++ ++ if (ngx_handle_write_event(wev, clcf->send_lowat) != NGX_OK) { ++ goto error; ++ } ++ ++ if (c->tcp_nopush == NGX_TCP_NOPUSH_SET) { ++ if (ngx_tcp_push(c->fd) == -1) { ++ ngx_connection_error(c, ngx_socket_errno, ngx_tcp_push_n " failed"); ++ goto error; ++ } ++ ++ c->tcp_nopush = NGX_TCP_NOPUSH_UNSET; ++ tcp_nodelay = ngx_tcp_nodelay_and_tcp_nopush ? 1 : 0; ++ ++ } else { ++ tcp_nodelay = 1; ++ } ++ ++ if (tcp_nodelay ++ && clcf->tcp_nodelay ++ && c->tcp_nodelay == NGX_TCP_NODELAY_UNSET) ++ { ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "tcp_nodelay"); ++ ++ if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY, ++ (const void *) &tcp_nodelay, sizeof(int)) ++ == -1) ++ { ++#if (NGX_SOLARIS) ++ /* Solaris returns EINVAL if a socket has been shut down */ ++ c->log_error = NGX_ERROR_IGNORE_EINVAL; ++#endif ++ ++ ngx_connection_error(c, ngx_socket_errno, ++ "setsockopt(TCP_NODELAY) failed"); ++ ++ c->log_error = NGX_ERROR_INFO; ++ goto error; ++ } ++ ++ c->tcp_nodelay = NGX_TCP_NODELAY_SET; ++ } ++ ++ if (cl) { ++ ngx_add_timer(wev, clcf->send_timeout); ++ ++ } else { ++ if (wev->timer_set) { ++ ngx_del_timer(wev); ++ } ++ } ++ ++ for ( /* void */ ; out; out = fn) { ++ fn = out->next; ++ ++ if (out->handler(sc, out) != NGX_OK) { ++ out->blocked = 1; ++ out->priority = NGX_SPDY_HIGHEST_PRIORITY; ++ break; ++ } ++ ++ ngx_log_debug4(NGX_LOG_DEBUG_HTTP, c->log, 0, ++ "spdy frame sent: %p sid:%ui bl:%d len:%uz", ++ out, out->stream ? out->stream->id : 0, ++ out->blocked, out->length); ++ } ++ ++ frame = NULL; ++ ++ for ( /* void */ ; out; out = fn) { ++ fn = out->next; ++ out->next = frame; ++ frame = out; ++ } ++ ++ sc->last_out = frame; ++ ++ return NGX_OK; ++ ++error: ++ ++ c->error = 1; ++ ++ if (!sc->blocked) { ++ ngx_post_event(wev, &ngx_posted_events); ++ } ++ ++ return NGX_ERROR; ++} ++ ++ ++static void ++ngx_http_spdy_handle_connection(ngx_http_spdy_connection_t *sc) ++{ ++ ngx_connection_t *c; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ++ if (sc->last_out || sc->processing) { ++ return; ++ } ++ ++ c = sc->connection; ++ ++ if (c->error) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ if (c->buffered) { ++ return; ++ } ++ ++ sscf = ngx_http_get_module_srv_conf(sc->http_connection->conf_ctx, ++ ngx_http_spdy_module); ++ if (sc->incomplete) { ++ ngx_add_timer(c->read, sscf->recv_timeout); ++ return; ++ } ++ ++ if (ngx_terminate || ngx_exiting) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ ngx_destroy_pool(sc->pool); ++ ++ sc->pool = NULL; ++ sc->free_ctl_frames = NULL; ++ sc->free_fake_connections = NULL; ++ ++#if (NGX_HTTP_SSL) ++ if (c->ssl) { ++ ngx_ssl_free_buffer(c); ++ } ++#endif ++ ++ c->destroyed = 1; ++ c->idle = 1; ++ ngx_reusable_connection(c, 1); ++ ++ c->write->handler = ngx_http_empty_handler; ++ c->read->handler = ngx_http_spdy_keepalive_handler; ++ ++ if (c->write->timer_set) { ++ ngx_del_timer(c->write); ++ } ++ ++ ngx_add_timer(c->read, sscf->keepalive_timeout); ++} ++ ++ ++static u_char * ++ngx_http_spdy_proxy_protocol(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ ngx_log_t *log; ++ ++ log = sc->connection->log; ++ log->action = "reading PROXY protocol"; ++ ++ pos = ngx_proxy_protocol_read(sc->connection, pos, end); ++ ++ log->action = "processing SPDY"; ++ ++ if (pos == NULL) { ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_head(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ uint32_t head, flen; ++ ngx_uint_t type; ++ ++ if (end - pos < NGX_SPDY_FRAME_HEADER_SIZE) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_head); ++ } ++ ++ head = ngx_spdy_frame_parse_uint32(pos); ++ ++ pos += sizeof(uint32_t); ++ ++ flen = ngx_spdy_frame_parse_uint32(pos); ++ ++ sc->flags = ngx_spdy_frame_flags(flen); ++ sc->length = ngx_spdy_frame_length(flen); ++ ++ pos += sizeof(uint32_t); ++ ++ ngx_log_debug3(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "process spdy frame head:%08XD f:%Xd l:%uz", ++ head, sc->flags, sc->length); ++ ++ if (ngx_spdy_ctl_frame_check(head)) { ++ type = ngx_spdy_ctl_frame_type(head); ++ ++ switch (type) { ++ ++ case NGX_SPDY_SYN_STREAM: ++ return ngx_http_spdy_state_syn_stream(sc, pos, end); ++ ++ case NGX_SPDY_SYN_REPLY: ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent unexpected SYN_REPLY frame"); ++ return ngx_http_spdy_state_protocol_error(sc); ++ ++ case NGX_SPDY_RST_STREAM: ++ return ngx_http_spdy_state_rst_stream(sc, pos, end); ++ ++ case NGX_SPDY_SETTINGS: ++ return ngx_http_spdy_state_settings(sc, pos, end); ++ ++ case NGX_SPDY_PING: ++ return ngx_http_spdy_state_ping(sc, pos, end); ++ ++ case NGX_SPDY_GOAWAY: ++ return ngx_http_spdy_state_skip(sc, pos, end); /* TODO */ ++ ++ case NGX_SPDY_HEADERS: ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent unexpected HEADERS frame"); ++ return ngx_http_spdy_state_protocol_error(sc); ++ ++ case NGX_SPDY_WINDOW_UPDATE: ++ return ngx_http_spdy_state_window_update(sc, pos, end); ++ ++ default: ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy control frame with unknown type %ui", type); ++ return ngx_http_spdy_state_skip(sc, pos, end); ++ } ++ } ++ ++ if (ngx_spdy_data_frame_check(head)) { ++ sc->stream = ngx_http_spdy_get_stream_by_id(sc, head); ++ return ngx_http_spdy_state_data(sc, pos, end); ++ } ++ ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent invalid frame"); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_syn_stream(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ ngx_uint_t sid, prio; ++ ngx_http_spdy_stream_t *stream; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ++ if (end - pos < NGX_SPDY_SYN_STREAM_SIZE) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_syn_stream); ++ } ++ ++ if (sc->length <= NGX_SPDY_SYN_STREAM_SIZE) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent SYN_STREAM frame with incorrect length %uz", ++ sc->length); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ sc->length -= NGX_SPDY_SYN_STREAM_SIZE; ++ ++ sid = ngx_spdy_frame_parse_sid(pos); ++ prio = pos[8] >> 5; ++ ++ pos += NGX_SPDY_SYN_STREAM_SIZE; ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy SYN_STREAM frame sid:%ui prio:%ui", sid, prio); ++ ++ if (sid % 2 == 0 || sid <= sc->last_sid) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent SYN_STREAM frame " ++ "with invalid Stream-ID %ui", sid); ++ ++ stream = ngx_http_spdy_get_stream_by_id(sc, sid); ++ ++ if (stream) { ++ if (ngx_http_spdy_terminate_stream(sc, stream, ++ NGX_SPDY_PROTOCOL_ERROR) ++ != NGX_OK) ++ { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ } ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ sc->last_sid = sid; ++ ++ sscf = ngx_http_get_module_srv_conf(sc->http_connection->conf_ctx, ++ ngx_http_spdy_module); ++ ++ if (sc->processing >= sscf->concurrent_streams) { ++ ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "concurrent streams exceeded %ui", sc->processing); ++ ++ if (ngx_http_spdy_send_rst_stream(sc, sid, NGX_SPDY_REFUSED_STREAM, ++ prio) ++ != NGX_OK) ++ { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ return ngx_http_spdy_state_headers_skip(sc, pos, end); ++ } ++ ++ stream = ngx_http_spdy_create_stream(sc, sid, prio); ++ if (stream == NULL) { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ stream->in_closed = (sc->flags & NGX_SPDY_FLAG_FIN) ? 1 : 0; ++ ++ stream->request->request_length = NGX_SPDY_FRAME_HEADER_SIZE ++ + NGX_SPDY_SYN_STREAM_SIZE ++ + sc->length; ++ ++ sc->stream = stream; ++ ++ return ngx_http_spdy_state_headers(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_headers(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ int z; ++ size_t size; ++ ngx_buf_t *buf; ++ ngx_int_t rc; ++ ngx_http_request_t *r; ++ ++ size = end - pos; ++ ++ if (size == 0) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_headers); ++ } ++ ++ if (size > sc->length) { ++ size = sc->length; ++ } ++ ++ r = sc->stream->request; ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "process spdy header block %uz of %uz", size, sc->length); ++ ++ buf = r->header_in; ++ ++ sc->zstream_in.next_in = pos; ++ sc->zstream_in.avail_in = size; ++ sc->zstream_in.next_out = buf->last; ++ ++ /* one byte is reserved for null-termination of the last header value */ ++ sc->zstream_in.avail_out = buf->end - buf->last - 1; ++ ++ z = inflate(&sc->zstream_in, Z_NO_FLUSH); ++ ++ if (z == Z_NEED_DICT) { ++ z = inflateSetDictionary(&sc->zstream_in, ngx_http_spdy_dict, ++ sizeof(ngx_http_spdy_dict)); ++ ++ if (z != Z_OK) { ++ if (z == Z_DATA_ERROR) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent SYN_STREAM frame with header " ++ "block encoded using wrong dictionary: %ul", ++ (u_long) sc->zstream_in.adler); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, ++ "inflateSetDictionary() failed: %d", z); ++ ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy inflateSetDictionary(): %d", z); ++ ++ z = sc->zstream_in.avail_in ? inflate(&sc->zstream_in, Z_NO_FLUSH) ++ : Z_OK; ++ } ++ ++ ngx_log_debug5(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy inflate out: ni:%p no:%p ai:%ud ao:%ud rc:%d", ++ sc->zstream_in.next_in, sc->zstream_in.next_out, ++ sc->zstream_in.avail_in, sc->zstream_in.avail_out, ++ z); ++ ++ if (z != Z_OK) { ++ return ngx_http_spdy_state_inflate_error(sc, z); ++ } ++ ++ sc->length -= sc->zstream_in.next_in - pos; ++ pos = sc->zstream_in.next_in; ++ ++ buf->last = sc->zstream_in.next_out; ++ ++ if (r->headers_in.headers.part.elts == NULL) { ++ ++ if (buf->last - buf->pos < NGX_SPDY_NV_NUM_SIZE) { ++ ++ if (sc->length == 0) { ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "premature end of spdy header block"); ++ ++ return ngx_http_spdy_state_headers_error(sc, pos, end); ++ } ++ ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_headers); ++ } ++ ++ sc->entries = ngx_spdy_frame_parse_uint32(buf->pos); ++ ++ buf->pos += NGX_SPDY_NV_NUM_SIZE; ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy header block has %ui entries", ++ sc->entries); ++ ++ if (ngx_list_init(&r->headers_in.headers, r->pool, 20, ++ sizeof(ngx_table_elt_t)) ++ != NGX_OK) ++ { ++ ngx_http_spdy_close_stream(sc->stream, ++ NGX_HTTP_INTERNAL_SERVER_ERROR); ++ return ngx_http_spdy_state_headers_skip(sc, pos, end); ++ } ++ ++ if (ngx_array_init(&r->headers_in.cookies, r->pool, 2, ++ sizeof(ngx_table_elt_t *)) ++ != NGX_OK) ++ { ++ ngx_http_spdy_close_stream(sc->stream, ++ NGX_HTTP_INTERNAL_SERVER_ERROR); ++ return ngx_http_spdy_state_headers_skip(sc, pos, end); ++ } ++ } ++ ++ while (sc->entries) { ++ ++ rc = ngx_http_spdy_parse_header(r); ++ ++ switch (rc) { ++ ++ case NGX_DONE: ++ sc->entries--; ++ ++ case NGX_OK: ++ break; ++ ++ case NGX_AGAIN: ++ ++ if (sc->zstream_in.avail_in) { ++ ++ rc = ngx_http_spdy_alloc_large_header_buffer(r); ++ ++ if (rc == NGX_DECLINED) { ++ ngx_http_finalize_request(r, ++ NGX_HTTP_REQUEST_HEADER_TOO_LARGE); ++ return ngx_http_spdy_state_headers_skip(sc, pos, end); ++ } ++ ++ if (rc != NGX_OK) { ++ ngx_http_spdy_close_stream(sc->stream, ++ NGX_HTTP_INTERNAL_SERVER_ERROR); ++ return ngx_http_spdy_state_headers_skip(sc, pos, end); ++ } ++ ++ /* null-terminate the last processed header name or value */ ++ *buf->pos = '\0'; ++ ++ buf = r->header_in; ++ ++ sc->zstream_in.next_out = buf->last; ++ ++ /* one byte is reserved for null-termination */ ++ sc->zstream_in.avail_out = buf->end - buf->last - 1; ++ ++ z = inflate(&sc->zstream_in, Z_NO_FLUSH); ++ ++ ngx_log_debug5(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy inflate out: ni:%p no:%p ai:%ud ao:%ud rc:%d", ++ sc->zstream_in.next_in, sc->zstream_in.next_out, ++ sc->zstream_in.avail_in, sc->zstream_in.avail_out, ++ z); ++ ++ if (z != Z_OK) { ++ return ngx_http_spdy_state_inflate_error(sc, z); ++ } ++ ++ sc->length -= sc->zstream_in.next_in - pos; ++ pos = sc->zstream_in.next_in; ++ ++ buf->last = sc->zstream_in.next_out; ++ ++ continue; ++ } ++ ++ if (sc->length == 0) { ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "premature end of spdy header block"); ++ ++ return ngx_http_spdy_state_headers_error(sc, pos, end); ++ } ++ ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_headers); ++ ++ case NGX_HTTP_PARSE_INVALID_HEADER: ++ ngx_http_finalize_request(r, NGX_HTTP_BAD_REQUEST); ++ return ngx_http_spdy_state_headers_skip(sc, pos, end); ++ ++ default: /* NGX_ERROR */ ++ return ngx_http_spdy_state_headers_error(sc, pos, end); ++ } ++ ++ /* a header line has been parsed successfully */ ++ ++ rc = ngx_http_spdy_handle_request_header(r); ++ ++ if (rc != NGX_OK) { ++ if (rc == NGX_HTTP_PARSE_INVALID_HEADER) { ++ ngx_http_finalize_request(r, NGX_HTTP_BAD_REQUEST); ++ return ngx_http_spdy_state_headers_skip(sc, pos, end); ++ } ++ ++ if (rc != NGX_ABORT) { ++ ngx_http_spdy_close_stream(sc->stream, ++ NGX_HTTP_INTERNAL_SERVER_ERROR); ++ } ++ ++ return ngx_http_spdy_state_headers_skip(sc, pos, end); ++ } ++ } ++ ++ if (buf->pos != buf->last || sc->zstream_in.avail_in) { ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "incorrect number of spdy header block entries"); ++ ++ return ngx_http_spdy_state_headers_error(sc, pos, end); ++ } ++ ++ if (sc->length) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_headers); ++ } ++ ++ /* null-terminate the last header value */ ++ *buf->pos = '\0'; ++ ++ ngx_http_spdy_run_request(r); ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_headers_skip(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ int n; ++ size_t size; ++ u_char buffer[NGX_SPDY_SKIP_HEADERS_BUFFER_SIZE]; ++ ++ if (sc->length == 0) { ++ return ngx_http_spdy_state_complete(sc, pos, end); ++ } ++ ++ size = end - pos; ++ ++ if (size == 0) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_headers_skip); ++ } ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy header block skip %uz of %uz", size, sc->length); ++ ++ sc->zstream_in.next_in = pos; ++ sc->zstream_in.avail_in = (size < sc->length) ? size : sc->length; ++ ++ while (sc->zstream_in.avail_in) { ++ sc->zstream_in.next_out = buffer; ++ sc->zstream_in.avail_out = NGX_SPDY_SKIP_HEADERS_BUFFER_SIZE; ++ ++ n = inflate(&sc->zstream_in, Z_NO_FLUSH); ++ ++ ngx_log_debug5(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy inflate out: ni:%p no:%p ai:%ud ao:%ud rc:%d", ++ sc->zstream_in.next_in, sc->zstream_in.next_out, ++ sc->zstream_in.avail_in, sc->zstream_in.avail_out, ++ n); ++ ++ if (n != Z_OK) { ++ return ngx_http_spdy_state_inflate_error(sc, n); ++ } ++ } ++ ++ pos = sc->zstream_in.next_in; ++ ++ if (size < sc->length) { ++ sc->length -= size; ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_headers_skip); ++ } ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_headers_error(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ ngx_http_spdy_stream_t *stream; ++ ++ stream = sc->stream; ++ ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent SYN_STREAM frame for stream %ui " ++ "with invalid header block", stream->id); ++ ++ if (ngx_http_spdy_send_rst_stream(sc, stream->id, NGX_SPDY_PROTOCOL_ERROR, ++ stream->priority) ++ != NGX_OK) ++ { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ stream->out_closed = 1; ++ ++ ngx_http_spdy_close_stream(stream, NGX_HTTP_BAD_REQUEST); ++ ++ return ngx_http_spdy_state_headers_skip(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_window_update(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ size_t delta; ++ ngx_uint_t sid; ++ ngx_event_t *wev; ++ ngx_queue_t *q; ++ ngx_http_spdy_stream_t *stream; ++ ++ if (end - pos < NGX_SPDY_WINDOW_UPDATE_SIZE) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_window_update); ++ } ++ ++ if (sc->length != NGX_SPDY_WINDOW_UPDATE_SIZE) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent WINDOW_UPDATE frame " ++ "with incorrect length %uz", sc->length); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ sid = ngx_spdy_frame_parse_sid(pos); ++ ++ pos += NGX_SPDY_SID_SIZE; ++ ++ delta = ngx_spdy_frame_parse_delta(pos); ++ ++ pos += NGX_SPDY_DELTA_SIZE; ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy WINDOW_UPDATE sid:%ui delta:%uz", sid, delta); ++ ++ if (sid) { ++ stream = ngx_http_spdy_get_stream_by_id(sc, sid); ++ ++ if (stream == NULL) { ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "unknown spdy stream"); ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++ } ++ ++ if (stream->send_window > (ssize_t) (NGX_SPDY_MAX_WINDOW - delta)) { ++ ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client violated flow control for stream %ui: " ++ "received WINDOW_UPDATE frame with delta %uz " ++ "not allowed for window %z", ++ sid, delta, stream->send_window); ++ ++ if (ngx_http_spdy_terminate_stream(sc, stream, ++ NGX_SPDY_FLOW_CONTROL_ERROR) ++ == NGX_ERROR) ++ { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++ } ++ ++ stream->send_window += delta; ++ ++ if (stream->exhausted) { ++ stream->exhausted = 0; ++ ++ wev = stream->request->connection->write; ++ ++ if (!wev->timer_set) { ++ wev->delayed = 0; ++ wev->handler(wev); ++ } ++ } ++ ++ } else { ++ sc->send_window += delta; ++ ++ if (sc->send_window > NGX_SPDY_MAX_WINDOW) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client violated connection flow control: " ++ "received WINDOW_UPDATE frame with delta %uz " ++ "not allowed for window %uz", ++ delta, sc->send_window); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ while (!ngx_queue_empty(&sc->waiting)) { ++ q = ngx_queue_head(&sc->waiting); ++ ++ ngx_queue_remove(q); ++ ++ stream = ngx_queue_data(q, ngx_http_spdy_stream_t, queue); ++ ++ stream->handled = 0; ++ ++ wev = stream->request->connection->write; ++ ++ if (!wev->timer_set) { ++ wev->delayed = 0; ++ wev->handler(wev); ++ ++ if (sc->send_window == 0) { ++ break; ++ } ++ } ++ } ++ } ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_data(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ ngx_http_spdy_stream_t *stream; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy DATA frame"); ++ ++ if (sc->length > sc->recv_window) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client violated connection flow control: " ++ "received DATA frame length %uz, available window %uz", ++ sc->length, sc->recv_window); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ sc->recv_window -= sc->length; ++ ++ if (sc->recv_window < NGX_SPDY_MAX_WINDOW / 4) { ++ ++ if (ngx_http_spdy_send_window_update(sc, 0, ++ NGX_SPDY_MAX_WINDOW ++ - sc->recv_window) ++ == NGX_ERROR) ++ { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ sc->recv_window = NGX_SPDY_MAX_WINDOW; ++ } ++ ++ stream = sc->stream; ++ ++ if (stream == NULL) { ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "unknown spdy stream"); ++ ++ return ngx_http_spdy_state_skip(sc, pos, end); ++ } ++ ++ if (sc->length > stream->recv_window) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client violated flow control for stream %ui: " ++ "received DATA frame length %uz, available window %uz", ++ stream->id, sc->length, stream->recv_window); ++ ++ if (ngx_http_spdy_terminate_stream(sc, stream, ++ NGX_SPDY_FLOW_CONTROL_ERROR) ++ == NGX_ERROR) ++ { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ return ngx_http_spdy_state_skip(sc, pos, end); ++ } ++ ++ stream->recv_window -= sc->length; ++ ++ if (stream->recv_window < NGX_SPDY_STREAM_WINDOW / 4) { ++ ++ if (ngx_http_spdy_send_window_update(sc, stream->id, ++ NGX_SPDY_STREAM_WINDOW ++ - stream->recv_window) ++ == NGX_ERROR) ++ { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ stream->recv_window = NGX_SPDY_STREAM_WINDOW; ++ } ++ ++ if (stream->in_closed) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent DATA frame for half-closed stream %ui", ++ stream->id); ++ ++ if (ngx_http_spdy_terminate_stream(sc, stream, ++ NGX_SPDY_STREAM_ALREADY_CLOSED) ++ == NGX_ERROR) ++ { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ return ngx_http_spdy_state_skip(sc, pos, end); ++ } ++ ++ return ngx_http_spdy_state_read_data(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_read_data(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ size_t size; ++ ssize_t n; ++ ngx_buf_t *buf; ++ ngx_int_t rc; ++ ngx_temp_file_t *tf; ++ ngx_http_request_t *r; ++ ngx_http_spdy_stream_t *stream; ++ ngx_http_request_body_t *rb; ++ ngx_http_core_loc_conf_t *clcf; ++ ++ stream = sc->stream; ++ ++ if (stream == NULL) { ++ return ngx_http_spdy_state_skip(sc, pos, end); ++ } ++ ++ if (stream->skip_data) { ++ ++ if (sc->flags & NGX_SPDY_FLAG_FIN) { ++ stream->in_closed = 1; ++ } ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "skipping spdy DATA frame, reason: %d", ++ stream->skip_data); ++ ++ return ngx_http_spdy_state_skip(sc, pos, end); ++ } ++ ++ size = end - pos; ++ ++ if (size > sc->length) { ++ size = sc->length; ++ } ++ ++ r = stream->request; ++ ++ if (r->request_body == NULL ++ && ngx_http_spdy_init_request_body(r) != NGX_OK) ++ { ++ stream->skip_data = NGX_SPDY_DATA_INTERNAL_ERROR; ++ return ngx_http_spdy_state_skip(sc, pos, end); ++ } ++ ++ rb = r->request_body; ++ tf = rb->temp_file; ++ buf = rb->buf; ++ ++ if (size) { ++ rb->rest += size; ++ ++ if (r->headers_in.content_length_n != -1 ++ && r->headers_in.content_length_n < rb->rest) ++ { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client intended to send body data " ++ "larger than declared"); ++ ++ stream->skip_data = NGX_SPDY_DATA_ERROR; ++ goto error; ++ ++ } else { ++ clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); ++ ++ if (clcf->client_max_body_size ++ && clcf->client_max_body_size < rb->rest) ++ { ++ ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, ++ "client intended to send " ++ "too large chunked body: %O bytes", rb->rest); ++ ++ stream->skip_data = NGX_SPDY_DATA_ERROR; ++ goto error; ++ } ++ } ++ ++ sc->length -= size; ++ ++ if (tf) { ++ buf->start = pos; ++ buf->pos = pos; ++ ++ pos += size; ++ ++ buf->end = pos; ++ buf->last = pos; ++ ++ n = ngx_write_chain_to_temp_file(tf, rb->bufs); ++ ++ /* TODO: n == 0 or not complete and level event */ ++ ++ if (n == NGX_ERROR) { ++ stream->skip_data = NGX_SPDY_DATA_INTERNAL_ERROR; ++ goto error; ++ } ++ ++ tf->offset += n; ++ ++ } else { ++ buf->last = ngx_cpymem(buf->last, pos, size); ++ pos += size; ++ } ++ ++ r->request_length += size; ++ } ++ ++ if (sc->length) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_read_data); ++ } ++ ++ if (sc->flags & NGX_SPDY_FLAG_FIN) { ++ ++ stream->in_closed = 1; ++ ++ if (r->headers_in.content_length_n < 0) { ++ r->headers_in.content_length_n = rb->rest; ++ ++ } else if (r->headers_in.content_length_n != rb->rest) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client prematurely closed stream: " ++ "only %O out of %O bytes of request body received", ++ rb->rest, r->headers_in.content_length_n); ++ ++ stream->skip_data = NGX_SPDY_DATA_ERROR; ++ goto error; ++ } ++ ++ if (tf) { ++ ngx_memzero(buf, sizeof(ngx_buf_t)); ++ ++ buf->in_file = 1; ++ buf->file_last = tf->file.offset; ++ buf->file = &tf->file; ++ ++ rb->buf = NULL; ++ } ++ ++ if (rb->post_handler) { ++ r->read_event_handler = ngx_http_block_reading; ++ rb->post_handler(r); ++ } ++ } ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++ ++error: ++ ++ if (rb->post_handler) { ++ ++ if (stream->skip_data == NGX_SPDY_DATA_ERROR) { ++ rc = (r->headers_in.content_length_n == -1) ++ ? NGX_HTTP_REQUEST_ENTITY_TOO_LARGE ++ : NGX_HTTP_BAD_REQUEST; ++ ++ } else { ++ rc = NGX_HTTP_INTERNAL_SERVER_ERROR; ++ } ++ ++ ngx_http_finalize_request(r, rc); ++ } ++ ++ return ngx_http_spdy_state_skip(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_rst_stream(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ ngx_uint_t sid, status; ++ ngx_event_t *ev; ++ ngx_connection_t *fc; ++ ngx_http_spdy_stream_t *stream; ++ ++ if (end - pos < NGX_SPDY_RST_STREAM_SIZE) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_rst_stream); ++ } ++ ++ if (sc->length != NGX_SPDY_RST_STREAM_SIZE) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent RST_STREAM frame with incorrect length %uz", ++ sc->length); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ sid = ngx_spdy_frame_parse_sid(pos); ++ ++ pos += NGX_SPDY_SID_SIZE; ++ ++ status = ngx_spdy_frame_parse_uint32(pos); ++ ++ pos += sizeof(uint32_t); ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy RST_STREAM sid:%ui st:%ui", sid, status); ++ ++ stream = ngx_http_spdy_get_stream_by_id(sc, sid); ++ ++ if (stream == NULL) { ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "unknown spdy stream"); ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++ } ++ ++ stream->in_closed = 1; ++ stream->out_closed = 1; ++ ++ fc = stream->request->connection; ++ fc->error = 1; ++ ++ switch (status) { ++ ++ case NGX_SPDY_CANCEL: ++ ngx_log_error(NGX_LOG_INFO, fc->log, 0, ++ "client canceled stream %ui", sid); ++ break; ++ ++ case NGX_SPDY_INTERNAL_ERROR: ++ ngx_log_error(NGX_LOG_INFO, fc->log, 0, ++ "client terminated stream %ui due to internal error", ++ sid); ++ break; ++ ++ default: ++ ngx_log_error(NGX_LOG_INFO, fc->log, 0, ++ "client terminated stream %ui with status %ui", ++ sid, status); ++ break; ++ } ++ ++ ev = fc->read; ++ ev->handler(ev); ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_ping(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ u_char *p; ++ ngx_buf_t *buf; ++ ngx_http_spdy_out_frame_t *frame; ++ ++ if (end - pos < NGX_SPDY_PING_SIZE) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_ping); ++ } ++ ++ if (sc->length != NGX_SPDY_PING_SIZE) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent PING frame with incorrect length %uz", ++ sc->length); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy PING frame"); ++ ++ frame = ngx_http_spdy_get_ctl_frame(sc, NGX_SPDY_PING_SIZE, ++ NGX_SPDY_HIGHEST_PRIORITY); ++ if (frame == NULL) { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ buf = frame->first->buf; ++ ++ p = buf->pos; ++ ++ p = ngx_spdy_frame_write_head(p, NGX_SPDY_PING); ++ p = ngx_spdy_frame_write_flags_and_len(p, 0, NGX_SPDY_PING_SIZE); ++ ++ p = ngx_cpymem(p, pos, NGX_SPDY_PING_SIZE); ++ ++ buf->last = p; ++ ++ ngx_http_spdy_queue_frame(sc, frame); ++ ++ pos += NGX_SPDY_PING_SIZE; ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_skip(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ size_t size; ++ ++ size = end - pos; ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy frame skip %uz of %uz", size, sc->length); ++ ++ if (size < sc->length) { ++ sc->length -= size; ++ return ngx_http_spdy_state_save(sc, end, end, ++ ngx_http_spdy_state_skip); ++ } ++ ++ return ngx_http_spdy_state_complete(sc, pos + sc->length, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_settings(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ ngx_uint_t fid, val; ++ ++ if (sc->entries == 0) { ++ ++ if (end - pos < NGX_SPDY_SETTINGS_NUM_SIZE) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_settings); ++ } ++ ++ sc->entries = ngx_spdy_frame_parse_uint32(pos); ++ ++ pos += NGX_SPDY_SETTINGS_NUM_SIZE; ++ sc->length -= NGX_SPDY_SETTINGS_NUM_SIZE; ++ ++ if (sc->length < sc->entries * NGX_SPDY_SETTINGS_PAIR_SIZE) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent SETTINGS frame with incorrect " ++ "length %uz or number of entries %ui", ++ sc->length, sc->entries); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy SETTINGS frame has %ui entries", sc->entries); ++ } ++ ++ while (sc->entries) { ++ if (end - pos < NGX_SPDY_SETTINGS_PAIR_SIZE) { ++ return ngx_http_spdy_state_save(sc, pos, end, ++ ngx_http_spdy_state_settings); ++ } ++ ++ sc->entries--; ++ sc->length -= NGX_SPDY_SETTINGS_PAIR_SIZE; ++ ++ fid = ngx_spdy_frame_parse_uint32(pos); ++ ++ pos += NGX_SPDY_SETTINGS_FID_SIZE; ++ ++ val = ngx_spdy_frame_parse_uint32(pos); ++ ++ pos += NGX_SPDY_SETTINGS_VAL_SIZE; ++ ++ ngx_log_debug3(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy SETTINGS entry fl:%ui id:%ui val:%ui", ++ ngx_spdy_frame_flags(fid), ngx_spdy_frame_id(fid), val); ++ ++ if (ngx_spdy_frame_flags(fid) == NGX_SPDY_SETTINGS_FLAG_PERSISTED) { ++ continue; ++ } ++ ++ switch (ngx_spdy_frame_id(fid)) { ++ ++ case NGX_SPDY_SETTINGS_INIT_WINDOW: ++ ++ if (val > NGX_SPDY_MAX_WINDOW) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent SETTINGS frame with " ++ "incorrect INIT_WINDOW value: %ui", val); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ if (ngx_http_spdy_adjust_windows(sc, val - sc->init_window) ++ != NGX_OK) ++ { ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ sc->init_window = val; ++ ++ continue; ++ } ++ } ++ ++ return ngx_http_spdy_state_complete(sc, pos, end); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_complete(ngx_http_spdy_connection_t *sc, u_char *pos, ++ u_char *end) ++{ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy frame complete pos:%p end:%p", pos, end); ++ ++ if (pos > end) { ++ ngx_log_error(NGX_LOG_ALERT, sc->connection->log, 0, ++ "receive buffer overrun"); ++ ++ ngx_debug_point(); ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ sc->handler = ngx_http_spdy_state_head; ++ sc->stream = NULL; ++ ++ return pos; ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_save(ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end, ngx_http_spdy_handler_pt handler) ++{ ++ size_t size; ++ ++ ngx_log_debug3(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy frame state save pos:%p end:%p handler:%p", ++ pos, end, handler); ++ ++ size = end - pos; ++ ++ if (size > NGX_SPDY_STATE_BUFFER_SIZE) { ++ ngx_log_error(NGX_LOG_ALERT, sc->connection->log, 0, ++ "state buffer overflow: %uz bytes required", size); ++ ++ ngx_debug_point(); ++ return ngx_http_spdy_state_internal_error(sc); ++ } ++ ++ ngx_memcpy(sc->buffer, pos, NGX_SPDY_STATE_BUFFER_SIZE); ++ ++ sc->buffer_used = size; ++ sc->handler = handler; ++ sc->incomplete = 1; ++ ++ return end; ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_inflate_error(ngx_http_spdy_connection_t *sc, int rc) ++{ ++ if (rc == Z_DATA_ERROR || rc == Z_STREAM_END) { ++ ngx_log_error(NGX_LOG_INFO, sc->connection->log, 0, ++ "client sent SYN_STREAM frame with " ++ "corrupted header block, inflate() failed: %d", rc); ++ ++ return ngx_http_spdy_state_protocol_error(sc); ++ } ++ ++ ngx_log_error(NGX_LOG_ERR, sc->connection->log, 0, ++ "inflate() failed: %d", rc); ++ ++ return ngx_http_spdy_state_internal_error(sc); ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_protocol_error(ngx_http_spdy_connection_t *sc) ++{ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy state protocol error"); ++ ++ if (sc->stream) { ++ sc->stream->out_closed = 1; ++ ngx_http_spdy_close_stream(sc->stream, NGX_HTTP_BAD_REQUEST); ++ } ++ ++ ngx_http_spdy_finalize_connection(sc, NGX_HTTP_CLIENT_CLOSED_REQUEST); ++ ++ return NULL; ++} ++ ++ ++static u_char * ++ngx_http_spdy_state_internal_error(ngx_http_spdy_connection_t *sc) ++{ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy state internal error"); ++ ++ if (sc->stream) { ++ sc->stream->out_closed = 1; ++ ngx_http_spdy_close_stream(sc->stream, NGX_HTTP_INTERNAL_SERVER_ERROR); ++ } ++ ++ ngx_http_spdy_finalize_connection(sc, NGX_HTTP_INTERNAL_SERVER_ERROR); ++ ++ return NULL; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_send_window_update(ngx_http_spdy_connection_t *sc, ngx_uint_t sid, ++ ngx_uint_t delta) ++{ ++ u_char *p; ++ ngx_buf_t *buf; ++ ngx_http_spdy_out_frame_t *frame; ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy send WINDOW_UPDATE sid:%ui delta:%ui", sid, delta); ++ ++ frame = ngx_http_spdy_get_ctl_frame(sc, NGX_SPDY_WINDOW_UPDATE_SIZE, ++ NGX_SPDY_HIGHEST_PRIORITY); ++ if (frame == NULL) { ++ return NGX_ERROR; ++ } ++ ++ buf = frame->first->buf; ++ ++ p = buf->pos; ++ ++ p = ngx_spdy_frame_write_head(p, NGX_SPDY_WINDOW_UPDATE); ++ p = ngx_spdy_frame_write_flags_and_len(p, 0, NGX_SPDY_WINDOW_UPDATE_SIZE); ++ ++ p = ngx_spdy_frame_write_sid(p, sid); ++ p = ngx_spdy_frame_aligned_write_uint32(p, delta); ++ ++ buf->last = p; ++ ++ ngx_http_spdy_queue_frame(sc, frame); ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_send_rst_stream(ngx_http_spdy_connection_t *sc, ngx_uint_t sid, ++ ngx_uint_t status, ngx_uint_t priority) ++{ ++ u_char *p; ++ ngx_buf_t *buf; ++ ngx_http_spdy_out_frame_t *frame; ++ ++ if (sc->connection->error) { ++ return NGX_OK; ++ } ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy send RST_STREAM sid:%ui st:%ui", sid, status); ++ ++ frame = ngx_http_spdy_get_ctl_frame(sc, NGX_SPDY_RST_STREAM_SIZE, ++ priority); ++ if (frame == NULL) { ++ return NGX_ERROR; ++ } ++ ++ buf = frame->first->buf; ++ ++ p = buf->pos; ++ ++ p = ngx_spdy_frame_write_head(p, NGX_SPDY_RST_STREAM); ++ p = ngx_spdy_frame_write_flags_and_len(p, 0, NGX_SPDY_RST_STREAM_SIZE); ++ ++ p = ngx_spdy_frame_write_sid(p, sid); ++ p = ngx_spdy_frame_aligned_write_uint32(p, status); ++ ++ buf->last = p; ++ ++ ngx_http_spdy_queue_frame(sc, frame); ++ ++ return NGX_OK; ++} ++ ++ ++#if 0 ++static ngx_int_t ++ngx_http_spdy_send_goaway(ngx_http_spdy_connection_t *sc) ++{ ++ u_char *p; ++ ngx_buf_t *buf; ++ ngx_http_spdy_out_frame_t *frame; ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy send GOAWAY sid:%ui", sc->last_sid); ++ ++ frame = ngx_http_spdy_get_ctl_frame(sc, NGX_SPDY_GOAWAY_SIZE, ++ NGX_SPDY_HIGHEST_PRIORITY); ++ if (frame == NULL) { ++ return NGX_ERROR; ++ } ++ ++ buf = frame->first->buf; ++ ++ p = buf->pos; ++ ++ p = ngx_spdy_frame_write_head(p, NGX_SPDY_GOAWAY); ++ p = ngx_spdy_frame_write_flags_and_len(p, 0, NGX_SPDY_GOAWAY_SIZE); ++ ++ p = ngx_spdy_frame_write_sid(p, sc->last_sid); ++ ++ buf->last = p; ++ ++ ngx_http_spdy_queue_frame(sc, frame); ++ ++ return NGX_OK; ++} ++#endif ++ ++ ++static ngx_int_t ++ngx_http_spdy_send_settings(ngx_http_spdy_connection_t *sc) ++{ ++ u_char *p; ++ ngx_buf_t *buf; ++ ngx_chain_t *cl; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ngx_http_spdy_out_frame_t *frame; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy send SETTINGS frame"); ++ ++ frame = ngx_palloc(sc->pool, sizeof(ngx_http_spdy_out_frame_t)); ++ if (frame == NULL) { ++ return NGX_ERROR; ++ } ++ ++ cl = ngx_alloc_chain_link(sc->pool); ++ if (cl == NULL) { ++ return NGX_ERROR; ++ } ++ ++ buf = ngx_create_temp_buf(sc->pool, NGX_SPDY_FRAME_HEADER_SIZE ++ + NGX_SPDY_SETTINGS_NUM_SIZE ++ + 2 * NGX_SPDY_SETTINGS_PAIR_SIZE); ++ if (buf == NULL) { ++ return NGX_ERROR; ++ } ++ ++ buf->last_buf = 1; ++ ++ cl->buf = buf; ++ cl->next = NULL; ++ ++ frame->first = cl; ++ frame->last = cl; ++ frame->handler = ngx_http_spdy_settings_frame_handler; ++ frame->stream = NULL; ++#if (NGX_DEBUG) ++ frame->length = NGX_SPDY_SETTINGS_NUM_SIZE ++ + 2 * NGX_SPDY_SETTINGS_PAIR_SIZE; ++#endif ++ frame->priority = NGX_SPDY_HIGHEST_PRIORITY; ++ frame->blocked = 0; ++ ++ p = buf->pos; ++ ++ p = ngx_spdy_frame_write_head(p, NGX_SPDY_SETTINGS); ++ p = ngx_spdy_frame_write_flags_and_len(p, NGX_SPDY_FLAG_CLEAR_SETTINGS, ++ NGX_SPDY_SETTINGS_NUM_SIZE ++ + 2 * NGX_SPDY_SETTINGS_PAIR_SIZE); ++ ++ p = ngx_spdy_frame_aligned_write_uint32(p, 2); ++ ++ sscf = ngx_http_get_module_srv_conf(sc->http_connection->conf_ctx, ++ ngx_http_spdy_module); ++ ++ p = ngx_spdy_frame_write_flags_and_id(p, 0, NGX_SPDY_SETTINGS_MAX_STREAMS); ++ p = ngx_spdy_frame_aligned_write_uint32(p, sscf->concurrent_streams); ++ ++ p = ngx_spdy_frame_write_flags_and_id(p, 0, NGX_SPDY_SETTINGS_INIT_WINDOW); ++ p = ngx_spdy_frame_aligned_write_uint32(p, NGX_SPDY_STREAM_WINDOW); ++ ++ buf->last = p; ++ ++ ngx_http_spdy_queue_frame(sc, frame); ++ ++ return NGX_OK; ++} ++ ++ ++ngx_int_t ++ngx_http_spdy_settings_frame_handler(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_out_frame_t *frame) ++{ ++ ngx_buf_t *buf; ++ ++ buf = frame->first->buf; ++ ++ if (buf->pos != buf->last) { ++ return NGX_AGAIN; ++ } ++ ++ ngx_free_chain(sc->pool, frame->first); ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_http_spdy_out_frame_t * ++ngx_http_spdy_get_ctl_frame(ngx_http_spdy_connection_t *sc, size_t length, ++ ngx_uint_t priority) ++{ ++ ngx_chain_t *cl; ++ ngx_http_spdy_out_frame_t *frame; ++ ++ frame = sc->free_ctl_frames; ++ ++ if (frame) { ++ sc->free_ctl_frames = frame->next; ++ ++ cl = frame->first; ++ cl->buf->pos = cl->buf->start; ++ ++ } else { ++ frame = ngx_palloc(sc->pool, sizeof(ngx_http_spdy_out_frame_t)); ++ if (frame == NULL) { ++ return NULL; ++ } ++ ++ cl = ngx_alloc_chain_link(sc->pool); ++ if (cl == NULL) { ++ return NULL; ++ } ++ ++ cl->buf = ngx_create_temp_buf(sc->pool, ++ NGX_SPDY_CTL_FRAME_BUFFER_SIZE); ++ if (cl->buf == NULL) { ++ return NULL; ++ } ++ ++ cl->buf->last_buf = 1; ++ ++ frame->first = cl; ++ frame->last = cl; ++ frame->handler = ngx_http_spdy_ctl_frame_handler; ++ frame->stream = NULL; ++ } ++ ++#if (NGX_DEBUG) ++ if (length > NGX_SPDY_CTL_FRAME_BUFFER_SIZE - NGX_SPDY_FRAME_HEADER_SIZE) { ++ ngx_log_error(NGX_LOG_ALERT, sc->pool->log, 0, ++ "requested control frame is too large: %uz", length); ++ return NULL; ++ } ++ ++ frame->length = length; ++#endif ++ ++ frame->priority = priority; ++ frame->blocked = 0; ++ ++ return frame; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_ctl_frame_handler(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_out_frame_t *frame) ++{ ++ ngx_buf_t *buf; ++ ++ buf = frame->first->buf; ++ ++ if (buf->pos != buf->last) { ++ return NGX_AGAIN; ++ } ++ ++ frame->next = sc->free_ctl_frames; ++ sc->free_ctl_frames = frame; ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_http_spdy_stream_t * ++ngx_http_spdy_create_stream(ngx_http_spdy_connection_t *sc, ngx_uint_t id, ++ ngx_uint_t priority) ++{ ++ ngx_log_t *log; ++ ngx_uint_t index; ++ ngx_event_t *rev, *wev; ++ ngx_connection_t *fc; ++ ngx_http_log_ctx_t *ctx; ++ ngx_http_request_t *r; ++ ngx_http_spdy_stream_t *stream; ++ ngx_http_core_srv_conf_t *cscf; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ++ fc = sc->free_fake_connections; ++ ++ if (fc) { ++ sc->free_fake_connections = fc->data; ++ ++ rev = fc->read; ++ wev = fc->write; ++ log = fc->log; ++ ctx = log->data; ++ ++ } else { ++ fc = ngx_palloc(sc->pool, sizeof(ngx_connection_t)); ++ if (fc == NULL) { ++ return NULL; ++ } ++ ++ rev = ngx_palloc(sc->pool, sizeof(ngx_event_t)); ++ if (rev == NULL) { ++ return NULL; ++ } ++ ++ wev = ngx_palloc(sc->pool, sizeof(ngx_event_t)); ++ if (wev == NULL) { ++ return NULL; ++ } ++ ++ log = ngx_palloc(sc->pool, sizeof(ngx_log_t)); ++ if (log == NULL) { ++ return NULL; ++ } ++ ++ ctx = ngx_palloc(sc->pool, sizeof(ngx_http_log_ctx_t)); ++ if (ctx == NULL) { ++ return NULL; ++ } ++ ++ ctx->connection = fc; ++ ctx->request = NULL; ++ } ++ ++ ngx_memcpy(log, sc->connection->log, sizeof(ngx_log_t)); ++ ++ log->data = ctx; ++ ++ ngx_memzero(rev, sizeof(ngx_event_t)); ++ ++ rev->data = fc; ++ rev->ready = 1; ++ rev->handler = ngx_http_spdy_close_stream_handler; ++ rev->log = log; ++ ++ ngx_memcpy(wev, rev, sizeof(ngx_event_t)); ++ ++ wev->write = 1; ++ ++ ngx_memcpy(fc, sc->connection, sizeof(ngx_connection_t)); ++ ++ fc->data = sc->http_connection; ++ fc->read = rev; ++ fc->write = wev; ++ fc->sent = 0; ++ fc->log = log; ++ fc->buffered = 0; ++ fc->sndlowat = 1; ++ fc->tcp_nodelay = NGX_TCP_NODELAY_DISABLED; ++ ++ r = ngx_http_create_request(fc); ++ if (r == NULL) { ++ return NULL; ++ } ++ ++ r->valid_location = 1; ++ ++ fc->data = r; ++ sc->connection->requests++; ++ ++ cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module); ++ ++ r->header_in = ngx_create_temp_buf(r->pool, ++ cscf->client_header_buffer_size); ++ if (r->header_in == NULL) { ++ ngx_http_free_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR); ++ return NULL; ++ } ++ ++ r->headers_in.connection_type = NGX_HTTP_CONNECTION_CLOSE; ++ ++ stream = ngx_pcalloc(r->pool, sizeof(ngx_http_spdy_stream_t)); ++ if (stream == NULL) { ++ ngx_http_free_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR); ++ return NULL; ++ } ++ ++ r->spdy_stream = stream; ++ ++ stream->id = id; ++ stream->request = r; ++ stream->connection = sc; ++ ++ stream->send_window = sc->init_window; ++ stream->recv_window = NGX_SPDY_STREAM_WINDOW; ++ ++ stream->priority = priority; ++ ++ sscf = ngx_http_get_module_srv_conf(r, ngx_http_spdy_module); ++ ++ index = ngx_http_spdy_stream_index(sscf, id); ++ ++ stream->index = sc->streams_index[index]; ++ sc->streams_index[index] = stream; ++ ++ sc->processing++; ++ ++ return stream; ++} ++ ++ ++static ngx_http_spdy_stream_t * ++ngx_http_spdy_get_stream_by_id(ngx_http_spdy_connection_t *sc, ++ ngx_uint_t sid) ++{ ++ ngx_http_spdy_stream_t *stream; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ++ sscf = ngx_http_get_module_srv_conf(sc->http_connection->conf_ctx, ++ ngx_http_spdy_module); ++ ++ stream = sc->streams_index[ngx_http_spdy_stream_index(sscf, sid)]; ++ ++ while (stream) { ++ if (stream->id == sid) { ++ return stream; ++ } ++ ++ stream = stream->index; ++ } ++ ++ return NULL; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_parse_header(ngx_http_request_t *r) ++{ ++ u_char *p, *end, ch; ++ ngx_uint_t hash; ++ ngx_http_core_srv_conf_t *cscf; ++ ++ enum { ++ sw_name_len = 0, ++ sw_name, ++ sw_value_len, ++ sw_value ++ } state; ++ ++ state = r->state; ++ ++ p = r->header_in->pos; ++ end = r->header_in->last; ++ ++ switch (state) { ++ ++ case sw_name_len: ++ ++ if (end - p < NGX_SPDY_NV_NLEN_SIZE) { ++ return NGX_AGAIN; ++ } ++ ++ r->lowcase_index = ngx_spdy_frame_parse_uint32(p); ++ ++ if (r->lowcase_index == 0) { ++ return NGX_ERROR; ++ } ++ ++ /* null-terminate the previous header value */ ++ *p = '\0'; ++ ++ p += NGX_SPDY_NV_NLEN_SIZE; ++ ++ r->invalid_header = 0; ++ ++ state = sw_name; ++ ++ /* fall through */ ++ ++ case sw_name: ++ ++ if ((ngx_uint_t) (end - p) < r->lowcase_index) { ++ break; ++ } ++ ++ cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module); ++ ++ r->header_name_start = p; ++ r->header_name_end = p + r->lowcase_index; ++ ++ if (p[0] == ':') { ++ p++; ++ } ++ ++ hash = 0; ++ ++ for ( /* void */ ; p != r->header_name_end; p++) { ++ ++ ch = *p; ++ ++ hash = ngx_hash(hash, ch); ++ ++ if ((ch >= 'a' && ch <= 'z') ++ || (ch == '-') ++ || (ch >= '0' && ch <= '9') ++ || (ch == '_' && cscf->underscores_in_headers)) ++ { ++ continue; ++ } ++ ++ switch (ch) { ++ case '\0': ++ case LF: ++ case CR: ++ case ':': ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent invalid header name: \"%*s\"", ++ r->lowcase_index, r->header_name_start); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ if (ch >= 'A' && ch <= 'Z') { ++ return NGX_ERROR; ++ } ++ ++ r->invalid_header = 1; ++ } ++ ++ r->header_hash = hash; ++ ++ state = sw_value_len; ++ ++ /* fall through */ ++ ++ case sw_value_len: ++ ++ if (end - p < NGX_SPDY_NV_VLEN_SIZE) { ++ break; ++ } ++ ++ r->lowcase_index = ngx_spdy_frame_parse_uint32(p); ++ ++ /* null-terminate header name */ ++ *p = '\0'; ++ ++ p += NGX_SPDY_NV_VLEN_SIZE; ++ ++ state = sw_value; ++ ++ /* fall through */ ++ ++ case sw_value: ++ ++ if ((ngx_uint_t) (end - p) < r->lowcase_index) { ++ break; ++ } ++ ++ r->header_start = p; ++ ++ while (r->lowcase_index--) { ++ ch = *p; ++ ++ if (ch == '\0') { ++ ++ if (p == r->header_start) { ++ return NGX_ERROR; ++ } ++ ++ r->header_end = p; ++ r->header_in->pos = p + 1; ++ ++ r->state = sw_value; ++ ++ return NGX_OK; ++ } ++ ++ if (ch == CR || ch == LF) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent header \"%*s\" with " ++ "invalid value: \"%*s\\%c...\"", ++ r->header_name_end - r->header_name_start, ++ r->header_name_start, ++ p - r->header_start, ++ r->header_start, ++ ch == CR ? 'r' : 'n'); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ p++; ++ } ++ ++ r->header_end = p; ++ r->header_in->pos = p; ++ ++ r->state = 0; ++ ++ return NGX_DONE; ++ } ++ ++ r->header_in->pos = p; ++ r->state = state; ++ ++ return NGX_AGAIN; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_alloc_large_header_buffer(ngx_http_request_t *r) ++{ ++ u_char *old, *new, *p; ++ size_t rest; ++ ngx_buf_t *buf; ++ ngx_http_spdy_stream_t *stream; ++ ngx_http_core_srv_conf_t *cscf; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy alloc large header buffer"); ++ ++ stream = r->spdy_stream; ++ ++ cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module); ++ ++ if (stream->header_buffers ++ == (ngx_uint_t) cscf->large_client_header_buffers.num) ++ { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent too large request"); ++ ++ return NGX_DECLINED; ++ } ++ ++ rest = r->header_in->last - r->header_in->pos; ++ ++ /* ++ * One more byte is needed for null-termination ++ * and another one for further progress. ++ */ ++ if (rest > cscf->large_client_header_buffers.size - 2) { ++ p = r->header_in->pos; ++ ++ if (rest > NGX_MAX_ERROR_STR - 300) { ++ rest = NGX_MAX_ERROR_STR - 300; ++ } ++ ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent too long header name or value: \"%*s...\"", ++ rest, p); ++ ++ return NGX_DECLINED; ++ } ++ ++ buf = ngx_create_temp_buf(r->pool, cscf->large_client_header_buffers.size); ++ if (buf == NULL) { ++ return NGX_ERROR; ++ } ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy large header alloc: %p %uz", ++ buf->pos, buf->end - buf->last); ++ ++ old = r->header_in->pos; ++ new = buf->pos; ++ ++ if (rest) { ++ buf->last = ngx_cpymem(new, old, rest); ++ } ++ ++ r->header_in = buf; ++ ++ stream->header_buffers++; ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_handle_request_header(ngx_http_request_t *r) ++{ ++ ngx_uint_t i; ++ ngx_table_elt_t *h; ++ ngx_http_core_srv_conf_t *cscf; ++ ngx_http_spdy_request_header_t *sh; ++ ++ if (r->invalid_header) { ++ cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module); ++ ++ if (cscf->ignore_invalid_headers) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent invalid header: \"%*s\"", ++ r->header_end - r->header_name_start, ++ r->header_name_start); ++ return NGX_OK; ++ } ++ ++ } ++ ++ if (r->header_name_start[0] == ':') { ++ r->header_name_start++; ++ ++ for (i = 0; i < NGX_SPDY_REQUEST_HEADERS; i++) { ++ sh = &ngx_http_spdy_request_headers[i]; ++ ++ if (sh->hash != r->header_hash ++ || sh->len != r->header_name_end - r->header_name_start ++ || ngx_strncmp(sh->header, r->header_name_start, sh->len) != 0) ++ { ++ continue; ++ } ++ ++ return sh->handler(r); ++ } ++ ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent invalid header name: \":%*s\"", ++ r->header_end - r->header_name_start, ++ r->header_name_start); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ h = ngx_list_push(&r->headers_in.headers); ++ if (h == NULL) { ++ return NGX_ERROR; ++ } ++ ++ h->hash = r->header_hash; ++ ++ h->key.len = r->header_name_end - r->header_name_start; ++ h->key.data = r->header_name_start; ++ ++ h->value.len = r->header_end - r->header_start; ++ h->value.data = r->header_start; ++ ++ h->lowcase_key = h->key.data; ++ ++ return NGX_OK; ++} ++ ++ ++void ++ngx_http_spdy_request_headers_init(void) ++{ ++ ngx_uint_t i; ++ ngx_http_spdy_request_header_t *h; ++ ++ for (i = 0; i < NGX_SPDY_REQUEST_HEADERS; i++) { ++ h = &ngx_http_spdy_request_headers[i]; ++ h->hash = ngx_hash_key(h->header, h->len); ++ } ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_parse_method(ngx_http_request_t *r) ++{ ++ size_t k, len; ++ ngx_uint_t n; ++ const u_char *p, *m; ++ ++ /* ++ * This array takes less than 256 sequential bytes, ++ * and if typical CPU cache line size is 64 bytes, ++ * it is prefetched for 4 load operations. ++ */ ++ static const struct { ++ u_char len; ++ const u_char method[11]; ++ uint32_t value; ++ } tests[] = { ++ { 3, "GET", NGX_HTTP_GET }, ++ { 4, "POST", NGX_HTTP_POST }, ++ { 4, "HEAD", NGX_HTTP_HEAD }, ++ { 7, "OPTIONS", NGX_HTTP_OPTIONS }, ++ { 8, "PROPFIND", NGX_HTTP_PROPFIND }, ++ { 3, "PUT", NGX_HTTP_PUT }, ++ { 5, "MKCOL", NGX_HTTP_MKCOL }, ++ { 6, "DELETE", NGX_HTTP_DELETE }, ++ { 4, "COPY", NGX_HTTP_COPY }, ++ { 4, "MOVE", NGX_HTTP_MOVE }, ++ { 9, "PROPPATCH", NGX_HTTP_PROPPATCH }, ++ { 4, "LOCK", NGX_HTTP_LOCK }, ++ { 6, "UNLOCK", NGX_HTTP_UNLOCK }, ++ { 5, "PATCH", NGX_HTTP_PATCH }, ++ { 5, "TRACE", NGX_HTTP_TRACE } ++ }, *test; ++ ++ if (r->method_name.len) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent duplicate :method header"); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ len = r->header_end - r->header_start; ++ ++ r->method_name.len = len; ++ r->method_name.data = r->header_start; ++ ++ test = tests; ++ n = sizeof(tests) / sizeof(tests[0]); ++ ++ do { ++ if (len == test->len) { ++ p = r->method_name.data; ++ m = test->method; ++ k = len; ++ ++ do { ++ if (*p++ != *m++) { ++ goto next; ++ } ++ } while (--k); ++ ++ r->method = test->value; ++ return NGX_OK; ++ } ++ ++ next: ++ test++; ++ ++ } while (--n); ++ ++ p = r->method_name.data; ++ ++ do { ++ if ((*p < 'A' || *p > 'Z') && *p != '_') { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent invalid method: \"%V\"", ++ &r->method_name); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ p++; ++ ++ } while (--len); ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_parse_scheme(ngx_http_request_t *r) ++{ ++ if (r->schema_start) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent duplicate :schema header"); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ r->schema_start = r->header_start; ++ r->schema_end = r->header_end; ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_parse_host(ngx_http_request_t *r) ++{ ++ ngx_table_elt_t *h; ++ ++ if (r->headers_in.host) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent duplicate :host header"); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ h = ngx_list_push(&r->headers_in.headers); ++ if (h == NULL) { ++ return NGX_ERROR; ++ } ++ ++ r->headers_in.host = h; ++ ++ h->hash = r->header_hash; ++ ++ h->key.len = r->header_name_end - r->header_name_start; ++ h->key.data = r->header_name_start; ++ ++ h->value.len = r->header_end - r->header_start; ++ h->value.data = r->header_start; ++ ++ h->lowcase_key = h->key.data; ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_parse_path(ngx_http_request_t *r) ++{ ++ if (r->unparsed_uri.len) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent duplicate :path header"); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ r->uri_start = r->header_start; ++ r->uri_end = r->header_end; ++ ++ if (ngx_http_parse_uri(r) != NGX_OK) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent invalid URI: \"%*s\"", ++ r->uri_end - r->uri_start, r->uri_start); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ if (ngx_http_process_request_uri(r) != NGX_OK) { ++ /* ++ * request has been finalized already ++ * in ngx_http_process_request_uri() ++ */ ++ return NGX_ABORT; ++ } ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_parse_version(ngx_http_request_t *r) ++{ ++ u_char *p, ch; ++ ++ if (r->http_protocol.len) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent duplicate :version header"); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++ } ++ ++ p = r->header_start; ++ ++ if (r->header_end - p < 8 || !(ngx_str5cmp(p, 'H', 'T', 'T', 'P', '/'))) { ++ goto invalid; ++ } ++ ++ ch = *(p + 5); ++ ++ if (ch < '1' || ch > '9') { ++ goto invalid; ++ } ++ ++ r->http_major = ch - '0'; ++ ++ for (p += 6; p != r->header_end - 2; p++) { ++ ++ ch = *p; ++ ++ if (ch == '.') { ++ break; ++ } ++ ++ if (ch < '0' || ch > '9') { ++ goto invalid; ++ } ++ ++ r->http_major = r->http_major * 10 + ch - '0'; ++ } ++ ++ if (*p != '.') { ++ goto invalid; ++ } ++ ++ ch = *(p + 1); ++ ++ if (ch < '0' || ch > '9') { ++ goto invalid; ++ } ++ ++ r->http_minor = ch - '0'; ++ ++ for (p += 2; p != r->header_end; p++) { ++ ++ ch = *p; ++ ++ if (ch < '0' || ch > '9') { ++ goto invalid; ++ } ++ ++ r->http_minor = r->http_minor * 10 + ch - '0'; ++ } ++ ++ r->http_protocol.len = r->header_end - r->header_start; ++ r->http_protocol.data = r->header_start; ++ r->http_version = r->http_major * 1000 + r->http_minor; ++ ++ return NGX_OK; ++ ++invalid: ++ ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client sent invalid http version: \"%*s\"", ++ r->header_end - r->header_start, r->header_start); ++ ++ return NGX_HTTP_PARSE_INVALID_HEADER; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_construct_request_line(ngx_http_request_t *r) ++{ ++ u_char *p; ++ ++ if (r->method_name.len == 0 ++ || r->unparsed_uri.len == 0 ++ || r->http_protocol.len == 0) ++ { ++ ngx_http_finalize_request(r, NGX_HTTP_BAD_REQUEST); ++ return NGX_ERROR; ++ } ++ ++ r->request_line.len = r->method_name.len + 1 ++ + r->unparsed_uri.len + 1 ++ + r->http_protocol.len; ++ ++ p = ngx_pnalloc(r->pool, r->request_line.len + 1); ++ if (p == NULL) { ++ ngx_http_spdy_close_stream(r->spdy_stream, ++ NGX_HTTP_INTERNAL_SERVER_ERROR); ++ return NGX_ERROR; ++ } ++ ++ r->request_line.data = p; ++ ++ p = ngx_cpymem(p, r->method_name.data, r->method_name.len); ++ ++ *p++ = ' '; ++ ++ p = ngx_cpymem(p, r->unparsed_uri.data, r->unparsed_uri.len); ++ ++ *p++ = ' '; ++ ++ ngx_memcpy(p, r->http_protocol.data, r->http_protocol.len + 1); ++ ++ /* some modules expect the space character after method name */ ++ r->method_name.data = r->request_line.data; ++ ++ return NGX_OK; ++} ++ ++ ++static void ++ngx_http_spdy_run_request(ngx_http_request_t *r) ++{ ++ ngx_uint_t i; ++ ngx_list_part_t *part; ++ ngx_table_elt_t *h; ++ ngx_http_header_t *hh; ++ ngx_http_core_main_conf_t *cmcf; ++ ++ if (ngx_http_spdy_construct_request_line(r) != NGX_OK) { ++ return; ++ } ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy http request line: \"%V\"", &r->request_line); ++ ++ cmcf = ngx_http_get_module_main_conf(r, ngx_http_core_module); ++ ++ part = &r->headers_in.headers.part; ++ h = part->elts; ++ ++ for (i = 0 ;; i++) { ++ ++ if (i >= part->nelts) { ++ if (part->next == NULL) { ++ break; ++ } ++ ++ part = part->next; ++ h = part->elts; ++ i = 0; ++ } ++ ++ hh = ngx_hash_find(&cmcf->headers_in_hash, h[i].hash, ++ h[i].lowcase_key, h[i].key.len); ++ ++ if (hh && hh->handler(r, &h[i], hh->offset) != NGX_OK) { ++ return; ++ } ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy http header: \"%V: %V\"", &h[i].key, &h[i].value); ++ } ++ ++ r->http_state = NGX_HTTP_PROCESS_REQUEST_STATE; ++ ++ if (ngx_http_process_request_header(r) != NGX_OK) { ++ return; ++ } ++ ++ if (r->headers_in.content_length_n > 0 && r->spdy_stream->in_closed) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "client prematurely closed stream"); ++ ++ r->spdy_stream->skip_data = NGX_SPDY_DATA_ERROR; ++ ++ ngx_http_finalize_request(r, NGX_HTTP_BAD_REQUEST); ++ return; ++ } ++ ++ ngx_http_process_request(r); ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_init_request_body(ngx_http_request_t *r) ++{ ++ ngx_buf_t *buf; ++ ngx_temp_file_t *tf; ++ ngx_http_request_body_t *rb; ++ ngx_http_core_loc_conf_t *clcf; ++ ++ rb = ngx_pcalloc(r->pool, sizeof(ngx_http_request_body_t)); ++ if (rb == NULL) { ++ return NGX_ERROR; ++ } ++ ++ r->request_body = rb; ++ ++ if (r->spdy_stream->in_closed) { ++ return NGX_OK; ++ } ++ ++ rb->rest = r->headers_in.content_length_n; ++ ++ clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); ++ ++ if (r->request_body_in_file_only ++ || rb->rest > (off_t) clcf->client_body_buffer_size ++ || rb->rest < 0) ++ { ++ tf = ngx_pcalloc(r->pool, sizeof(ngx_temp_file_t)); ++ if (tf == NULL) { ++ return NGX_ERROR; ++ } ++ ++ tf->file.fd = NGX_INVALID_FILE; ++ tf->file.log = r->connection->log; ++ tf->path = clcf->client_body_temp_path; ++ tf->pool = r->pool; ++ tf->warn = "a client request body is buffered to a temporary file"; ++ tf->log_level = r->request_body_file_log_level; ++ tf->persistent = r->request_body_in_persistent_file; ++ tf->clean = r->request_body_in_clean_file; ++ ++ if (r->request_body_file_group_access) { ++ tf->access = 0660; ++ } ++ ++ rb->temp_file = tf; ++ ++ if (r->spdy_stream->in_closed ++ && ngx_create_temp_file(&tf->file, tf->path, tf->pool, ++ tf->persistent, tf->clean, tf->access) ++ != NGX_OK) ++ { ++ return NGX_ERROR; ++ } ++ ++ buf = ngx_calloc_buf(r->pool); ++ if (buf == NULL) { ++ return NGX_ERROR; ++ } ++ ++ } else { ++ ++ if (rb->rest == 0) { ++ return NGX_OK; ++ } ++ ++ buf = ngx_create_temp_buf(r->pool, (size_t) rb->rest); ++ if (buf == NULL) { ++ return NGX_ERROR; ++ } ++ } ++ ++ rb->buf = buf; ++ ++ rb->bufs = ngx_alloc_chain_link(r->pool); ++ if (rb->bufs == NULL) { ++ return NGX_ERROR; ++ } ++ ++ rb->bufs->buf = buf; ++ rb->bufs->next = NULL; ++ ++ rb->rest = 0; ++ ++ return NGX_OK; ++} ++ ++ ++ngx_int_t ++ngx_http_spdy_read_request_body(ngx_http_request_t *r, ++ ngx_http_client_body_handler_pt post_handler) ++{ ++ ngx_http_spdy_stream_t *stream; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy read request body"); ++ ++ stream = r->spdy_stream; ++ ++ switch (stream->skip_data) { ++ ++ case NGX_SPDY_DATA_DISCARD: ++ post_handler(r); ++ return NGX_OK; ++ ++ case NGX_SPDY_DATA_ERROR: ++ if (r->headers_in.content_length_n == -1) { ++ return NGX_HTTP_REQUEST_ENTITY_TOO_LARGE; ++ } else { ++ return NGX_HTTP_BAD_REQUEST; ++ } ++ ++ case NGX_SPDY_DATA_INTERNAL_ERROR: ++ return NGX_HTTP_INTERNAL_SERVER_ERROR; ++ } ++ ++ if (!r->request_body && ngx_http_spdy_init_request_body(r) != NGX_OK) { ++ stream->skip_data = NGX_SPDY_DATA_INTERNAL_ERROR; ++ return NGX_HTTP_INTERNAL_SERVER_ERROR; ++ } ++ ++ if (stream->in_closed) { ++ post_handler(r); ++ return NGX_OK; ++ } ++ ++ r->request_body->post_handler = post_handler; ++ ++ r->read_event_handler = ngx_http_test_reading; ++ r->write_event_handler = ngx_http_request_empty_handler; ++ ++ return NGX_AGAIN; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_terminate_stream(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_stream_t *stream, ngx_uint_t status) ++{ ++ ngx_event_t *rev; ++ ngx_connection_t *fc; ++ ++ if (ngx_http_spdy_send_rst_stream(sc, stream->id, status, ++ NGX_SPDY_HIGHEST_PRIORITY) ++ == NGX_ERROR) ++ { ++ return NGX_ERROR; ++ } ++ ++ stream->out_closed = 1; ++ ++ fc = stream->request->connection; ++ fc->error = 1; ++ ++ rev = fc->read; ++ rev->handler(rev); ++ ++ return NGX_OK; ++} ++ ++ ++static void ++ngx_http_spdy_close_stream_handler(ngx_event_t *ev) ++{ ++ ngx_connection_t *fc; ++ ngx_http_request_t *r; ++ ++ fc = ev->data; ++ r = fc->data; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy close stream handler"); ++ ++ ngx_http_spdy_close_stream(r->spdy_stream, 0); ++} ++ ++ ++void ++ngx_http_spdy_close_stream(ngx_http_spdy_stream_t *stream, ngx_int_t rc) ++{ ++ ngx_event_t *ev; ++ ngx_connection_t *fc; ++ ngx_http_spdy_stream_t **index, *s; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ngx_http_spdy_connection_t *sc; ++ ++ sc = stream->connection; ++ ++ ngx_log_debug3(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy close stream %ui, queued %ui, processing %ui", ++ stream->id, stream->queued, sc->processing); ++ ++ fc = stream->request->connection; ++ ++ if (stream->queued) { ++ fc->write->handler = ngx_http_spdy_close_stream_handler; ++ return; ++ } ++ ++ if (!stream->out_closed) { ++ if (ngx_http_spdy_send_rst_stream(sc, stream->id, ++ NGX_SPDY_INTERNAL_ERROR, ++ stream->priority) ++ != NGX_OK) ++ { ++ sc->connection->error = 1; ++ } ++ } ++ ++ if (sc->stream == stream) { ++ sc->stream = NULL; ++ } ++ ++ sscf = ngx_http_get_module_srv_conf(sc->http_connection->conf_ctx, ++ ngx_http_spdy_module); ++ ++ index = sc->streams_index + ngx_http_spdy_stream_index(sscf, stream->id); ++ ++ for ( ;; ) { ++ s = *index; ++ ++ if (s == NULL) { ++ break; ++ } ++ ++ if (s == stream) { ++ *index = s->index; ++ break; ++ } ++ ++ index = &s->index; ++ } ++ ++ ngx_http_free_request(stream->request, rc); ++ ++ ev = fc->read; ++ ++ if (ev->active || ev->disabled) { ++ ngx_log_error(NGX_LOG_ALERT, sc->connection->log, 0, ++ "fake read event was activated"); ++ } ++ ++ if (ev->timer_set) { ++ ngx_del_timer(ev); ++ } ++ ++ if (ev->posted) { ++ ngx_delete_posted_event(ev); ++ } ++ ++ ev = fc->write; ++ ++ if (ev->active || ev->disabled) { ++ ngx_log_error(NGX_LOG_ALERT, sc->connection->log, 0, ++ "fake write event was activated"); ++ } ++ ++ if (ev->timer_set) { ++ ngx_del_timer(ev); ++ } ++ ++ if (ev->posted) { ++ ngx_delete_posted_event(ev); ++ } ++ ++ fc->data = sc->free_fake_connections; ++ sc->free_fake_connections = fc; ++ ++ sc->processing--; ++ ++ if (sc->processing || sc->blocked) { ++ return; ++ } ++ ++ ev = sc->connection->read; ++ ++ ev->handler = ngx_http_spdy_handle_connection_handler; ++ ngx_post_event(ev, &ngx_posted_events); ++} ++ ++ ++static void ++ngx_http_spdy_handle_connection_handler(ngx_event_t *rev) ++{ ++ ngx_connection_t *c; ++ ++ rev->handler = ngx_http_spdy_read_handler; ++ ++ if (rev->ready) { ++ ngx_http_spdy_read_handler(rev); ++ return; ++ } ++ ++ c = rev->data; ++ ++ ngx_http_spdy_handle_connection(c->data); ++} ++ ++ ++static void ++ngx_http_spdy_keepalive_handler(ngx_event_t *rev) ++{ ++ ngx_connection_t *c; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ngx_http_spdy_connection_t *sc; ++ ++ c = rev->data; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "spdy keepalive handler"); ++ ++ if (rev->timedout || c->close) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++#if (NGX_HAVE_KQUEUE) ++ ++ if (ngx_event_flags & NGX_USE_KQUEUE_EVENT) { ++ if (rev->pending_eof) { ++ c->log->handler = NULL; ++ ngx_log_error(NGX_LOG_INFO, c->log, rev->kq_errno, ++ "kevent() reported that client %V closed " ++ "keepalive connection", &c->addr_text); ++#if (NGX_HTTP_SSL) ++ if (c->ssl) { ++ c->ssl->no_send_shutdown = 1; ++ } ++#endif ++ ngx_http_close_connection(c); ++ return; ++ } ++ } ++ ++#endif ++ ++ c->destroyed = 0; ++ c->idle = 0; ++ ngx_reusable_connection(c, 0); ++ ++ sc = c->data; ++ ++ sscf = ngx_http_get_module_srv_conf(sc->http_connection->conf_ctx, ++ ngx_http_spdy_module); ++ ++ sc->pool = ngx_create_pool(sscf->pool_size, sc->connection->log); ++ if (sc->pool == NULL) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ sc->streams_index = ngx_pcalloc(sc->pool, ++ ngx_http_spdy_streams_index_size(sscf) ++ * sizeof(ngx_http_spdy_stream_t *)); ++ if (sc->streams_index == NULL) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ c->write->handler = ngx_http_spdy_write_handler; ++ ++ rev->handler = ngx_http_spdy_read_handler; ++ ngx_http_spdy_read_handler(rev); ++} ++ ++ ++static void ++ngx_http_spdy_finalize_connection(ngx_http_spdy_connection_t *sc, ++ ngx_int_t rc) ++{ ++ ngx_uint_t i, size; ++ ngx_event_t *ev; ++ ngx_connection_t *c, *fc; ++ ngx_http_request_t *r; ++ ngx_http_spdy_stream_t *stream; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ++ c = sc->connection; ++ ++ if (!sc->processing) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ c->error = 1; ++ c->read->handler = ngx_http_empty_handler; ++ c->write->handler = ngx_http_empty_handler; ++ ++ sc->last_out = NULL; ++ ++ sc->blocked = 1; ++ ++ sscf = ngx_http_get_module_srv_conf(sc->http_connection->conf_ctx, ++ ngx_http_spdy_module); ++ ++ size = ngx_http_spdy_streams_index_size(sscf); ++ ++ for (i = 0; i < size; i++) { ++ stream = sc->streams_index[i]; ++ ++ while (stream) { ++ stream->handled = 0; ++ ++ r = stream->request; ++ fc = r->connection; ++ ++ fc->error = 1; ++ ++ if (stream->queued) { ++ stream->queued = 0; ++ ++ ev = fc->write; ++ ev->delayed = 0; ++ ++ } else { ++ ev = fc->read; ++ } ++ ++ stream = stream->index; ++ ++ ev->eof = 1; ++ ev->handler(ev); ++ } ++ } ++ ++ sc->blocked = 0; ++ ++ if (sc->processing) { ++ return; ++ } ++ ++ ngx_http_close_connection(c); ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_adjust_windows(ngx_http_spdy_connection_t *sc, ssize_t delta) ++{ ++ ngx_uint_t i, size; ++ ngx_event_t *wev; ++ ngx_http_spdy_stream_t *stream, *sn; ++ ngx_http_spdy_srv_conf_t *sscf; ++ ++ sscf = ngx_http_get_module_srv_conf(sc->http_connection->conf_ctx, ++ ngx_http_spdy_module); ++ ++ size = ngx_http_spdy_streams_index_size(sscf); ++ ++ for (i = 0; i < size; i++) { ++ ++ for (stream = sc->streams_index[i]; stream; stream = sn) { ++ sn = stream->index; ++ ++ if (delta > 0 ++ && stream->send_window ++ > (ssize_t) (NGX_SPDY_MAX_WINDOW - delta)) ++ { ++ if (ngx_http_spdy_terminate_stream(sc, stream, ++ NGX_SPDY_FLOW_CONTROL_ERROR) ++ == NGX_ERROR) ++ { ++ return NGX_ERROR; ++ } ++ ++ continue; ++ } ++ ++ stream->send_window += delta; ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy:%ui adjust window:%z", ++ stream->id, stream->send_window); ++ ++ if (stream->send_window > 0 && stream->exhausted) { ++ stream->exhausted = 0; ++ ++ wev = stream->request->connection->write; ++ ++ if (!wev->timer_set) { ++ wev->delayed = 0; ++ wev->handler(wev); ++ } ++ } ++ } ++ } ++ ++ return NGX_OK; ++} ++ ++ ++static void ++ngx_http_spdy_pool_cleanup(void *data) ++{ ++ ngx_http_spdy_connection_t *sc = data; ++ ++ if (sc->pool) { ++ ngx_destroy_pool(sc->pool); ++ } ++} ++ ++ ++static void * ++ngx_http_spdy_zalloc(void *opaque, u_int items, u_int size) ++{ ++ ngx_http_spdy_connection_t *sc = opaque; ++ ++ return ngx_palloc(sc->connection->pool, items * size); ++} ++ ++ ++static void ++ngx_http_spdy_zfree(void *opaque, void *address) ++{ ++#if 0 ++ ngx_http_spdy_connection_t *sc = opaque; ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy zfree: %p", address); ++#endif ++} +diff --git a/src/http/ngx_http_spdy.h b/src/http/ngx_http_spdy.h +new file mode 100644 +index 0000000..df24495 +--- /dev/null ++++ b/src/http/ngx_http_spdy.h +@@ -0,0 +1,261 @@ ++/* ++ * Copyright (C) Nginx, Inc. ++ * Copyright (C) Valentin V. Bartenev ++ */ ++ ++ ++#ifndef _NGX_HTTP_SPDY_H_INCLUDED_ ++#define _NGX_HTTP_SPDY_H_INCLUDED_ ++ ++ ++#include ++#include ++#include ++ ++#include ++ ++ ++#define NGX_SPDY_VERSION 3 ++ ++#define NGX_SPDY_NPN_ADVERTISE "\x08spdy/3.1" ++#define NGX_SPDY_NPN_NEGOTIATED "spdy/3.1" ++ ++#define NGX_SPDY_STATE_BUFFER_SIZE 16 ++ ++#define NGX_SPDY_CTL_BIT 1 ++ ++#define NGX_SPDY_SYN_STREAM 1 ++#define NGX_SPDY_SYN_REPLY 2 ++#define NGX_SPDY_RST_STREAM 3 ++#define NGX_SPDY_SETTINGS 4 ++#define NGX_SPDY_PING 6 ++#define NGX_SPDY_GOAWAY 7 ++#define NGX_SPDY_HEADERS 8 ++#define NGX_SPDY_WINDOW_UPDATE 9 ++ ++#define NGX_SPDY_FRAME_HEADER_SIZE 8 ++ ++#define NGX_SPDY_SID_SIZE 4 ++#define NGX_SPDY_DELTA_SIZE 4 ++ ++#define NGX_SPDY_SYN_STREAM_SIZE 10 ++#define NGX_SPDY_SYN_REPLY_SIZE 4 ++#define NGX_SPDY_RST_STREAM_SIZE 8 ++#define NGX_SPDY_PING_SIZE 4 ++#define NGX_SPDY_GOAWAY_SIZE 8 ++#define NGX_SPDY_WINDOW_UPDATE_SIZE 8 ++#define NGX_SPDY_NV_NUM_SIZE 4 ++#define NGX_SPDY_NV_NLEN_SIZE 4 ++#define NGX_SPDY_NV_VLEN_SIZE 4 ++#define NGX_SPDY_SETTINGS_NUM_SIZE 4 ++#define NGX_SPDY_SETTINGS_FID_SIZE 4 ++#define NGX_SPDY_SETTINGS_VAL_SIZE 4 ++ ++#define NGX_SPDY_SETTINGS_PAIR_SIZE \ ++ (NGX_SPDY_SETTINGS_FID_SIZE + NGX_SPDY_SETTINGS_VAL_SIZE) ++ ++#define NGX_SPDY_HIGHEST_PRIORITY 0 ++#define NGX_SPDY_LOWEST_PRIORITY 7 ++ ++#define NGX_SPDY_FLAG_FIN 0x01 ++#define NGX_SPDY_FLAG_UNIDIRECTIONAL 0x02 ++#define NGX_SPDY_FLAG_CLEAR_SETTINGS 0x01 ++ ++#define NGX_SPDY_MAX_FRAME_SIZE ((1 << 24) - 1) ++ ++#define NGX_SPDY_DATA_DISCARD 1 ++#define NGX_SPDY_DATA_ERROR 2 ++#define NGX_SPDY_DATA_INTERNAL_ERROR 3 ++ ++ ++typedef struct ngx_http_spdy_connection_s ngx_http_spdy_connection_t; ++typedef struct ngx_http_spdy_out_frame_s ngx_http_spdy_out_frame_t; ++ ++ ++typedef u_char *(*ngx_http_spdy_handler_pt) (ngx_http_spdy_connection_t *sc, ++ u_char *pos, u_char *end); ++ ++struct ngx_http_spdy_connection_s { ++ ngx_connection_t *connection; ++ ngx_http_connection_t *http_connection; ++ ++ ngx_uint_t processing; ++ ++ size_t send_window; ++ size_t recv_window; ++ size_t init_window; ++ ++ ngx_queue_t waiting; ++ ++ u_char buffer[NGX_SPDY_STATE_BUFFER_SIZE]; ++ size_t buffer_used; ++ ngx_http_spdy_handler_pt handler; ++ ++ z_stream zstream_in; ++ z_stream zstream_out; ++ ++ ngx_pool_t *pool; ++ ++ ngx_http_spdy_out_frame_t *free_ctl_frames; ++ ngx_connection_t *free_fake_connections; ++ ++ ngx_http_spdy_stream_t **streams_index; ++ ++ ngx_http_spdy_out_frame_t *last_out; ++ ++ ngx_queue_t posted; ++ ++ ngx_http_spdy_stream_t *stream; ++ ++ ngx_uint_t entries; ++ size_t length; ++ u_char flags; ++ ++ ngx_uint_t last_sid; ++ ++ unsigned blocked:1; ++ unsigned incomplete:1; ++}; ++ ++ ++struct ngx_http_spdy_stream_s { ++ ngx_uint_t id; ++ ngx_http_request_t *request; ++ ngx_http_spdy_connection_t *connection; ++ ngx_http_spdy_stream_t *index; ++ ++ ngx_uint_t header_buffers; ++ ngx_uint_t queued; ++ ++ /* ++ * A change to SETTINGS_INITIAL_WINDOW_SIZE could cause the ++ * send_window to become negative, hence it's signed. ++ */ ++ ssize_t send_window; ++ size_t recv_window; ++ ++ ngx_http_spdy_out_frame_t *free_frames; ++ ngx_chain_t *free_data_headers; ++ ngx_chain_t *free_bufs; ++ ++ ngx_queue_t queue; ++ ++ unsigned priority:3; ++ unsigned handled:1; ++ unsigned blocked:1; ++ unsigned exhausted:1; ++ unsigned in_closed:1; ++ unsigned out_closed:1; ++ unsigned skip_data:2; ++}; ++ ++ ++struct ngx_http_spdy_out_frame_s { ++ ngx_http_spdy_out_frame_t *next; ++ ngx_chain_t *first; ++ ngx_chain_t *last; ++ ngx_int_t (*handler)(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_out_frame_t *frame); ++ ++ ngx_http_spdy_stream_t *stream; ++ size_t length; ++ ++ ngx_uint_t priority; ++ unsigned blocked:1; ++ unsigned fin:1; ++}; ++ ++ ++static ngx_inline void ++ngx_http_spdy_queue_frame(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_out_frame_t *frame) ++{ ++ ngx_http_spdy_out_frame_t **out; ++ ++ for (out = &sc->last_out; *out; out = &(*out)->next) ++ { ++ /* ++ * NB: higher values represent lower priorities. ++ */ ++ if (frame->priority >= (*out)->priority) { ++ break; ++ } ++ } ++ ++ frame->next = *out; ++ *out = frame; ++} ++ ++ ++static ngx_inline void ++ngx_http_spdy_queue_blocked_frame(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_out_frame_t *frame) ++{ ++ ngx_http_spdy_out_frame_t **out; ++ ++ for (out = &sc->last_out; *out; out = &(*out)->next) ++ { ++ if ((*out)->blocked) { ++ break; ++ } ++ } ++ ++ frame->next = *out; ++ *out = frame; ++} ++ ++ ++void ngx_http_spdy_init(ngx_event_t *rev); ++void ngx_http_spdy_request_headers_init(void); ++ ++ngx_int_t ngx_http_spdy_read_request_body(ngx_http_request_t *r, ++ ngx_http_client_body_handler_pt post_handler); ++ ++void ngx_http_spdy_close_stream(ngx_http_spdy_stream_t *stream, ngx_int_t rc); ++ ++ngx_int_t ngx_http_spdy_send_output_queue(ngx_http_spdy_connection_t *sc); ++ ++ ++#define ngx_spdy_frame_aligned_write_uint16(p, s) \ ++ (*(uint16_t *) (p) = htons((uint16_t) (s)), (p) + sizeof(uint16_t)) ++ ++#define ngx_spdy_frame_aligned_write_uint32(p, s) \ ++ (*(uint32_t *) (p) = htonl((uint32_t) (s)), (p) + sizeof(uint32_t)) ++ ++#if (NGX_HAVE_NONALIGNED) ++ ++#define ngx_spdy_frame_write_uint16 ngx_spdy_frame_aligned_write_uint16 ++#define ngx_spdy_frame_write_uint32 ngx_spdy_frame_aligned_write_uint32 ++ ++#else ++ ++#define ngx_spdy_frame_write_uint16(p, s) \ ++ ((p)[0] = (u_char) ((s) >> 8), \ ++ (p)[1] = (u_char) (s), \ ++ (p) + sizeof(uint16_t)) ++ ++#define ngx_spdy_frame_write_uint32(p, s) \ ++ ((p)[0] = (u_char) ((s) >> 24), \ ++ (p)[1] = (u_char) ((s) >> 16), \ ++ (p)[2] = (u_char) ((s) >> 8), \ ++ (p)[3] = (u_char) (s), \ ++ (p) + sizeof(uint32_t)) ++ ++#endif ++ ++ ++#define ngx_spdy_ctl_frame_head(t) \ ++ ((uint32_t) NGX_SPDY_CTL_BIT << 31 | NGX_SPDY_VERSION << 16 | (t)) ++ ++#define ngx_spdy_frame_write_head(p, t) \ ++ ngx_spdy_frame_aligned_write_uint32(p, ngx_spdy_ctl_frame_head(t)) ++ ++#define ngx_spdy_frame_write_flags_and_len(p, f, l) \ ++ ngx_spdy_frame_aligned_write_uint32(p, (f) << 24 | (l)) ++#define ngx_spdy_frame_write_flags_and_id(p, f, i) \ ++ ngx_spdy_frame_aligned_write_uint32(p, (f) << 24 | (i)) ++ ++#define ngx_spdy_frame_write_sid ngx_spdy_frame_aligned_write_uint32 ++#define ngx_spdy_frame_write_window ngx_spdy_frame_aligned_write_uint32 ++ ++#endif /* _NGX_HTTP_SPDY_H_INCLUDED_ */ +diff --git a/src/http/ngx_http_spdy_filter_module.c b/src/http/ngx_http_spdy_filter_module.c +new file mode 100644 +index 0000000..377e935 +--- /dev/null ++++ b/src/http/ngx_http_spdy_filter_module.c +@@ -0,0 +1,1222 @@ ++ ++/* ++ * Copyright (C) Nginx, Inc. ++ * Copyright (C) Valentin V. Bartenev ++ */ ++ ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++ ++#define ngx_http_spdy_nv_nsize(h) (NGX_SPDY_NV_NLEN_SIZE + sizeof(h) - 1) ++#define ngx_http_spdy_nv_vsize(h) (NGX_SPDY_NV_VLEN_SIZE + sizeof(h) - 1) ++ ++#define ngx_http_spdy_nv_write_num ngx_spdy_frame_write_uint32 ++#define ngx_http_spdy_nv_write_nlen ngx_spdy_frame_write_uint32 ++#define ngx_http_spdy_nv_write_vlen ngx_spdy_frame_write_uint32 ++ ++#define ngx_http_spdy_nv_write_name(p, h) \ ++ ngx_cpymem(ngx_http_spdy_nv_write_nlen(p, sizeof(h) - 1), h, sizeof(h) - 1) ++ ++#define ngx_http_spdy_nv_write_val(p, h) \ ++ ngx_cpymem(ngx_http_spdy_nv_write_vlen(p, sizeof(h) - 1), h, sizeof(h) - 1) ++ ++ ++static ngx_chain_t *ngx_http_spdy_send_chain(ngx_connection_t *fc, ++ ngx_chain_t *in, off_t limit); ++ ++static ngx_inline ngx_int_t ngx_http_spdy_filter_send( ++ ngx_connection_t *fc, ngx_http_spdy_stream_t *stream); ++static ngx_inline ngx_int_t ngx_http_spdy_flow_control( ++ ngx_http_spdy_connection_t *sc, ngx_http_spdy_stream_t *stream); ++static void ngx_http_spdy_waiting_queue(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_stream_t *stream); ++ ++static ngx_chain_t *ngx_http_spdy_filter_get_shadow( ++ ngx_http_spdy_stream_t *stream, ngx_buf_t *buf, off_t offset, off_t size); ++static ngx_http_spdy_out_frame_t *ngx_http_spdy_filter_get_data_frame( ++ ngx_http_spdy_stream_t *stream, size_t len, ngx_chain_t *first, ++ ngx_chain_t *last); ++ ++static ngx_int_t ngx_http_spdy_syn_frame_handler( ++ ngx_http_spdy_connection_t *sc, ngx_http_spdy_out_frame_t *frame); ++static ngx_int_t ngx_http_spdy_data_frame_handler( ++ ngx_http_spdy_connection_t *sc, ngx_http_spdy_out_frame_t *frame); ++static ngx_inline void ngx_http_spdy_handle_frame( ++ ngx_http_spdy_stream_t *stream, ngx_http_spdy_out_frame_t *frame); ++static ngx_inline void ngx_http_spdy_handle_stream( ++ ngx_http_spdy_connection_t *sc, ngx_http_spdy_stream_t *stream); ++ ++static void ngx_http_spdy_filter_cleanup(void *data); ++ ++static ngx_int_t ngx_http_spdy_filter_init(ngx_conf_t *cf); ++ ++ ++static ngx_http_module_t ngx_http_spdy_filter_module_ctx = { ++ NULL, /* preconfiguration */ ++ ngx_http_spdy_filter_init, /* postconfiguration */ ++ ++ NULL, /* create main configuration */ ++ NULL, /* init main configuration */ ++ ++ NULL, /* create server configuration */ ++ NULL, /* merge server configuration */ ++ ++ NULL, /* create location configuration */ ++ NULL /* merge location configuration */ ++}; ++ ++ ++ngx_module_t ngx_http_spdy_filter_module = { ++ NGX_MODULE_V1, ++ &ngx_http_spdy_filter_module_ctx, /* module context */ ++ NULL, /* module directives */ ++ NGX_HTTP_MODULE, /* module type */ ++ NULL, /* init master */ ++ NULL, /* init module */ ++ NULL, /* init process */ ++ NULL, /* init thread */ ++ NULL, /* exit thread */ ++ NULL, /* exit process */ ++ NULL, /* exit master */ ++ NGX_MODULE_V1_PADDING ++}; ++ ++ ++static ngx_http_output_header_filter_pt ngx_http_next_header_filter; ++ ++ ++static ngx_int_t ++ngx_http_spdy_header_filter(ngx_http_request_t *r) ++{ ++ int rc; ++ size_t len; ++ u_char *p, *buf, *last; ++ ngx_buf_t *b; ++ ngx_str_t host; ++ ngx_uint_t i, j, count, port; ++ ngx_chain_t *cl; ++ ngx_list_part_t *part, *pt; ++ ngx_table_elt_t *header, *h; ++ ngx_connection_t *c; ++ ngx_http_cleanup_t *cln; ++ ngx_http_core_loc_conf_t *clcf; ++ ngx_http_core_srv_conf_t *cscf; ++ ngx_http_spdy_stream_t *stream; ++ ngx_http_spdy_out_frame_t *frame; ++ ngx_http_spdy_connection_t *sc; ++ struct sockaddr_in *sin; ++#if (NGX_HAVE_INET6) ++ struct sockaddr_in6 *sin6; ++#endif ++ u_char addr[NGX_SOCKADDR_STRLEN]; ++ ++ if (!r->spdy_stream) { ++ return ngx_http_next_header_filter(r); ++ } ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, ++ "spdy header filter"); ++ ++ if (r->header_sent) { ++ return NGX_OK; ++ } ++ ++ r->header_sent = 1; ++ ++ if (r != r->main) { ++ return NGX_OK; ++ } ++ ++ c = r->connection; ++ ++ if (r->method == NGX_HTTP_HEAD) { ++ r->header_only = 1; ++ } ++ ++ switch (r->headers_out.status) { ++ ++ case NGX_HTTP_OK: ++ case NGX_HTTP_PARTIAL_CONTENT: ++ break; ++ ++ case NGX_HTTP_NOT_MODIFIED: ++ r->header_only = 1; ++ break; ++ ++ case NGX_HTTP_NO_CONTENT: ++ r->header_only = 1; ++ ++ ngx_str_null(&r->headers_out.content_type); ++ ++ r->headers_out.content_length = NULL; ++ r->headers_out.content_length_n = -1; ++ ++ /* fall through */ ++ ++ default: ++ r->headers_out.last_modified_time = -1; ++ r->headers_out.last_modified = NULL; ++ } ++ ++ len = NGX_SPDY_NV_NUM_SIZE ++ + ngx_http_spdy_nv_nsize(":version") ++ + ngx_http_spdy_nv_vsize("HTTP/1.1") ++ + ngx_http_spdy_nv_nsize(":status") ++ + (r->headers_out.status_line.len ++ ? NGX_SPDY_NV_VLEN_SIZE + r->headers_out.status_line.len ++ : ngx_http_spdy_nv_vsize("418")); ++ ++ clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); ++ ++ if (r->headers_out.server == NULL) { ++ len += ngx_http_spdy_nv_nsize("server"); ++ len += clcf->server_tokens ? ngx_http_spdy_nv_vsize(NGINX_VER) ++ : ngx_http_spdy_nv_vsize("nginx"); ++ } ++ ++ if (r->headers_out.date == NULL) { ++ len += ngx_http_spdy_nv_nsize("date") ++ + ngx_http_spdy_nv_vsize("Wed, 31 Dec 1986 10:00:00 GMT"); ++ } ++ ++ if (r->headers_out.content_type.len) { ++ len += ngx_http_spdy_nv_nsize("content-type") ++ + NGX_SPDY_NV_VLEN_SIZE + r->headers_out.content_type.len; ++ ++ if (r->headers_out.content_type_len == r->headers_out.content_type.len ++ && r->headers_out.charset.len) ++ { ++ len += sizeof("; charset=") - 1 + r->headers_out.charset.len; ++ } ++ } ++ ++ if (r->headers_out.content_length == NULL ++ && r->headers_out.content_length_n >= 0) ++ { ++ len += ngx_http_spdy_nv_nsize("content-length") ++ + NGX_SPDY_NV_VLEN_SIZE + NGX_OFF_T_LEN; ++ } ++ ++ if (r->headers_out.last_modified == NULL ++ && r->headers_out.last_modified_time != -1) ++ { ++ len += ngx_http_spdy_nv_nsize("last-modified") ++ + ngx_http_spdy_nv_vsize("Wed, 31 Dec 1986 10:00:00 GMT"); ++ } ++ ++ if (r->headers_out.location ++ && r->headers_out.location->value.len ++ && r->headers_out.location->value.data[0] == '/') ++ { ++ r->headers_out.location->hash = 0; ++ ++ if (clcf->server_name_in_redirect) { ++ cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module); ++ host = cscf->server_name; ++ ++ } else if (r->headers_in.server.len) { ++ host = r->headers_in.server; ++ ++ } else { ++ host.len = NGX_SOCKADDR_STRLEN; ++ host.data = addr; ++ ++ if (ngx_connection_local_sockaddr(c, &host, 0) != NGX_OK) { ++ return NGX_ERROR; ++ } ++ } ++ ++ switch (c->local_sockaddr->sa_family) { ++ ++#if (NGX_HAVE_INET6) ++ case AF_INET6: ++ sin6 = (struct sockaddr_in6 *) c->local_sockaddr; ++ port = ntohs(sin6->sin6_port); ++ break; ++#endif ++#if (NGX_HAVE_UNIX_DOMAIN) ++ case AF_UNIX: ++ port = 0; ++ break; ++#endif ++ default: /* AF_INET */ ++ sin = (struct sockaddr_in *) c->local_sockaddr; ++ port = ntohs(sin->sin_port); ++ break; ++ } ++ ++ len += ngx_http_spdy_nv_nsize("location") ++ + ngx_http_spdy_nv_vsize("https://") ++ + host.len ++ + r->headers_out.location->value.len; ++ ++ if (clcf->port_in_redirect) { ++ ++#if (NGX_HTTP_SSL) ++ if (c->ssl) ++ port = (port == 443) ? 0 : port; ++ else ++#endif ++ port = (port == 80) ? 0 : port; ++ ++ } else { ++ port = 0; ++ } ++ ++ if (port) { ++ len += sizeof(":65535") - 1; ++ } ++ ++ } else { ++ ngx_str_null(&host); ++ port = 0; ++ } ++ ++ part = &r->headers_out.headers.part; ++ header = part->elts; ++ ++ for (i = 0; /* void */; i++) { ++ ++ if (i >= part->nelts) { ++ if (part->next == NULL) { ++ break; ++ } ++ ++ part = part->next; ++ header = part->elts; ++ i = 0; ++ } ++ ++ if (header[i].hash == 0) { ++ continue; ++ } ++ ++ len += NGX_SPDY_NV_NLEN_SIZE + header[i].key.len ++ + NGX_SPDY_NV_VLEN_SIZE + header[i].value.len; ++ } ++ ++ buf = ngx_alloc(len, r->pool->log); ++ if (buf == NULL) { ++ return NGX_ERROR; ++ } ++ ++ last = buf + NGX_SPDY_NV_NUM_SIZE; ++ ++ last = ngx_http_spdy_nv_write_name(last, ":version"); ++ last = ngx_http_spdy_nv_write_val(last, "HTTP/1.1"); ++ ++ last = ngx_http_spdy_nv_write_name(last, ":status"); ++ ++ if (r->headers_out.status_line.len) { ++ last = ngx_http_spdy_nv_write_vlen(last, ++ r->headers_out.status_line.len); ++ last = ngx_cpymem(last, r->headers_out.status_line.data, ++ r->headers_out.status_line.len); ++ } else { ++ last = ngx_http_spdy_nv_write_vlen(last, 3); ++ last = ngx_sprintf(last, "%03ui", r->headers_out.status); ++ } ++ ++ count = 2; ++ ++ if (r->headers_out.server == NULL) { ++ last = ngx_http_spdy_nv_write_name(last, "server"); ++ last = clcf->server_tokens ++ ? ngx_http_spdy_nv_write_val(last, NGINX_VER) ++ : ngx_http_spdy_nv_write_val(last, "nginx"); ++ ++ count++; ++ } ++ ++ if (r->headers_out.date == NULL) { ++ last = ngx_http_spdy_nv_write_name(last, "date"); ++ ++ last = ngx_http_spdy_nv_write_vlen(last, ngx_cached_http_time.len); ++ ++ last = ngx_cpymem(last, ngx_cached_http_time.data, ++ ngx_cached_http_time.len); ++ ++ count++; ++ } ++ ++ if (r->headers_out.content_type.len) { ++ ++ last = ngx_http_spdy_nv_write_name(last, "content-type"); ++ ++ p = last + NGX_SPDY_NV_VLEN_SIZE; ++ ++ last = ngx_cpymem(p, r->headers_out.content_type.data, ++ r->headers_out.content_type.len); ++ ++ if (r->headers_out.content_type_len == r->headers_out.content_type.len ++ && r->headers_out.charset.len) ++ { ++ last = ngx_cpymem(last, "; charset=", sizeof("; charset=") - 1); ++ ++ last = ngx_cpymem(last, r->headers_out.charset.data, ++ r->headers_out.charset.len); ++ ++ /* update r->headers_out.content_type for possible logging */ ++ ++ r->headers_out.content_type.len = last - p; ++ r->headers_out.content_type.data = p; ++ } ++ ++ (void) ngx_http_spdy_nv_write_vlen(p - NGX_SPDY_NV_VLEN_SIZE, ++ r->headers_out.content_type.len); ++ ++ count++; ++ } ++ ++ if (r->headers_out.content_length == NULL ++ && r->headers_out.content_length_n >= 0) ++ { ++ last = ngx_http_spdy_nv_write_name(last, "content-length"); ++ ++ p = last + NGX_SPDY_NV_VLEN_SIZE; ++ ++ last = ngx_sprintf(p, "%O", r->headers_out.content_length_n); ++ ++ (void) ngx_http_spdy_nv_write_vlen(p - NGX_SPDY_NV_VLEN_SIZE, ++ last - p); ++ ++ count++; ++ } ++ ++ if (r->headers_out.last_modified == NULL ++ && r->headers_out.last_modified_time != -1) ++ { ++ last = ngx_http_spdy_nv_write_name(last, "last-modified"); ++ ++ p = last + NGX_SPDY_NV_VLEN_SIZE; ++ ++ last = ngx_http_time(p, r->headers_out.last_modified_time); ++ ++ (void) ngx_http_spdy_nv_write_vlen(p - NGX_SPDY_NV_VLEN_SIZE, ++ last - p); ++ ++ count++; ++ } ++ ++ if (host.data) { ++ ++ last = ngx_http_spdy_nv_write_name(last, "location"); ++ ++ p = last + NGX_SPDY_NV_VLEN_SIZE; ++ ++ last = ngx_cpymem(p, "http", sizeof("http") - 1); ++ ++#if (NGX_HTTP_SSL) ++ if (c->ssl) { ++ *last++ ='s'; ++ } ++#endif ++ ++ *last++ = ':'; *last++ = '/'; *last++ = '/'; ++ ++ last = ngx_cpymem(last, host.data, host.len); ++ ++ if (port) { ++ last = ngx_sprintf(last, ":%ui", port); ++ } ++ ++ last = ngx_cpymem(last, r->headers_out.location->value.data, ++ r->headers_out.location->value.len); ++ ++ /* update r->headers_out.location->value for possible logging */ ++ ++ r->headers_out.location->value.len = last - p; ++ r->headers_out.location->value.data = p; ++ ngx_str_set(&r->headers_out.location->key, "location"); ++ ++ (void) ngx_http_spdy_nv_write_vlen(p - NGX_SPDY_NV_VLEN_SIZE, ++ r->headers_out.location->value.len); ++ ++ count++; ++ } ++ ++ part = &r->headers_out.headers.part; ++ header = part->elts; ++ ++ for (i = 0; /* void */; i++) { ++ ++ if (i >= part->nelts) { ++ if (part->next == NULL) { ++ break; ++ } ++ ++ part = part->next; ++ header = part->elts; ++ i = 0; ++ } ++ ++ if (header[i].hash == 0 || header[i].hash == 2) { ++ continue; ++ } ++ ++ last = ngx_http_spdy_nv_write_nlen(last, header[i].key.len); ++ ++ ngx_strlow(last, header[i].key.data, header[i].key.len); ++ last += header[i].key.len; ++ ++ p = last + NGX_SPDY_NV_VLEN_SIZE; ++ ++ last = ngx_cpymem(p, header[i].value.data, header[i].value.len); ++ ++ pt = part; ++ h = header; ++ ++ for (j = i + 1; /* void */; j++) { ++ ++ if (j >= pt->nelts) { ++ if (pt->next == NULL) { ++ break; ++ } ++ ++ pt = pt->next; ++ h = pt->elts; ++ j = 0; ++ } ++ ++ if (h[j].hash == 0 || h[j].hash == 2 ++ || h[j].key.len != header[i].key.len ++ || ngx_strncasecmp(header[i].key.data, h[j].key.data, ++ header[i].key.len)) ++ { ++ continue; ++ } ++ ++ if (h[j].value.len) { ++ if (last != p) { ++ *last++ = '\0'; ++ } ++ ++ last = ngx_cpymem(last, h[j].value.data, h[j].value.len); ++ } ++ ++ h[j].hash = 2; ++ } ++ ++ (void) ngx_http_spdy_nv_write_vlen(p - NGX_SPDY_NV_VLEN_SIZE, ++ last - p); ++ ++ count++; ++ } ++ ++ (void) ngx_http_spdy_nv_write_num(buf, count); ++ ++ stream = r->spdy_stream; ++ sc = stream->connection; ++ ++ len = last - buf; ++ ++ b = ngx_create_temp_buf(r->pool, NGX_SPDY_FRAME_HEADER_SIZE ++ + NGX_SPDY_SYN_REPLY_SIZE ++ + deflateBound(&sc->zstream_out, len)); ++ if (b == NULL) { ++ ngx_free(buf); ++ return NGX_ERROR; ++ } ++ ++ b->last += NGX_SPDY_FRAME_HEADER_SIZE + NGX_SPDY_SYN_REPLY_SIZE; ++ ++ sc->zstream_out.next_in = buf; ++ sc->zstream_out.avail_in = len; ++ sc->zstream_out.next_out = b->last; ++ sc->zstream_out.avail_out = b->end - b->last; ++ ++ rc = deflate(&sc->zstream_out, Z_SYNC_FLUSH); ++ ++ ngx_free(buf); ++ ++ if (rc != Z_OK) { ++ ngx_log_error(NGX_LOG_ALERT, c->log, 0, "deflate() failed: %d", rc); ++ return NGX_ERROR; ++ } ++ ++ ngx_log_debug5(NGX_LOG_DEBUG_HTTP, c->log, 0, ++ "spdy deflate out: ni:%p no:%p ai:%ud ao:%ud rc:%d", ++ sc->zstream_out.next_in, sc->zstream_out.next_out, ++ sc->zstream_out.avail_in, sc->zstream_out.avail_out, ++ rc); ++ ++ b->last = sc->zstream_out.next_out; ++ ++ p = b->pos; ++ p = ngx_spdy_frame_write_head(p, NGX_SPDY_SYN_REPLY); ++ ++ len = b->last - b->pos; ++ ++ r->header_size = len; ++ ++ len -= NGX_SPDY_FRAME_HEADER_SIZE; ++ ++ if (r->header_only) { ++ b->last_buf = 1; ++ p = ngx_spdy_frame_write_flags_and_len(p, NGX_SPDY_FLAG_FIN, len); ++ ++ } else { ++ p = ngx_spdy_frame_write_flags_and_len(p, 0, len); ++ } ++ ++ (void) ngx_spdy_frame_write_sid(p, stream->id); ++ ++ cl = ngx_alloc_chain_link(r->pool); ++ if (cl == NULL) { ++ return NGX_ERROR; ++ } ++ ++ cl->buf = b; ++ cl->next = NULL; ++ ++ frame = ngx_palloc(r->pool, sizeof(ngx_http_spdy_out_frame_t)); ++ if (frame == NULL) { ++ return NGX_ERROR; ++ } ++ ++ frame->first = cl; ++ frame->last = cl; ++ frame->handler = ngx_http_spdy_syn_frame_handler; ++ frame->stream = stream; ++ frame->length = len; ++ frame->priority = stream->priority; ++ frame->blocked = 1; ++ frame->fin = r->header_only; ++ ++ ngx_log_debug3(NGX_LOG_DEBUG_HTTP, stream->request->connection->log, 0, ++ "spdy:%ui create SYN_REPLY frame %p: len:%uz", ++ stream->id, frame, frame->length); ++ ++ ngx_http_spdy_queue_blocked_frame(sc, frame); ++ ++ cln = ngx_http_cleanup_add(r, 0); ++ if (cln == NULL) { ++ return NGX_ERROR; ++ } ++ ++ cln->handler = ngx_http_spdy_filter_cleanup; ++ cln->data = stream; ++ ++ stream->queued = 1; ++ ++ c->send_chain = ngx_http_spdy_send_chain; ++ c->need_last_buf = 1; ++ ++ return ngx_http_spdy_filter_send(c, stream); ++} ++ ++ ++static ngx_chain_t * ++ngx_http_spdy_send_chain(ngx_connection_t *fc, ngx_chain_t *in, off_t limit) ++{ ++ off_t size, offset; ++ size_t rest, frame_size; ++ ngx_chain_t *cl, *out, **ln; ++ ngx_http_request_t *r; ++ ngx_http_spdy_stream_t *stream; ++ ngx_http_spdy_loc_conf_t *slcf; ++ ngx_http_spdy_out_frame_t *frame; ++ ngx_http_spdy_connection_t *sc; ++ ++ r = fc->data; ++ stream = r->spdy_stream; ++ ++#if (NGX_SUPPRESS_WARN) ++ size = 0; ++#endif ++ ++ while (in) { ++ size = ngx_buf_size(in->buf); ++ ++ if (size || in->buf->last_buf) { ++ break; ++ } ++ ++ in = in->next; ++ } ++ ++ if (in == NULL) { ++ ++ if (stream->queued) { ++ fc->write->delayed = 1; ++ } else { ++ fc->buffered &= ~NGX_SPDY_BUFFERED; ++ } ++ ++ return NULL; ++ } ++ ++ sc = stream->connection; ++ ++ if (size && ngx_http_spdy_flow_control(sc, stream) == NGX_DECLINED) { ++ fc->write->delayed = 1; ++ return in; ++ } ++ ++ if (limit == 0 || limit > (off_t) sc->send_window) { ++ limit = sc->send_window; ++ } ++ ++ if (limit > stream->send_window) { ++ limit = (stream->send_window > 0) ? stream->send_window : 0; ++ } ++ ++ if (in->buf->tag == (ngx_buf_tag_t) &ngx_http_spdy_filter_get_shadow) { ++ cl = ngx_alloc_chain_link(r->pool); ++ if (cl == NULL) { ++ return NGX_CHAIN_ERROR; ++ } ++ ++ cl->buf = in->buf; ++ in->buf = cl->buf->shadow; ++ ++ offset = ngx_buf_in_memory(in->buf) ++ ? (cl->buf->pos - in->buf->pos) ++ : (cl->buf->file_pos - in->buf->file_pos); ++ ++ cl->next = stream->free_bufs; ++ stream->free_bufs = cl; ++ ++ } else { ++ offset = 0; ++ } ++ ++#if (NGX_SUPPRESS_WARN) ++ cl = NULL; ++#endif ++ ++ slcf = ngx_http_get_module_loc_conf(r, ngx_http_spdy_module); ++ ++ frame_size = (limit <= (off_t) slcf->chunk_size) ? (size_t) limit ++ : slcf->chunk_size; ++ ++ for ( ;; ) { ++ ln = &out; ++ rest = frame_size; ++ ++ while ((off_t) rest >= size) { ++ ++ if (offset) { ++ cl = ngx_http_spdy_filter_get_shadow(stream, in->buf, ++ offset, size); ++ if (cl == NULL) { ++ return NGX_CHAIN_ERROR; ++ } ++ ++ offset = 0; ++ ++ } else { ++ cl = ngx_alloc_chain_link(r->pool); ++ if (cl == NULL) { ++ return NGX_CHAIN_ERROR; ++ } ++ ++ cl->buf = in->buf; ++ } ++ ++ *ln = cl; ++ ln = &cl->next; ++ ++ rest -= (size_t) size; ++ in = in->next; ++ ++ if (in == NULL) { ++ frame_size -= rest; ++ rest = 0; ++ break; ++ } ++ ++ size = ngx_buf_size(in->buf); ++ } ++ ++ if (rest) { ++ cl = ngx_http_spdy_filter_get_shadow(stream, in->buf, ++ offset, rest); ++ if (cl == NULL) { ++ return NGX_CHAIN_ERROR; ++ } ++ ++ cl->buf->flush = 0; ++ cl->buf->last_buf = 0; ++ ++ *ln = cl; ++ ++ offset += rest; ++ size -= rest; ++ } ++ ++ frame = ngx_http_spdy_filter_get_data_frame(stream, frame_size, ++ out, cl); ++ if (frame == NULL) { ++ return NGX_CHAIN_ERROR; ++ } ++ ++ ngx_http_spdy_queue_frame(sc, frame); ++ ++ sc->send_window -= frame_size; ++ ++ stream->send_window -= frame_size; ++ stream->queued++; ++ ++ if (in == NULL) { ++ break; ++ } ++ ++ limit -= frame_size; ++ ++ if (limit == 0) { ++ break; ++ } ++ ++ if (limit < (off_t) slcf->chunk_size) { ++ frame_size = (size_t) limit; ++ } ++ } ++ ++ if (offset) { ++ cl = ngx_http_spdy_filter_get_shadow(stream, in->buf, offset, size); ++ if (cl == NULL) { ++ return NGX_CHAIN_ERROR; ++ } ++ ++ in->buf = cl->buf; ++ ngx_free_chain(r->pool, cl); ++ } ++ ++ if (ngx_http_spdy_filter_send(fc, stream) == NGX_ERROR) { ++ return NGX_CHAIN_ERROR; ++ } ++ ++ if (in && ngx_http_spdy_flow_control(sc, stream) == NGX_DECLINED) { ++ fc->write->delayed = 1; ++ } ++ ++ return in; ++} ++ ++ ++static ngx_chain_t * ++ngx_http_spdy_filter_get_shadow(ngx_http_spdy_stream_t *stream, ngx_buf_t *buf, ++ off_t offset, off_t size) ++{ ++ ngx_buf_t *chunk; ++ ngx_chain_t *cl; ++ ++ cl = ngx_chain_get_free_buf(stream->request->pool, &stream->free_bufs); ++ if (cl == NULL) { ++ return NULL; ++ } ++ ++ chunk = cl->buf; ++ ++ ngx_memcpy(chunk, buf, sizeof(ngx_buf_t)); ++ ++ chunk->tag = (ngx_buf_tag_t) &ngx_http_spdy_filter_get_shadow; ++ chunk->shadow = buf; ++ ++ if (ngx_buf_in_memory(chunk)) { ++ chunk->pos += offset; ++ chunk->last = chunk->pos + size; ++ } ++ ++ if (chunk->in_file) { ++ chunk->file_pos += offset; ++ chunk->file_last = chunk->file_pos + size; ++ } ++ ++ return cl; ++} ++ ++ ++static ngx_http_spdy_out_frame_t * ++ngx_http_spdy_filter_get_data_frame(ngx_http_spdy_stream_t *stream, ++ size_t len, ngx_chain_t *first, ngx_chain_t *last) ++{ ++ u_char *p; ++ ngx_buf_t *buf; ++ ngx_uint_t flags; ++ ngx_chain_t *cl; ++ ngx_http_spdy_out_frame_t *frame; ++ ++ ++ frame = stream->free_frames; ++ ++ if (frame) { ++ stream->free_frames = frame->next; ++ ++ } else { ++ frame = ngx_palloc(stream->request->pool, ++ sizeof(ngx_http_spdy_out_frame_t)); ++ if (frame == NULL) { ++ return NULL; ++ } ++ } ++ ++ flags = last->buf->last_buf ? NGX_SPDY_FLAG_FIN : 0; ++ ++ ngx_log_debug4(NGX_LOG_DEBUG_HTTP, stream->request->connection->log, 0, ++ "spdy:%ui create DATA frame %p: len:%uz flags:%ui", ++ stream->id, frame, len, flags); ++ ++ cl = ngx_chain_get_free_buf(stream->request->pool, ++ &stream->free_data_headers); ++ if (cl == NULL) { ++ return NULL; ++ } ++ ++ buf = cl->buf; ++ ++ if (buf->start) { ++ p = buf->start; ++ buf->pos = p; ++ ++ p += NGX_SPDY_SID_SIZE; ++ ++ (void) ngx_spdy_frame_write_flags_and_len(p, flags, len); ++ ++ } else { ++ p = ngx_palloc(stream->request->pool, NGX_SPDY_FRAME_HEADER_SIZE); ++ if (p == NULL) { ++ return NULL; ++ } ++ ++ buf->pos = p; ++ buf->start = p; ++ ++ p = ngx_spdy_frame_write_sid(p, stream->id); ++ p = ngx_spdy_frame_write_flags_and_len(p, flags, len); ++ ++ buf->last = p; ++ buf->end = p; ++ ++ buf->tag = (ngx_buf_tag_t) &ngx_http_spdy_filter_get_data_frame; ++ buf->memory = 1; ++ } ++ ++ cl->next = first; ++ first = cl; ++ ++ last->buf->flush = 1; ++ ++ frame->first = first; ++ frame->last = last; ++ frame->handler = ngx_http_spdy_data_frame_handler; ++ frame->stream = stream; ++ frame->length = len; ++ frame->priority = stream->priority; ++ frame->blocked = 0; ++ frame->fin = last->buf->last_buf; ++ ++ return frame; ++} ++ ++ ++static ngx_inline ngx_int_t ++ngx_http_spdy_filter_send(ngx_connection_t *fc, ngx_http_spdy_stream_t *stream) ++{ ++ stream->blocked = 1; ++ ++ if (ngx_http_spdy_send_output_queue(stream->connection) == NGX_ERROR) { ++ fc->error = 1; ++ return NGX_ERROR; ++ } ++ ++ stream->blocked = 0; ++ ++ if (stream->queued) { ++ fc->buffered |= NGX_SPDY_BUFFERED; ++ fc->write->delayed = 1; ++ return NGX_AGAIN; ++ } ++ ++ fc->buffered &= ~NGX_SPDY_BUFFERED; ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_inline ngx_int_t ++ngx_http_spdy_flow_control(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_stream_t *stream) ++{ ++ if (stream->send_window <= 0) { ++ stream->exhausted = 1; ++ return NGX_DECLINED; ++ } ++ ++ if (sc->send_window == 0) { ++ ngx_http_spdy_waiting_queue(sc, stream); ++ return NGX_DECLINED; ++ } ++ ++ return NGX_OK; ++} ++ ++ ++static void ++ngx_http_spdy_waiting_queue(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_stream_t *stream) ++{ ++ ngx_queue_t *q; ++ ngx_http_spdy_stream_t *s; ++ ++ if (stream->handled) { ++ return; ++ } ++ ++ stream->handled = 1; ++ ++ for (q = ngx_queue_last(&sc->waiting); ++ q != ngx_queue_sentinel(&sc->waiting); ++ q = ngx_queue_prev(q)) ++ { ++ s = ngx_queue_data(q, ngx_http_spdy_stream_t, queue); ++ ++ /* ++ * NB: higher values represent lower priorities. ++ */ ++ if (stream->priority >= s->priority) { ++ break; ++ } ++ } ++ ++ ngx_queue_insert_after(q, &stream->queue); ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_syn_frame_handler(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_out_frame_t *frame) ++{ ++ ngx_buf_t *buf; ++ ngx_http_spdy_stream_t *stream; ++ ++ buf = frame->first->buf; ++ ++ if (buf->pos != buf->last) { ++ return NGX_AGAIN; ++ } ++ ++ stream = frame->stream; ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy:%ui SYN_REPLY frame %p was sent", stream->id, frame); ++ ++ ngx_free_chain(stream->request->pool, frame->first); ++ ++ ngx_http_spdy_handle_frame(stream, frame); ++ ++ ngx_http_spdy_handle_stream(sc, stream); ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_data_frame_handler(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_out_frame_t *frame) ++{ ++ ngx_buf_t *buf; ++ ngx_chain_t *cl, *ln; ++ ngx_http_spdy_stream_t *stream; ++ ++ stream = frame->stream; ++ ++ cl = frame->first; ++ ++ if (cl->buf->tag == (ngx_buf_tag_t) &ngx_http_spdy_filter_get_data_frame) { ++ ++ if (cl->buf->pos != cl->buf->last) { ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy:%ui DATA frame %p was sent partially", ++ stream->id, frame); ++ ++ return NGX_AGAIN; ++ } ++ ++ ln = cl->next; ++ ++ cl->next = stream->free_data_headers; ++ stream->free_data_headers = cl; ++ ++ if (cl == frame->last) { ++ goto done; ++ } ++ ++ cl = ln; ++ } ++ ++ for ( ;; ) { ++ if (cl->buf->tag == (ngx_buf_tag_t) &ngx_http_spdy_filter_get_shadow) { ++ buf = cl->buf->shadow; ++ ++ if (ngx_buf_in_memory(buf)) { ++ buf->pos = cl->buf->pos; ++ } ++ ++ if (buf->in_file) { ++ buf->file_pos = cl->buf->file_pos; ++ } ++ } ++ ++ if (ngx_buf_size(cl->buf) != 0) { ++ ++ if (cl != frame->first) { ++ frame->first = cl; ++ ngx_http_spdy_handle_stream(sc, stream); ++ } ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy:%ui DATA frame %p was sent partially", ++ stream->id, frame); ++ ++ return NGX_AGAIN; ++ } ++ ++ ln = cl->next; ++ ++ if (cl->buf->tag == (ngx_buf_tag_t) &ngx_http_spdy_filter_get_shadow) { ++ cl->next = stream->free_bufs; ++ stream->free_bufs = cl; ++ ++ } else { ++ ngx_free_chain(stream->request->pool, cl); ++ } ++ ++ if (cl == frame->last) { ++ goto done; ++ } ++ ++ cl = ln; ++ } ++ ++done: ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, ++ "spdy:%ui DATA frame %p was sent", stream->id, frame); ++ ++ stream->request->header_size += NGX_SPDY_FRAME_HEADER_SIZE; ++ ++ ngx_http_spdy_handle_frame(stream, frame); ++ ++ ngx_http_spdy_handle_stream(sc, stream); ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_inline void ++ngx_http_spdy_handle_frame(ngx_http_spdy_stream_t *stream, ++ ngx_http_spdy_out_frame_t *frame) ++{ ++ ngx_http_request_t *r; ++ ++ r = stream->request; ++ ++ r->connection->sent += NGX_SPDY_FRAME_HEADER_SIZE + frame->length; ++ ++ if (frame->fin) { ++ stream->out_closed = 1; ++ } ++ ++ frame->next = stream->free_frames; ++ stream->free_frames = frame; ++ ++ stream->queued--; ++} ++ ++ ++static ngx_inline void ++ngx_http_spdy_handle_stream(ngx_http_spdy_connection_t *sc, ++ ngx_http_spdy_stream_t *stream) ++{ ++ ngx_event_t *wev; ++ ++ if (stream->handled || stream->blocked || stream->exhausted) { ++ return; ++ } ++ ++ wev = stream->request->connection->write; ++ ++ /* ++ * This timer can only be set if the stream was delayed because of rate ++ * limit. In that case the event should be triggered by the timer. ++ */ ++ ++ if (!wev->timer_set) { ++ wev->delayed = 0; ++ ++ stream->handled = 1; ++ ngx_queue_insert_tail(&sc->posted, &stream->queue); ++ } ++} ++ ++ ++static void ++ngx_http_spdy_filter_cleanup(void *data) ++{ ++ ngx_http_spdy_stream_t *stream = data; ++ ++ size_t delta; ++ ngx_http_spdy_out_frame_t *frame, **fn; ++ ngx_http_spdy_connection_t *sc; ++ ++ if (stream->handled) { ++ stream->handled = 0; ++ ngx_queue_remove(&stream->queue); ++ } ++ ++ if (stream->queued == 0) { ++ return; ++ } ++ ++ delta = 0; ++ sc = stream->connection; ++ fn = &sc->last_out; ++ ++ for ( ;; ) { ++ frame = *fn; ++ ++ if (frame == NULL) { ++ break; ++ } ++ ++ if (frame->stream == stream && !frame->blocked) { ++ *fn = frame->next; ++ ++ delta += frame->length; ++ ++ if (--stream->queued == 0) { ++ break; ++ } ++ ++ continue; ++ } ++ ++ fn = &frame->next; ++ } ++ ++ if (sc->send_window == 0 && delta && !ngx_queue_empty(&sc->waiting)) { ++ ngx_queue_add(&sc->posted, &sc->waiting); ++ ngx_queue_init(&sc->waiting); ++ } ++ ++ sc->send_window += delta; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_filter_init(ngx_conf_t *cf) ++{ ++ ngx_http_next_header_filter = ngx_http_top_header_filter; ++ ngx_http_top_header_filter = ngx_http_spdy_header_filter; ++ ++ return NGX_OK; ++} +diff --git a/src/http/ngx_http_spdy_module.c b/src/http/ngx_http_spdy_module.c +new file mode 100644 +index 0000000..5178a36 +--- /dev/null ++++ b/src/http/ngx_http_spdy_module.c +@@ -0,0 +1,408 @@ ++ ++/* ++ * Copyright (C) Nginx, Inc. ++ * Copyright (C) Valentin V. Bartenev ++ */ ++ ++ ++#include ++#include ++#include ++#include ++ ++ ++static ngx_int_t ngx_http_spdy_add_variables(ngx_conf_t *cf); ++ ++static ngx_int_t ngx_http_spdy_variable(ngx_http_request_t *r, ++ ngx_http_variable_value_t *v, uintptr_t data); ++static ngx_int_t ngx_http_spdy_request_priority_variable(ngx_http_request_t *r, ++ ngx_http_variable_value_t *v, uintptr_t data); ++ ++static ngx_int_t ngx_http_spdy_module_init(ngx_cycle_t *cycle); ++ ++static void *ngx_http_spdy_create_main_conf(ngx_conf_t *cf); ++static char *ngx_http_spdy_init_main_conf(ngx_conf_t *cf, void *conf); ++static void *ngx_http_spdy_create_srv_conf(ngx_conf_t *cf); ++static char *ngx_http_spdy_merge_srv_conf(ngx_conf_t *cf, void *parent, ++ void *child); ++static void *ngx_http_spdy_create_loc_conf(ngx_conf_t *cf); ++static char *ngx_http_spdy_merge_loc_conf(ngx_conf_t *cf, void *parent, ++ void *child); ++ ++static char *ngx_http_spdy_recv_buffer_size(ngx_conf_t *cf, void *post, ++ void *data); ++static char *ngx_http_spdy_pool_size(ngx_conf_t *cf, void *post, void *data); ++static char *ngx_http_spdy_streams_index_mask(ngx_conf_t *cf, void *post, ++ void *data); ++static char *ngx_http_spdy_chunk_size(ngx_conf_t *cf, void *post, void *data); ++ ++ ++static ngx_conf_num_bounds_t ngx_http_spdy_headers_comp_bounds = { ++ ngx_conf_check_num_bounds, 0, 9 ++}; ++ ++static ngx_conf_post_t ngx_http_spdy_recv_buffer_size_post = ++ { ngx_http_spdy_recv_buffer_size }; ++static ngx_conf_post_t ngx_http_spdy_pool_size_post = ++ { ngx_http_spdy_pool_size }; ++static ngx_conf_post_t ngx_http_spdy_streams_index_mask_post = ++ { ngx_http_spdy_streams_index_mask }; ++static ngx_conf_post_t ngx_http_spdy_chunk_size_post = ++ { ngx_http_spdy_chunk_size }; ++ ++ ++static ngx_command_t ngx_http_spdy_commands[] = { ++ ++ { ngx_string("spdy_recv_buffer_size"), ++ NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1, ++ ngx_conf_set_size_slot, ++ NGX_HTTP_MAIN_CONF_OFFSET, ++ offsetof(ngx_http_spdy_main_conf_t, recv_buffer_size), ++ &ngx_http_spdy_recv_buffer_size_post }, ++ ++ { ngx_string("spdy_pool_size"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, ++ ngx_conf_set_size_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_spdy_srv_conf_t, pool_size), ++ &ngx_http_spdy_pool_size_post }, ++ ++ { ngx_string("spdy_max_concurrent_streams"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, ++ ngx_conf_set_num_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_spdy_srv_conf_t, concurrent_streams), ++ NULL }, ++ ++ { ngx_string("spdy_streams_index_size"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, ++ ngx_conf_set_num_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_spdy_srv_conf_t, streams_index_mask), ++ &ngx_http_spdy_streams_index_mask_post }, ++ ++ { ngx_string("spdy_recv_timeout"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, ++ ngx_conf_set_msec_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_spdy_srv_conf_t, recv_timeout), ++ NULL }, ++ ++ { ngx_string("spdy_keepalive_timeout"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, ++ ngx_conf_set_msec_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_spdy_srv_conf_t, keepalive_timeout), ++ NULL }, ++ ++ { ngx_string("spdy_headers_comp"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, ++ ngx_conf_set_num_slot, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_spdy_srv_conf_t, headers_comp), ++ &ngx_http_spdy_headers_comp_bounds }, ++ ++ { ngx_string("spdy_chunk_size"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, ++ ngx_conf_set_size_slot, ++ NGX_HTTP_LOC_CONF_OFFSET, ++ offsetof(ngx_http_spdy_loc_conf_t, chunk_size), ++ &ngx_http_spdy_chunk_size_post }, ++ ++ ngx_null_command ++}; ++ ++ ++static ngx_http_module_t ngx_http_spdy_module_ctx = { ++ ngx_http_spdy_add_variables, /* preconfiguration */ ++ NULL, /* postconfiguration */ ++ ++ ngx_http_spdy_create_main_conf, /* create main configuration */ ++ ngx_http_spdy_init_main_conf, /* init main configuration */ ++ ++ ngx_http_spdy_create_srv_conf, /* create server configuration */ ++ ngx_http_spdy_merge_srv_conf, /* merge server configuration */ ++ ++ ngx_http_spdy_create_loc_conf, /* create location configuration */ ++ ngx_http_spdy_merge_loc_conf /* merge location configuration */ ++}; ++ ++ ++ngx_module_t ngx_http_spdy_module = { ++ NGX_MODULE_V1, ++ &ngx_http_spdy_module_ctx, /* module context */ ++ ngx_http_spdy_commands, /* module directives */ ++ NGX_HTTP_MODULE, /* module type */ ++ NULL, /* init master */ ++ ngx_http_spdy_module_init, /* init module */ ++ NULL, /* init process */ ++ NULL, /* init thread */ ++ NULL, /* exit thread */ ++ NULL, /* exit process */ ++ NULL, /* exit master */ ++ NGX_MODULE_V1_PADDING ++}; ++ ++ ++static ngx_http_variable_t ngx_http_spdy_vars[] = { ++ ++ { ngx_string("spdy"), NULL, ++ ngx_http_spdy_variable, 0, 0, 0 }, ++ ++ { ngx_string("spdy_request_priority"), NULL, ++ ngx_http_spdy_request_priority_variable, 0, 0, 0 }, ++ ++ { ngx_null_string, NULL, NULL, 0, 0, 0 } ++}; ++ ++ ++static ngx_int_t ++ngx_http_spdy_add_variables(ngx_conf_t *cf) ++{ ++ ngx_http_variable_t *var, *v; ++ ++ for (v = ngx_http_spdy_vars; v->name.len; v++) { ++ var = ngx_http_add_variable(cf, &v->name, v->flags); ++ if (var == NULL) { ++ return NGX_ERROR; ++ } ++ ++ var->get_handler = v->get_handler; ++ var->data = v->data; ++ } ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_variable(ngx_http_request_t *r, ++ ngx_http_variable_value_t *v, uintptr_t data) ++{ ++ if (r->spdy_stream) { ++ v->len = sizeof("3.1") - 1; ++ v->valid = 1; ++ v->no_cacheable = 0; ++ v->not_found = 0; ++ v->data = (u_char *) "3.1"; ++ ++ return NGX_OK; ++ } ++ ++ *v = ngx_http_variable_null_value; ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_request_priority_variable(ngx_http_request_t *r, ++ ngx_http_variable_value_t *v, uintptr_t data) ++{ ++ if (r->spdy_stream) { ++ v->len = 1; ++ v->valid = 1; ++ v->no_cacheable = 0; ++ v->not_found = 0; ++ ++ v->data = ngx_pnalloc(r->pool, 1); ++ if (v->data == NULL) { ++ return NGX_ERROR; ++ } ++ ++ v->data[0] = '0' + (u_char) r->spdy_stream->priority; ++ ++ return NGX_OK; ++ } ++ ++ *v = ngx_http_variable_null_value; ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_http_spdy_module_init(ngx_cycle_t *cycle) ++{ ++ ngx_http_spdy_request_headers_init(); ++ ++ return NGX_OK; ++} ++ ++ ++static void * ++ngx_http_spdy_create_main_conf(ngx_conf_t *cf) ++{ ++ ngx_http_spdy_main_conf_t *smcf; ++ ++ smcf = ngx_pcalloc(cf->pool, sizeof(ngx_http_spdy_main_conf_t)); ++ if (smcf == NULL) { ++ return NULL; ++ } ++ ++ smcf->recv_buffer_size = NGX_CONF_UNSET_SIZE; ++ ++ return smcf; ++} ++ ++ ++static char * ++ngx_http_spdy_init_main_conf(ngx_conf_t *cf, void *conf) ++{ ++ ngx_http_spdy_main_conf_t *smcf = conf; ++ ++ ngx_conf_init_size_value(smcf->recv_buffer_size, 256 * 1024); ++ ++ return NGX_CONF_OK; ++} ++ ++ ++static void * ++ngx_http_spdy_create_srv_conf(ngx_conf_t *cf) ++{ ++ ngx_http_spdy_srv_conf_t *sscf; ++ ++ sscf = ngx_pcalloc(cf->pool, sizeof(ngx_http_spdy_srv_conf_t)); ++ if (sscf == NULL) { ++ return NULL; ++ } ++ ++ sscf->pool_size = NGX_CONF_UNSET_SIZE; ++ ++ sscf->concurrent_streams = NGX_CONF_UNSET_UINT; ++ sscf->streams_index_mask = NGX_CONF_UNSET_UINT; ++ ++ sscf->recv_timeout = NGX_CONF_UNSET_MSEC; ++ sscf->keepalive_timeout = NGX_CONF_UNSET_MSEC; ++ ++ sscf->headers_comp = NGX_CONF_UNSET; ++ ++ return sscf; ++} ++ ++ ++static char * ++ngx_http_spdy_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) ++{ ++ ngx_http_spdy_srv_conf_t *prev = parent; ++ ngx_http_spdy_srv_conf_t *conf = child; ++ ++ ngx_conf_merge_size_value(conf->pool_size, prev->pool_size, 4096); ++ ++ ngx_conf_merge_uint_value(conf->concurrent_streams, ++ prev->concurrent_streams, 100); ++ ++ ngx_conf_merge_uint_value(conf->streams_index_mask, ++ prev->streams_index_mask, 32 - 1); ++ ++ ngx_conf_merge_msec_value(conf->recv_timeout, ++ prev->recv_timeout, 30000); ++ ngx_conf_merge_msec_value(conf->keepalive_timeout, ++ prev->keepalive_timeout, 180000); ++ ++ ngx_conf_merge_value(conf->headers_comp, prev->headers_comp, 0); ++ ++ return NGX_CONF_OK; ++} ++ ++ ++static void * ++ngx_http_spdy_create_loc_conf(ngx_conf_t *cf) ++{ ++ ngx_http_spdy_loc_conf_t *slcf; ++ ++ slcf = ngx_pcalloc(cf->pool, sizeof(ngx_http_spdy_loc_conf_t)); ++ if (slcf == NULL) { ++ return NULL; ++ } ++ ++ slcf->chunk_size = NGX_CONF_UNSET_SIZE; ++ ++ return slcf; ++} ++ ++ ++static char * ++ngx_http_spdy_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) ++{ ++ ngx_http_spdy_loc_conf_t *prev = parent; ++ ngx_http_spdy_loc_conf_t *conf = child; ++ ++ ngx_conf_merge_size_value(conf->chunk_size, prev->chunk_size, 8 * 1024); ++ ++ return NGX_CONF_OK; ++} ++ ++ ++static char * ++ngx_http_spdy_recv_buffer_size(ngx_conf_t *cf, void *post, void *data) ++{ ++ size_t *sp = data; ++ ++ if (*sp <= 2 * NGX_SPDY_STATE_BUFFER_SIZE) { ++ return "value is too small"; ++ } ++ ++ return NGX_CONF_OK; ++} ++ ++ ++static char * ++ngx_http_spdy_pool_size(ngx_conf_t *cf, void *post, void *data) ++{ ++ size_t *sp = data; ++ ++ if (*sp < NGX_MIN_POOL_SIZE) { ++ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, ++ "the pool size must be no less than %uz", ++ NGX_MIN_POOL_SIZE); ++ return NGX_CONF_ERROR; ++ } ++ ++ if (*sp % NGX_POOL_ALIGNMENT) { ++ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, ++ "the pool size must be a multiple of %uz", ++ NGX_POOL_ALIGNMENT); ++ return NGX_CONF_ERROR; ++ } ++ ++ return NGX_CONF_OK; ++} ++ ++ ++static char * ++ngx_http_spdy_streams_index_mask(ngx_conf_t *cf, void *post, void *data) ++{ ++ ngx_uint_t *np = data; ++ ++ ngx_uint_t mask; ++ ++ mask = *np - 1; ++ ++ if (*np == 0 || (*np & mask)) { ++ return "must be a power of two"; ++ } ++ ++ *np = mask; ++ ++ return NGX_CONF_OK; ++} ++ ++ ++static char * ++ngx_http_spdy_chunk_size(ngx_conf_t *cf, void *post, void *data) ++{ ++ size_t *sp = data; ++ ++ if (*sp == 0) { ++ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, ++ "the spdy chunk size cannot be zero"); ++ return NGX_CONF_ERROR; ++ } ++ ++ if (*sp > NGX_SPDY_MAX_FRAME_SIZE) { ++ *sp = NGX_SPDY_MAX_FRAME_SIZE; ++ } ++ ++ return NGX_CONF_OK; ++} +diff --git a/src/http/ngx_http_spdy_module.h b/src/http/ngx_http_spdy_module.h +new file mode 100644 +index 0000000..5242322 +--- /dev/null ++++ b/src/http/ngx_http_spdy_module.h +@@ -0,0 +1,41 @@ ++ ++/* ++ * Copyright (C) Nginx, Inc. ++ * Copyright (C) Valentin V. Bartenev ++ */ ++ ++ ++#ifndef _NGX_HTTP_SPDY_MODULE_H_INCLUDED_ ++#define _NGX_HTTP_SPDY_MODULE_H_INCLUDED_ ++ ++ ++#include ++#include ++#include ++ ++ ++typedef struct { ++ size_t recv_buffer_size; ++ u_char *recv_buffer; ++} ngx_http_spdy_main_conf_t; ++ ++ ++typedef struct { ++ size_t pool_size; ++ ngx_uint_t concurrent_streams; ++ ngx_uint_t streams_index_mask; ++ ngx_msec_t recv_timeout; ++ ngx_msec_t keepalive_timeout; ++ ngx_int_t headers_comp; ++} ngx_http_spdy_srv_conf_t; ++ ++ ++typedef struct { ++ size_t chunk_size; ++} ngx_http_spdy_loc_conf_t; ++ ++ ++extern ngx_module_t ngx_http_spdy_module; ++ ++ ++#endif /* _NGX_HTTP_SPDY_MODULE_H_INCLUDED_ */ +diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c +index fcfa2ad..aac19d6 100644 +--- a/src/http/ngx_http_upstream.c ++++ b/src/http/ngx_http_upstream.c +@@ -512,6 +512,12 @@ ngx_http_upstream_init(ngx_http_request_t *r) + return; + } + #endif ++#if (NGX_HTTP_SPDY) ++ if (r->spdy_stream) { ++ ngx_http_upstream_init_request(r); ++ return; ++ } ++#endif + + if (c->read->timer_set) { + ngx_del_timer(c->read); +@@ -1306,6 +1312,11 @@ ngx_http_upstream_check_broken_connection(ngx_http_request_t *r, + return; + } + #endif ++#if (NGX_HTTP_SPDY) ++ if (r->spdy_stream) { ++ return; ++ } ++#endif + + #if (NGX_HAVE_KQUEUE) + +diff --git a/src/http/v2/ngx_http_v2_module.c b/src/http/v2/ngx_http_v2_module.c +index 032abcb..a9cb728 100644 +--- a/src/http/v2/ngx_http_v2_module.c ++++ b/src/http/v2/ngx_http_v2_module.c +@@ -34,8 +34,6 @@ static char *ngx_http_v2_preread_size(ngx_conf_t *cf, void *post, void *data); + static char *ngx_http_v2_streams_index_mask(ngx_conf_t *cf, void *post, + void *data); + static char *ngx_http_v2_chunk_size(ngx_conf_t *cf, void *post, void *data); +-static char *ngx_http_v2_spdy_deprecated(ngx_conf_t *cf, ngx_command_t *cmd, +- void *conf); + + + static ngx_conf_post_t ngx_http_v2_recv_buffer_size_post = +@@ -129,62 +127,6 @@ static ngx_command_t ngx_http_v2_commands[] = { + offsetof(ngx_http_v2_loc_conf_t, chunk_size), + &ngx_http_v2_chunk_size_post }, + +- { ngx_string("spdy_recv_buffer_size"), +- NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1, +- ngx_http_v2_spdy_deprecated, +- NGX_HTTP_MAIN_CONF_OFFSET, +- 0, +- NULL }, +- +- { ngx_string("spdy_pool_size"), +- NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, +- ngx_http_v2_spdy_deprecated, +- NGX_HTTP_SRV_CONF_OFFSET, +- 0, +- NULL }, +- +- { ngx_string("spdy_max_concurrent_streams"), +- NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, +- ngx_http_v2_spdy_deprecated, +- NGX_HTTP_SRV_CONF_OFFSET, +- 0, +- NULL }, +- +- { ngx_string("spdy_streams_index_size"), +- NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, +- ngx_http_v2_spdy_deprecated, +- NGX_HTTP_SRV_CONF_OFFSET, +- 0, +- NULL }, +- +- { ngx_string("spdy_recv_timeout"), +- NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, +- ngx_http_v2_spdy_deprecated, +- NGX_HTTP_SRV_CONF_OFFSET, +- 0, +- NULL }, +- +- { ngx_string("spdy_keepalive_timeout"), +- NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, +- ngx_http_v2_spdy_deprecated, +- NGX_HTTP_SRV_CONF_OFFSET, +- 0, +- NULL }, +- +- { ngx_string("spdy_headers_comp"), +- NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, +- ngx_http_v2_spdy_deprecated, +- NGX_HTTP_SRV_CONF_OFFSET, +- 0, +- NULL }, +- +- { ngx_string("spdy_chunk_size"), +- NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, +- ngx_http_v2_spdy_deprecated, +- NGX_HTTP_LOC_CONF_OFFSET, +- 0, +- NULL }, +- + ngx_null_command + }; + +@@ -496,14 +438,3 @@ ngx_http_v2_chunk_size(ngx_conf_t *cf, void *post, void *data) + + return NGX_CONF_OK; + } +- +- +-static char * +-ngx_http_v2_spdy_deprecated(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +-{ +- ngx_conf_log_error(NGX_LOG_WARN, cf, 0, +- "invalid directive \"%V\": ngx_http_spdy_module " +- "was superseded by ngx_http_v2_module", &cmd->name); +- +- return NGX_CONF_OK; +-} From f70c96a6dfaef03bb066687b80dfdef3b1f837e9 Mon Sep 17 00:00:00 2001 From: Alessandro Ghedini Date: Mon, 5 Jun 2017 15:12:18 +0100 Subject: [PATCH 09/16] s/CloudFlare/Cloudflare/ --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 7e1367e..b8dcf26 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,14 @@ sslconfig ========= -CloudFlare's Internet facing SSL cipher configuration +Cloudflare's Internet facing SSL cipher configuration This repository tracks the history of the SSL cipher configuration used for -CloudFlare's public-facing SSL web servers. The repository tracks an internal -CloudFlare repository, but dates may not exactly match when changes are made. +Cloudflare's public-facing SSL web servers. The repository tracks an internal +Cloudflare repository, but dates may not exactly match when changes are made. There is a single file called conf which contains the configuration used in -CloudFlare's NGINX servers. This is only a fragment of the configuration. +Cloudflare's NGINX servers. This is only a fragment of the configuration. We currently use OpenSSL 1.0.2-stable (+ patches). @@ -16,7 +16,7 @@ We currently use OpenSSL 1.0.2-stable (+ patches). ChaCha20/Poly1305 patch ----------------------- -CloudFlare uses [a patch](patches/openssl__chacha20_poly1305_cf.patch) for +Cloudflare uses [a patch](patches/openssl__chacha20_poly1305_cf.patch) for OpenSSL that enables the ChaCha20/Poly1305 cipher suites and implements special logic to ensure it is only taken if it is the client's top cipher choice. Without this patch, the cipher suite choice in the configuration From d42811d5b67b7eac40016d5c6e170fe245886b84 Mon Sep 17 00:00:00 2001 From: Alessandro Ghedini Date: Mon, 5 Jun 2017 15:15:00 +0100 Subject: [PATCH 10/16] Update to latest SSL config This adds TLS 1.3, X25519 and the equal preferences cipher config for ChaCha20. --- conf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/conf b/conf index 9caa46e..af8942c 100644 --- a/conf +++ b/conf @@ -1,3 +1,4 @@ -ssl_protocols TLSv1 TLSv1.1 TLSv1.2; -ssl_ciphers EECDH+CHACHA20:EECDH+CHACHA20-draft:EECDH+AES128:RSA+AES128:EECDH+AES256:RSA+AES256:EECDH+3DES:RSA+3DES:!MD5; +ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; +ssl_ecdh_curve X25519:P-256:P-384:P-224:P-521; +ssl_ciphers '[ECDHE-ECDSA-AES128-GCM-SHA256|ECDHE-ECDSA-CHACHA20-POLY1305|ECDHE-RSA-AES128-GCM-SHA256|ECDHE-RSA-CHACHA20-POLY1305]:ECDHE+AES128:RSA+AES128:ECDHE+AES256:RSA+AES256:ECDHE+3DES:RSA+3DES'; ssl_prefer_server_ciphers on; From 30025979eb0fcbadf0f11672cb6b4ea726af0434 Mon Sep 17 00:00:00 2001 From: Alessandro Ghedini Date: Tue, 6 Jun 2017 12:41:08 +0100 Subject: [PATCH 11/16] Don't mention OpenSSL --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index b8dcf26..fcc069f 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,6 @@ Cloudflare repository, but dates may not exactly match when changes are made. There is a single file called conf which contains the configuration used in Cloudflare's NGINX servers. This is only a fragment of the configuration. -We currently use OpenSSL 1.0.2-stable (+ patches). - - ChaCha20/Poly1305 patch ----------------------- From c8fca695e1b422a0054fc3ee27f33d68c0c1bbff Mon Sep 17 00:00:00 2001 From: Vlad Krasnov Date: Thu, 22 Jun 2017 18:27:42 -0700 Subject: [PATCH 12/16] HTTP/2 full HPACK encoding implementation for nginx Apply patch, configure with --with-http_v2_hpack_enc --- patches/nginx_http2_hpack.patch | 1031 +++++++++++++++++++++++++++++++ 1 file changed, 1031 insertions(+) create mode 100644 patches/nginx_http2_hpack.patch diff --git a/patches/nginx_http2_hpack.patch b/patches/nginx_http2_hpack.patch new file mode 100644 index 0000000..4168701 --- /dev/null +++ b/patches/nginx_http2_hpack.patch @@ -0,0 +1,1031 @@ +# HG changeset patch +# User Vlad Krasnov +# Date 1498167669 25200 +# Thu Jun 22 14:41:09 2017 -0700 +# Node ID 895cea03ac21fb18d2c2ba32389cd67dc74ddbd0 +# Parent a39bc74873faf9e5bea616561b43f6ecc55229f9 +HTTP/2: add support for HPACK encoding + +Add support for full HPACK encoding as per RFC7541. +This modification improves header compression ratio by 5-10% for the first +response, and by 40-95% for consequential responses on the connection. +The implementation is similar to the one used by Cloudflare. + +diff -r a39bc74873fa -r 895cea03ac21 auto/modules +--- a/auto/modules Mon Jun 19 14:25:42 2017 +0300 ++++ b/auto/modules Thu Jun 22 14:41:09 2017 -0700 +@@ -436,6 +436,10 @@ + . auto/module + fi + ++ if [ $HTTP_V2_HPACK_ENC = YES ]; then ++ have=NGX_HTTP_V2_HPACK_ENC . auto/have ++ fi ++ + if :; then + ngx_module_name=ngx_http_static_module + ngx_module_incs= +diff -r a39bc74873fa -r 895cea03ac21 auto/options +--- a/auto/options Mon Jun 19 14:25:42 2017 +0300 ++++ b/auto/options Thu Jun 22 14:41:09 2017 -0700 +@@ -59,6 +59,7 @@ + HTTP_GZIP=YES + HTTP_SSL=NO + HTTP_V2=NO ++HTTP_V2_HPACK_ENC=NO + HTTP_SSI=YES + HTTP_POSTPONE=NO + HTTP_REALIP=NO +@@ -221,6 +222,7 @@ + + --with-http_ssl_module) HTTP_SSL=YES ;; + --with-http_v2_module) HTTP_V2=YES ;; ++ --with-http_v2_hpack_enc) HTTP_V2_HPACK_ENC=YES ;; + --with-http_realip_module) HTTP_REALIP=YES ;; + --with-http_addition_module) HTTP_ADDITION=YES ;; + --with-http_xslt_module) HTTP_XSLT=YES ;; +@@ -430,6 +432,7 @@ + + --with-http_ssl_module enable ngx_http_ssl_module + --with-http_v2_module enable ngx_http_v2_module ++ --with-http_v2_hpack_enc enable ngx_http_v2_hpack_enc + --with-http_realip_module enable ngx_http_realip_module + --with-http_addition_module enable ngx_http_addition_module + --with-http_xslt_module enable ngx_http_xslt_module +diff -r a39bc74873fa -r 895cea03ac21 src/core/ngx_murmurhash.c +--- a/src/core/ngx_murmurhash.c Mon Jun 19 14:25:42 2017 +0300 ++++ b/src/core/ngx_murmurhash.c Thu Jun 22 14:41:09 2017 -0700 +@@ -50,3 +50,63 @@ + + return h; + } ++ ++ ++uint64_t ++ngx_murmur_hash2_64(u_char *data, size_t len, uint64_t seed) ++{ ++ uint64_t h, k; ++ ++ h = seed ^ len; ++ ++ while (len >= 8) { ++ k = data[0]; ++ k |= data[1] << 8; ++ k |= data[2] << 16; ++ k |= data[3] << 24; ++ k |= (uint64_t)data[4] << 32; ++ k |= (uint64_t)data[5] << 40; ++ k |= (uint64_t)data[6] << 48; ++ k |= (uint64_t)data[7] << 56; ++ ++ k *= 0xc6a4a7935bd1e995ull; ++ k ^= k >> 47; ++ k *= 0xc6a4a7935bd1e995ull; ++ ++ h ^= k; ++ h *= 0xc6a4a7935bd1e995ull; ++ ++ data += 8; ++ len -= 8; ++ } ++ ++ switch (len) { ++ case 7: ++ h ^= (uint64_t)data[6] << 48; ++ /* fall through */ ++ case 6: ++ h ^= (uint64_t)data[5] << 40; ++ /* fall through */ ++ case 5: ++ h ^= (uint64_t)data[4] << 32; ++ /* fall through */ ++ case 4: ++ h ^= data[3] << 24; ++ /* fall through */ ++ case 3: ++ h ^= data[2] << 16; ++ /* fall through */ ++ case 2: ++ h ^= data[1] << 8; ++ /* fall through */ ++ case 1: ++ h ^= data[0]; ++ h *= 0xc6a4a7935bd1e995ull; ++ } ++ ++ h ^= h >> 47; ++ h *= 0xc6a4a7935bd1e995ull; ++ h ^= h >> 47; ++ ++ return h; ++} +diff -r a39bc74873fa -r 895cea03ac21 src/core/ngx_murmurhash.h +--- a/src/core/ngx_murmurhash.h Mon Jun 19 14:25:42 2017 +0300 ++++ b/src/core/ngx_murmurhash.h Thu Jun 22 14:41:09 2017 -0700 +@@ -15,5 +15,7 @@ + + uint32_t ngx_murmur_hash2(u_char *data, size_t len); + ++uint64_t ngx_murmur_hash2_64(u_char *data, size_t len, uint64_t seed); ++ + + #endif /* _NGX_MURMURHASH_H_INCLUDED_ */ +diff -r a39bc74873fa -r 895cea03ac21 src/http/v2/ngx_http_v2.c +--- a/src/http/v2/ngx_http_v2.c Mon Jun 19 14:25:42 2017 +0300 ++++ b/src/http/v2/ngx_http_v2.c Thu Jun 22 14:41:09 2017 -0700 +@@ -245,6 +245,8 @@ + + h2c->frame_size = NGX_HTTP_V2_DEFAULT_FRAME_SIZE; + ++ h2c->max_hpack_table_size = NGX_HTTP_V2_DEFAULT_HPACK_TABLE_SIZE; ++ + h2scf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_v2_module); + + h2c->pool = ngx_create_pool(h2scf->pool_size, h2c->connection->log); +@@ -2018,6 +2020,17 @@ + h2c->frame_size = value; + break; + ++ case NGX_HTTP_V2_HEADER_TABLE_SIZE_SETTING: ++ ++ if (value > NGX_HTTP_V2_MAX_HPACK_TABLE_SIZE) { ++ h2c->max_hpack_table_size = NGX_HTTP_V2_MAX_HPACK_TABLE_SIZE; ++ } else { ++ h2c->max_hpack_table_size = value; ++ } ++ ++ h2c->indicate_resize = 1; ++ break; ++ + default: + break; + } +diff -r a39bc74873fa -r 895cea03ac21 src/http/v2/ngx_http_v2.h +--- a/src/http/v2/ngx_http_v2.h Mon Jun 19 14:25:42 2017 +0300 ++++ b/src/http/v2/ngx_http_v2.h Thu Jun 22 14:41:09 2017 -0700 +@@ -49,6 +49,13 @@ + #define NGX_HTTP_V2_MAX_WINDOW ((1U << 31) - 1) + #define NGX_HTTP_V2_DEFAULT_WINDOW 65535 + ++#define HPACK_ENC_HTABLE_SZ 128 /* better to keep a PoT < 64k */ ++#define HPACK_ENC_HTABLE_ENTRIES ((HPACK_ENC_HTABLE_SZ * 100) / 128) ++#define HPACK_ENC_DYNAMIC_KEY_TBL_SZ 10 /* 10 is sufficient for most */ ++#define HPACK_ENC_MAX_ENTRY 512 /* longest header size to match */ ++ ++#define NGX_HTTP_V2_DEFAULT_HPACK_TABLE_SIZE 4096 ++#define NGX_HTTP_V2_MAX_HPACK_TABLE_SIZE 16384 /* < 64k */ + + typedef struct ngx_http_v2_connection_s ngx_http_v2_connection_t; + typedef struct ngx_http_v2_node_s ngx_http_v2_node_t; +@@ -110,6 +117,46 @@ + } ngx_http_v2_hpack_t; + + ++#if (NGX_HTTP_V2_HPACK_ENC) ++typedef struct { ++ uint64_t hash_val; ++ uint32_t index; ++ uint16_t pos; ++ uint16_t klen, vlen; ++ uint16_t size; ++ uint16_t next; ++} ngx_http_v2_hpack_enc_entry_t; ++ ++ ++typedef struct { ++ uint64_t hash_val; ++ uint32_t index; ++ uint16_t pos; ++ uint16_t klen; ++} ngx_http_v2_hpack_name_entry_t; ++ ++ ++typedef struct { ++ size_t size; /* size as defined in RFC 7541 */ ++ uint32_t top; /* the last entry */ ++ uint32_t pos; ++ uint16_t n_elems; /* number of elements */ ++ uint16_t base; /* index of the oldest entry */ ++ uint16_t last; /* index of the newest entry */ ++ ++ /* hash table for dynamic entries, instead using a generic hash table, ++ which would be too slow to process a significant amount of headers, ++ this table is not determenistic, and might ocasionally fail to insert ++ a value, at the cost of slightly worse compression, but significantly ++ faster performance */ ++ ngx_http_v2_hpack_enc_entry_t htable[HPACK_ENC_HTABLE_SZ]; ++ ngx_http_v2_hpack_name_entry_t heads[HPACK_ENC_DYNAMIC_KEY_TBL_SZ]; ++ u_char storage[NGX_HTTP_V2_MAX_HPACK_TABLE_SIZE + ++ HPACK_ENC_MAX_ENTRY]; ++} ngx_http_v2_hpack_enc_t; ++#endif ++ ++ + struct ngx_http_v2_connection_s { + ngx_connection_t *connection; + ngx_http_connection_t *http_connection; +@@ -122,6 +169,8 @@ + + size_t frame_size; + ++ size_t max_hpack_table_size; ++ + ngx_queue_t waiting; + + ngx_http_v2_state_t state; +@@ -146,6 +195,11 @@ + unsigned settings_ack:1; + unsigned blocked:1; + unsigned goaway:1; ++ unsigned indicate_resize:1; ++ ++#if (NGX_HTTP_V2_HPACK_ENC) ++ ngx_http_v2_hpack_enc_t hpack_enc; ++#endif + }; + + +@@ -347,4 +401,31 @@ + + #define ngx_http_v2_write_sid ngx_http_v2_write_uint32 + ++u_char *ngx_http_v2_string_encode(u_char *dst, u_char *src, size_t len, ++ u_char *tmp, ngx_uint_t lower); ++ ++u_char * ++ngx_http_v2_write_int(u_char *pos, ngx_uint_t prefix, ngx_uint_t value); ++ ++#define ngx_http_v2_write_name(dst, src, len, tmp) \ ++ ngx_http_v2_string_encode(dst, src, len, tmp, 1) ++#define ngx_http_v2_write_value(dst, src, len, tmp) \ ++ ngx_http_v2_string_encode(dst, src, len, tmp, 0) ++ ++u_char * ++ngx_http_v2_write_header(ngx_http_v2_connection_t *h2c, u_char *pos, ++ u_char *key, size_t key_len, u_char *value, size_t value_len, ++ u_char *tmp); ++ ++void ++ngx_http_v2_table_resize(ngx_http_v2_connection_t *h2c); ++ ++#define ngx_http_v2_write_header_str(key, value) \ ++ ngx_http_v2_write_header(h2c, pos, (u_char *) key, sizeof(key) - 1, \ ++ (u_char *) value, sizeof(value) - 1, tmp); ++ ++#define ngx_http_v2_write_header_tbl(key, val) \ ++ ngx_http_v2_write_header(h2c, pos, (u_char *) key, sizeof(key) - 1, \ ++ val.data, val.len, tmp); ++ + #endif /* _NGX_HTTP_V2_H_INCLUDED_ */ +diff -r a39bc74873fa -r 895cea03ac21 src/http/v2/ngx_http_v2_filter_module.c +--- a/src/http/v2/ngx_http_v2_filter_module.c Mon Jun 19 14:25:42 2017 +0300 ++++ b/src/http/v2/ngx_http_v2_filter_module.c Thu Jun 22 14:41:09 2017 -0700 +@@ -25,11 +25,6 @@ + #define ngx_http_v2_indexed(i) (128 + (i)) + #define ngx_http_v2_inc_indexed(i) (64 + (i)) + +-#define ngx_http_v2_write_name(dst, src, len, tmp) \ +- ngx_http_v2_string_encode(dst, src, len, tmp, 1) +-#define ngx_http_v2_write_value(dst, src, len, tmp) \ +- ngx_http_v2_string_encode(dst, src, len, tmp, 0) +- + #define NGX_HTTP_V2_ENCODE_RAW 0 + #define NGX_HTTP_V2_ENCODE_HUFF 0x80 + +@@ -53,10 +48,6 @@ + #define NGX_HTTP_V2_NO_TRAILERS (ngx_http_v2_out_frame_t *) -1 + + +-static u_char *ngx_http_v2_string_encode(u_char *dst, u_char *src, size_t len, +- u_char *tmp, ngx_uint_t lower); +-static u_char *ngx_http_v2_write_int(u_char *pos, ngx_uint_t prefix, +- ngx_uint_t value); + static ngx_http_v2_out_frame_t *ngx_http_v2_create_headers_frame( + ngx_http_request_t *r, u_char *pos, u_char *end, ngx_uint_t fin); + static ngx_http_v2_out_frame_t *ngx_http_v2_create_trailers_frame( +@@ -142,6 +133,7 @@ + ngx_http_core_loc_conf_t *clcf; + ngx_http_core_srv_conf_t *cscf; + u_char addr[NGX_SOCKADDR_STRLEN]; ++ ngx_http_v2_connection_t *h2c; + + static const u_char nginx[5] = "\x84\xaa\x63\x55\xe7"; + #if (NGX_HTTP_GZIP) +@@ -150,11 +142,9 @@ + #endif + + static size_t nginx_ver_len = ngx_http_v2_literal_size(NGINX_VER); +- static u_char nginx_ver[ngx_http_v2_literal_size(NGINX_VER)]; + + static size_t nginx_ver_build_len = + ngx_http_v2_literal_size(NGINX_VER_BUILD); +- static u_char nginx_ver_build[ngx_http_v2_literal_size(NGINX_VER_BUILD)]; + + if (!r->stream) { + return ngx_http_next_header_filter(r); +@@ -415,7 +405,7 @@ + } + + tmp = ngx_palloc(r->pool, tmp_len); +- pos = ngx_pnalloc(r->pool, len); ++ pos = ngx_pnalloc(r->pool, len + 15 + 1); + + if (pos == NULL || tmp == NULL) { + return NGX_ERROR; +@@ -423,6 +413,18 @@ + + start = pos; + ++ h2c = r->stream->connection; ++ ++ if (h2c->indicate_resize) { ++ *pos = 32; ++ pos = ngx_http_v2_write_int(pos, ngx_http_v2_prefix(5), ++ h2c->max_hpack_table_size); ++ h2c->indicate_resize = 0; ++#if (NGX_HTTP_V2_HPACK_ENC) ++ ngx_http_v2_table_resize(h2c); ++#endif ++ } ++ + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, fc->log, 0, + "http2 output header: \":status: %03ui\"", + r->headers_out.status); +@@ -431,67 +433,28 @@ + *pos++ = status; + + } else { +- *pos++ = ngx_http_v2_inc_indexed(NGX_HTTP_V2_STATUS_INDEX); +- *pos++ = NGX_HTTP_V2_ENCODE_RAW | 3; +- pos = ngx_sprintf(pos, "%03ui", r->headers_out.status); ++ ngx_sprintf(pos + 8, "%O3ui", r->headers_out.status); ++ pos = ngx_http_v2_write_header(h2c, pos, (u_char *)":status", ++ sizeof(":status") - 1, pos + 8, 3, tmp); + } + + if (r->headers_out.server == NULL) { +- + if (clcf->server_tokens == NGX_HTTP_SERVER_TOKENS_ON) { +- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, fc->log, 0, +- "http2 output header: \"server: %s\"", +- NGINX_VER); ++ pos = ngx_http_v2_write_header_str("server", NGINX_VER); + + } else if (clcf->server_tokens == NGX_HTTP_SERVER_TOKENS_BUILD) { +- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, fc->log, 0, +- "http2 output header: \"server: %s\"", +- NGINX_VER_BUILD); ++ pos = ngx_http_v2_write_header_str("server", NGINX_VER_BUILD); + + } else { +- ngx_log_debug0(NGX_LOG_DEBUG_HTTP, fc->log, 0, +- "http2 output header: \"server: nginx\""); +- } +- +- *pos++ = ngx_http_v2_inc_indexed(NGX_HTTP_V2_SERVER_INDEX); +- +- if (clcf->server_tokens == NGX_HTTP_SERVER_TOKENS_ON) { +- if (nginx_ver[0] == '\0') { +- p = ngx_http_v2_write_value(nginx_ver, (u_char *) NGINX_VER, +- sizeof(NGINX_VER) - 1, tmp); +- nginx_ver_len = p - nginx_ver; +- } +- +- pos = ngx_cpymem(pos, nginx_ver, nginx_ver_len); +- +- } else if (clcf->server_tokens == NGX_HTTP_SERVER_TOKENS_BUILD) { +- if (nginx_ver_build[0] == '\0') { +- p = ngx_http_v2_write_value(nginx_ver_build, +- (u_char *) NGINX_VER_BUILD, +- sizeof(NGINX_VER_BUILD) - 1, tmp); +- nginx_ver_build_len = p - nginx_ver_build; +- } +- +- pos = ngx_cpymem(pos, nginx_ver_build, nginx_ver_build_len); +- +- } else { +- pos = ngx_cpymem(pos, nginx, sizeof(nginx)); ++ pos = ngx_http_v2_write_header_str("server", "nginx"); + } + } + + if (r->headers_out.date == NULL) { +- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, fc->log, 0, +- "http2 output header: \"date: %V\"", +- &ngx_cached_http_time); +- +- *pos++ = ngx_http_v2_inc_indexed(NGX_HTTP_V2_DATE_INDEX); +- pos = ngx_http_v2_write_value(pos, ngx_cached_http_time.data, +- ngx_cached_http_time.len, tmp); ++ pos = ngx_http_v2_write_header_tbl("date", ngx_cached_http_time); + } + + if (r->headers_out.content_type.len) { +- *pos++ = ngx_http_v2_inc_indexed(NGX_HTTP_V2_CONTENT_TYPE_INDEX); +- + if (r->headers_out.content_type_len == r->headers_out.content_type.len + && r->headers_out.charset.len) + { +@@ -517,64 +480,36 @@ + r->headers_out.content_type.data = p - len; + } + +- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, fc->log, 0, +- "http2 output header: \"content-type: %V\"", +- &r->headers_out.content_type); +- +- pos = ngx_http_v2_write_value(pos, r->headers_out.content_type.data, +- r->headers_out.content_type.len, tmp); ++ pos = ngx_http_v2_write_header_tbl("content-type", ++ r->headers_out.content_type); + } + + if (r->headers_out.content_length == NULL + && r->headers_out.content_length_n >= 0) + { +- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, fc->log, 0, +- "http2 output header: \"content-length: %O\"", +- r->headers_out.content_length_n); +- +- *pos++ = ngx_http_v2_inc_indexed(NGX_HTTP_V2_CONTENT_LENGTH_INDEX); +- +- p = pos; +- pos = ngx_sprintf(pos + 1, "%O", r->headers_out.content_length_n); +- *p = NGX_HTTP_V2_ENCODE_RAW | (u_char) (pos - p - 1); ++ p = ngx_sprintf(pos + 15, "%O", r->headers_out.content_length_n); ++ pos = ngx_http_v2_write_header(h2c, pos, (u_char *)"content-length", ++ sizeof("content-length") - 1, pos + 15, ++ p - (pos + 15), tmp); + } + + if (r->headers_out.last_modified == NULL + && r->headers_out.last_modified_time != -1) + { +- *pos++ = ngx_http_v2_inc_indexed(NGX_HTTP_V2_LAST_MODIFIED_INDEX); +- +- ngx_http_time(pos, r->headers_out.last_modified_time); ++ ngx_http_time(pos + 14, r->headers_out.last_modified_time); + len = sizeof("Wed, 31 Dec 1986 18:00:00 GMT") - 1; +- +- ngx_log_debug2(NGX_LOG_DEBUG_HTTP, fc->log, 0, +- "http2 output header: \"last-modified: %*s\"", +- len, pos); +- +- /* +- * Date will always be encoded using huffman in the temporary buffer, +- * so it's safe here to use src and dst pointing to the same address. +- */ +- pos = ngx_http_v2_write_value(pos, pos, len, tmp); ++ pos = ngx_http_v2_write_header(h2c, pos, (u_char *)"last-modified", ++ sizeof("last-modified") - 1, pos + 14, ++ len, tmp); + } + + if (r->headers_out.location && r->headers_out.location->value.len) { +- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, fc->log, 0, +- "http2 output header: \"location: %V\"", +- &r->headers_out.location->value); +- +- *pos++ = ngx_http_v2_inc_indexed(NGX_HTTP_V2_LOCATION_INDEX); +- pos = ngx_http_v2_write_value(pos, r->headers_out.location->value.data, +- r->headers_out.location->value.len, tmp); ++ pos = ngx_http_v2_write_header_tbl("location", r->headers_out.location->value); + } + + #if (NGX_HTTP_GZIP) + if (r->gzip_vary) { +- ngx_log_debug0(NGX_LOG_DEBUG_HTTP, fc->log, 0, +- "http2 output header: \"vary: Accept-Encoding\""); +- +- *pos++ = ngx_http_v2_inc_indexed(NGX_HTTP_V2_VARY_INDEX); +- pos = ngx_cpymem(pos, accept_encoding, sizeof(accept_encoding)); ++ pos = ngx_http_v2_write_header_str("vary", "Accept-Encoding"); + } + #endif + +@@ -597,23 +532,9 @@ + continue; + } + +-#if (NGX_DEBUG) +- if (fc->log->log_level & NGX_LOG_DEBUG_HTTP) { +- ngx_strlow(tmp, header[i].key.data, header[i].key.len); +- +- ngx_log_debug3(NGX_LOG_DEBUG_HTTP, fc->log, 0, +- "http2 output header: \"%*s: %V\"", +- header[i].key.len, tmp, &header[i].value); +- } +-#endif +- +- *pos++ = 0; +- +- pos = ngx_http_v2_write_name(pos, header[i].key.data, +- header[i].key.len, tmp); +- +- pos = ngx_http_v2_write_value(pos, header[i].value.data, +- header[i].value.len, tmp); ++ pos = ngx_http_v2_write_header(h2c, pos, header[i].key.data, ++ header[i].key.len, header[i].value.data, ++ header[i].value.len, tmp); + } + + frame = ngx_http_v2_create_headers_frame(r, start, pos, r->header_only); +@@ -643,11 +564,12 @@ + static ngx_http_v2_out_frame_t * + ngx_http_v2_create_trailers_frame(ngx_http_request_t *r) + { +- u_char *pos, *start, *tmp; +- size_t len, tmp_len; +- ngx_uint_t i; +- ngx_list_part_t *part; +- ngx_table_elt_t *header; ++ u_char *pos, *start, *tmp; ++ size_t len, tmp_len; ++ ngx_uint_t i; ++ ngx_list_part_t *part; ++ ngx_table_elt_t *header; ++ ngx_http_v2_connection_t *h2c; + + len = 0; + tmp_len = 0; +@@ -713,6 +635,8 @@ + part = &r->headers_out.trailers.part; + header = part->elts; + ++ h2c = r->stream->connection; ++ + for (i = 0; /* void */; i++) { + + if (i >= part->nelts) { +@@ -739,20 +663,16 @@ + } + #endif + +- *pos++ = 0; +- +- pos = ngx_http_v2_write_name(pos, header[i].key.data, +- header[i].key.len, tmp); +- +- pos = ngx_http_v2_write_value(pos, header[i].value.data, +- header[i].value.len, tmp); ++ pos = ngx_http_v2_write_header(h2c, pos, header[i].key.data, ++ header[i].key.len, header[i].value.data, ++ header[i].value.len, tmp); + } + + return ngx_http_v2_create_headers_frame(r, start, pos, 1); + } + + +-static u_char * ++u_char * + ngx_http_v2_string_encode(u_char *dst, u_char *src, size_t len, u_char *tmp, + ngx_uint_t lower) + { +@@ -778,7 +698,7 @@ + } + + +-static u_char * ++u_char * + ngx_http_v2_write_int(u_char *pos, ngx_uint_t prefix, ngx_uint_t value) + { + if (value < prefix) { +diff -r a39bc74873fa -r 895cea03ac21 src/http/v2/ngx_http_v2_table.c +--- a/src/http/v2/ngx_http_v2_table.c Mon Jun 19 14:25:42 2017 +0300 ++++ b/src/http/v2/ngx_http_v2_table.c Thu Jun 22 14:41:09 2017 -0700 +@@ -347,3 +347,434 @@ + + return NGX_OK; + } ++ ++ ++#if (NGX_HTTP_V2_HPACK_ENC) ++ ++static ngx_int_t ++hpack_get_static_index(ngx_http_v2_connection_t *h2c, u_char *val, size_t len); ++ ++static ngx_int_t ++hpack_get_dynamic_index(ngx_http_v2_connection_t *h2c, uint64_t key_hash, ++ uint8_t *key, size_t key_len); ++ ++ ++void ++ngx_http_v2_table_resize(ngx_http_v2_connection_t *h2c) ++{ ++ ngx_http_v2_hpack_enc_entry_t *table; ++ uint64_t idx; ++ ++ table = h2c->hpack_enc.htable; ++ ++ while (h2c->hpack_enc.size > h2c->max_hpack_table_size) { ++ idx = h2c->hpack_enc.base; ++ h2c->hpack_enc.base = table[idx].next; ++ h2c->hpack_enc.size -= table[idx].size; ++ table[idx].hash_val = 0; ++ h2c->hpack_enc.n_elems--; ++ } ++} ++ ++ ++/* checks if a header is in the hpack table - if so returns the table entry, ++ otherwise encodes and inserts into the table and returns 0, ++ if failed to insert into table, returns -1 */ ++static ngx_int_t ++ngx_http_v2_table_encode_strings(ngx_http_v2_connection_t *h2c, ++ size_t key_len, size_t val_len, uint8_t *key, uint8_t *val, ++ ngx_int_t *header_idx) ++{ ++ uint64_t hash_val, key_hash, idx, lru; ++ int i; ++ size_t size = key_len + val_len + 32; ++ uint8_t *storage = h2c->hpack_enc.storage; ++ ++ ngx_http_v2_hpack_enc_entry_t *table; ++ ngx_http_v2_hpack_name_entry_t *name; ++ ++ *header_idx = NGX_ERROR; ++ /* step 1: compute the hash value of header */ ++ if (size > HPACK_ENC_MAX_ENTRY || size > h2c->max_hpack_table_size) { ++ return NGX_ERROR; ++ } ++ ++ key_hash = ngx_murmur_hash2_64(key, key_len, 0x01234); ++ hash_val = ngx_murmur_hash2_64(val, val_len, key_hash); ++ ++ if (hash_val == 0) { ++ return NGX_ERROR; ++ } ++ ++ /* step 2: check if full header in the table */ ++ idx = hash_val; ++ i = -1; ++ while (idx) { ++ /* at most 8 locations are checked, but most will be done in 1 or 2 */ ++ table = &h2c->hpack_enc.htable[idx % HPACK_ENC_HTABLE_SZ]; ++ if (table->hash_val == hash_val ++ && table->klen == key_len ++ && table->vlen == val_len ++ && ngx_memcmp(key, storage + table->pos, key_len) == 0 ++ && ngx_memcmp(val, storage + table->pos + key_len, val_len) == 0) ++ { ++ return (h2c->hpack_enc.top - table->index) + 61; ++ } ++ ++ if (table->hash_val == 0 && i == -1) { ++ i = idx % HPACK_ENC_HTABLE_SZ; ++ break; ++ } ++ ++ idx >>= 8; ++ } ++ ++ /* step 3: check if key is in one of the tables */ ++ *header_idx = hpack_get_static_index(h2c, key, key_len); ++ ++ if (i == -1) { ++ return NGX_ERROR; ++ } ++ ++ if (*header_idx == NGX_ERROR) { ++ *header_idx = hpack_get_dynamic_index(h2c, key_hash, key, key_len); ++ } ++ ++ /* step 4: store the new entry */ ++ table = h2c->hpack_enc.htable; ++ ++ if (h2c->hpack_enc.top == 0xffffffff) { ++ /* just to be on the safe side, avoid overflow */ ++ ngx_memset(&h2c->hpack_enc, 0, sizeof(ngx_http_v2_hpack_enc_t)); ++ } ++ ++ while ((h2c->hpack_enc.size + size > h2c->max_hpack_table_size) ++ || h2c->hpack_enc.n_elems == HPACK_ENC_HTABLE_ENTRIES) { ++ /* make space for the new entry first */ ++ idx = h2c->hpack_enc.base; ++ h2c->hpack_enc.base = table[idx].next; ++ h2c->hpack_enc.size -= table[idx].size; ++ table[idx].hash_val = 0; ++ h2c->hpack_enc.n_elems--; ++ } ++ ++ table[i] = (ngx_http_v2_hpack_enc_entry_t){.hash_val = hash_val, ++ .index = h2c->hpack_enc.top, ++ .pos = h2c->hpack_enc.pos, ++ .klen = key_len, ++ .vlen = val_len, ++ .size = size, ++ .next = 0}; ++ ++ table[h2c->hpack_enc.last].next = i; ++ if (h2c->hpack_enc.n_elems == 0) { ++ h2c->hpack_enc.base = i; ++ } ++ ++ h2c->hpack_enc.last = i; ++ h2c->hpack_enc.top++; ++ h2c->hpack_enc.size += size; ++ h2c->hpack_enc.n_elems++; ++ ++ /* update header name lookup */ ++ if (*header_idx == NGX_ERROR ) { ++ lru = h2c->hpack_enc.top; ++ ++ for (i=0; ihpack_enc.heads[i]; ++ ++ if ( name->hash_val == 0 || (name->hash_val == key_hash ++ && ngx_memcmp(storage + name->pos, key, key_len) == 0) ) ++ { ++ name->hash_val = key_hash; ++ name->pos = h2c->hpack_enc.pos; ++ name->index = h2c->hpack_enc.top - 1; ++ break; ++ } ++ ++ if (lru > name->index) { ++ lru = name->index; ++ idx = i; ++ } ++ } ++ ++ if (i == HPACK_ENC_DYNAMIC_KEY_TBL_SZ) { ++ name = &h2c->hpack_enc.heads[idx]; ++ name->hash_val = hash_val; ++ name->pos = h2c->hpack_enc.pos; ++ name->index = h2c->hpack_enc.top - 1; ++ } ++ } ++ ++ ngx_memcpy(storage + h2c->hpack_enc.pos, key, key_len); ++ ngx_memcpy(storage + h2c->hpack_enc.pos + key_len, val, val_len); ++ ++ h2c->hpack_enc.pos += size; ++ if (h2c->hpack_enc.pos > NGX_HTTP_V2_MAX_HPACK_TABLE_SIZE) { ++ h2c->hpack_enc.pos = 0; ++ } ++ ++ return NGX_OK; ++} ++ ++ ++u_char * ++ngx_http_v2_write_header(ngx_http_v2_connection_t *h2c, u_char *pos, ++ u_char *key, size_t key_len, ++ u_char *value, size_t value_len, ++ u_char *tmp) ++{ ++ ngx_int_t idx, header_idx; ++ ++ ngx_log_debug4(NGX_LOG_DEBUG_HTTP, h2c->connection->log, 0, ++ "http2 output header: %*s: %*s", key_len, key, value_len, ++ value); ++ ++ /* attempt to find the value in the dynamic table */ ++ idx = ngx_http_v2_table_encode_strings(h2c, key_len, value_len, key, value, ++ &header_idx); ++ ++ if (idx > 0) { ++ /* positive index indicates success */ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, h2c->connection->log, 0, ++ "http2 hpack encode: Indexed Header Field: %ud", idx); ++ ++ *pos = 128; ++ pos = ngx_http_v2_write_int(pos, ngx_http_v2_prefix(7), idx); ++ ++ } else { ++ ++ if (header_idx == NGX_ERROR) { /* if key is not present */ ++ ++ if (idx == NGX_ERROR) { /* if header was not added */ ++ *pos++ = 0; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, h2c->connection->log, 0, ++ "http2 hpack encode: Literal Header Field without" ++ " Indexing — New Name"); ++ } else { /* if header was added */ ++ *pos++ = 64; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, h2c->connection->log, 0, ++ "http2 hpack encode: Literal Header Field with " ++ "Incremental Indexing — New Name"); ++ } ++ ++ pos = ngx_http_v2_write_name(pos, key, key_len, tmp); ++ ++ } else { /* if key is present */ ++ ++ if (idx == NGX_ERROR) { ++ *pos = 0; ++ pos = ngx_http_v2_write_int(pos, ngx_http_v2_prefix(4), header_idx); ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, h2c->connection->log, 0, ++ "http2 hpack encode: Literal Header Field without" ++ " Indexing — Indexed Name: %ud", header_idx); ++ } else { ++ *pos = 64; ++ pos = ngx_http_v2_write_int(pos, ngx_http_v2_prefix(6), header_idx); ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, h2c->connection->log, 0, ++ "http2 hpack encode: Literal Header Field with " ++ "Incremental Indexing — Indexed Name: %ud", header_idx); ++ } ++ } ++ ++ pos = ngx_http_v2_write_value(pos, value, value_len, tmp); ++ } ++ ++ return pos; ++} ++ ++ ++static ngx_int_t ++hpack_get_dynamic_index(ngx_http_v2_connection_t *h2c, uint64_t key_hash, ++ uint8_t *key, size_t key_len) ++{ ++ ngx_http_v2_hpack_name_entry_t *name; ++ int i; ++ ++ for (i=0; ihpack_enc.heads[i]; ++ ++ if (name->hash_val == key_hash ++ && ngx_memcmp(h2c->hpack_enc.storage + name->pos, key, key_len) == 0) ++ { ++ if (name->index >= h2c->hpack_enc.top - h2c->hpack_enc.n_elems) { ++ return (h2c->hpack_enc.top - name->index) + 61; ++ } ++ break; ++ } ++ } ++ ++ return NGX_ERROR; ++} ++ ++ ++/* decide if a given header is present in the static dictionary, this could be ++ done in several ways, but it seems the fastest one is "exhaustive" search */ ++static ngx_int_t ++hpack_get_static_index(ngx_http_v2_connection_t *h2c, u_char *val, size_t len) ++{ ++ /* the static dictionary of response only headers, ++ although response headers can be put by origin, ++ that would be rare */ ++ static const struct { ++ u_char len; ++ const u_char val[28]; ++ u_char idx; ++ } server_headers[] = { ++ { 3, "age", 21},//0 ++ { 3, "via", 60}, ++ { 4, "date", 33},//2 ++ { 4, "etag", 34}, ++ { 4, "link", 45}, ++ { 4, "vary", 59}, ++ { 5, "allow", 22},//6 ++ { 6, "server", 54},//7 ++ { 7, "expires", 36},//8 ++ { 7, "refresh", 52}, ++ { 8, "location", 46},//10 ++ {10, "set-cookie", 55},//11 ++ {11, "retry-after", 53},//12 ++ {12, "content-type", 31},//13 ++ {13, "content-range", 30},//14 ++ {13, "accept-ranges", 18}, ++ {13, "cache-control", 24}, ++ {13, "last-modified", 44}, ++ {14, "content-length", 28},//18 ++ {16, "content-encoding", 26},//19 ++ {16, "content-language", 27}, ++ {16, "content-location", 29}, ++ {16, "www-authenticate", 61}, ++ {17, "transfer-encoding", 57},//23 ++ {18, "proxy-authenticate", 48},//24 ++ {19, "content-disposition", 25},//25 ++ {25, "strict-transport-security", 56},//26 ++ {27, "access-control-allow-origin", 20},//27 ++ {99, "", 99}, ++ }, *header; ++ ++ /* for a given length, where to start the search ++ since minimal length is 3, the table has a -3 ++ offset */ ++ static const int8_t start_at[] = { ++ [3-3] = 0, ++ [4-3] = 2, ++ [5-3] = 6, ++ [6-3] = 7, ++ [7-3] = 8, ++ [8-3] = 10, ++ [9-3] = -1, ++ [10-3] = 11, ++ [11-3] = 12, ++ [12-3] = 13, ++ [13-3] = 14, ++ [14-3] = 18, ++ [15-3] = -1, ++ [16-3] = 19, ++ [17-3] = 23, ++ [18-3] = 24, ++ [19-3] = 25, ++ [20-3] = -1, ++ [21-3] = -1, ++ [22-3] = -1, ++ [23-3] = -1, ++ [24-3] = -1, ++ [25-3] = 26, ++ [26-3] = -1, ++ [27-3] = 27, ++ }; ++ ++ uint64_t pref; ++ size_t save_len = len, i; ++ int8_t start; ++ ++ /* early exit for out of bounds lengths */ ++ if (len < 3 || len > 27) { ++ return NGX_ERROR; ++ } ++ ++ start = start_at[len - 3]; ++ if (start == -1) { ++ /* exit for non existent lengths */ ++ return NGX_ERROR; ++ } ++ ++ header = &server_headers[start_at[len - 3]]; ++ ++ /* load first 8 bytes of key, for fast comparison */ ++ if (len < 8) { ++ pref = 0; ++ if (len >= 4) { ++ pref = *(uint32_t *)(val + len - 4) | 0x20202020; ++ len -= 4; ++ } ++ while (len > 0) { /* 3 iterations at most */ ++ pref = (pref << 8) ^ (val[len - 1] | 0x20); ++ len--; ++ } ++ } else { ++ pref = *(uint64_t *)val | 0x2020202020202020; ++ len -= 8; ++ } ++ ++ /* iterate over headers with the right length */ ++ while (header->len == save_len) { ++ /* quickly compare the first 8 bytes, most tests will end here */ ++ if (pref != *(uint64_t *) header->val) { ++ header++; ++ continue; ++ } ++ ++ if (len == 0) { ++ /* len == 0, indicates prefix held the entire key */ ++ return header->idx; ++ } ++ /* for longer keys compare the rest */ ++ i = 1 + (save_len + 7) % 8; /* align so we can compare in quadwords */ ++ ++ while (i + 8 <= save_len) { /* 3 iterations at most */ ++ if ( *(uint64_t *)&header->val[i] ++ != (*(uint64_t *) &val[i]| 0x2020202020202020) ) ++ { ++ header++; ++ i = 0; ++ break; ++ } ++ i += 8; ++ } ++ ++ if (i == 0) { ++ continue; ++ } ++ ++ /* found the corresponding entry in the static dictionary */ ++ return header->idx; ++ } ++ ++ return NGX_ERROR; ++} ++ ++#else ++ ++u_char * ++ngx_http_v2_write_header(ngx_http_v2_connection_t *h2c, u_char *pos, ++ u_char *key, size_t key_len, ++ u_char *value, size_t value_len, ++ u_char *tmp) ++{ ++ ngx_log_debug4(NGX_LOG_DEBUG_HTTP, h2c->connection->log, 0, ++ "http2 output header: %*s: %*s", key_len, key, value_len, ++ value); ++ ++ *pos++ = 64; ++ pos = ngx_http_v2_write_name(pos, key, key_len, tmp); ++ pos = ngx_http_v2_write_value(pos, value, value_len, tmp); ++ ++ return pos; ++} ++ ++#endif From b64360c20c19ba5c45c64478a7eeb7ea22abd35d Mon Sep 17 00:00:00 2001 From: Vlad Krasnov Date: Thu, 22 Jun 2017 19:03:48 -0700 Subject: [PATCH 13/16] HTTP/2 HPACK for nginx 1.13.1 --- patches/nginx_1.13.1_http2_hpack.patch | 1006 ++++++++++++++++++++++++ 1 file changed, 1006 insertions(+) create mode 100644 patches/nginx_1.13.1_http2_hpack.patch diff --git a/patches/nginx_1.13.1_http2_hpack.patch b/patches/nginx_1.13.1_http2_hpack.patch new file mode 100644 index 0000000..fbba8b2 --- /dev/null +++ b/patches/nginx_1.13.1_http2_hpack.patch @@ -0,0 +1,1006 @@ +From d8fc95eb37e1e8e2ab49696b43bad931a6e08456 Mon Sep 17 00:00:00 2001 +From: Vlad Krasnov +Date: Thu, 22 Jun 2017 19:01:04 -0700 +Subject: [PATCH 1/1] HTTP/2: add support for HPACK encoding + +Add support for full HPACK encoding as per RFC7541. +This modification improves header compression ratio by 5-10% for the first +response, and by 40-95% for consequential responses on the connection. +The implementation is similar to the one used by Cloudflare. +--- + auto/modules | 4 + + auto/options | 3 + + src/core/ngx_murmurhash.c | 60 +++++ + src/core/ngx_murmurhash.h | 2 + + src/http/v2/ngx_http_v2.c | 13 + + src/http/v2/ngx_http_v2.h | 81 ++++++ + src/http/v2/ngx_http_v2_filter_module.c | 154 +++--------- + src/http/v2/ngx_http_v2_table.c | 431 ++++++++++++++++++++++++++++++++ + 8 files changed, 632 insertions(+), 116 deletions(-) + +diff --git a/auto/modules b/auto/modules +index be3561e6..1be97dd1 100644 +--- a/auto/modules ++++ b/auto/modules +@@ -436,6 +436,10 @@ if [ $HTTP = YES ]; then + . auto/module + fi + ++ if [ $HTTP_V2_HPACK_ENC = YES ]; then ++ have=NGX_HTTP_V2_HPACK_ENC . auto/have ++ fi ++ + if :; then + ngx_module_name=ngx_http_static_module + ngx_module_incs= +diff --git a/auto/options b/auto/options +index 66b822a7..9e58e7dd 100644 +--- a/auto/options ++++ b/auto/options +@@ -59,6 +59,7 @@ HTTP_CHARSET=YES + HTTP_GZIP=YES + HTTP_SSL=NO + HTTP_V2=NO ++HTTP_V2_HPACK_ENC=NO + HTTP_SSI=YES + HTTP_POSTPONE=NO + HTTP_REALIP=NO +@@ -221,6 +222,7 @@ $0: warning: the \"--with-ipv6\" option is deprecated" + + --with-http_ssl_module) HTTP_SSL=YES ;; + --with-http_v2_module) HTTP_V2=YES ;; ++ --with-http_v2_hpack_enc) HTTP_V2_HPACK_ENC=YES ;; + --with-http_realip_module) HTTP_REALIP=YES ;; + --with-http_addition_module) HTTP_ADDITION=YES ;; + --with-http_xslt_module) HTTP_XSLT=YES ;; +@@ -430,6 +432,7 @@ cat << END + + --with-http_ssl_module enable ngx_http_ssl_module + --with-http_v2_module enable ngx_http_v2_module ++ --with-http_v2_hpack_enc enable ngx_http_v2_hpack_enc + --with-http_realip_module enable ngx_http_realip_module + --with-http_addition_module enable ngx_http_addition_module + --with-http_xslt_module enable ngx_http_xslt_module +diff --git a/src/core/ngx_murmurhash.c b/src/core/ngx_murmurhash.c +index 5ade658d..4932f20d 100644 +--- a/src/core/ngx_murmurhash.c ++++ b/src/core/ngx_murmurhash.c +@@ -50,3 +50,63 @@ ngx_murmur_hash2(u_char *data, size_t len) + + return h; + } ++ ++ ++uint64_t ++ngx_murmur_hash2_64(u_char *data, size_t len, uint64_t seed) ++{ ++ uint64_t h, k; ++ ++ h = seed ^ len; ++ ++ while (len >= 8) { ++ k = data[0]; ++ k |= data[1] << 8; ++ k |= data[2] << 16; ++ k |= data[3] << 24; ++ k |= (uint64_t)data[4] << 32; ++ k |= (uint64_t)data[5] << 40; ++ k |= (uint64_t)data[6] << 48; ++ k |= (uint64_t)data[7] << 56; ++ ++ k *= 0xc6a4a7935bd1e995ull; ++ k ^= k >> 47; ++ k *= 0xc6a4a7935bd1e995ull; ++ ++ h ^= k; ++ h *= 0xc6a4a7935bd1e995ull; ++ ++ data += 8; ++ len -= 8; ++ } ++ ++ switch (len) { ++ case 7: ++ h ^= (uint64_t)data[6] << 48; ++ /* fall through */ ++ case 6: ++ h ^= (uint64_t)data[5] << 40; ++ /* fall through */ ++ case 5: ++ h ^= (uint64_t)data[4] << 32; ++ /* fall through */ ++ case 4: ++ h ^= data[3] << 24; ++ /* fall through */ ++ case 3: ++ h ^= data[2] << 16; ++ /* fall through */ ++ case 2: ++ h ^= data[1] << 8; ++ /* fall through */ ++ case 1: ++ h ^= data[0]; ++ h *= 0xc6a4a7935bd1e995ull; ++ } ++ ++ h ^= h >> 47; ++ h *= 0xc6a4a7935bd1e995ull; ++ h ^= h >> 47; ++ ++ return h; ++} +diff --git a/src/core/ngx_murmurhash.h b/src/core/ngx_murmurhash.h +index 54e867d3..322b3df9 100644 +--- a/src/core/ngx_murmurhash.h ++++ b/src/core/ngx_murmurhash.h +@@ -15,5 +15,7 @@ + + uint32_t ngx_murmur_hash2(u_char *data, size_t len); + ++uint64_t ngx_murmur_hash2_64(u_char *data, size_t len, uint64_t seed); ++ + + #endif /* _NGX_MURMURHASH_H_INCLUDED_ */ +diff --git a/src/http/v2/ngx_http_v2.c b/src/http/v2/ngx_http_v2.c +index ed78638c..11d31e5a 100644 +--- a/src/http/v2/ngx_http_v2.c ++++ b/src/http/v2/ngx_http_v2.c +@@ -245,6 +245,8 @@ ngx_http_v2_init(ngx_event_t *rev) + + h2c->frame_size = NGX_HTTP_V2_DEFAULT_FRAME_SIZE; + ++ h2c->max_hpack_table_size = NGX_HTTP_V2_DEFAULT_HPACK_TABLE_SIZE; ++ + h2scf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_v2_module); + + h2c->pool = ngx_create_pool(h2scf->pool_size, h2c->connection->log); +@@ -2013,6 +2015,17 @@ ngx_http_v2_state_settings_params(ngx_http_v2_connection_t *h2c, u_char *pos, + h2c->frame_size = value; + break; + ++ case NGX_HTTP_V2_HEADER_TABLE_SIZE_SETTING: ++ ++ if (value > NGX_HTTP_V2_MAX_HPACK_TABLE_SIZE) { ++ h2c->max_hpack_table_size = NGX_HTTP_V2_MAX_HPACK_TABLE_SIZE; ++ } else { ++ h2c->max_hpack_table_size = value; ++ } ++ ++ h2c->indicate_resize = 1; ++ break; ++ + default: + break; + } +diff --git a/src/http/v2/ngx_http_v2.h b/src/http/v2/ngx_http_v2.h +index be34a093..cfd6d0ed 100644 +--- a/src/http/v2/ngx_http_v2.h ++++ b/src/http/v2/ngx_http_v2.h +@@ -49,6 +49,13 @@ + #define NGX_HTTP_V2_MAX_WINDOW ((1U << 31) - 1) + #define NGX_HTTP_V2_DEFAULT_WINDOW 65535 + ++#define HPACK_ENC_HTABLE_SZ 128 /* better to keep a PoT < 64k */ ++#define HPACK_ENC_HTABLE_ENTRIES ((HPACK_ENC_HTABLE_SZ * 100) / 128) ++#define HPACK_ENC_DYNAMIC_KEY_TBL_SZ 10 /* 10 is sufficient for most */ ++#define HPACK_ENC_MAX_ENTRY 512 /* longest header size to match */ ++ ++#define NGX_HTTP_V2_DEFAULT_HPACK_TABLE_SIZE 4096 ++#define NGX_HTTP_V2_MAX_HPACK_TABLE_SIZE 16384 /* < 64k */ + + typedef struct ngx_http_v2_connection_s ngx_http_v2_connection_t; + typedef struct ngx_http_v2_node_s ngx_http_v2_node_t; +@@ -110,6 +117,46 @@ typedef struct { + } ngx_http_v2_hpack_t; + + ++#if (NGX_HTTP_V2_HPACK_ENC) ++typedef struct { ++ uint64_t hash_val; ++ uint32_t index; ++ uint16_t pos; ++ uint16_t klen, vlen; ++ uint16_t size; ++ uint16_t next; ++} ngx_http_v2_hpack_enc_entry_t; ++ ++ ++typedef struct { ++ uint64_t hash_val; ++ uint32_t index; ++ uint16_t pos; ++ uint16_t klen; ++} ngx_http_v2_hpack_name_entry_t; ++ ++ ++typedef struct { ++ size_t size; /* size as defined in RFC 7541 */ ++ uint32_t top; /* the last entry */ ++ uint32_t pos; ++ uint16_t n_elems; /* number of elements */ ++ uint16_t base; /* index of the oldest entry */ ++ uint16_t last; /* index of the newest entry */ ++ ++ /* hash table for dynamic entries, instead using a generic hash table, ++ which would be too slow to process a significant amount of headers, ++ this table is not determenistic, and might ocasionally fail to insert ++ a value, at the cost of slightly worse compression, but significantly ++ faster performance */ ++ ngx_http_v2_hpack_enc_entry_t htable[HPACK_ENC_HTABLE_SZ]; ++ ngx_http_v2_hpack_name_entry_t heads[HPACK_ENC_DYNAMIC_KEY_TBL_SZ]; ++ u_char storage[NGX_HTTP_V2_MAX_HPACK_TABLE_SIZE + ++ HPACK_ENC_MAX_ENTRY]; ++} ngx_http_v2_hpack_enc_t; ++#endif ++ ++ + struct ngx_http_v2_connection_s { + ngx_connection_t *connection; + ngx_http_connection_t *http_connection; +@@ -122,6 +169,8 @@ struct ngx_http_v2_connection_s { + + size_t frame_size; + ++ size_t max_hpack_table_size; ++ + ngx_queue_t waiting; + + ngx_http_v2_state_t state; +@@ -146,6 +195,11 @@ struct ngx_http_v2_connection_s { + unsigned settings_ack:1; + unsigned blocked:1; + unsigned goaway:1; ++ unsigned indicate_resize:1; ++ ++#if (NGX_HTTP_V2_HPACK_ENC) ++ ngx_http_v2_hpack_enc_t hpack_enc; ++#endif + }; + + +@@ -338,4 +392,31 @@ size_t ngx_http_v2_huff_encode(u_char *src, size_t len, u_char *dst, + + #define ngx_http_v2_write_sid ngx_http_v2_write_uint32 + ++u_char *ngx_http_v2_string_encode(u_char *dst, u_char *src, size_t len, ++ u_char *tmp, ngx_uint_t lower); ++ ++u_char * ++ngx_http_v2_write_int(u_char *pos, ngx_uint_t prefix, ngx_uint_t value); ++ ++#define ngx_http_v2_write_name(dst, src, len, tmp) \ ++ ngx_http_v2_string_encode(dst, src, len, tmp, 1) ++#define ngx_http_v2_write_value(dst, src, len, tmp) \ ++ ngx_http_v2_string_encode(dst, src, len, tmp, 0) ++ ++u_char * ++ngx_http_v2_write_header(ngx_http_v2_connection_t *h2c, u_char *pos, ++ u_char *key, size_t key_len, u_char *value, size_t value_len, ++ u_char *tmp); ++ ++void ++ngx_http_v2_table_resize(ngx_http_v2_connection_t *h2c); ++ ++#define ngx_http_v2_write_header_str(key, value) \ ++ ngx_http_v2_write_header(h2c, pos, (u_char *) key, sizeof(key) - 1, \ ++ (u_char *) value, sizeof(value) - 1, tmp); ++ ++#define ngx_http_v2_write_header_tbl(key, val) \ ++ ngx_http_v2_write_header(h2c, pos, (u_char *) key, sizeof(key) - 1, \ ++ val.data, val.len, tmp); ++ + #endif /* _NGX_HTTP_V2_H_INCLUDED_ */ +diff --git a/src/http/v2/ngx_http_v2_filter_module.c b/src/http/v2/ngx_http_v2_filter_module.c +index 7276531f..6bb33aa0 100644 +--- a/src/http/v2/ngx_http_v2_filter_module.c ++++ b/src/http/v2/ngx_http_v2_filter_module.c +@@ -25,11 +25,6 @@ + #define ngx_http_v2_indexed(i) (128 + (i)) + #define ngx_http_v2_inc_indexed(i) (64 + (i)) + +-#define ngx_http_v2_write_name(dst, src, len, tmp) \ +- ngx_http_v2_string_encode(dst, src, len, tmp, 1) +-#define ngx_http_v2_write_value(dst, src, len, tmp) \ +- ngx_http_v2_string_encode(dst, src, len, tmp, 0) +- + #define NGX_HTTP_V2_ENCODE_RAW 0 + #define NGX_HTTP_V2_ENCODE_HUFF 0x80 + +@@ -51,10 +46,6 @@ + #define NGX_HTTP_V2_VARY_INDEX 59 + + +-static u_char *ngx_http_v2_string_encode(u_char *dst, u_char *src, size_t len, +- u_char *tmp, ngx_uint_t lower); +-static u_char *ngx_http_v2_write_int(u_char *pos, ngx_uint_t prefix, +- ngx_uint_t value); + static ngx_http_v2_out_frame_t *ngx_http_v2_create_headers_frame( + ngx_http_request_t *r, u_char *pos, u_char *end); + +@@ -138,6 +129,7 @@ ngx_http_v2_header_filter(ngx_http_request_t *r) + ngx_http_core_loc_conf_t *clcf; + ngx_http_core_srv_conf_t *cscf; + u_char addr[NGX_SOCKADDR_STRLEN]; ++ ngx_http_v2_connection_t *h2c; + + static const u_char nginx[5] = "\x84\xaa\x63\x55\xe7"; + #if (NGX_HTTP_GZIP) +@@ -146,11 +138,9 @@ ngx_http_v2_header_filter(ngx_http_request_t *r) + #endif + + static size_t nginx_ver_len = ngx_http_v2_literal_size(NGINX_VER); +- static u_char nginx_ver[ngx_http_v2_literal_size(NGINX_VER)]; + + static size_t nginx_ver_build_len = + ngx_http_v2_literal_size(NGINX_VER_BUILD); +- static u_char nginx_ver_build[ngx_http_v2_literal_size(NGINX_VER_BUILD)]; + + if (!r->stream) { + return ngx_http_next_header_filter(r); +@@ -411,7 +401,7 @@ ngx_http_v2_header_filter(ngx_http_request_t *r) + } + + tmp = ngx_palloc(r->pool, tmp_len); +- pos = ngx_pnalloc(r->pool, len); ++ pos = ngx_pnalloc(r->pool, len + 15 + 1); + + if (pos == NULL || tmp == NULL) { + return NGX_ERROR; +@@ -419,6 +409,18 @@ ngx_http_v2_header_filter(ngx_http_request_t *r) + + start = pos; + ++ h2c = r->stream->connection; ++ ++ if (h2c->indicate_resize) { ++ *pos = 32; ++ pos = ngx_http_v2_write_int(pos, ngx_http_v2_prefix(5), ++ h2c->max_hpack_table_size); ++ h2c->indicate_resize = 0; ++#if (NGX_HTTP_V2_HPACK_ENC) ++ ngx_http_v2_table_resize(h2c); ++#endif ++ } ++ + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, fc->log, 0, + "http2 output header: \":status: %03ui\"", + r->headers_out.status); +@@ -427,67 +429,28 @@ ngx_http_v2_header_filter(ngx_http_request_t *r) + *pos++ = status; + + } else { +- *pos++ = ngx_http_v2_inc_indexed(NGX_HTTP_V2_STATUS_INDEX); +- *pos++ = NGX_HTTP_V2_ENCODE_RAW | 3; +- pos = ngx_sprintf(pos, "%03ui", r->headers_out.status); ++ ngx_sprintf(pos + 8, "%O3ui", r->headers_out.status); ++ pos = ngx_http_v2_write_header(h2c, pos, (u_char *)":status", ++ sizeof(":status") - 1, pos + 8, 3, tmp); + } + + if (r->headers_out.server == NULL) { +- + if (clcf->server_tokens == NGX_HTTP_SERVER_TOKENS_ON) { +- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, fc->log, 0, +- "http2 output header: \"server: %s\"", +- NGINX_VER); +- +- } else if (clcf->server_tokens == NGX_HTTP_SERVER_TOKENS_BUILD) { +- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, fc->log, 0, +- "http2 output header: \"server: %s\"", +- NGINX_VER_BUILD); +- +- } else { +- ngx_log_debug0(NGX_LOG_DEBUG_HTTP, fc->log, 0, +- "http2 output header: \"server: nginx\""); +- } +- +- *pos++ = ngx_http_v2_inc_indexed(NGX_HTTP_V2_SERVER_INDEX); +- +- if (clcf->server_tokens == NGX_HTTP_SERVER_TOKENS_ON) { +- if (nginx_ver[0] == '\0') { +- p = ngx_http_v2_write_value(nginx_ver, (u_char *) NGINX_VER, +- sizeof(NGINX_VER) - 1, tmp); +- nginx_ver_len = p - nginx_ver; +- } +- +- pos = ngx_cpymem(pos, nginx_ver, nginx_ver_len); ++ pos = ngx_http_v2_write_header_str("server", NGINX_VER); + + } else if (clcf->server_tokens == NGX_HTTP_SERVER_TOKENS_BUILD) { +- if (nginx_ver_build[0] == '\0') { +- p = ngx_http_v2_write_value(nginx_ver_build, +- (u_char *) NGINX_VER_BUILD, +- sizeof(NGINX_VER_BUILD) - 1, tmp); +- nginx_ver_build_len = p - nginx_ver_build; +- } +- +- pos = ngx_cpymem(pos, nginx_ver_build, nginx_ver_build_len); ++ pos = ngx_http_v2_write_header_str("server", NGINX_VER_BUILD); + + } else { +- pos = ngx_cpymem(pos, nginx, sizeof(nginx)); ++ pos = ngx_http_v2_write_header_str("server", "nginx"); + } + } + + if (r->headers_out.date == NULL) { +- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, fc->log, 0, +- "http2 output header: \"date: %V\"", +- &ngx_cached_http_time); +- +- *pos++ = ngx_http_v2_inc_indexed(NGX_HTTP_V2_DATE_INDEX); +- pos = ngx_http_v2_write_value(pos, ngx_cached_http_time.data, +- ngx_cached_http_time.len, tmp); ++ pos = ngx_http_v2_write_header_tbl("date", ngx_cached_http_time); + } + + if (r->headers_out.content_type.len) { +- *pos++ = ngx_http_v2_inc_indexed(NGX_HTTP_V2_CONTENT_TYPE_INDEX); +- + if (r->headers_out.content_type_len == r->headers_out.content_type.len + && r->headers_out.charset.len) + { +@@ -513,64 +476,36 @@ ngx_http_v2_header_filter(ngx_http_request_t *r) + r->headers_out.content_type.data = p - len; + } + +- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, fc->log, 0, +- "http2 output header: \"content-type: %V\"", +- &r->headers_out.content_type); +- +- pos = ngx_http_v2_write_value(pos, r->headers_out.content_type.data, +- r->headers_out.content_type.len, tmp); ++ pos = ngx_http_v2_write_header_tbl("content-type", ++ r->headers_out.content_type); + } + + if (r->headers_out.content_length == NULL + && r->headers_out.content_length_n >= 0) + { +- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, fc->log, 0, +- "http2 output header: \"content-length: %O\"", +- r->headers_out.content_length_n); +- +- *pos++ = ngx_http_v2_inc_indexed(NGX_HTTP_V2_CONTENT_LENGTH_INDEX); +- +- p = pos; +- pos = ngx_sprintf(pos + 1, "%O", r->headers_out.content_length_n); +- *p = NGX_HTTP_V2_ENCODE_RAW | (u_char) (pos - p - 1); ++ p = ngx_sprintf(pos + 15, "%O", r->headers_out.content_length_n); ++ pos = ngx_http_v2_write_header(h2c, pos, (u_char *)"content-length", ++ sizeof("content-length") - 1, pos + 15, ++ p - (pos + 15), tmp); + } + + if (r->headers_out.last_modified == NULL + && r->headers_out.last_modified_time != -1) + { +- *pos++ = ngx_http_v2_inc_indexed(NGX_HTTP_V2_LAST_MODIFIED_INDEX); +- +- ngx_http_time(pos, r->headers_out.last_modified_time); ++ ngx_http_time(pos + 14, r->headers_out.last_modified_time); + len = sizeof("Wed, 31 Dec 1986 18:00:00 GMT") - 1; +- +- ngx_log_debug2(NGX_LOG_DEBUG_HTTP, fc->log, 0, +- "http2 output header: \"last-modified: %*s\"", +- len, pos); +- +- /* +- * Date will always be encoded using huffman in the temporary buffer, +- * so it's safe here to use src and dst pointing to the same address. +- */ +- pos = ngx_http_v2_write_value(pos, pos, len, tmp); ++ pos = ngx_http_v2_write_header(h2c, pos, (u_char *)"last-modified", ++ sizeof("last-modified") - 1, pos + 14, ++ len, tmp); + } + + if (r->headers_out.location && r->headers_out.location->value.len) { +- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, fc->log, 0, +- "http2 output header: \"location: %V\"", +- &r->headers_out.location->value); +- +- *pos++ = ngx_http_v2_inc_indexed(NGX_HTTP_V2_LOCATION_INDEX); +- pos = ngx_http_v2_write_value(pos, r->headers_out.location->value.data, +- r->headers_out.location->value.len, tmp); ++ pos = ngx_http_v2_write_header_tbl("location", r->headers_out.location->value); + } + + #if (NGX_HTTP_GZIP) + if (r->gzip_vary) { +- ngx_log_debug0(NGX_LOG_DEBUG_HTTP, fc->log, 0, +- "http2 output header: \"vary: Accept-Encoding\""); +- +- *pos++ = ngx_http_v2_inc_indexed(NGX_HTTP_V2_VARY_INDEX); +- pos = ngx_cpymem(pos, accept_encoding, sizeof(accept_encoding)); ++ pos = ngx_http_v2_write_header_str("vary", "Accept-Encoding"); + } + #endif + +@@ -593,23 +528,10 @@ ngx_http_v2_header_filter(ngx_http_request_t *r) + continue; + } + +-#if (NGX_DEBUG) +- if (fc->log->log_level & NGX_LOG_DEBUG_HTTP) { +- ngx_strlow(tmp, header[i].key.data, header[i].key.len); +- +- ngx_log_debug3(NGX_LOG_DEBUG_HTTP, fc->log, 0, +- "http2 output header: \"%*s: %V\"", +- header[i].key.len, tmp, &header[i].value); +- } +-#endif +- +- *pos++ = 0; +- +- pos = ngx_http_v2_write_name(pos, header[i].key.data, +- header[i].key.len, tmp); ++ pos = ngx_http_v2_write_header(h2c, pos, header[i].key.data, ++ header[i].key.len, header[i].value.data, ++ header[i].value.len, tmp); + +- pos = ngx_http_v2_write_value(pos, header[i].value.data, +- header[i].value.len, tmp); + } + + frame = ngx_http_v2_create_headers_frame(r, start, pos); +@@ -636,7 +558,7 @@ ngx_http_v2_header_filter(ngx_http_request_t *r) + } + + +-static u_char * ++u_char * + ngx_http_v2_string_encode(u_char *dst, u_char *src, size_t len, u_char *tmp, + ngx_uint_t lower) + { +@@ -662,7 +584,7 @@ ngx_http_v2_string_encode(u_char *dst, u_char *src, size_t len, u_char *tmp, + } + + +-static u_char * ++u_char * + ngx_http_v2_write_int(u_char *pos, ngx_uint_t prefix, ngx_uint_t value) + { + if (value < prefix) { +diff --git a/src/http/v2/ngx_http_v2_table.c b/src/http/v2/ngx_http_v2_table.c +index a73748a9..ed64d406 100644 +--- a/src/http/v2/ngx_http_v2_table.c ++++ b/src/http/v2/ngx_http_v2_table.c +@@ -347,3 +347,434 @@ ngx_http_v2_table_size(ngx_http_v2_connection_t *h2c, size_t size) + + return NGX_OK; + } ++ ++ ++#if (NGX_HTTP_V2_HPACK_ENC) ++ ++static ngx_int_t ++hpack_get_static_index(ngx_http_v2_connection_t *h2c, u_char *val, size_t len); ++ ++static ngx_int_t ++hpack_get_dynamic_index(ngx_http_v2_connection_t *h2c, uint64_t key_hash, ++ uint8_t *key, size_t key_len); ++ ++ ++void ++ngx_http_v2_table_resize(ngx_http_v2_connection_t *h2c) ++{ ++ ngx_http_v2_hpack_enc_entry_t *table; ++ uint64_t idx; ++ ++ table = h2c->hpack_enc.htable; ++ ++ while (h2c->hpack_enc.size > h2c->max_hpack_table_size) { ++ idx = h2c->hpack_enc.base; ++ h2c->hpack_enc.base = table[idx].next; ++ h2c->hpack_enc.size -= table[idx].size; ++ table[idx].hash_val = 0; ++ h2c->hpack_enc.n_elems--; ++ } ++} ++ ++ ++/* checks if a header is in the hpack table - if so returns the table entry, ++ otherwise encodes and inserts into the table and returns 0, ++ if failed to insert into table, returns -1 */ ++static ngx_int_t ++ngx_http_v2_table_encode_strings(ngx_http_v2_connection_t *h2c, ++ size_t key_len, size_t val_len, uint8_t *key, uint8_t *val, ++ ngx_int_t *header_idx) ++{ ++ uint64_t hash_val, key_hash, idx, lru; ++ int i; ++ size_t size = key_len + val_len + 32; ++ uint8_t *storage = h2c->hpack_enc.storage; ++ ++ ngx_http_v2_hpack_enc_entry_t *table; ++ ngx_http_v2_hpack_name_entry_t *name; ++ ++ *header_idx = NGX_ERROR; ++ /* step 1: compute the hash value of header */ ++ if (size > HPACK_ENC_MAX_ENTRY || size > h2c->max_hpack_table_size) { ++ return NGX_ERROR; ++ } ++ ++ key_hash = ngx_murmur_hash2_64(key, key_len, 0x01234); ++ hash_val = ngx_murmur_hash2_64(val, val_len, key_hash); ++ ++ if (hash_val == 0) { ++ return NGX_ERROR; ++ } ++ ++ /* step 2: check if full header in the table */ ++ idx = hash_val; ++ i = -1; ++ while (idx) { ++ /* at most 8 locations are checked, but most will be done in 1 or 2 */ ++ table = &h2c->hpack_enc.htable[idx % HPACK_ENC_HTABLE_SZ]; ++ if (table->hash_val == hash_val ++ && table->klen == key_len ++ && table->vlen == val_len ++ && ngx_memcmp(key, storage + table->pos, key_len) == 0 ++ && ngx_memcmp(val, storage + table->pos + key_len, val_len) == 0) ++ { ++ return (h2c->hpack_enc.top - table->index) + 61; ++ } ++ ++ if (table->hash_val == 0 && i == -1) { ++ i = idx % HPACK_ENC_HTABLE_SZ; ++ break; ++ } ++ ++ idx >>= 8; ++ } ++ ++ /* step 3: check if key is in one of the tables */ ++ *header_idx = hpack_get_static_index(h2c, key, key_len); ++ ++ if (i == -1) { ++ return NGX_ERROR; ++ } ++ ++ if (*header_idx == NGX_ERROR) { ++ *header_idx = hpack_get_dynamic_index(h2c, key_hash, key, key_len); ++ } ++ ++ /* step 4: store the new entry */ ++ table = h2c->hpack_enc.htable; ++ ++ if (h2c->hpack_enc.top == 0xffffffff) { ++ /* just to be on the safe side, avoid overflow */ ++ ngx_memset(&h2c->hpack_enc, 0, sizeof(ngx_http_v2_hpack_enc_t)); ++ } ++ ++ while ((h2c->hpack_enc.size + size > h2c->max_hpack_table_size) ++ || h2c->hpack_enc.n_elems == HPACK_ENC_HTABLE_ENTRIES) { ++ /* make space for the new entry first */ ++ idx = h2c->hpack_enc.base; ++ h2c->hpack_enc.base = table[idx].next; ++ h2c->hpack_enc.size -= table[idx].size; ++ table[idx].hash_val = 0; ++ h2c->hpack_enc.n_elems--; ++ } ++ ++ table[i] = (ngx_http_v2_hpack_enc_entry_t){.hash_val = hash_val, ++ .index = h2c->hpack_enc.top, ++ .pos = h2c->hpack_enc.pos, ++ .klen = key_len, ++ .vlen = val_len, ++ .size = size, ++ .next = 0}; ++ ++ table[h2c->hpack_enc.last].next = i; ++ if (h2c->hpack_enc.n_elems == 0) { ++ h2c->hpack_enc.base = i; ++ } ++ ++ h2c->hpack_enc.last = i; ++ h2c->hpack_enc.top++; ++ h2c->hpack_enc.size += size; ++ h2c->hpack_enc.n_elems++; ++ ++ /* update header name lookup */ ++ if (*header_idx == NGX_ERROR ) { ++ lru = h2c->hpack_enc.top; ++ ++ for (i=0; ihpack_enc.heads[i]; ++ ++ if ( name->hash_val == 0 || (name->hash_val == key_hash ++ && ngx_memcmp(storage + name->pos, key, key_len) == 0) ) ++ { ++ name->hash_val = key_hash; ++ name->pos = h2c->hpack_enc.pos; ++ name->index = h2c->hpack_enc.top - 1; ++ break; ++ } ++ ++ if (lru > name->index) { ++ lru = name->index; ++ idx = i; ++ } ++ } ++ ++ if (i == HPACK_ENC_DYNAMIC_KEY_TBL_SZ) { ++ name = &h2c->hpack_enc.heads[idx]; ++ name->hash_val = hash_val; ++ name->pos = h2c->hpack_enc.pos; ++ name->index = h2c->hpack_enc.top - 1; ++ } ++ } ++ ++ ngx_memcpy(storage + h2c->hpack_enc.pos, key, key_len); ++ ngx_memcpy(storage + h2c->hpack_enc.pos + key_len, val, val_len); ++ ++ h2c->hpack_enc.pos += size; ++ if (h2c->hpack_enc.pos > NGX_HTTP_V2_MAX_HPACK_TABLE_SIZE) { ++ h2c->hpack_enc.pos = 0; ++ } ++ ++ return NGX_OK; ++} ++ ++ ++u_char * ++ngx_http_v2_write_header(ngx_http_v2_connection_t *h2c, u_char *pos, ++ u_char *key, size_t key_len, ++ u_char *value, size_t value_len, ++ u_char *tmp) ++{ ++ ngx_int_t idx, header_idx; ++ ++ ngx_log_debug4(NGX_LOG_DEBUG_HTTP, h2c->connection->log, 0, ++ "http2 output header: %*s: %*s", key_len, key, value_len, ++ value); ++ ++ /* attempt to find the value in the dynamic table */ ++ idx = ngx_http_v2_table_encode_strings(h2c, key_len, value_len, key, value, ++ &header_idx); ++ ++ if (idx > 0) { ++ /* positive index indicates success */ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, h2c->connection->log, 0, ++ "http2 hpack encode: Indexed Header Field: %ud", idx); ++ ++ *pos = 128; ++ pos = ngx_http_v2_write_int(pos, ngx_http_v2_prefix(7), idx); ++ ++ } else { ++ ++ if (header_idx == NGX_ERROR) { /* if key is not present */ ++ ++ if (idx == NGX_ERROR) { /* if header was not added */ ++ *pos++ = 0; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, h2c->connection->log, 0, ++ "http2 hpack encode: Literal Header Field without" ++ " Indexing — New Name"); ++ } else { /* if header was added */ ++ *pos++ = 64; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, h2c->connection->log, 0, ++ "http2 hpack encode: Literal Header Field with " ++ "Incremental Indexing — New Name"); ++ } ++ ++ pos = ngx_http_v2_write_name(pos, key, key_len, tmp); ++ ++ } else { /* if key is present */ ++ ++ if (idx == NGX_ERROR) { ++ *pos = 0; ++ pos = ngx_http_v2_write_int(pos, ngx_http_v2_prefix(4), header_idx); ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, h2c->connection->log, 0, ++ "http2 hpack encode: Literal Header Field without" ++ " Indexing — Indexed Name: %ud", header_idx); ++ } else { ++ *pos = 64; ++ pos = ngx_http_v2_write_int(pos, ngx_http_v2_prefix(6), header_idx); ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, h2c->connection->log, 0, ++ "http2 hpack encode: Literal Header Field with " ++ "Incremental Indexing — Indexed Name: %ud", header_idx); ++ } ++ } ++ ++ pos = ngx_http_v2_write_value(pos, value, value_len, tmp); ++ } ++ ++ return pos; ++} ++ ++ ++static ngx_int_t ++hpack_get_dynamic_index(ngx_http_v2_connection_t *h2c, uint64_t key_hash, ++ uint8_t *key, size_t key_len) ++{ ++ ngx_http_v2_hpack_name_entry_t *name; ++ int i; ++ ++ for (i=0; ihpack_enc.heads[i]; ++ ++ if (name->hash_val == key_hash ++ && ngx_memcmp(h2c->hpack_enc.storage + name->pos, key, key_len) == 0) ++ { ++ if (name->index >= h2c->hpack_enc.top - h2c->hpack_enc.n_elems) { ++ return (h2c->hpack_enc.top - name->index) + 61; ++ } ++ break; ++ } ++ } ++ ++ return NGX_ERROR; ++} ++ ++ ++/* decide if a given header is present in the static dictionary, this could be ++ done in several ways, but it seems the fastest one is "exhaustive" search */ ++static ngx_int_t ++hpack_get_static_index(ngx_http_v2_connection_t *h2c, u_char *val, size_t len) ++{ ++ /* the static dictionary of response only headers, ++ although response headers can be put by origin, ++ that would be rare */ ++ static const struct { ++ u_char len; ++ const u_char val[28]; ++ u_char idx; ++ } server_headers[] = { ++ { 3, "age", 21},//0 ++ { 3, "via", 60}, ++ { 4, "date", 33},//2 ++ { 4, "etag", 34}, ++ { 4, "link", 45}, ++ { 4, "vary", 59}, ++ { 5, "allow", 22},//6 ++ { 6, "server", 54},//7 ++ { 7, "expires", 36},//8 ++ { 7, "refresh", 52}, ++ { 8, "location", 46},//10 ++ {10, "set-cookie", 55},//11 ++ {11, "retry-after", 53},//12 ++ {12, "content-type", 31},//13 ++ {13, "content-range", 30},//14 ++ {13, "accept-ranges", 18}, ++ {13, "cache-control", 24}, ++ {13, "last-modified", 44}, ++ {14, "content-length", 28},//18 ++ {16, "content-encoding", 26},//19 ++ {16, "content-language", 27}, ++ {16, "content-location", 29}, ++ {16, "www-authenticate", 61}, ++ {17, "transfer-encoding", 57},//23 ++ {18, "proxy-authenticate", 48},//24 ++ {19, "content-disposition", 25},//25 ++ {25, "strict-transport-security", 56},//26 ++ {27, "access-control-allow-origin", 20},//27 ++ {99, "", 99}, ++ }, *header; ++ ++ /* for a given length, where to start the search ++ since minimal length is 3, the table has a -3 ++ offset */ ++ static const int8_t start_at[] = { ++ [3-3] = 0, ++ [4-3] = 2, ++ [5-3] = 6, ++ [6-3] = 7, ++ [7-3] = 8, ++ [8-3] = 10, ++ [9-3] = -1, ++ [10-3] = 11, ++ [11-3] = 12, ++ [12-3] = 13, ++ [13-3] = 14, ++ [14-3] = 18, ++ [15-3] = -1, ++ [16-3] = 19, ++ [17-3] = 23, ++ [18-3] = 24, ++ [19-3] = 25, ++ [20-3] = -1, ++ [21-3] = -1, ++ [22-3] = -1, ++ [23-3] = -1, ++ [24-3] = -1, ++ [25-3] = 26, ++ [26-3] = -1, ++ [27-3] = 27, ++ }; ++ ++ uint64_t pref; ++ size_t save_len = len, i; ++ int8_t start; ++ ++ /* early exit for out of bounds lengths */ ++ if (len < 3 || len > 27) { ++ return NGX_ERROR; ++ } ++ ++ start = start_at[len - 3]; ++ if (start == -1) { ++ /* exit for non existent lengths */ ++ return NGX_ERROR; ++ } ++ ++ header = &server_headers[start_at[len - 3]]; ++ ++ /* load first 8 bytes of key, for fast comparison */ ++ if (len < 8) { ++ pref = 0; ++ if (len >= 4) { ++ pref = *(uint32_t *)(val + len - 4) | 0x20202020; ++ len -= 4; ++ } ++ while (len > 0) { /* 3 iterations at most */ ++ pref = (pref << 8) ^ (val[len - 1] | 0x20); ++ len--; ++ } ++ } else { ++ pref = *(uint64_t *)val | 0x2020202020202020; ++ len -= 8; ++ } ++ ++ /* iterate over headers with the right length */ ++ while (header->len == save_len) { ++ /* quickly compare the first 8 bytes, most tests will end here */ ++ if (pref != *(uint64_t *) header->val) { ++ header++; ++ continue; ++ } ++ ++ if (len == 0) { ++ /* len == 0, indicates prefix held the entire key */ ++ return header->idx; ++ } ++ /* for longer keys compare the rest */ ++ i = 1 + (save_len + 7) % 8; /* align so we can compare in quadwords */ ++ ++ while (i + 8 <= save_len) { /* 3 iterations at most */ ++ if ( *(uint64_t *)&header->val[i] ++ != (*(uint64_t *) &val[i]| 0x2020202020202020) ) ++ { ++ header++; ++ i = 0; ++ break; ++ } ++ i += 8; ++ } ++ ++ if (i == 0) { ++ continue; ++ } ++ ++ /* found the corresponding entry in the static dictionary */ ++ return header->idx; ++ } ++ ++ return NGX_ERROR; ++} ++ ++#else ++ ++u_char * ++ngx_http_v2_write_header(ngx_http_v2_connection_t *h2c, u_char *pos, ++ u_char *key, size_t key_len, ++ u_char *value, size_t value_len, ++ u_char *tmp) ++{ ++ ngx_log_debug4(NGX_LOG_DEBUG_HTTP, h2c->connection->log, 0, ++ "http2 output header: %*s: %*s", key_len, key, value_len, ++ value); ++ ++ *pos++ = 64; ++ pos = ngx_http_v2_write_name(pos, key, key_len, tmp); ++ pos = ngx_http_v2_write_value(pos, value, value_len, tmp); ++ ++ return pos; ++} ++ ++#endif +-- +2.11.0 (Apple Git-81) + From 33dbff6c5c0109612f58e4337234bd7674c21de8 Mon Sep 17 00:00:00 2001 From: Alessandro Ghedini Date: Fri, 18 Oct 2019 15:51:06 -0700 Subject: [PATCH 14/16] Remove P-224 --- conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf b/conf index af8942c..93a2e13 100644 --- a/conf +++ b/conf @@ -1,4 +1,4 @@ ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; -ssl_ecdh_curve X25519:P-256:P-384:P-224:P-521; +ssl_ecdh_curve X25519:P-256:P-384:P-521; ssl_ciphers '[ECDHE-ECDSA-AES128-GCM-SHA256|ECDHE-ECDSA-CHACHA20-POLY1305|ECDHE-RSA-AES128-GCM-SHA256|ECDHE-RSA-CHACHA20-POLY1305]:ECDHE+AES128:RSA+AES128:ECDHE+AES256:RSA+AES256:ECDHE+3DES:RSA+3DES'; ssl_prefer_server_ciphers on; From 869088b2f26e423004c14e07e6f6df6b2d37b9ae Mon Sep 17 00:00:00 2001 From: Hrushikesh Deshpande Date: Mon, 16 Sep 2024 10:05:42 -0400 Subject: [PATCH 15/16] Adding semgrep yaml file --- .github/workflows/semgrep.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/semgrep.yml diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml new file mode 100644 index 0000000..ede31f5 --- /dev/null +++ b/.github/workflows/semgrep.yml @@ -0,0 +1,23 @@ + +on: + pull_request: {} + workflow_dispatch: {} + push: + branches: + - main + - master +name: Semgrep config +jobs: + semgrep: + name: semgrep/ci + runs-on: ubuntu-20.04 + env: + SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }} + SEMGREP_URL: https://cloudflare.semgrep.dev + SEMGREP_APP_URL: https://cloudflare.semgrep.dev + SEMGREP_VERSION_CHECK_URL: https://cloudflare.semgrep.dev/api/check-version + container: + image: returntocorp/semgrep + steps: + - uses: actions/checkout@v3 + - run: semgrep ci From c1be76a225900fc59e4c6db08530845ec3ea6227 Mon Sep 17 00:00:00 2001 From: Hrushikesh Deshpande Date: Mon, 16 Sep 2024 10:06:27 -0400 Subject: [PATCH 16/16] Adding semgrep yaml file