HTTPS support for the Lwt bindings

based on the comment in #77 regarding TLS integration, this PR adds
HTTPS support to the Lwt bindings.

Async support will be added later - @seliopou, would you like a
different PR?

The approach I took in this PR allows downstream users of http/af to use
either `ocaml-tls` or `lwt_ssl` (build time disambiguation based on
installed libraries - preference is given to `ocaml-tls`). The reason
why I did the extra work of including support for `lwt_ssl` is due to
the fact that `ocaml-tls` doesn't yet include support for elliptic curve
ciphersuites (upstream issue:
mirleft/ocaml-tls#362).

.travis.yml

@@ -12,10 +12,22 @@ env:
   - PACKAGE="httpaf-async" DISTRO="ubuntu-16.04"    OCAML_VERSION="4.07.1"
   - PACKAGE="httpaf"       DISTRO="ubuntu-16.04"    OCAML_VERSION="4.06.0"
   - PACKAGE="httpaf-async" DISTRO="ubuntu-16.04"    OCAML_VERSION="4.06.0"
-  - PACKAGE="httpaf-lwt"   DISTRO="ubuntu-16.04"    OCAML_VERSION="4.06.0"
+  - |
+    PRE_INSTALL_HOOK="sudo apt-get install -y libgmp-dev; opam install tls"
+    PACKAGE="httpaf-lwt"
+    DISTRO="ubuntu-16.04"
+    OCAML_VERSION="4.06.0"
   - PACKAGE="httpaf"       DISTRO="ubuntu-16.04"    OCAML_VERSION="4.04.2"
   - PACKAGE="httpaf-async" DISTRO="ubuntu-16.04"    OCAML_VERSION="4.04.2"
-  - PACKAGE="httpaf-lwt"   DISTRO="ubuntu-16.04"    OCAML_VERSION="4.04.2"
+  - |
+    PRE_INSTALL_HOOK="sudo apt-get install -y libgmp-dev; opam install tls"
+    PACKAGE="httpaf-lwt"
+    DISTRO="ubuntu-16.04"
+    OCAML_VERSION="4.04.2"
   - PACKAGE="httpaf"       DISTRO="debian-unstable" OCAML_VERSION="4.03.0"
   - PACKAGE="httpaf-async" DISTRO="debian-unstable" OCAML_VERSION="4.03.0"
-  - PACKAGE="httpaf-lwt"   DISTRO="debian-unstable" OCAML_VERSION="4.03.0"
+  - |
+    PRE_INSTALL_HOOK="sudo apt-get install -y libgmp-dev; opam install tls"
+    PACKAGE="httpaf-lwt"
+    DISTRO="debian-unstable"
+    OCAML_VERSION="4.03.0"

certificates/server.key

@@ -0,0 +1,15 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIICXQIBAAKBgQC2QEje5rwhlD2iq162+Ng3AH9BfA/jNJLDqi9VPk1eMUNGicJv
+K+aOANKIsOOr9v4RiEXZSYmFEvGSy+Sf1bCDHwHLLSdNs6Y49b77POgatrVZOTRE
+BE/t1soVT3a/vVJWCLtVCjm70u0S5tcfn4S6IapeIYAVAmcaqwSa+GQNoQIDAQAB
+AoGAd/CShG8g/JBMh9Nz/8KAuKHRHc2BvysIM1C62cSosgaFmdRrazJfBrEv3Nlc
+2/0uc2dVYIxuvm8bIFqi2TWOdX9jWJf6oXwEPXCD0SaDbJTaoh0b+wjyHuaGlttY
+Ztvmf8mK1BOhyl3vNMxh/8Re0dGvGgPZHpn8zanaqfGVz+ECQQDngieUpwzxA0QZ
+GZKRYhHoLEaPiQzBaXphqWcCLLN7oAKxZlUCUckxRRe0tKINf0cB3Kr9gGQjPpm0
+YoqXo8mNAkEAyYgdd+JDi9FH3Cz6ijvPU0hYkriwTii0V09+Ar5DvYQNzNEIEJu8
+Q3Yte/TPRuK8zhnp97Bsy9v/Ji/LSWbtZQJBAJe9y8u3otfmWCBLjrIUIcCYJLe4
+ENBFHp4ctxPJ0Ora+mjkthuLF+BfdSZQr1dBcX1a8giuuvQO+Bgv7r9t75ECQC7F
+omEyaA7JEW5uGe9/Fgz0G2ph5rkdBU3GKy6jzcDsJu/EC6UfH8Bgawn7tSd0c/E5
+Xm2Xyog9lKfeK8XrV2kCQQCTico5lQPjfIwjhvn45ALc/0OrkaK0hQNpXgUNFJFQ
+tuX2WMD5flMyA5PCx5XBU8gEMHYa8Kr5d6uoixnbS0cZ
+-----END RSA PRIVATE KEY-----

certificates/server.pem

@@ -0,0 +1,15 @@
+-----BEGIN CERTIFICATE-----
+MIICYzCCAcwCCQDLbE6ES1ih1DANBgkqhkiG9w0BAQUFADB2MQswCQYDVQQGEwJB
+VTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0
+cyBQdHkgTHRkMRUwEwYDVQQDDAxZT1VSIE5BTUUhISExGDAWBgkqhkiG9w0BCQEW
+CW1lQGJhci5kZTAeFw0xNDAyMTcyMjA4NDVaFw0xNTAyMTcyMjA4NDVaMHYxCzAJ
+BgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5l
+dCBXaWRnaXRzIFB0eSBMdGQxFTATBgNVBAMMDFlPVVIgTkFNRSEhITEYMBYGCSqG
+SIb3DQEJARYJbWVAYmFyLmRlMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC2
+QEje5rwhlD2iq162+Ng3AH9BfA/jNJLDqi9VPk1eMUNGicJvK+aOANKIsOOr9v4R
+iEXZSYmFEvGSy+Sf1bCDHwHLLSdNs6Y49b77POgatrVZOTREBE/t1soVT3a/vVJW
+CLtVCjm70u0S5tcfn4S6IapeIYAVAmcaqwSa+GQNoQIDAQABMA0GCSqGSIb3DQEB
+BQUAA4GBAIo4ZppIlp3JRyltRC1/AyCC0tsh5TdM3W7258wdoP3lEe08UlLwpnPc
+aJ/cX8rMG4Xf4it77yrbVrU3MumBEGN5TW4jn4+iZyFbp6TT3OUF55nsXDjNHBbu
+deDVpGuPTI6CZQVhU5qEMF3xmlokG+VV+HCDTglNQc+fdLM0LoNF
+-----END CERTIFICATE-----

examples/lwt/jbuild

@@ -1,5 +1,5 @@
 (jbuild_version 1)
 
 (executables
- ((names (lwt_get lwt_post lwt_echo_server))
+ ((names (lwt_get lwt_post lwt_echo_server lwt_https_get lwt_https_server))
   (libraries (httpaf httpaf-lwt lwt lwt.unix))))

examples/lwt/lwt_echo_server.ml

@@ -5,7 +5,7 @@ let connection_handler : Unix.sockaddr -> Lwt_unix.file_descr -> unit Lwt.t =
   let module Response = Httpaf.Response in
   let module Status = Httpaf.Status in
 
-  let request_handler : Unix.sockaddr -> _ Reqd.t -> unit =
+  let request_handler : Unix.sockaddr -> Reqd.t -> unit =
       fun _client_address request_descriptor ->
 
     let request = Reqd.request request_descriptor in

examples/lwt/lwt_https_get.ml

@@ -0,0 +1,73 @@
+module Body = Httpaf.Body
+
+let response_handler notify_response_received response response_body =
+  let module Response = Httpaf.Response in
+  match Response.(response.status) with
+  | `OK ->
+    let rec read_response () =
+      Body.schedule_read
+        response_body
+        ~on_eof:(fun () -> Lwt.wakeup_later notify_response_received ())
+        ~on_read:(fun response_fragment ~off ~len ->
+          let response_fragment_string = Bytes.create len in
+          Lwt_bytes.blit_to_bytes
+            response_fragment off
+            response_fragment_string 0
+            len;
+          print_string (Bytes.unsafe_to_string response_fragment_string);
+
+          read_response ())
+    in
+    read_response ()
+
+  | _ ->
+    Format.fprintf Format.err_formatter "%a\n%!" Response.pp_hum response;
+    exit 1
+
+let error_handler _ =
+  assert false
+
+open Lwt.Infix
+
+let () =
+  let host = ref None in
+  let port = ref 443 in
+
+  Arg.parse
+    ["-p", Set_int port, " Port number (443 by default)"]
+    (fun host_argument -> host := Some host_argument)
+    "lwt_https_get.exe [-p N] HOST";
+
+  let host =
+    match !host with
+    | None -> failwith "No hostname provided"
+    | Some host -> host
+  in
+
+  Lwt_main.run begin
+    Lwt_unix.getaddrinfo host (string_of_int !port) [Unix.(AI_FAMILY PF_INET)]
+    >>= fun addresses ->
+
+    let socket = Lwt_unix.socket Unix.PF_INET Unix.SOCK_STREAM 0 in
+    Lwt_unix.connect socket (List.hd addresses).Unix.ai_addr
+    >>= fun () ->
+
+    let request_headers =
+      Httpaf.Request.create
+        `GET "/" ~headers:(Httpaf.Headers.of_list ["Host", host])
+    in
+
+    let response_received, notify_response_received = Lwt.wait () in
+    let response_handler = response_handler notify_response_received in
+
+    let request_body =
+      Httpaf_lwt.Client.request_tls
+        socket
+        request_headers
+        ~error_handler
+        ~response_handler
+    in
+    Body.close_writer request_body;
+
+    response_received
+  end

examples/lwt/lwt_https_server.ml

@@ -0,0 +1,106 @@
+let connection_handler : Unix.sockaddr -> Lwt_unix.file_descr -> unit Lwt.t =
+  let module Body = Httpaf.Body in
+  let module Headers = Httpaf.Headers in
+  let module Reqd = Httpaf.Reqd in
+  let module Response = Httpaf.Response in
+  let module Status = Httpaf.Status in
+
+  let request_handler : Unix.sockaddr -> Reqd.t -> unit =
+      fun _client_address request_descriptor ->
+
+    let request = Reqd.request request_descriptor in
+    match request.meth with
+    | `POST ->
+      let request_body = Reqd.request_body request_descriptor in
+
+      let response_content_type =
+        match Headers.get request.headers "Content-Type" with
+        | Some request_content_type -> request_content_type
+        | None -> "application/octet-stream"
+      in
+
+      let response =
+        Response.create
+          ~headers:(Headers.of_list [
+            "Content-Type", response_content_type;
+            "Connection", "close";
+          ])
+          `OK
+      in
+
+      let response_body =
+        Reqd.respond_with_streaming request_descriptor response in
+
+      let rec respond () =
+        Body.schedule_read
+          request_body
+          ~on_eof:(fun () -> Body.close_writer response_body)
+          ~on_read:(fun request_data ~off ~len ->
+            Body.write_bigstring response_body request_data ~off ~len;
+            respond ())
+      in
+      respond ()
+
+    | _ ->
+      Reqd.respond_with_string
+        request_descriptor (Response.create `Method_not_allowed) ""
+  in
+
+  let error_handler :
+      Unix.sockaddr ->
+      ?request:Httpaf.Request.t ->
+      _ ->
+      (Headers.t -> [`write] Body.t) ->
+        unit =
+      fun _client_address ?request:_ error start_response ->
+
+    let response_body = start_response Headers.empty in
+
+    begin match error with
+    | `Exn exn ->
+      Body.write_string response_body (Printexc.to_string exn);
+      Body.write_string response_body "\n";
+
+    | #Status.standard as error ->
+      Body.write_string response_body (Status.default_reason_phrase error)
+    end;
+
+    Body.close_writer response_body
+  in
+
+  let certfile = "./certificates/server.pem" in
+  let keyfile = "./certificates/server.key" in
+  Httpaf_lwt.Server.create_tls_connection_handler
+    ?server:None
+    ~certfile
+    ~keyfile
+    ?config:None
+    ~request_handler
+    ~error_handler
+
+
+
+let () =
+  let open Lwt.Infix in
+
+  let port = ref 8080 in
+  Arg.parse
+    ["-p", Arg.Set_int port, " Listening port number (8080 by default)"]
+    ignore
+    "Echoes POST requests. Runs forever.";
+
+  let listen_address = Unix.(ADDR_INET (inet_addr_loopback, !port)) in
+
+  Lwt.async begin fun () ->
+    Lwt_io.establish_server_with_client_socket
+      listen_address connection_handler
+    >>= fun _server ->
+      Printf.printf "Listening on port %i and echoing POST requests.\n" !port;
+      print_string "To send a POST request, try\n\n";
+      print_string "  curl https://localhost:8080 -k -X POST -d foo\n\n";
+      flush stdout;
+      Lwt.return_unit
+  end;
+
+  let forever, _ = Lwt.wait () in
+  Lwt_main.run forever

lwt/buffer.ml

@@ -0,0 +1,37 @@
+open Lwt.Infix
+
+(* Based on the Buffer module in httpaf_async.ml. *)
+type t =
+  { buffer      : Lwt_bytes.t
+  ; mutable off : int
+  ; mutable len : int }
+
+let create size =
+  let buffer = Lwt_bytes.create size in
+  { buffer; off = 0; len = 0 }
+
+let compress t =
+  if t.len = 0
+  then begin
+    t.off <- 0;
+    t.len <- 0;
+  end else if t.off > 0
+  then begin
+    Lwt_bytes.blit t.buffer t.off t.buffer 0 t.len;
+    t.off <- 0;
+  end
+
+let get t ~f =
+  let n = f t.buffer ~off:t.off ~len:t.len in
+  t.off <- t.off + n;
+  t.len <- t.len - n;
+  if t.len = 0
+  then t.off <- 0;
+  n
+
+let put t ~f =
+  compress t;
+  f t.buffer ~off:(t.off + t.len) ~len:(Lwt_bytes.length t.buffer - t.len)
+  >>= fun n ->
+  t.len <- t.len + n;
+  Lwt.return n

lwt/buffer.mli

@@ -0,0 +1,6 @@
+type t
+
+val create : int -> t
+
+val get : t -> f:(Lwt_bytes.t -> off:int -> len:int -> int) -> int
+val put : t -> f:(Lwt_bytes.t -> off:int -> len:int -> int Lwt.t) -> int Lwt.t