From bc9dfb2cb50aaa7837de64bc3989ae107a12cca8 Mon Sep 17 00:00:00 2001 From: Scott Feldman Date: Mon, 27 Mar 2023 14:15:21 -0700 Subject: [PATCH 01/11] Add network device driver model, netdev This PR adds a network device driver model called netdev. There will be a companion PR for TinyGo drivers to update the netdev drivers and network examples. This PR covers the core "net" package. An RFC for the work is here: #tinygo-org/drivers#487. Some things have changed from the RFC, but nothing major. The "net" package is a partial port of Go's "net" package, version 1.19.3. The src/net/README file has details on what is modified from Go's "net" package. Most "net" features are working as they would in normal Go. TCP/UDP/TLS protocol support is there. As well as HTTP client and server support. Standard Go network packages such as golang.org/x/net/websockets and Paho MQTT client work as-is. Other packages are likely to work as-is. Testing results are here (https://docs.google.com/spreadsheets/d/e/2PACX-1vT0cCjBvwXf9HJf6aJV2Sw198F2ief02gmbMV0sQocKT4y4RpfKv3dh6Jyew8lQW64FouZ8GwA2yjxI/pubhtml?gid=1013173032&single=true). --- loader/goroot.go | 2 + src/crypto/tls/common.go | 12 + src/crypto/tls/tls.go | 63 + src/net/README.md | 107 + src/net/conn_test.go | 478 --- src/net/dial.go | 158 +- src/net/errors.go | 10 - src/net/http/client.go | 523 ++++ src/net/http/clone.go | 76 + src/net/http/cookie.go | 470 +++ src/net/http/fs.go | 974 ++++++ src/net/http/header.go | 275 ++ src/net/http/http.go | 161 + src/net/http/internal/ascii/print.go | 63 + src/net/http/internal/ascii/print_test.go | 95 + src/net/http/internal/chunked.go | 264 ++ src/net/http/internal/chunked_test.go | 241 ++ src/net/http/jar.go | 29 + src/net/http/method.go | 22 + src/net/http/request.go | 1447 +++++++++ src/net/http/response.go | 373 +++ src/net/http/server.go | 3261 +++++++++++++++++++++ src/net/http/sniff.go | 306 ++ src/net/http/status.go | 212 ++ src/net/http/transfer.go | 1127 +++++++ src/net/http/transport.go | 22 + src/net/interface.go | 253 -- src/net/interface_tinygo.go | 53 - src/net/ip.go | 30 +- src/net/iprawsock.go | 20 +- src/net/ipsock.go | 6 +- src/net/mac.go | 22 +- src/net/mac_test.go | 109 + src/net/net.go | 91 +- src/net/netdev.go | 46 + src/net/parse.go | 257 +- src/net/pipe.go | 2 +- src/net/pipe_test.go | 48 - src/net/tcpsock.go | 233 +- src/net/tlssock.go | 154 + src/net/udpsock.go | 211 ++ src/net/writev_test.go | 132 - src/os/file_other.go | 8 + src/syscall/net.go | 19 + src/syscall/syscall_libc_darwin.go | 1 - src/syscall/syscall_libc_wasi.go | 1 - 46 files changed, 11374 insertions(+), 1093 deletions(-) create mode 100644 src/crypto/tls/common.go create mode 100644 src/crypto/tls/tls.go create mode 100644 src/net/README.md delete mode 100644 src/net/conn_test.go delete mode 100644 src/net/errors.go create mode 100644 src/net/http/client.go create mode 100644 src/net/http/clone.go create mode 100644 src/net/http/cookie.go create mode 100644 src/net/http/fs.go create mode 100644 src/net/http/header.go create mode 100644 src/net/http/http.go create mode 100644 src/net/http/internal/ascii/print.go create mode 100644 src/net/http/internal/ascii/print_test.go create mode 100644 src/net/http/internal/chunked.go create mode 100644 src/net/http/internal/chunked_test.go create mode 100644 src/net/http/jar.go create mode 100644 src/net/http/method.go create mode 100644 src/net/http/request.go create mode 100644 src/net/http/response.go create mode 100644 src/net/http/server.go create mode 100644 src/net/http/sniff.go create mode 100644 src/net/http/status.go create mode 100644 src/net/http/transfer.go create mode 100644 src/net/http/transport.go delete mode 100644 src/net/interface.go delete mode 100644 src/net/interface_tinygo.go create mode 100644 src/net/mac_test.go create mode 100644 src/net/netdev.go delete mode 100644 src/net/pipe_test.go create mode 100644 src/net/tlssock.go delete mode 100644 src/net/writev_test.go diff --git a/loader/goroot.go b/loader/goroot.go index b6812badc4..d1d8e044dd 100644 --- a/loader/goroot.go +++ b/loader/goroot.go @@ -228,6 +228,7 @@ func pathsToOverride(goMinor int, needsSyscallPackage bool) map[string]bool { "": true, "crypto/": true, "crypto/rand/": false, + "crypto/tls/": false, "device/": false, "examples/": false, "internal/": true, @@ -237,6 +238,7 @@ func pathsToOverride(goMinor int, needsSyscallPackage bool) map[string]bool { "internal/task/": false, "machine/": false, "net/": true, + "net/http/": false, "os/": true, "reflect/": false, "runtime/": false, diff --git a/src/crypto/tls/common.go b/src/crypto/tls/common.go new file mode 100644 index 0000000000..f97c47e19c --- /dev/null +++ b/src/crypto/tls/common.go @@ -0,0 +1,12 @@ +// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. + +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +// ConnectionState records basic TLS details about the connection. +type ConnectionState struct { + // TINYGO: empty; TLS connection offloaded to device +} diff --git a/src/crypto/tls/tls.go b/src/crypto/tls/tls.go new file mode 100644 index 0000000000..1d1eee105c --- /dev/null +++ b/src/crypto/tls/tls.go @@ -0,0 +1,63 @@ +// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. + +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package tls partially implements TLS 1.2, as specified in RFC 5246, +// and TLS 1.3, as specified in RFC 8446. +package tls + +// BUG(agl): The crypto/tls package only implements some countermeasures +// against Lucky13 attacks on CBC-mode encryption, and only on SHA1 +// variants. See http://www.isg.rhul.ac.uk/tls/TLStiming.pdf and +// https://www.imperialviolet.org/2013/02/04/luckythirteen.html. + +import ( + "fmt" + "net" +) + +// Client returns a new TLS client side connection +// using conn as the underlying transport. +// The config cannot be nil: users must set either ServerName or +// InsecureSkipVerify in the config. +func Client(conn net.Conn, config *Config) *net.TLSConn { + panic("tls.Client() not implemented") + return nil +} + +// DialWithDialer connects to the given network address using dialer.Dial and +// then initiates a TLS handshake, returning the resulting TLS connection. Any +// timeout or deadline given in the dialer apply to connection and TLS +// handshake as a whole. +// +// DialWithDialer interprets a nil configuration as equivalent to the zero +// configuration; see the documentation of Config for the defaults. +// +// DialWithDialer uses context.Background internally; to specify the context, +// use Dialer.DialContext with NetDialer set to the desired dialer. +func DialWithDialer(dialer *net.Dialer, network, addr string, config *Config) (*net.TLSConn, error) { + switch network { + case "tcp", "tcp4": + default: + return nil, fmt.Errorf("Network %s not supported", network) + } + + return net.DialTLS(addr) +} + +// Dial connects to the given network address using net.Dial +// and then initiates a TLS handshake, returning the resulting +// TLS connection. +// Dial interprets a nil configuration as equivalent to +// the zero configuration; see the documentation of Config +// for the defaults. +func Dial(network, addr string, config *Config) (*net.TLSConn, error) { + return DialWithDialer(new(net.Dialer), network, addr, config) +} + +// Config is a placeholder for future compatibility with +// tls.Config. +type Config struct { +} diff --git a/src/net/README.md b/src/net/README.md new file mode 100644 index 0000000000..bba8fdbba8 --- /dev/null +++ b/src/net/README.md @@ -0,0 +1,107 @@ +This is a port of Go's "net" package. The port offers a subset of Go's "net" +package. The subset maintains Go 1 compatiblity guarantee. + +The "net" package is modified to use netdev, TinyGo's network device driver interface. +Netdev replaces the OS syscall interface for I/O access to the networking +device. + +#### Table of Contents + +- ["net" Package](#net-package) +- [Netdev and Netlink](#netdev-and-netlink) +- [Using "net" and "net/http" Packages](#using-net-and-nethttp-packages) + +## "net" Package + +The "net" package is ported from Go 1.19.3. The tree listings below shows the +files copied. If the file is marked with an '\*', it is copied _and_ modified +to work with netdev. If the file is marked with an '+', the file is new. If +there is no mark, it is a straight copy. + +``` +src/net +├── dial.go * +├── http +│   ├── client.go * +│   ├── clone.go +│   ├── cookie.go +│   ├── fs.go +│   ├── header.go * +│   ├── http.go +│   ├── internal +│   │   ├── ascii +│   │   │   ├── print.go +│   │   │   └── print_test.go +│   │   ├── chunked.go +│   │   └── chunked_test.go +│   ├── jar.go +│   ├── method.go +│   ├── request.go * +│   ├── response.go * +│   ├── server.go * +│   ├── sniff.go +│   ├── status.go +│   ├── transfer.go * +│   └── transport.go * +├── ip.go +├── iprawsock.go * +├── ipsock.go * +├── mac.go +├── mac_test.go +├── netdev.go + +├── net.go * +├── parse.go +├── pipe.go +├── README.md +├── tcpsock.go * +├── tlssock.go + +└── udpsock.go * + +src/crypto/tls/ +├── common.go * +└── tls.go * +``` + +The modifications to "net" are to basically wrap TCPConn, UDPConn, and TLSConn +around netdev socket calls. In Go, these net.Conns call out to OS syscalls for +the socket operations. In TinyGo, the OS syscalls aren't available, so netdev +socket calls are substituted. + +The modifications to "net/http" are on the client and the server side. On the +client side, the TinyGo code changes remove the back-end round-tripper code and +replaces it with direct calls to TCPConns/TLSConns. All of Go's http +request/response handling code is intact and operational in TinyGo. Same holds +true for the server side. The server side supports the normal server features +like ServeMux and Hijacker (for websockets). + +### Maintaining "net" + +As Go progresses, changes to the "net" package need to be periodically +back-ported to TinyGo's "net" package. This is to pick up any upstream bug +fixes or security fixes. + +Changes "net" package files are marked with // TINYGO comments. + +The files that are marked modified * may contain only a subset of the original +file. Basically only the parts necessary to compile and run the example/net +examples are copied (and maybe modified). + +## Netdev and Netlink + +Netdev is TinyGo's network device driver model. Network drivers implement the +netdever interface, providing a common network I/O interface to TinyGo's "net" +package. The interface is modeled after the BSD socket interface. net.Conn +implementations (TCPConn, UDPConn, and TLSConn) use the netdev interface for +device I/O access. + +Network drivers also (optionally) implement the Netlinker interface. This +interface is not used by TinyGo's "net" package, but rather provides the TinyGo +application direct access to the network device for common settings and control +that fall outside of netdev's socket interface. + +See the README-net.md in drivers repo for more details on netdev and netlink. + +## Using "net" and "net/http" Packages + +See README-net.md in drivers repo to more details on using "net" and "net/http" +packages in a TinyGo application. diff --git a/src/net/conn_test.go b/src/net/conn_test.go deleted file mode 100644 index 4e1ac28c43..0000000000 --- a/src/net/conn_test.go +++ /dev/null @@ -1,478 +0,0 @@ -// The following is copied from x/net official implementation. -// Source: https://cs.opensource.google/go/x/net/+/f15817d1:nettest/conntest.go -// Changes from original the file: -// - Some variables are pulled in from nettest/nettest.go file. -// - The implementation of checkForTimeoutError() function is changed in -// accordance with error returned by the Pipe implementation. - -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package net - -import ( - "bytes" - "encoding/binary" - "io" - "io/ioutil" - "math/rand" - "os" - "runtime" - "sync" - "testing" - "time" -) - -// The following variables are copied from nettest/nettest.go file -var ( - aLongTimeAgo = time.Unix(233431200, 0) - neverTimeout = time.Time{} -) - -// MakePipe creates a connection between two endpoints and returns the pair -// as c1 and c2, such that anything written to c1 is read by c2 and vice-versa. -// The stop function closes all resources, including c1, c2, and the underlying -// Listener (if there is one), and should not be nil. -type MakePipe func() (c1, c2 Conn, stop func(), err error) - -// testConn tests that a Conn implementation properly satisfies the interface. -// The tests should not produce any false positives, but may experience -// false negatives. Thus, some issues may only be detected when the test is -// run multiple times. For maximal effectiveness, run the tests under the -// race detector. -func testConn(t *testing.T, mp MakePipe) { - t.Run("BasicIO", func(t *testing.T) { timeoutWrapper(t, mp, testBasicIO) }) - t.Run("PingPong", func(t *testing.T) { timeoutWrapper(t, mp, testPingPong) }) - t.Run("RacyRead", func(t *testing.T) { timeoutWrapper(t, mp, testRacyRead) }) - t.Run("RacyWrite", func(t *testing.T) { timeoutWrapper(t, mp, testRacyWrite) }) - t.Run("ReadTimeout", func(t *testing.T) { timeoutWrapper(t, mp, testReadTimeout) }) - t.Run("WriteTimeout", func(t *testing.T) { timeoutWrapper(t, mp, testWriteTimeout) }) - t.Run("PastTimeout", func(t *testing.T) { timeoutWrapper(t, mp, testPastTimeout) }) - t.Run("PresentTimeout", func(t *testing.T) { timeoutWrapper(t, mp, testPresentTimeout) }) - t.Run("FutureTimeout", func(t *testing.T) { timeoutWrapper(t, mp, testFutureTimeout) }) - t.Run("CloseTimeout", func(t *testing.T) { timeoutWrapper(t, mp, testCloseTimeout) }) - t.Run("ConcurrentMethods", func(t *testing.T) { timeoutWrapper(t, mp, testConcurrentMethods) }) -} - -type connTester func(t *testing.T, c1, c2 Conn) - -func timeoutWrapper(t *testing.T, mp MakePipe, f connTester) { - t.Helper() - c1, c2, stop, err := mp() - if err != nil { - t.Fatalf("unable to make pipe: %v", err) - } - var once sync.Once - defer once.Do(func() { stop() }) - timer := time.AfterFunc(time.Minute, func() { - once.Do(func() { - t.Error("test timed out; terminating pipe") - stop() - }) - }) - defer timer.Stop() - f(t, c1, c2) -} - -// testBasicIO tests that the data sent on c1 is properly received on c2. -func testBasicIO(t *testing.T, c1, c2 Conn) { - want := make([]byte, 1<<20) - rand.New(rand.NewSource(0)).Read(want) - - dataCh := make(chan []byte) - go func() { - rd := bytes.NewReader(want) - if err := chunkedCopy(c1, rd); err != nil { - t.Errorf("unexpected c1.Write error: %v", err) - } - if err := c1.Close(); err != nil { - t.Errorf("unexpected c1.Close error: %v", err) - } - }() - - go func() { - wr := new(bytes.Buffer) - if err := chunkedCopy(wr, c2); err != nil { - t.Errorf("unexpected c2.Read error: %v", err) - } - if err := c2.Close(); err != nil { - t.Errorf("unexpected c2.Close error: %v", err) - } - dataCh <- wr.Bytes() - }() - - if got := <-dataCh; !bytes.Equal(got, want) { - t.Error("transmitted data differs") - } -} - -// testPingPong tests that the two endpoints can synchronously send data to -// each other in a typical request-response pattern. -func testPingPong(t *testing.T, c1, c2 Conn) { - var wg sync.WaitGroup - defer wg.Wait() - - pingPonger := func(c Conn) { - defer wg.Done() - buf := make([]byte, 8) - var prev uint64 - for { - if _, err := io.ReadFull(c, buf); err != nil { - if err == io.EOF { - break - } - t.Errorf("unexpected Read error: %v", err) - } - - v := binary.LittleEndian.Uint64(buf) - binary.LittleEndian.PutUint64(buf, v+1) - if prev != 0 && prev+2 != v { - t.Errorf("mismatching value: got %d, want %d", v, prev+2) - } - prev = v - if v == 1000 { - break - } - - if _, err := c.Write(buf); err != nil { - t.Errorf("unexpected Write error: %v", err) - break - } - } - if err := c.Close(); err != nil { - t.Errorf("unexpected Close error: %v", err) - } - } - - wg.Add(2) - go pingPonger(c1) - go pingPonger(c2) - - // Start off the chain reaction. - if _, err := c1.Write(make([]byte, 8)); err != nil { - t.Errorf("unexpected c1.Write error: %v", err) - } -} - -// testRacyRead tests that it is safe to mutate the input Read buffer -// immediately after cancelation has occurred. -func testRacyRead(t *testing.T, c1, c2 Conn) { - go chunkedCopy(c2, rand.New(rand.NewSource(0))) - - var wg sync.WaitGroup - defer wg.Wait() - - c1.SetReadDeadline(time.Now().Add(time.Millisecond)) - for i := 0; i < 10; i++ { - wg.Add(1) - go func() { - defer wg.Done() - - b1 := make([]byte, 1024) - b2 := make([]byte, 1024) - for j := 0; j < 100; j++ { - _, err := c1.Read(b1) - copy(b1, b2) // Mutate b1 to trigger potential race - if err != nil { - checkForTimeoutError(t, err) - c1.SetReadDeadline(time.Now().Add(time.Millisecond)) - } - } - }() - } -} - -// testRacyWrite tests that it is safe to mutate the input Write buffer -// immediately after cancelation has occurred. -func testRacyWrite(t *testing.T, c1, c2 Conn) { - go chunkedCopy(ioutil.Discard, c2) - - var wg sync.WaitGroup - defer wg.Wait() - - c1.SetWriteDeadline(time.Now().Add(time.Millisecond)) - for i := 0; i < 10; i++ { - wg.Add(1) - go func() { - defer wg.Done() - - b1 := make([]byte, 1024) - b2 := make([]byte, 1024) - for j := 0; j < 100; j++ { - _, err := c1.Write(b1) - copy(b1, b2) // Mutate b1 to trigger potential race - if err != nil { - checkForTimeoutError(t, err) - c1.SetWriteDeadline(time.Now().Add(time.Millisecond)) - } - } - }() - } -} - -// testReadTimeout tests that Read timeouts do not affect Write. -func testReadTimeout(t *testing.T, c1, c2 Conn) { - go chunkedCopy(ioutil.Discard, c2) - - c1.SetReadDeadline(aLongTimeAgo) - _, err := c1.Read(make([]byte, 1024)) - checkForTimeoutError(t, err) - if _, err := c1.Write(make([]byte, 1024)); err != nil { - t.Errorf("unexpected Write error: %v", err) - } -} - -// testWriteTimeout tests that Write timeouts do not affect Read. -func testWriteTimeout(t *testing.T, c1, c2 Conn) { - go chunkedCopy(c2, rand.New(rand.NewSource(0))) - - c1.SetWriteDeadline(aLongTimeAgo) - _, err := c1.Write(make([]byte, 1024)) - checkForTimeoutError(t, err) - if _, err := c1.Read(make([]byte, 1024)); err != nil { - t.Errorf("unexpected Read error: %v", err) - } -} - -// testPastTimeout tests that a deadline set in the past immediately times out -// Read and Write requests. -func testPastTimeout(t *testing.T, c1, c2 Conn) { - go chunkedCopy(c2, c2) - - testRoundtrip(t, c1) - - c1.SetDeadline(aLongTimeAgo) - n, err := c1.Write(make([]byte, 1024)) - if n != 0 { - t.Errorf("unexpected Write count: got %d, want 0", n) - } - checkForTimeoutError(t, err) - n, err = c1.Read(make([]byte, 1024)) - if n != 0 { - t.Errorf("unexpected Read count: got %d, want 0", n) - } - checkForTimeoutError(t, err) - - testRoundtrip(t, c1) -} - -// testPresentTimeout tests that a past deadline set while there are pending -// Read and Write operations immediately times out those operations. -func testPresentTimeout(t *testing.T, c1, c2 Conn) { - var wg sync.WaitGroup - defer wg.Wait() - wg.Add(3) - - deadlineSet := make(chan bool, 1) - go func() { - defer wg.Done() - time.Sleep(100 * time.Millisecond) - deadlineSet <- true - c1.SetReadDeadline(aLongTimeAgo) - c1.SetWriteDeadline(aLongTimeAgo) - }() - go func() { - defer wg.Done() - n, err := c1.Read(make([]byte, 1024)) - if n != 0 { - t.Errorf("unexpected Read count: got %d, want 0", n) - } - checkForTimeoutError(t, err) - if len(deadlineSet) == 0 { - t.Error("Read timed out before deadline is set") - } - }() - go func() { - defer wg.Done() - var err error - for err == nil { - _, err = c1.Write(make([]byte, 1024)) - } - checkForTimeoutError(t, err) - if len(deadlineSet) == 0 { - t.Error("Write timed out before deadline is set") - } - }() -} - -// testFutureTimeout tests that a future deadline will eventually time out -// Read and Write operations. -func testFutureTimeout(t *testing.T, c1, c2 Conn) { - var wg sync.WaitGroup - wg.Add(2) - - c1.SetDeadline(time.Now().Add(100 * time.Millisecond)) - go func() { - defer wg.Done() - _, err := c1.Read(make([]byte, 1024)) - checkForTimeoutError(t, err) - }() - go func() { - defer wg.Done() - var err error - for err == nil { - _, err = c1.Write(make([]byte, 1024)) - } - checkForTimeoutError(t, err) - }() - wg.Wait() - - go chunkedCopy(c2, c2) - resyncConn(t, c1) - testRoundtrip(t, c1) -} - -// testCloseTimeout tests that calling Close immediately times out pending -// Read and Write operations. -func testCloseTimeout(t *testing.T, c1, c2 Conn) { - go chunkedCopy(c2, c2) - - var wg sync.WaitGroup - defer wg.Wait() - wg.Add(3) - - // Test for cancelation upon connection closure. - c1.SetDeadline(neverTimeout) - go func() { - defer wg.Done() - time.Sleep(100 * time.Millisecond) - c1.Close() - }() - go func() { - defer wg.Done() - var err error - buf := make([]byte, 1024) - for err == nil { - _, err = c1.Read(buf) - } - }() - go func() { - defer wg.Done() - var err error - buf := make([]byte, 1024) - for err == nil { - _, err = c1.Write(buf) - } - }() -} - -// testConcurrentMethods tests that the methods of Conn can safely -// be called concurrently. -func testConcurrentMethods(t *testing.T, c1, c2 Conn) { - if runtime.GOOS == "plan9" { - t.Skip("skipping on plan9; see https://golang.org/issue/20489") - } - go chunkedCopy(c2, c2) - - // The results of the calls may be nonsensical, but this should - // not trigger a race detector warning. - var wg sync.WaitGroup - for i := 0; i < 100; i++ { - wg.Add(7) - go func() { - defer wg.Done() - c1.Read(make([]byte, 1024)) - }() - go func() { - defer wg.Done() - c1.Write(make([]byte, 1024)) - }() - go func() { - defer wg.Done() - c1.SetDeadline(time.Now().Add(10 * time.Millisecond)) - }() - go func() { - defer wg.Done() - c1.SetReadDeadline(aLongTimeAgo) - }() - go func() { - defer wg.Done() - c1.SetWriteDeadline(aLongTimeAgo) - }() - go func() { - defer wg.Done() - c1.LocalAddr() - }() - go func() { - defer wg.Done() - c1.RemoteAddr() - }() - } - wg.Wait() // At worst, the deadline is set 10ms into the future - - resyncConn(t, c1) - testRoundtrip(t, c1) -} - -// checkForTimeoutError checks that the error satisfies the OpError interface -// and that underlying Err is os.ErrDeadlineExceeded -func checkForTimeoutError(t *testing.T, err error) { - t.Helper() - operr, ok := err.(*OpError) - if !ok { - t.Errorf("got %T: %v, want OpError", err, err) - return - } - if operr.Err != os.ErrDeadlineExceeded { - t.Errorf("got %T: %v, want os.ErrDeadlineExceeded", err, err) - } -} - -// testRoundtrip writes something into c and reads it back. -// It assumes that everything written into c is echoed back to itself. -func testRoundtrip(t *testing.T, c Conn) { - t.Helper() - if err := c.SetDeadline(neverTimeout); err != nil { - t.Errorf("roundtrip SetDeadline error: %v", err) - } - - const s = "Hello, world!" - buf := []byte(s) - if _, err := c.Write(buf); err != nil { - t.Errorf("roundtrip Write error: %v", err) - } - if _, err := io.ReadFull(c, buf); err != nil { - t.Errorf("roundtrip Read error: %v", err) - } - if string(buf) != s { - t.Errorf("roundtrip data mismatch: got %q, want %q", buf, s) - } -} - -// resyncConn resynchronizes the connection into a sane state. -// It assumes that everything written into c is echoed back to itself. -// It assumes that 0xff is not currently on the wire or in the read buffer. -func resyncConn(t *testing.T, c Conn) { - t.Helper() - c.SetDeadline(neverTimeout) - errCh := make(chan error) - go func() { - _, err := c.Write([]byte{0xff}) - errCh <- err - }() - buf := make([]byte, 1024) - for { - n, err := c.Read(buf) - if n > 0 && bytes.IndexByte(buf[:n], 0xff) == n-1 { - break - } - if err != nil { - t.Errorf("unexpected Read error: %v", err) - break - } - } - if err := <-errCh; err != nil { - t.Errorf("unexpected Write error: %v", err) - } -} - -// chunkedCopy copies from r to w in fixed-width chunks to avoid -// causing a Write that exceeds the maximum packet size for packet-based -// connections like "unixpacket". -// We assume that the maximum packet size is at least 1024. -func chunkedCopy(w io.Writer, r io.Reader) error { - b := make([]byte, 1024) - _, err := io.CopyBuffer(struct{ io.Writer }{w}, struct{ io.Reader }{r}, b) - return err -} diff --git a/src/net/dial.go b/src/net/dial.go index a1cb75d87f..ac32e62182 100644 --- a/src/net/dial.go +++ b/src/net/dial.go @@ -1,25 +1,169 @@ +// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. + +// TINYGO: Omit DualStack support +// TINYGO: Omit Fast Fallback support +// TINYGO: Don't allow alternate resolver +// TINYGO: Omit DialTimeout + +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package net import ( "context" + "fmt" "time" ) +// defaultTCPKeepAlive is a default constant value for TCPKeepAlive times +// See golang.org/issue/31510 +const ( + defaultTCPKeepAlive = 15 * time.Second +) + +// A Dialer contains options for connecting to an address. +// +// The zero value for each field is equivalent to dialing +// without that option. Dialing with the zero value of Dialer +// is therefore equivalent to just calling the Dial function. +// +// It is safe to call Dialer's methods concurrently. type Dialer struct { - Timeout time.Duration - Deadline time.Time - DualStack bool + // Timeout is the maximum amount of time a dial will wait for + // a connect to complete. If Deadline is also set, it may fail + // earlier. + // + // The default is no timeout. + // + // When using TCP and dialing a host name with multiple IP + // addresses, the timeout may be divided between them. + // + // With or without a timeout, the operating system may impose + // its own earlier timeout. For instance, TCP timeouts are + // often around 3 minutes. + Timeout time.Duration + + // Deadline is the absolute point in time after which dials + // will fail. If Timeout is set, it may fail earlier. + // Zero means no deadline, or dependent on the operating system + // as with the Timeout option. + Deadline time.Time + + // LocalAddr is the local address to use when dialing an + // address. The address must be of a compatible type for the + // network being dialed. + // If nil, a local address is automatically chosen. + LocalAddr Addr + + // KeepAlive specifies the interval between keep-alive + // probes for an active network connection. + // If zero, keep-alive probes are sent with a default value + // (currently 15 seconds), if supported by the protocol and operating + // system. Network protocols or operating systems that do + // not support keep-alives ignore this field. + // If negative, keep-alive probes are disabled. KeepAlive time.Duration } +// Dial connects to the address on the named network. +// +// See Go "net" package Dial() for more information. +// +// Note: Tinygo Dial supports a subset of networks supported by Go Dial, +// specifically: "tcp", "tcp4", "udp", and "udp4". IP and unix networks are +// not supported. func Dial(network, address string) (Conn, error) { - return nil, ErrNotImplemented + var d Dialer + return d.Dial(network, address) } -func Listen(network, address string) (Listener, error) { - return nil, ErrNotImplemented +// DialTimeout acts like Dial but takes a timeout. +// +// The timeout includes name resolution, if required. +// When using TCP, and the host in the address parameter resolves to +// multiple IP addresses, the timeout is spread over each consecutive +// dial, such that each is given an appropriate fraction of the time +// to connect. +// +// See func Dial for a description of the network and address +// parameters. +func DialTimeout(network, address string, timeout time.Duration) (Conn, error) { + d := Dialer{Timeout: timeout} + return d.Dial(network, address) +} + +// Dial connects to the address on the named network. +// +// See func Dial for a description of the network and address +// parameters. +// +// Dial uses context.Background internally; to specify the context, use +// DialContext. +func (d *Dialer) Dial(network, address string) (Conn, error) { + return d.DialContext(context.Background(), network, address) } +// DialContext connects to the address on the named network using +// the provided context. +// +// The provided Context must be non-nil. If the context expires before +// the connection is complete, an error is returned. Once successfully +// connected, any expiration of the context will not affect the +// connection. +// +// When using TCP, and the host in the address parameter resolves to multiple +// network addresses, any dial timeout (from d.Timeout or ctx) is spread +// over each consecutive dial, such that each is given an appropriate +// fraction of the time to connect. +// For example, if a host has 4 IP addresses and the timeout is 1 minute, +// the connect to each single address will be given 15 seconds to complete +// before trying the next one. +// +// See func Dial for a description of the network and address +// parameters. func (d *Dialer) DialContext(ctx context.Context, network, address string) (Conn, error) { - return nil, ErrNotImplemented + + // TINYGO: Ignoring context + + switch network { + case "tcp", "tcp4": + raddr, err := ResolveTCPAddr(network, address) + if err != nil { + return nil, err + } + return DialTCP(network, nil, raddr) + case "udp", "udp4": + raddr, err := ResolveUDPAddr(network, address) + if err != nil { + return nil, err + } + return DialUDP(network, nil, raddr) + } + + return nil, fmt.Errorf("Network %s not supported", network) +} + +// Listen announces on the local network address. +// +// See Go "net" package Listen() for more information. +// +// Note: Tinygo Listen supports a subset of networks supported by Go Listen, +// specifically: "tcp", "tcp4". "tcp6" and unix networks are not supported. +func Listen(network, address string) (Listener, error) { + + // println("Listen", address) + switch network { + case "tcp", "tcp4": + default: + return nil, fmt.Errorf("Network %s not supported", network) + } + + laddr, err := ResolveTCPAddr(network, address) + if err != nil { + return nil, err + } + + return listenTCP(laddr) } diff --git a/src/net/errors.go b/src/net/errors.go deleted file mode 100644 index c1dc7b31c8..0000000000 --- a/src/net/errors.go +++ /dev/null @@ -1,10 +0,0 @@ -package net - -import "errors" - -var ( - // copied from poll.ErrNetClosing - errClosed = errors.New("use of closed network connection") - - ErrNotImplemented = errors.New("operation not implemented") -) diff --git a/src/net/http/client.go b/src/net/http/client.go new file mode 100644 index 0000000000..8bfef71efb --- /dev/null +++ b/src/net/http/client.go @@ -0,0 +1,523 @@ +// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. + +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// HTTP client. See RFC 7230 through 7235. +// +// This is the high-level Client interface. +// The low-level implementation is in transport.go. + +package http + +import ( + "bufio" + "crypto/tls" + "encoding/base64" + "errors" + "fmt" + "io" + "net" + "net/http/internal/ascii" + "net/url" + "strings" + "time" + + "golang.org/x/net/http/httpguts" +) + +// A Client is an HTTP client. Its zero value (DefaultClient) is a +// usable client that uses DefaultTransport. +// +// The Client's Transport typically has internal state (cached TCP +// connections), so Clients should be reused instead of created as +// needed. Clients are safe for concurrent use by multiple goroutines. +// +// A Client is higher-level than a RoundTripper (such as Transport) +// and additionally handles HTTP details such as cookies and +// redirects. +// +// When following redirects, the Client will forward all headers set on the +// initial Request except: +// +// • when forwarding sensitive headers like "Authorization", +// "WWW-Authenticate", and "Cookie" to untrusted targets. +// These headers will be ignored when following a redirect to a domain +// that is not a subdomain match or exact match of the initial domain. +// For example, a redirect from "foo.com" to either "foo.com" or "sub.foo.com" +// will forward the sensitive headers, but a redirect to "bar.com" will not. +// +// • when forwarding the "Cookie" header with a non-nil cookie Jar. +// Since each redirect may mutate the state of the cookie jar, +// a redirect may possibly alter a cookie set in the initial request. +// When forwarding the "Cookie" header, any mutated cookies will be omitted, +// with the expectation that the Jar will insert those mutated cookies +// with the updated values (assuming the origin matches). +// If Jar is nil, the initial cookies are forwarded without change. +type Client struct { + // Jar specifies the cookie jar. + // + // The Jar is used to insert relevant cookies into every + // outbound Request and is updated with the cookie values + // of every inbound Response. The Jar is consulted for every + // redirect that the Client follows. + // + // If Jar is nil, cookies are only sent if they are explicitly + // set on the Request. + Jar CookieJar + + // Timeout specifies a time limit for requests made by this + // Client. The timeout includes connection time, any + // redirects, and reading the response body. The timer remains + // running after Get, Head, Post, or Do return and will + // interrupt reading of the Response.Body. + // + // A Timeout of zero means no timeout. + // + // The Client cancels requests to the underlying Transport + // as if the Request's Context ended. + // + // For compatibility, the Client will also use the deprecated + // CancelRequest method on Transport if found. New + // RoundTripper implementations should use the Request's Context + // for cancellation instead of implementing CancelRequest. + Timeout time.Duration +} + +// DefaultClient is the default Client and is used by Get, Head, and Post. +var DefaultClient = &Client{} + +// didTimeout is non-nil only if err != nil. +func (c *Client) send(req *Request, deadline time.Time) (resp *Response, didTimeout func() bool, err error) { + if c.Jar != nil { + for _, cookie := range c.Jar.Cookies(req.URL) { + req.AddCookie(cookie) + } + } + resp, didTimeout, err = send(req, deadline) + if err != nil { + return nil, didTimeout, err + } + if c.Jar != nil { + if rc := resp.Cookies(); len(rc) > 0 { + c.Jar.SetCookies(req.URL, rc) + } + } + return resp, nil, nil +} + +func (c *Client) deadline() time.Time { + if c.Timeout > 0 { + return time.Now().Add(c.Timeout) + } + return time.Time{} +} + +// send issues an HTTP request. +// Caller should close resp.Body when done reading from it. +func send(req *Request, deadline time.Time) (resp *Response, didTimeout func() bool, err error) { + + // TINYGO: Removed round tripper + + if req.URL == nil { + req.closeBody() + return nil, alwaysFalse, errors.New("http: nil Request.URL") + } + + if req.RequestURI != "" { + req.closeBody() + return nil, alwaysFalse, errors.New("http: Request.RequestURI can't be set in client requests") + } + + // TINYGO: Removed forkReq stuff + + // Most the callers of send (Get, Post, et al) don't need + // Headers, leaving it uninitialized. We guarantee to the + // Transport that this has been initialized, though. + if req.Header == nil { + req.Header = make(Header) + } + + if u := req.URL.User; u != nil && req.Header.Get("Authorization") == "" { + username := u.Username() + password, _ := u.Password() + req.Header.Set("Authorization", "Basic "+basicAuth(username, password)) + } + + resp, err = roundTrip(req) + if err != nil { + + // TINYGO: Remove TLS error check + + return nil, didTimeout, err + } + if resp == nil { + return nil, didTimeout, fmt.Errorf("http: sendit returned a nil *Response with a nil error") + } + + // TINYGO: Skip check for resp.Body == nil since we'll set it in roundTrip + + return resp, nil, nil +} + +func roundTrip(req *Request) (*Response, error) { + + // TINYGO: This is an approximation of Transport.roudTrip() + + if req.URL == nil { + req.closeBody() + return nil, errors.New("http: nil Request.URL") + } + if req.Header == nil { + req.closeBody() + return nil, errors.New("http: nil Request.Header") + } + scheme := req.URL.Scheme + isHTTP := scheme == "http" || scheme == "https" + if isHTTP { + for k, vv := range req.Header { + if !httpguts.ValidHeaderFieldName(k) { + req.closeBody() + return nil, fmt.Errorf("net/http: invalid header field name %q", k) + } + for _, v := range vv { + if !httpguts.ValidHeaderFieldValue(v) { + req.closeBody() + // Don't include the value in the error, because it may be sensitive. + return nil, fmt.Errorf("net/http: invalid header field value for %q", k) + } + } + } + } + + // TINYGO: Skipping alternate round tripper + + if !isHTTP { + req.closeBody() + return nil, badStringError("unsupported protocol scheme", scheme) + } + if req.Method != "" && !validMethod(req.Method) { + req.closeBody() + return nil, fmt.Errorf("net/http: invalid method %q", req.Method) + } + if req.URL.Host == "" { + req.closeBody() + return nil, errors.New("http: no Host in request URL") + } + + // TINYGO: From here on just brute force dial a connection, + // TINYGO: send the request, read and return the response. + // TINYGO: The connection is closed when resp body is closed. + + var conn net.Conn + var err error + + host := req.Host + missingPort := !strings.Contains(host, ":") + + switch scheme { + case "http": + if missingPort { + host = host + ":80" + } + conn, err = net.Dial("tcp", host) + case "https": + if missingPort { + host = host + ":443" + } + conn, err = tls.Dial("tcp", host, nil) + } + if err != nil { + req.closeBody() + return nil, err + } + + // TINYGO: TODO handle timeouts + + writer := bufio.NewWriter(conn) + if err = req.Write(writer); err != nil { + req.closeBody() + return nil, err + } + req.closeBody() + if err = writer.Flush(); err != nil { + return nil, err + } + + req.onEOF = func() { conn.Close() } + + reader := bufio.NewReader(conn) + return ReadResponse(reader, req) +} + +// See 2 (end of page 4) https://www.ietf.org/rfc/rfc2617.txt +// "To receive authorization, the client sends the userid and password, +// separated by a single colon (":") character, within a base64 +// encoded string in the credentials." +// It is not meant to be urlencoded. +func basicAuth(username, password string) string { + auth := username + ":" + password + return base64.StdEncoding.EncodeToString([]byte(auth)) +} + +// Get issues a GET to the specified URL. If the response is one of +// the following redirect codes, Get follows the redirect, up to a +// maximum of 10 redirects: +// +// 301 (Moved Permanently) +// 302 (Found) +// 303 (See Other) +// 307 (Temporary Redirect) +// 308 (Permanent Redirect) +// +// An error is returned if there were too many redirects or if there +// was an HTTP protocol error. A non-2xx response doesn't cause an +// error. Any returned error will be of type *url.Error. The url.Error +// value's Timeout method will report true if the request timed out. +// +// When err is nil, resp always contains a non-nil resp.Body. +// Caller should close resp.Body when done reading from it. +// +// Get is a wrapper around DefaultClient.Get. +// +// To make a request with custom headers, use NewRequest and +// DefaultClient.Do. +// +// To make a request with a specified context.Context, use NewRequestWithContext +// and DefaultClient.Do. +func Get(url string) (resp *Response, err error) { + return DefaultClient.Get(url) +} + +// Get issues a GET to the specified URL. If the response is one of the +// following redirect codes, Get follows the redirect after calling the +// Client's CheckRedirect function: +// +// 301 (Moved Permanently) +// 302 (Found) +// 303 (See Other) +// 307 (Temporary Redirect) +// 308 (Permanent Redirect) +// +// An error is returned if the Client's CheckRedirect function fails +// or if there was an HTTP protocol error. A non-2xx response doesn't +// cause an error. Any returned error will be of type *url.Error. The +// url.Error value's Timeout method will report true if the request +// timed out. +// +// When err is nil, resp always contains a non-nil resp.Body. +// Caller should close resp.Body when done reading from it. +// +// To make a request with custom headers, use NewRequest and Client.Do. +// +// To make a request with a specified context.Context, use NewRequestWithContext +// and Client.Do. +func (c *Client) Get(url string) (resp *Response, err error) { + req, err := NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + return c.Do(req) +} + +func alwaysFalse() bool { return false } + +// urlErrorOp returns the (*url.Error).Op value to use for the +// provided (*Request).Method value. +func urlErrorOp(method string) string { + if method == "" { + return "Get" + } + if lowerMethod, ok := ascii.ToLower(method); ok { + return method[:1] + lowerMethod[1:] + } + return method +} + +// Do sends an HTTP request and returns an HTTP response, following +// policy (such as redirects, cookies, auth) as configured on the +// client. +// +// An error is returned if caused by client policy (such as +// CheckRedirect), or failure to speak HTTP (such as a network +// connectivity problem). A non-2xx status code doesn't cause an +// error. +// +// If the returned error is nil, the Response will contain a non-nil +// Body which the user is expected to close. If the Body is not both +// read to EOF and closed, the Client's underlying RoundTripper +// (typically Transport) may not be able to re-use a persistent TCP +// connection to the server for a subsequent "keep-alive" request. +// +// The request Body, if non-nil, will be closed by the underlying +// Transport, even on errors. +// +// On error, any Response can be ignored. A non-nil Response with a +// non-nil error only occurs when CheckRedirect fails, and even then +// the returned Response.Body is already closed. +// +// Generally Get, Post, or PostForm will be used instead of Do. +// +// If the server replies with a redirect, the Client first uses the +// CheckRedirect function to determine whether the redirect should be +// followed. If permitted, a 301, 302, or 303 redirect causes +// subsequent requests to use HTTP method GET +// (or HEAD if the original request was HEAD), with no body. +// A 307 or 308 redirect preserves the original HTTP method and body, +// provided that the Request.GetBody function is defined. +// The NewRequest function automatically sets GetBody for common +// standard library body types. +// +// Any returned error will be of type *url.Error. The url.Error +// value's Timeout method will report true if the request timed out. +func (c *Client) Do(req *Request) (*Response, error) { + return c.do(req) +} + +func (c *Client) do(req *Request) (retres *Response, reterr error) { + if req.URL == nil { + req.closeBody() + return nil, &url.Error{ + Op: urlErrorOp(req.Method), + Err: errors.New("http: nil Request.URL"), + } + } + + var err error + var didTimeout func() bool + var resp *Response + var deadline = c.deadline() + + // TINYGO: lots removed here, mostly handling multiple requests. + // TINYGO: we just want simple GET, POST, etc. In and out. + + if resp, didTimeout, err = c.send(req, deadline); err != nil { + // c.send() always closes req.Body + if !deadline.IsZero() && didTimeout() { + return nil, fmt.Errorf("%s (Client.Timeout exceeded while awaiting headers)", err.Error()) + } + return nil, err + } + + return resp, nil +} + +// Post issues a POST to the specified URL. +// +// Caller should close resp.Body when done reading from it. +// +// If the provided body is an io.Closer, it is closed after the +// request. +// +// Post is a wrapper around DefaultClient.Post. +// +// To set custom headers, use NewRequest and DefaultClient.Do. +// +// See the Client.Do method documentation for details on how redirects +// are handled. +// +// To make a request with a specified context.Context, use NewRequestWithContext +// and DefaultClient.Do. +func Post(url, contentType string, body io.Reader) (resp *Response, err error) { + return DefaultClient.Post(url, contentType, body) +} + +// Post issues a POST to the specified URL. +// +// Caller should close resp.Body when done reading from it. +// +// If the provided body is an io.Closer, it is closed after the +// request. +// +// To set custom headers, use NewRequest and Client.Do. +// +// To make a request with a specified context.Context, use NewRequestWithContext +// and Client.Do. +// +// See the Client.Do method documentation for details on how redirects +// are handled. +func (c *Client) Post(url, contentType string, body io.Reader) (resp *Response, err error) { + req, err := NewRequest("POST", url, body) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", contentType) + return c.Do(req) +} + +// PostForm issues a POST to the specified URL, with data's keys and +// values URL-encoded as the request body. +// +// The Content-Type header is set to application/x-www-form-urlencoded. +// To set other headers, use NewRequest and DefaultClient.Do. +// +// When err is nil, resp always contains a non-nil resp.Body. +// Caller should close resp.Body when done reading from it. +// +// PostForm is a wrapper around DefaultClient.PostForm. +// +// See the Client.Do method documentation for details on how redirects +// are handled. +// +// To make a request with a specified context.Context, use NewRequestWithContext +// and DefaultClient.Do. +func PostForm(url string, data url.Values) (resp *Response, err error) { + return DefaultClient.PostForm(url, data) +} + +// PostForm issues a POST to the specified URL, +// with data's keys and values URL-encoded as the request body. +// +// The Content-Type header is set to application/x-www-form-urlencoded. +// To set other headers, use NewRequest and Client.Do. +// +// When err is nil, resp always contains a non-nil resp.Body. +// Caller should close resp.Body when done reading from it. +// +// See the Client.Do method documentation for details on how redirects +// are handled. +// +// To make a request with a specified context.Context, use NewRequestWithContext +// and Client.Do. +func (c *Client) PostForm(url string, data url.Values) (resp *Response, err error) { + return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) +} + +// Head issues a HEAD to the specified URL. If the response is one of +// the following redirect codes, Head follows the redirect, up to a +// maximum of 10 redirects: +// +// 301 (Moved Permanently) +// 302 (Found) +// 303 (See Other) +// 307 (Temporary Redirect) +// 308 (Permanent Redirect) +// +// Head is a wrapper around DefaultClient.Head. +// +// To make a request with a specified context.Context, use NewRequestWithContext +// and DefaultClient.Do. +func Head(url string) (resp *Response, err error) { + return DefaultClient.Head(url) +} + +// Head issues a HEAD to the specified URL. If the response is one of the +// following redirect codes, Head follows the redirect after calling the +// Client's CheckRedirect function: +// +// 301 (Moved Permanently) +// 302 (Found) +// 303 (See Other) +// 307 (Temporary Redirect) +// 308 (Permanent Redirect) +// +// To make a request with a specified context.Context, use NewRequestWithContext +// and Client.Do. +func (c *Client) Head(url string) (resp *Response, err error) { + req, err := NewRequest("HEAD", url, nil) + if err != nil { + return nil, err + } + return c.Do(req) +} diff --git a/src/net/http/clone.go b/src/net/http/clone.go new file mode 100644 index 0000000000..aa42a7e9c7 --- /dev/null +++ b/src/net/http/clone.go @@ -0,0 +1,76 @@ +// TINYGO: The following is copied from Go 1.19.3 official implementation. + +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "mime/multipart" + "net/textproto" + "net/url" +) + +func cloneURLValues(v url.Values) url.Values { + if v == nil { + return nil + } + // http.Header and url.Values have the same representation, so temporarily + // treat it like http.Header, which does have a clone: + return url.Values(Header(v).Clone()) +} + +func cloneURL(u *url.URL) *url.URL { + if u == nil { + return nil + } + u2 := new(url.URL) + *u2 = *u + if u.User != nil { + u2.User = new(url.Userinfo) + *u2.User = *u.User + } + return u2 +} + +func cloneMultipartForm(f *multipart.Form) *multipart.Form { + if f == nil { + return nil + } + f2 := &multipart.Form{ + Value: (map[string][]string)(Header(f.Value).Clone()), + } + if f.File != nil { + m := make(map[string][]*multipart.FileHeader) + for k, vv := range f.File { + vv2 := make([]*multipart.FileHeader, len(vv)) + for i, v := range vv { + vv2[i] = cloneMultipartFileHeader(v) + } + m[k] = vv2 + } + f2.File = m + } + return f2 +} + +func cloneMultipartFileHeader(fh *multipart.FileHeader) *multipart.FileHeader { + if fh == nil { + return nil + } + fh2 := new(multipart.FileHeader) + *fh2 = *fh + fh2.Header = textproto.MIMEHeader(Header(fh.Header).Clone()) + return fh2 +} + +// cloneOrMakeHeader invokes Header.Clone but if the +// result is nil, it'll instead make and return a non-nil Header. +func cloneOrMakeHeader(hdr Header) Header { + clone := hdr.Clone() + if clone == nil { + clone = make(Header) + } + return clone +} diff --git a/src/net/http/cookie.go b/src/net/http/cookie.go new file mode 100644 index 0000000000..24c938c3d4 --- /dev/null +++ b/src/net/http/cookie.go @@ -0,0 +1,470 @@ +// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. + +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "errors" + "fmt" + "log" + "net" + "net/http/internal/ascii" + "net/textproto" + "strconv" + "strings" + "time" +) + +// A Cookie represents an HTTP cookie as sent in the Set-Cookie header of an +// HTTP response or the Cookie header of an HTTP request. +// +// See https://tools.ietf.org/html/rfc6265 for details. +type Cookie struct { + Name string + Value string + + Path string // optional + Domain string // optional + Expires time.Time // optional + RawExpires string // for reading cookies only + + // MaxAge=0 means no 'Max-Age' attribute specified. + // MaxAge<0 means delete cookie now, equivalently 'Max-Age: 0' + // MaxAge>0 means Max-Age attribute present and given in seconds + MaxAge int + Secure bool + HttpOnly bool + SameSite SameSite + Raw string + Unparsed []string // Raw text of unparsed attribute-value pairs +} + +// SameSite allows a server to define a cookie attribute making it impossible for +// the browser to send this cookie along with cross-site requests. The main +// goal is to mitigate the risk of cross-origin information leakage, and provide +// some protection against cross-site request forgery attacks. +// +// See https://tools.ietf.org/html/draft-ietf-httpbis-cookie-same-site-00 for details. +type SameSite int + +const ( + SameSiteDefaultMode SameSite = iota + 1 + SameSiteLaxMode + SameSiteStrictMode + SameSiteNoneMode +) + +// readSetCookies parses all "Set-Cookie" values from +// the header h and returns the successfully parsed Cookies. +func readSetCookies(h Header) []*Cookie { + cookieCount := len(h["Set-Cookie"]) + if cookieCount == 0 { + return []*Cookie{} + } + cookies := make([]*Cookie, 0, cookieCount) + for _, line := range h["Set-Cookie"] { + parts := strings.Split(textproto.TrimString(line), ";") + if len(parts) == 1 && parts[0] == "" { + continue + } + parts[0] = textproto.TrimString(parts[0]) + name, value, ok := strings.Cut(parts[0], "=") + if !ok { + continue + } + name = textproto.TrimString(name) + if !isCookieNameValid(name) { + continue + } + value, ok = parseCookieValue(value, true) + if !ok { + continue + } + c := &Cookie{ + Name: name, + Value: value, + Raw: line, + } + for i := 1; i < len(parts); i++ { + parts[i] = textproto.TrimString(parts[i]) + if len(parts[i]) == 0 { + continue + } + + attr, val, _ := strings.Cut(parts[i], "=") + lowerAttr, isASCII := ascii.ToLower(attr) + if !isASCII { + continue + } + val, ok = parseCookieValue(val, false) + if !ok { + c.Unparsed = append(c.Unparsed, parts[i]) + continue + } + + switch lowerAttr { + case "samesite": + lowerVal, ascii := ascii.ToLower(val) + if !ascii { + c.SameSite = SameSiteDefaultMode + continue + } + switch lowerVal { + case "lax": + c.SameSite = SameSiteLaxMode + case "strict": + c.SameSite = SameSiteStrictMode + case "none": + c.SameSite = SameSiteNoneMode + default: + c.SameSite = SameSiteDefaultMode + } + continue + case "secure": + c.Secure = true + continue + case "httponly": + c.HttpOnly = true + continue + case "domain": + c.Domain = val + continue + case "max-age": + secs, err := strconv.Atoi(val) + if err != nil || secs != 0 && val[0] == '0' { + break + } + if secs <= 0 { + secs = -1 + } + c.MaxAge = secs + continue + case "expires": + c.RawExpires = val + exptime, err := time.Parse(time.RFC1123, val) + if err != nil { + exptime, err = time.Parse("Mon, 02-Jan-2006 15:04:05 MST", val) + if err != nil { + c.Expires = time.Time{} + break + } + } + c.Expires = exptime.UTC() + continue + case "path": + c.Path = val + continue + } + c.Unparsed = append(c.Unparsed, parts[i]) + } + cookies = append(cookies, c) + } + return cookies +} + +// SetCookie adds a Set-Cookie header to the provided ResponseWriter's headers. +// The provided cookie must have a valid Name. Invalid cookies may be +// silently dropped. +func SetCookie(w ResponseWriter, cookie *Cookie) { + if v := cookie.String(); v != "" { + w.Header().Add("Set-Cookie", v) + } +} + +// String returns the serialization of the cookie for use in a Cookie +// header (if only Name and Value are set) or a Set-Cookie response +// header (if other fields are set). +// If c is nil or c.Name is invalid, the empty string is returned. +func (c *Cookie) String() string { + if c == nil || !isCookieNameValid(c.Name) { + return "" + } + // extraCookieLength derived from typical length of cookie attributes + // see RFC 6265 Sec 4.1. + const extraCookieLength = 110 + var b strings.Builder + b.Grow(len(c.Name) + len(c.Value) + len(c.Domain) + len(c.Path) + extraCookieLength) + b.WriteString(c.Name) + b.WriteRune('=') + b.WriteString(sanitizeCookieValue(c.Value)) + + if len(c.Path) > 0 { + b.WriteString("; Path=") + b.WriteString(sanitizeCookiePath(c.Path)) + } + if len(c.Domain) > 0 { + if validCookieDomain(c.Domain) { + // A c.Domain containing illegal characters is not + // sanitized but simply dropped which turns the cookie + // into a host-only cookie. A leading dot is okay + // but won't be sent. + d := c.Domain + if d[0] == '.' { + d = d[1:] + } + b.WriteString("; Domain=") + b.WriteString(d) + } else { + log.Printf("net/http: invalid Cookie.Domain %q; dropping domain attribute", c.Domain) + } + } + var buf [len(TimeFormat)]byte + if validCookieExpires(c.Expires) { + b.WriteString("; Expires=") + b.Write(c.Expires.UTC().AppendFormat(buf[:0], TimeFormat)) + } + if c.MaxAge > 0 { + b.WriteString("; Max-Age=") + b.Write(strconv.AppendInt(buf[:0], int64(c.MaxAge), 10)) + } else if c.MaxAge < 0 { + b.WriteString("; Max-Age=0") + } + if c.HttpOnly { + b.WriteString("; HttpOnly") + } + if c.Secure { + b.WriteString("; Secure") + } + switch c.SameSite { + case SameSiteDefaultMode: + // Skip, default mode is obtained by not emitting the attribute. + case SameSiteNoneMode: + b.WriteString("; SameSite=None") + case SameSiteLaxMode: + b.WriteString("; SameSite=Lax") + case SameSiteStrictMode: + b.WriteString("; SameSite=Strict") + } + return b.String() +} + +// Valid reports whether the cookie is valid. +func (c *Cookie) Valid() error { + if c == nil { + return errors.New("http: nil Cookie") + } + if !isCookieNameValid(c.Name) { + return errors.New("http: invalid Cookie.Name") + } + if !c.Expires.IsZero() && !validCookieExpires(c.Expires) { + return errors.New("http: invalid Cookie.Expires") + } + for i := 0; i < len(c.Value); i++ { + if !validCookieValueByte(c.Value[i]) { + return fmt.Errorf("http: invalid byte %q in Cookie.Value", c.Value[i]) + } + } + if len(c.Path) > 0 { + for i := 0; i < len(c.Path); i++ { + if !validCookiePathByte(c.Path[i]) { + return fmt.Errorf("http: invalid byte %q in Cookie.Path", c.Path[i]) + } + } + } + if len(c.Domain) > 0 { + if !validCookieDomain(c.Domain) { + return errors.New("http: invalid Cookie.Domain") + } + } + return nil +} + +// readCookies parses all "Cookie" values from the header h and +// returns the successfully parsed Cookies. +// +// if filter isn't empty, only cookies of that name are returned. +func readCookies(h Header, filter string) []*Cookie { + lines := h["Cookie"] + if len(lines) == 0 { + return []*Cookie{} + } + + cookies := make([]*Cookie, 0, len(lines)+strings.Count(lines[0], ";")) + for _, line := range lines { + line = textproto.TrimString(line) + + var part string + for len(line) > 0 { // continue since we have rest + part, line, _ = strings.Cut(line, ";") + part = textproto.TrimString(part) + if part == "" { + continue + } + name, val, _ := strings.Cut(part, "=") + name = textproto.TrimString(name) + if !isCookieNameValid(name) { + continue + } + if filter != "" && filter != name { + continue + } + val, ok := parseCookieValue(val, true) + if !ok { + continue + } + cookies = append(cookies, &Cookie{Name: name, Value: val}) + } + } + return cookies +} + +// validCookieDomain reports whether v is a valid cookie domain-value. +func validCookieDomain(v string) bool { + if isCookieDomainName(v) { + return true + } + if net.ParseIP(v) != nil && !strings.Contains(v, ":") { + return true + } + return false +} + +// validCookieExpires reports whether v is a valid cookie expires-value. +func validCookieExpires(t time.Time) bool { + // IETF RFC 6265 Section 5.1.1.5, the year must not be less than 1601 + return t.Year() >= 1601 +} + +// isCookieDomainName reports whether s is a valid domain name or a valid +// domain name with a leading dot '.'. It is almost a direct copy of +// package net's isDomainName. +func isCookieDomainName(s string) bool { + if len(s) == 0 { + return false + } + if len(s) > 255 { + return false + } + + if s[0] == '.' { + // A cookie a domain attribute may start with a leading dot. + s = s[1:] + } + last := byte('.') + ok := false // Ok once we've seen a letter. + partlen := 0 + for i := 0; i < len(s); i++ { + c := s[i] + switch { + default: + return false + case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z': + // No '_' allowed here (in contrast to package net). + ok = true + partlen++ + case '0' <= c && c <= '9': + // fine + partlen++ + case c == '-': + // Byte before dash cannot be dot. + if last == '.' { + return false + } + partlen++ + case c == '.': + // Byte before dot cannot be dot, dash. + if last == '.' || last == '-' { + return false + } + if partlen > 63 || partlen == 0 { + return false + } + partlen = 0 + } + last = c + } + if last == '-' || partlen > 63 { + return false + } + + return ok +} + +var cookieNameSanitizer = strings.NewReplacer("\n", "-", "\r", "-") + +func sanitizeCookieName(n string) string { + return cookieNameSanitizer.Replace(n) +} + +// sanitizeCookieValue produces a suitable cookie-value from v. +// https://tools.ietf.org/html/rfc6265#section-4.1.1 +// +// cookie-value = *cookie-octet / ( DQUOTE *cookie-octet DQUOTE ) +// cookie-octet = %x21 / %x23-2B / %x2D-3A / %x3C-5B / %x5D-7E +// ; US-ASCII characters excluding CTLs, +// ; whitespace DQUOTE, comma, semicolon, +// ; and backslash +// +// We loosen this as spaces and commas are common in cookie values +// but we produce a quoted cookie-value if and only if v contains +// commas or spaces. +// See https://golang.org/issue/7243 for the discussion. +func sanitizeCookieValue(v string) string { + v = sanitizeOrWarn("Cookie.Value", validCookieValueByte, v) + if len(v) == 0 { + return v + } + if strings.ContainsAny(v, " ,") { + return `"` + v + `"` + } + return v +} + +func validCookieValueByte(b byte) bool { + return 0x20 <= b && b < 0x7f && b != '"' && b != ';' && b != '\\' +} + +// path-av = "Path=" path-value +// path-value = +func sanitizeCookiePath(v string) string { + return sanitizeOrWarn("Cookie.Path", validCookiePathByte, v) +} + +func validCookiePathByte(b byte) bool { + return 0x20 <= b && b < 0x7f && b != ';' +} + +func sanitizeOrWarn(fieldName string, valid func(byte) bool, v string) string { + ok := true + for i := 0; i < len(v); i++ { + if valid(v[i]) { + continue + } + log.Printf("net/http: invalid byte %q in %s; dropping invalid bytes", v[i], fieldName) + ok = false + break + } + if ok { + return v + } + buf := make([]byte, 0, len(v)) + for i := 0; i < len(v); i++ { + if b := v[i]; valid(b) { + buf = append(buf, b) + } + } + return string(buf) +} + +func parseCookieValue(raw string, allowDoubleQuote bool) (string, bool) { + // Strip the quotes, if present. + if allowDoubleQuote && len(raw) > 1 && raw[0] == '"' && raw[len(raw)-1] == '"' { + raw = raw[1 : len(raw)-1] + } + for i := 0; i < len(raw); i++ { + if !validCookieValueByte(raw[i]) { + return "", false + } + } + return raw, true +} + +func isCookieNameValid(raw string) bool { + if raw == "" { + return false + } + return strings.IndexFunc(raw, isNotToken) < 0 +} diff --git a/src/net/http/fs.go b/src/net/http/fs.go new file mode 100644 index 0000000000..3967045c2f --- /dev/null +++ b/src/net/http/fs.go @@ -0,0 +1,974 @@ +// TINYGO: The following is copied from Go 1.19.3 official implementation. + +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// HTTP file system request handler + +package http + +import ( + "errors" + "fmt" + "io" + "io/fs" + "mime" + "mime/multipart" + "net/textproto" + "net/url" + "os" + "path" + "path/filepath" + "sort" + "strconv" + "strings" + "time" +) + +// A Dir implements FileSystem using the native file system restricted to a +// specific directory tree. +// +// While the FileSystem.Open method takes '/'-separated paths, a Dir's string +// value is a filename on the native file system, not a URL, so it is separated +// by filepath.Separator, which isn't necessarily '/'. +// +// Note that Dir could expose sensitive files and directories. Dir will follow +// symlinks pointing out of the directory tree, which can be especially dangerous +// if serving from a directory in which users are able to create arbitrary symlinks. +// Dir will also allow access to files and directories starting with a period, +// which could expose sensitive directories like .git or sensitive files like +// .htpasswd. To exclude files with a leading period, remove the files/directories +// from the server or create a custom FileSystem implementation. +// +// An empty Dir is treated as ".". +type Dir string + +// mapOpenError maps the provided non-nil error from opening name +// to a possibly better non-nil error. In particular, it turns OS-specific errors +// about opening files in non-directories into fs.ErrNotExist. See Issues 18984 and 49552. +func mapOpenError(originalErr error, name string, sep rune, stat func(string) (fs.FileInfo, error)) error { + if errors.Is(originalErr, fs.ErrNotExist) || errors.Is(originalErr, fs.ErrPermission) { + return originalErr + } + + parts := strings.Split(name, string(sep)) + for i := range parts { + if parts[i] == "" { + continue + } + fi, err := stat(strings.Join(parts[:i+1], string(sep))) + if err != nil { + return originalErr + } + if !fi.IsDir() { + return fs.ErrNotExist + } + } + return originalErr +} + +// Open implements FileSystem using os.Open, opening files for reading rooted +// and relative to the directory d. +func (d Dir) Open(name string) (File, error) { + if filepath.Separator != '/' && strings.ContainsRune(name, filepath.Separator) { + return nil, errors.New("http: invalid character in file path") + } + dir := string(d) + if dir == "" { + dir = "." + } + fullName := filepath.Join(dir, filepath.FromSlash(path.Clean("/"+name))) + f, err := os.Open(fullName) + if err != nil { + return nil, mapOpenError(err, fullName, filepath.Separator, os.Stat) + } + return f, nil +} + +// A FileSystem implements access to a collection of named files. +// The elements in a file path are separated by slash ('/', U+002F) +// characters, regardless of host operating system convention. +// See the FileServer function to convert a FileSystem to a Handler. +// +// This interface predates the fs.FS interface, which can be used instead: +// the FS adapter function converts an fs.FS to a FileSystem. +type FileSystem interface { + Open(name string) (File, error) +} + +// A File is returned by a FileSystem's Open method and can be +// served by the FileServer implementation. +// +// The methods should behave the same as those on an *os.File. +type File interface { + io.Closer + io.Reader + io.Seeker + Readdir(count int) ([]fs.FileInfo, error) + Stat() (fs.FileInfo, error) +} + +type anyDirs interface { + len() int + name(i int) string + isDir(i int) bool +} + +type fileInfoDirs []fs.FileInfo + +func (d fileInfoDirs) len() int { return len(d) } +func (d fileInfoDirs) isDir(i int) bool { return d[i].IsDir() } +func (d fileInfoDirs) name(i int) string { return d[i].Name() } + +type dirEntryDirs []fs.DirEntry + +func (d dirEntryDirs) len() int { return len(d) } +func (d dirEntryDirs) isDir(i int) bool { return d[i].IsDir() } +func (d dirEntryDirs) name(i int) string { return d[i].Name() } + +func dirList(w ResponseWriter, r *Request, f File) { + // Prefer to use ReadDir instead of Readdir, + // because the former doesn't require calling + // Stat on every entry of a directory on Unix. + var dirs anyDirs + var err error + if d, ok := f.(fs.ReadDirFile); ok { + var list dirEntryDirs + list, err = d.ReadDir(-1) + dirs = list + } else { + var list fileInfoDirs + list, err = f.Readdir(-1) + dirs = list + } + + if err != nil { + logf(r, "http: error reading directory: %v", err) + Error(w, "Error reading directory", StatusInternalServerError) + return + } + sort.Slice(dirs, func(i, j int) bool { return dirs.name(i) < dirs.name(j) }) + + w.Header().Set("Content-Type", "text/html; charset=utf-8") + fmt.Fprintf(w, "
\n")
+	for i, n := 0, dirs.len(); i < n; i++ {
+		name := dirs.name(i)
+		if dirs.isDir(i) {
+			name += "/"
+		}
+		// name may contain '?' or '#', which must be escaped to remain
+		// part of the URL path, and not indicate the start of a query
+		// string or fragment.
+		url := url.URL{Path: name}
+		fmt.Fprintf(w, "%s\n", url.String(), htmlReplacer.Replace(name))
+	}
+	fmt.Fprintf(w, "
\n") +} + +// ServeContent replies to the request using the content in the +// provided ReadSeeker. The main benefit of ServeContent over io.Copy +// is that it handles Range requests properly, sets the MIME type, and +// handles If-Match, If-Unmodified-Since, If-None-Match, If-Modified-Since, +// and If-Range requests. +// +// If the response's Content-Type header is not set, ServeContent +// first tries to deduce the type from name's file extension and, +// if that fails, falls back to reading the first block of the content +// and passing it to DetectContentType. +// The name is otherwise unused; in particular it can be empty and is +// never sent in the response. +// +// If modtime is not the zero time or Unix epoch, ServeContent +// includes it in a Last-Modified header in the response. If the +// request includes an If-Modified-Since header, ServeContent uses +// modtime to decide whether the content needs to be sent at all. +// +// The content's Seek method must work: ServeContent uses +// a seek to the end of the content to determine its size. +// +// If the caller has set w's ETag header formatted per RFC 7232, section 2.3, +// ServeContent uses it to handle requests using If-Match, If-None-Match, or If-Range. +// +// Note that *os.File implements the io.ReadSeeker interface. +func ServeContent(w ResponseWriter, req *Request, name string, modtime time.Time, content io.ReadSeeker) { + sizeFunc := func() (int64, error) { + size, err := content.Seek(0, io.SeekEnd) + if err != nil { + return 0, errSeeker + } + _, err = content.Seek(0, io.SeekStart) + if err != nil { + return 0, errSeeker + } + return size, nil + } + serveContent(w, req, name, modtime, sizeFunc, content) +} + +// errSeeker is returned by ServeContent's sizeFunc when the content +// doesn't seek properly. The underlying Seeker's error text isn't +// included in the sizeFunc reply so it's not sent over HTTP to end +// users. +var errSeeker = errors.New("seeker can't seek") + +// errNoOverlap is returned by serveContent's parseRange if first-byte-pos of +// all of the byte-range-spec values is greater than the content size. +var errNoOverlap = errors.New("invalid range: failed to overlap") + +// if name is empty, filename is unknown. (used for mime type, before sniffing) +// if modtime.IsZero(), modtime is unknown. +// content must be seeked to the beginning of the file. +// The sizeFunc is called at most once. Its error, if any, is sent in the HTTP response. +func serveContent(w ResponseWriter, r *Request, name string, modtime time.Time, sizeFunc func() (int64, error), content io.ReadSeeker) { + setLastModified(w, modtime) + done, rangeReq := checkPreconditions(w, r, modtime) + if done { + return + } + + code := StatusOK + + // If Content-Type isn't set, use the file's extension to find it, but + // if the Content-Type is unset explicitly, do not sniff the type. + ctypes, haveType := w.Header()["Content-Type"] + var ctype string + if !haveType { + ctype = mime.TypeByExtension(filepath.Ext(name)) + if ctype == "" { + // read a chunk to decide between utf-8 text and binary + var buf [sniffLen]byte + n, _ := io.ReadFull(content, buf[:]) + ctype = DetectContentType(buf[:n]) + _, err := content.Seek(0, io.SeekStart) // rewind to output whole file + if err != nil { + Error(w, "seeker can't seek", StatusInternalServerError) + return + } + } + w.Header().Set("Content-Type", ctype) + } else if len(ctypes) > 0 { + ctype = ctypes[0] + } + + size, err := sizeFunc() + if err != nil { + Error(w, err.Error(), StatusInternalServerError) + return + } + + // handle Content-Range header. + sendSize := size + var sendContent io.Reader = content + if size >= 0 { + ranges, err := parseRange(rangeReq, size) + if err != nil { + if err == errNoOverlap { + w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", size)) + } + Error(w, err.Error(), StatusRequestedRangeNotSatisfiable) + return + } + if sumRangesSize(ranges) > size { + // The total number of bytes in all the ranges + // is larger than the size of the file by + // itself, so this is probably an attack, or a + // dumb client. Ignore the range request. + ranges = nil + } + switch { + case len(ranges) == 1: + // RFC 7233, Section 4.1: + // "If a single part is being transferred, the server + // generating the 206 response MUST generate a + // Content-Range header field, describing what range + // of the selected representation is enclosed, and a + // payload consisting of the range. + // ... + // A server MUST NOT generate a multipart response to + // a request for a single range, since a client that + // does not request multiple parts might not support + // multipart responses." + ra := ranges[0] + if _, err := content.Seek(ra.start, io.SeekStart); err != nil { + Error(w, err.Error(), StatusRequestedRangeNotSatisfiable) + return + } + sendSize = ra.length + code = StatusPartialContent + w.Header().Set("Content-Range", ra.contentRange(size)) + case len(ranges) > 1: + sendSize = rangesMIMESize(ranges, ctype, size) + code = StatusPartialContent + + pr, pw := io.Pipe() + mw := multipart.NewWriter(pw) + w.Header().Set("Content-Type", "multipart/byteranges; boundary="+mw.Boundary()) + sendContent = pr + defer pr.Close() // cause writing goroutine to fail and exit if CopyN doesn't finish. + go func() { + for _, ra := range ranges { + part, err := mw.CreatePart(ra.mimeHeader(ctype, size)) + if err != nil { + pw.CloseWithError(err) + return + } + if _, err := content.Seek(ra.start, io.SeekStart); err != nil { + pw.CloseWithError(err) + return + } + if _, err := io.CopyN(part, content, ra.length); err != nil { + pw.CloseWithError(err) + return + } + } + mw.Close() + pw.Close() + }() + } + + w.Header().Set("Accept-Ranges", "bytes") + if w.Header().Get("Content-Encoding") == "" { + w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10)) + } + } + + w.WriteHeader(code) + + if r.Method != "HEAD" { + io.CopyN(w, sendContent, sendSize) + } +} + +// scanETag determines if a syntactically valid ETag is present at s. If so, +// the ETag and remaining text after consuming ETag is returned. Otherwise, +// it returns "", "". +func scanETag(s string) (etag string, remain string) { + s = textproto.TrimString(s) + start := 0 + if strings.HasPrefix(s, "W/") { + start = 2 + } + if len(s[start:]) < 2 || s[start] != '"' { + return "", "" + } + // ETag is either W/"text" or "text". + // See RFC 7232 2.3. + for i := start + 1; i < len(s); i++ { + c := s[i] + switch { + // Character values allowed in ETags. + case c == 0x21 || c >= 0x23 && c <= 0x7E || c >= 0x80: + case c == '"': + return s[:i+1], s[i+1:] + default: + return "", "" + } + } + return "", "" +} + +// etagStrongMatch reports whether a and b match using strong ETag comparison. +// Assumes a and b are valid ETags. +func etagStrongMatch(a, b string) bool { + return a == b && a != "" && a[0] == '"' +} + +// etagWeakMatch reports whether a and b match using weak ETag comparison. +// Assumes a and b are valid ETags. +func etagWeakMatch(a, b string) bool { + return strings.TrimPrefix(a, "W/") == strings.TrimPrefix(b, "W/") +} + +// condResult is the result of an HTTP request precondition check. +// See https://tools.ietf.org/html/rfc7232 section 3. +type condResult int + +const ( + condNone condResult = iota + condTrue + condFalse +) + +func checkIfMatch(w ResponseWriter, r *Request) condResult { + im := r.Header.Get("If-Match") + if im == "" { + return condNone + } + for { + im = textproto.TrimString(im) + if len(im) == 0 { + break + } + if im[0] == ',' { + im = im[1:] + continue + } + if im[0] == '*' { + return condTrue + } + etag, remain := scanETag(im) + if etag == "" { + break + } + if etagStrongMatch(etag, w.Header().get("Etag")) { + return condTrue + } + im = remain + } + + return condFalse +} + +func checkIfUnmodifiedSince(r *Request, modtime time.Time) condResult { + ius := r.Header.Get("If-Unmodified-Since") + if ius == "" || isZeroTime(modtime) { + return condNone + } + t, err := ParseTime(ius) + if err != nil { + return condNone + } + + // The Last-Modified header truncates sub-second precision so + // the modtime needs to be truncated too. + modtime = modtime.Truncate(time.Second) + if modtime.Before(t) || modtime.Equal(t) { + return condTrue + } + return condFalse +} + +func checkIfNoneMatch(w ResponseWriter, r *Request) condResult { + inm := r.Header.get("If-None-Match") + if inm == "" { + return condNone + } + buf := inm + for { + buf = textproto.TrimString(buf) + if len(buf) == 0 { + break + } + if buf[0] == ',' { + buf = buf[1:] + continue + } + if buf[0] == '*' { + return condFalse + } + etag, remain := scanETag(buf) + if etag == "" { + break + } + if etagWeakMatch(etag, w.Header().get("Etag")) { + return condFalse + } + buf = remain + } + return condTrue +} + +func checkIfModifiedSince(r *Request, modtime time.Time) condResult { + if r.Method != "GET" && r.Method != "HEAD" { + return condNone + } + ims := r.Header.Get("If-Modified-Since") + if ims == "" || isZeroTime(modtime) { + return condNone + } + t, err := ParseTime(ims) + if err != nil { + return condNone + } + // The Last-Modified header truncates sub-second precision so + // the modtime needs to be truncated too. + modtime = modtime.Truncate(time.Second) + if modtime.Before(t) || modtime.Equal(t) { + return condFalse + } + return condTrue +} + +func checkIfRange(w ResponseWriter, r *Request, modtime time.Time) condResult { + if r.Method != "GET" && r.Method != "HEAD" { + return condNone + } + ir := r.Header.get("If-Range") + if ir == "" { + return condNone + } + etag, _ := scanETag(ir) + if etag != "" { + if etagStrongMatch(etag, w.Header().Get("Etag")) { + return condTrue + } else { + return condFalse + } + } + // The If-Range value is typically the ETag value, but it may also be + // the modtime date. See golang.org/issue/8367. + if modtime.IsZero() { + return condFalse + } + t, err := ParseTime(ir) + if err != nil { + return condFalse + } + if t.Unix() == modtime.Unix() { + return condTrue + } + return condFalse +} + +var unixEpochTime = time.Unix(0, 0) + +// isZeroTime reports whether t is obviously unspecified (either zero or Unix()=0). +func isZeroTime(t time.Time) bool { + return t.IsZero() || t.Equal(unixEpochTime) +} + +func setLastModified(w ResponseWriter, modtime time.Time) { + if !isZeroTime(modtime) { + w.Header().Set("Last-Modified", modtime.UTC().Format(TimeFormat)) + } +} + +func writeNotModified(w ResponseWriter) { + // RFC 7232 section 4.1: + // a sender SHOULD NOT generate representation metadata other than the + // above listed fields unless said metadata exists for the purpose of + // guiding cache updates (e.g., Last-Modified might be useful if the + // response does not have an ETag field). + h := w.Header() + delete(h, "Content-Type") + delete(h, "Content-Length") + delete(h, "Content-Encoding") + if h.Get("Etag") != "" { + delete(h, "Last-Modified") + } + w.WriteHeader(StatusNotModified) +} + +// checkPreconditions evaluates request preconditions and reports whether a precondition +// resulted in sending StatusNotModified or StatusPreconditionFailed. +func checkPreconditions(w ResponseWriter, r *Request, modtime time.Time) (done bool, rangeHeader string) { + // This function carefully follows RFC 7232 section 6. + ch := checkIfMatch(w, r) + if ch == condNone { + ch = checkIfUnmodifiedSince(r, modtime) + } + if ch == condFalse { + w.WriteHeader(StatusPreconditionFailed) + return true, "" + } + switch checkIfNoneMatch(w, r) { + case condFalse: + if r.Method == "GET" || r.Method == "HEAD" { + writeNotModified(w) + return true, "" + } else { + w.WriteHeader(StatusPreconditionFailed) + return true, "" + } + case condNone: + if checkIfModifiedSince(r, modtime) == condFalse { + writeNotModified(w) + return true, "" + } + } + + rangeHeader = r.Header.get("Range") + if rangeHeader != "" && checkIfRange(w, r, modtime) == condFalse { + rangeHeader = "" + } + return false, rangeHeader +} + +// name is '/'-separated, not filepath.Separator. +func serveFile(w ResponseWriter, r *Request, fs FileSystem, name string, redirect bool) { + const indexPage = "/index.html" + + // redirect .../index.html to .../ + // can't use Redirect() because that would make the path absolute, + // which would be a problem running under StripPrefix + if strings.HasSuffix(r.URL.Path, indexPage) { + localRedirect(w, r, "./") + return + } + + f, err := fs.Open(name) + if err != nil { + msg, code := toHTTPError(err) + Error(w, msg, code) + return + } + defer f.Close() + + d, err := f.Stat() + if err != nil { + msg, code := toHTTPError(err) + Error(w, msg, code) + return + } + + if redirect { + // redirect to canonical path: / at end of directory url + // r.URL.Path always begins with / + url := r.URL.Path + if d.IsDir() { + if url[len(url)-1] != '/' { + localRedirect(w, r, path.Base(url)+"/") + return + } + } else { + if url[len(url)-1] == '/' { + localRedirect(w, r, "../"+path.Base(url)) + return + } + } + } + + if d.IsDir() { + url := r.URL.Path + // redirect if the directory name doesn't end in a slash + if url == "" || url[len(url)-1] != '/' { + localRedirect(w, r, path.Base(url)+"/") + return + } + + // use contents of index.html for directory, if present + index := strings.TrimSuffix(name, "/") + indexPage + ff, err := fs.Open(index) + if err == nil { + defer ff.Close() + dd, err := ff.Stat() + if err == nil { + name = index + d = dd + f = ff + } + } + } + + // Still a directory? (we didn't find an index.html file) + if d.IsDir() { + if checkIfModifiedSince(r, d.ModTime()) == condFalse { + writeNotModified(w) + return + } + setLastModified(w, d.ModTime()) + dirList(w, r, f) + return + } + + // serveContent will check modification time + sizeFunc := func() (int64, error) { return d.Size(), nil } + serveContent(w, r, d.Name(), d.ModTime(), sizeFunc, f) +} + +// toHTTPError returns a non-specific HTTP error message and status code +// for a given non-nil error value. It's important that toHTTPError does not +// actually return err.Error(), since msg and httpStatus are returned to users, +// and historically Go's ServeContent always returned just "404 Not Found" for +// all errors. We don't want to start leaking information in error messages. +func toHTTPError(err error) (msg string, httpStatus int) { + if errors.Is(err, fs.ErrNotExist) { + return "404 page not found", StatusNotFound + } + if errors.Is(err, fs.ErrPermission) { + return "403 Forbidden", StatusForbidden + } + // Default: + return "500 Internal Server Error", StatusInternalServerError +} + +// localRedirect gives a Moved Permanently response. +// It does not convert relative paths to absolute paths like Redirect does. +func localRedirect(w ResponseWriter, r *Request, newPath string) { + if q := r.URL.RawQuery; q != "" { + newPath += "?" + q + } + w.Header().Set("Location", newPath) + w.WriteHeader(StatusMovedPermanently) +} + +// ServeFile replies to the request with the contents of the named +// file or directory. +// +// If the provided file or directory name is a relative path, it is +// interpreted relative to the current directory and may ascend to +// parent directories. If the provided name is constructed from user +// input, it should be sanitized before calling ServeFile. +// +// As a precaution, ServeFile will reject requests where r.URL.Path +// contains a ".." path element; this protects against callers who +// might unsafely use filepath.Join on r.URL.Path without sanitizing +// it and then use that filepath.Join result as the name argument. +// +// As another special case, ServeFile redirects any request where r.URL.Path +// ends in "/index.html" to the same path, without the final +// "index.html". To avoid such redirects either modify the path or +// use ServeContent. +// +// Outside of those two special cases, ServeFile does not use +// r.URL.Path for selecting the file or directory to serve; only the +// file or directory provided in the name argument is used. +func ServeFile(w ResponseWriter, r *Request, name string) { + if containsDotDot(r.URL.Path) { + // Too many programs use r.URL.Path to construct the argument to + // serveFile. Reject the request under the assumption that happened + // here and ".." may not be wanted. + // Note that name might not contain "..", for example if code (still + // incorrectly) used filepath.Join(myDir, r.URL.Path). + Error(w, "invalid URL path", StatusBadRequest) + return + } + dir, file := filepath.Split(name) + serveFile(w, r, Dir(dir), file, false) +} + +func containsDotDot(v string) bool { + if !strings.Contains(v, "..") { + return false + } + for _, ent := range strings.FieldsFunc(v, isSlashRune) { + if ent == ".." { + return true + } + } + return false +} + +func isSlashRune(r rune) bool { return r == '/' || r == '\\' } + +type fileHandler struct { + root FileSystem +} + +type ioFS struct { + fsys fs.FS +} + +type ioFile struct { + file fs.File +} + +func (f ioFS) Open(name string) (File, error) { + if name == "/" { + name = "." + } else { + name = strings.TrimPrefix(name, "/") + } + file, err := f.fsys.Open(name) + if err != nil { + return nil, mapOpenError(err, name, '/', func(path string) (fs.FileInfo, error) { + return fs.Stat(f.fsys, path) + }) + } + return ioFile{file}, nil +} + +func (f ioFile) Close() error { return f.file.Close() } +func (f ioFile) Read(b []byte) (int, error) { return f.file.Read(b) } +func (f ioFile) Stat() (fs.FileInfo, error) { return f.file.Stat() } + +var errMissingSeek = errors.New("io.File missing Seek method") +var errMissingReadDir = errors.New("io.File directory missing ReadDir method") + +func (f ioFile) Seek(offset int64, whence int) (int64, error) { + s, ok := f.file.(io.Seeker) + if !ok { + return 0, errMissingSeek + } + return s.Seek(offset, whence) +} + +func (f ioFile) ReadDir(count int) ([]fs.DirEntry, error) { + d, ok := f.file.(fs.ReadDirFile) + if !ok { + return nil, errMissingReadDir + } + return d.ReadDir(count) +} + +func (f ioFile) Readdir(count int) ([]fs.FileInfo, error) { + d, ok := f.file.(fs.ReadDirFile) + if !ok { + return nil, errMissingReadDir + } + var list []fs.FileInfo + for { + dirs, err := d.ReadDir(count - len(list)) + for _, dir := range dirs { + info, err := dir.Info() + if err != nil { + // Pretend it doesn't exist, like (*os.File).Readdir does. + continue + } + list = append(list, info) + } + if err != nil { + return list, err + } + if count < 0 || len(list) >= count { + break + } + } + return list, nil +} + +// FS converts fsys to a FileSystem implementation, +// for use with FileServer and NewFileTransport. +func FS(fsys fs.FS) FileSystem { + return ioFS{fsys} +} + +// FileServer returns a handler that serves HTTP requests +// with the contents of the file system rooted at root. +// +// As a special case, the returned file server redirects any request +// ending in "/index.html" to the same path, without the final +// "index.html". +// +// To use the operating system's file system implementation, +// use http.Dir: +// +// http.Handle("/", http.FileServer(http.Dir("/tmp"))) +// +// To use an fs.FS implementation, use http.FS to convert it: +// +// http.Handle("/", http.FileServer(http.FS(fsys))) +func FileServer(root FileSystem) Handler { + return &fileHandler{root} +} + +func (f *fileHandler) ServeHTTP(w ResponseWriter, r *Request) { + upath := r.URL.Path + if !strings.HasPrefix(upath, "/") { + upath = "/" + upath + r.URL.Path = upath + } + serveFile(w, r, f.root, path.Clean(upath), true) +} + +// httpRange specifies the byte range to be sent to the client. +type httpRange struct { + start, length int64 +} + +func (r httpRange) contentRange(size int64) string { + return fmt.Sprintf("bytes %d-%d/%d", r.start, r.start+r.length-1, size) +} + +func (r httpRange) mimeHeader(contentType string, size int64) textproto.MIMEHeader { + return textproto.MIMEHeader{ + "Content-Range": {r.contentRange(size)}, + "Content-Type": {contentType}, + } +} + +// parseRange parses a Range header string as per RFC 7233. +// errNoOverlap is returned if none of the ranges overlap. +func parseRange(s string, size int64) ([]httpRange, error) { + if s == "" { + return nil, nil // header not present + } + const b = "bytes=" + if !strings.HasPrefix(s, b) { + return nil, errors.New("invalid range") + } + var ranges []httpRange + noOverlap := false + for _, ra := range strings.Split(s[len(b):], ",") { + ra = textproto.TrimString(ra) + if ra == "" { + continue + } + start, end, ok := strings.Cut(ra, "-") + if !ok { + return nil, errors.New("invalid range") + } + start, end = textproto.TrimString(start), textproto.TrimString(end) + var r httpRange + if start == "" { + // If no start is specified, end specifies the + // range start relative to the end of the file, + // and we are dealing with + // which has to be a non-negative integer as per + // RFC 7233 Section 2.1 "Byte-Ranges". + if end == "" || end[0] == '-' { + return nil, errors.New("invalid range") + } + i, err := strconv.ParseInt(end, 10, 64) + if i < 0 || err != nil { + return nil, errors.New("invalid range") + } + if i > size { + i = size + } + r.start = size - i + r.length = size - r.start + } else { + i, err := strconv.ParseInt(start, 10, 64) + if err != nil || i < 0 { + return nil, errors.New("invalid range") + } + if i >= size { + // If the range begins after the size of the content, + // then it does not overlap. + noOverlap = true + continue + } + r.start = i + if end == "" { + // If no end is specified, range extends to end of the file. + r.length = size - r.start + } else { + i, err := strconv.ParseInt(end, 10, 64) + if err != nil || r.start > i { + return nil, errors.New("invalid range") + } + if i >= size { + i = size - 1 + } + r.length = i - r.start + 1 + } + } + ranges = append(ranges, r) + } + if noOverlap && len(ranges) == 0 { + // The specified ranges did not overlap with the content. + return nil, errNoOverlap + } + return ranges, nil +} + +// countingWriter counts how many bytes have been written to it. +type countingWriter int64 + +func (w *countingWriter) Write(p []byte) (n int, err error) { + *w += countingWriter(len(p)) + return len(p), nil +} + +// rangesMIMESize returns the number of bytes it takes to encode the +// provided ranges as a multipart response. +func rangesMIMESize(ranges []httpRange, contentType string, contentSize int64) (encSize int64) { + var w countingWriter + mw := multipart.NewWriter(&w) + for _, ra := range ranges { + mw.CreatePart(ra.mimeHeader(contentType, contentSize)) + encSize += ra.length + } + mw.Close() + encSize += int64(w) + return +} + +func sumRangesSize(ranges []httpRange) (size int64) { + for _, ra := range ranges { + size += ra.length + } + return +} diff --git a/src/net/http/header.go b/src/net/http/header.go new file mode 100644 index 0000000000..a5779f6132 --- /dev/null +++ b/src/net/http/header.go @@ -0,0 +1,275 @@ +// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. + +// TINYGO: Removed trace stuff + +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "io" + "net/http/internal/ascii" + "net/textproto" + "sort" + "strings" + "sync" + "time" + + "golang.org/x/net/http/httpguts" +) + +// A Header represents the key-value pairs in an HTTP header. +// +// The keys should be in canonical form, as returned by +// CanonicalHeaderKey. +type Header map[string][]string + +// Add adds the key, value pair to the header. +// It appends to any existing values associated with key. +// The key is case insensitive; it is canonicalized by +// CanonicalHeaderKey. +func (h Header) Add(key, value string) { + textproto.MIMEHeader(h).Add(key, value) +} + +// Set sets the header entries associated with key to the +// single element value. It replaces any existing values +// associated with key. The key is case insensitive; it is +// canonicalized by textproto.CanonicalMIMEHeaderKey. +// To use non-canonical keys, assign to the map directly. +func (h Header) Set(key, value string) { + textproto.MIMEHeader(h).Set(key, value) +} + +// Get gets the first value associated with the given key. If +// there are no values associated with the key, Get returns "". +// It is case insensitive; textproto.CanonicalMIMEHeaderKey is +// used to canonicalize the provided key. Get assumes that all +// keys are stored in canonical form. To use non-canonical keys, +// access the map directly. +func (h Header) Get(key string) string { + return textproto.MIMEHeader(h).Get(key) +} + +// Values returns all values associated with the given key. +// It is case insensitive; textproto.CanonicalMIMEHeaderKey is +// used to canonicalize the provided key. To use non-canonical +// keys, access the map directly. +// The returned slice is not a copy. +func (h Header) Values(key string) []string { + return textproto.MIMEHeader(h).Values(key) +} + +// get is like Get, but key must already be in CanonicalHeaderKey form. +func (h Header) get(key string) string { + if v := h[key]; len(v) > 0 { + return v[0] + } + return "" +} + +// has reports whether h has the provided key defined, even if it's +// set to 0-length slice. +func (h Header) has(key string) bool { + _, ok := h[key] + return ok +} + +// Del deletes the values associated with key. +// The key is case insensitive; it is canonicalized by +// CanonicalHeaderKey. +func (h Header) Del(key string) { + textproto.MIMEHeader(h).Del(key) +} + +// Write writes a header in wire format. +func (h Header) Write(w io.Writer) error { + return h.write(w) +} + +func (h Header) write(w io.Writer) error { + return h.writeSubset(w, nil) +} + +// Clone returns a copy of h or nil if h is nil. +func (h Header) Clone() Header { + if h == nil { + return nil + } + + // Find total number of values. + nv := 0 + for _, vv := range h { + nv += len(vv) + } + sv := make([]string, nv) // shared backing array for headers' values + h2 := make(Header, len(h)) + for k, vv := range h { + if vv == nil { + // Preserve nil values. ReverseProxy distinguishes + // between nil and zero-length header values. + h2[k] = nil + continue + } + n := copy(sv, vv) + h2[k] = sv[:n:n] + sv = sv[n:] + } + return h2 +} + +var timeFormats = []string{ + TimeFormat, + time.RFC850, + time.ANSIC, +} + +// ParseTime parses a time header (such as the Date: header), +// trying each of the three formats allowed by HTTP/1.1: +// TimeFormat, time.RFC850, and time.ANSIC. +func ParseTime(text string) (t time.Time, err error) { + for _, layout := range timeFormats { + t, err = time.Parse(layout, text) + if err == nil { + return + } + } + return +} + +var headerNewlineToSpace = strings.NewReplacer("\n", " ", "\r", " ") + +// stringWriter implements WriteString on a Writer. +type stringWriter struct { + w io.Writer +} + +func (w stringWriter) WriteString(s string) (n int, err error) { + return w.w.Write([]byte(s)) +} + +type keyValues struct { + key string + values []string +} + +// A headerSorter implements sort.Interface by sorting a []keyValues +// by key. It's used as a pointer, so it can fit in a sort.Interface +// interface value without allocation. +type headerSorter struct { + kvs []keyValues +} + +func (s *headerSorter) Len() int { return len(s.kvs) } +func (s *headerSorter) Swap(i, j int) { s.kvs[i], s.kvs[j] = s.kvs[j], s.kvs[i] } +func (s *headerSorter) Less(i, j int) bool { return s.kvs[i].key < s.kvs[j].key } + +var headerSorterPool = sync.Pool{ + New: func() any { return new(headerSorter) }, +} + +// sortedKeyValues returns h's keys sorted in the returned kvs +// slice. The headerSorter used to sort is also returned, for possible +// return to headerSorterCache. +func (h Header) sortedKeyValues(exclude map[string]bool) (kvs []keyValues, hs *headerSorter) { + hs = headerSorterPool.Get().(*headerSorter) + if cap(hs.kvs) < len(h) { + hs.kvs = make([]keyValues, 0, len(h)) + } + kvs = hs.kvs[:0] + for k, vv := range h { + if !exclude[k] { + kvs = append(kvs, keyValues{k, vv}) + } + } + hs.kvs = kvs + sort.Sort(hs) + return kvs, hs +} + +// WriteSubset writes a header in wire format. +// If exclude is not nil, keys where exclude[key] == true are not written. +// Keys are not canonicalized before checking the exclude map. +func (h Header) WriteSubset(w io.Writer, exclude map[string]bool) error { + return h.writeSubset(w, exclude) +} + +func (h Header) writeSubset(w io.Writer, exclude map[string]bool) error { + ws, ok := w.(io.StringWriter) + if !ok { + ws = stringWriter{w} + } + kvs, sorter := h.sortedKeyValues(exclude) + for _, kv := range kvs { + if !httpguts.ValidHeaderFieldName(kv.key) { + // This could be an error. In the common case of + // writing response headers, however, we have no good + // way to provide the error back to the server + // handler, so just drop invalid headers instead. + continue + } + for _, v := range kv.values { + v = headerNewlineToSpace.Replace(v) + v = textproto.TrimString(v) + for _, s := range []string{kv.key, ": ", v, "\r\n"} { + if _, err := ws.WriteString(s); err != nil { + headerSorterPool.Put(sorter) + return err + } + } + } + } + headerSorterPool.Put(sorter) + return nil +} + +// CanonicalHeaderKey returns the canonical format of the +// header key s. The canonicalization converts the first +// letter and any letter following a hyphen to upper case; +// the rest are converted to lowercase. For example, the +// canonical key for "accept-encoding" is "Accept-Encoding". +// If s contains a space or invalid header field bytes, it is +// returned without modifications. +func CanonicalHeaderKey(s string) string { return textproto.CanonicalMIMEHeaderKey(s) } + +// hasToken reports whether token appears with v, ASCII +// case-insensitive, with space or comma boundaries. +// token must be all lowercase. +// v may contain mixed cased. +func hasToken(v, token string) bool { + if len(token) > len(v) || token == "" { + return false + } + if v == token { + return true + } + for sp := 0; sp <= len(v)-len(token); sp++ { + // Check that first character is good. + // The token is ASCII, so checking only a single byte + // is sufficient. We skip this potential starting + // position if both the first byte and its potential + // ASCII uppercase equivalent (b|0x20) don't match. + // False positives ('^' => '~') are caught by EqualFold. + if b := v[sp]; b != token[0] && b|0x20 != token[0] { + continue + } + // Check that start pos is on a valid token boundary. + if sp > 0 && !isTokenBoundary(v[sp-1]) { + continue + } + // Check that end pos is on a valid token boundary. + if endPos := sp + len(token); endPos != len(v) && !isTokenBoundary(v[endPos]) { + continue + } + if ascii.EqualFold(v[sp:sp+len(token)], token) { + return true + } + } + return false +} + +func isTokenBoundary(b byte) bool { + return b == ' ' || b == ',' || b == '\t' +} diff --git a/src/net/http/http.go b/src/net/http/http.go new file mode 100644 index 0000000000..fc1db57f38 --- /dev/null +++ b/src/net/http/http.go @@ -0,0 +1,161 @@ +// TINYGO: The following is copied from Go 1.19.3 official implementation. + +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:generate bundle -o=h2_bundle.go -prefix=http2 -tags=!nethttpomithttp2 golang.org/x/net/http2 + +package http + +import ( + "io" + "strconv" + "strings" + "time" + "unicode/utf8" + + "golang.org/x/net/http/httpguts" +) + +// incomparable is a zero-width, non-comparable type. Adding it to a struct +// makes that struct also non-comparable, and generally doesn't add +// any size (as long as it's first). +type incomparable [0]func() + +// maxInt64 is the effective "infinite" value for the Server and +// Transport's byte-limiting readers. +const maxInt64 = 1<<63 - 1 + +// aLongTimeAgo is a non-zero time, far in the past, used for +// immediate cancellation of network operations. +var aLongTimeAgo = time.Unix(1, 0) + +// omitBundledHTTP2 is set by omithttp2.go when the nethttpomithttp2 +// build tag is set. That means h2_bundle.go isn't compiled in and we +// shouldn't try to use it. +var omitBundledHTTP2 bool + +// TODO(bradfitz): move common stuff here. The other files have accumulated +// generic http stuff in random places. + +// contextKey is a value for use with context.WithValue. It's used as +// a pointer so it fits in an interface{} without allocation. +type contextKey struct { + name string +} + +func (k *contextKey) String() string { return "net/http context value " + k.name } + +// Given a string of the form "host", "host:port", or "[ipv6::address]:port", +// return true if the string includes a port. +func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") } + +// removeEmptyPort strips the empty port in ":port" to "" +// as mandated by RFC 3986 Section 6.2.3. +func removeEmptyPort(host string) string { + if hasPort(host) { + return strings.TrimSuffix(host, ":") + } + return host +} + +func isNotToken(r rune) bool { + return !httpguts.IsTokenRune(r) +} + +// stringContainsCTLByte reports whether s contains any ASCII control character. +func stringContainsCTLByte(s string) bool { + for i := 0; i < len(s); i++ { + b := s[i] + if b < ' ' || b == 0x7f { + return true + } + } + return false +} + +func hexEscapeNonASCII(s string) string { + newLen := 0 + for i := 0; i < len(s); i++ { + if s[i] >= utf8.RuneSelf { + newLen += 3 + } else { + newLen++ + } + } + if newLen == len(s) { + return s + } + b := make([]byte, 0, newLen) + for i := 0; i < len(s); i++ { + if s[i] >= utf8.RuneSelf { + b = append(b, '%') + b = strconv.AppendInt(b, int64(s[i]), 16) + } else { + b = append(b, s[i]) + } + } + return string(b) +} + +// NoBody is an io.ReadCloser with no bytes. Read always returns EOF +// and Close always returns nil. It can be used in an outgoing client +// request to explicitly signal that a request has zero bytes. +// An alternative, however, is to simply set Request.Body to nil. +var NoBody = noBody{} + +type noBody struct{} + +func (noBody) Read([]byte) (int, error) { return 0, io.EOF } +func (noBody) Close() error { return nil } +func (noBody) WriteTo(io.Writer) (int64, error) { return 0, nil } + +var ( + // verify that an io.Copy from NoBody won't require a buffer: + _ io.WriterTo = NoBody + _ io.ReadCloser = NoBody +) + +// PushOptions describes options for Pusher.Push. +type PushOptions struct { + // Method specifies the HTTP method for the promised request. + // If set, it must be "GET" or "HEAD". Empty means "GET". + Method string + + // Header specifies additional promised request headers. This cannot + // include HTTP/2 pseudo header fields like ":path" and ":scheme", + // which will be added automatically. + Header Header +} + +// Pusher is the interface implemented by ResponseWriters that support +// HTTP/2 server push. For more background, see +// https://tools.ietf.org/html/rfc7540#section-8.2. +type Pusher interface { + // Push initiates an HTTP/2 server push. This constructs a synthetic + // request using the given target and options, serializes that request + // into a PUSH_PROMISE frame, then dispatches that request using the + // server's request handler. If opts is nil, default options are used. + // + // The target must either be an absolute path (like "/path") or an absolute + // URL that contains a valid host and the same scheme as the parent request. + // If the target is a path, it will inherit the scheme and host of the + // parent request. + // + // The HTTP/2 spec disallows recursive pushes and cross-authority pushes. + // Push may or may not detect these invalid pushes; however, invalid + // pushes will be detected and canceled by conforming clients. + // + // Handlers that wish to push URL X should call Push before sending any + // data that may trigger a request for URL X. This avoids a race where the + // client issues requests for X before receiving the PUSH_PROMISE for X. + // + // Push will run in a separate goroutine making the order of arrival + // non-deterministic. Any required synchronization needs to be implemented + // by the caller. + // + // Push returns ErrNotSupported if the client has disabled push or if push + // is not supported on the underlying connection. + Push(target string, opts *PushOptions) error +} diff --git a/src/net/http/internal/ascii/print.go b/src/net/http/internal/ascii/print.go new file mode 100644 index 0000000000..c2b3a9bda9 --- /dev/null +++ b/src/net/http/internal/ascii/print.go @@ -0,0 +1,63 @@ +// TINYGO: The following is copied from Go 1.19.3 official implementation. + +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ascii + +import ( + "strings" + "unicode" +) + +// EqualFold is strings.EqualFold, ASCII only. It reports whether s and t +// are equal, ASCII-case-insensitively. +func EqualFold(s, t string) bool { + if len(s) != len(t) { + return false + } + for i := 0; i < len(s); i++ { + if lower(s[i]) != lower(t[i]) { + return false + } + } + return true +} + +// lower returns the ASCII lowercase version of b. +func lower(b byte) byte { + if 'A' <= b && b <= 'Z' { + return b + ('a' - 'A') + } + return b +} + +// IsPrint returns whether s is ASCII and printable according to +// https://tools.ietf.org/html/rfc20#section-4.2. +func IsPrint(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] < ' ' || s[i] > '~' { + return false + } + } + return true +} + +// Is returns whether s is ASCII. +func Is(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] > unicode.MaxASCII { + return false + } + } + return true +} + +// ToLower returns the lowercase version of s if s is ASCII and printable. +func ToLower(s string) (lower string, ok bool) { + if !IsPrint(s) { + return "", false + } + return strings.ToLower(s), true +} diff --git a/src/net/http/internal/ascii/print_test.go b/src/net/http/internal/ascii/print_test.go new file mode 100644 index 0000000000..0b7767ca33 --- /dev/null +++ b/src/net/http/internal/ascii/print_test.go @@ -0,0 +1,95 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ascii + +import "testing" + +func TestEqualFold(t *testing.T) { + var tests = []struct { + name string + a, b string + want bool + }{ + { + name: "empty", + want: true, + }, + { + name: "simple match", + a: "CHUNKED", + b: "chunked", + want: true, + }, + { + name: "same string", + a: "chunked", + b: "chunked", + want: true, + }, + { + name: "Unicode Kelvin symbol", + a: "chunKed", // This "K" is 'KELVIN SIGN' (\u212A) + b: "chunked", + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := EqualFold(tt.a, tt.b); got != tt.want { + t.Errorf("AsciiEqualFold(%q,%q): got %v want %v", tt.a, tt.b, got, tt.want) + } + }) + } +} + +func TestIsPrint(t *testing.T) { + var tests = []struct { + name string + in string + want bool + }{ + { + name: "empty", + want: true, + }, + { + name: "ASCII low", + in: "This is a space: ' '", + want: true, + }, + { + name: "ASCII high", + in: "This is a tilde: '~'", + want: true, + }, + { + name: "ASCII low non-print", + in: "This is a unit separator: \x1F", + want: false, + }, + { + name: "Ascii high non-print", + in: "This is a Delete: \x7F", + want: false, + }, + { + name: "Unicode letter", + in: "Today it's 280K outside: it's freezing!", // This "K" is 'KELVIN SIGN' (\u212A) + want: false, + }, + { + name: "Unicode emoji", + in: "Gophers like 🧀", + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsPrint(tt.in); got != tt.want { + t.Errorf("IsASCIIPrint(%q): got %v want %v", tt.in, got, tt.want) + } + }) + } +} diff --git a/src/net/http/internal/chunked.go b/src/net/http/internal/chunked.go new file mode 100644 index 0000000000..34b533158d --- /dev/null +++ b/src/net/http/internal/chunked.go @@ -0,0 +1,264 @@ +// TINYGO: The following is copied from Go 1.19.3 official implementation. + +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The wire protocol for HTTP's "chunked" Transfer-Encoding. + +// Package internal contains HTTP internals shared by net/http and +// net/http/httputil. +package internal + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" +) + +const maxLineLength = 4096 // assumed <= bufio.defaultBufSize + +var ErrLineTooLong = errors.New("header line too long") + +// NewChunkedReader returns a new chunkedReader that translates the data read from r +// out of HTTP "chunked" format before returning it. +// The chunkedReader returns io.EOF when the final 0-length chunk is read. +// +// NewChunkedReader is not needed by normal applications. The http package +// automatically decodes chunking when reading response bodies. +func NewChunkedReader(r io.Reader) io.Reader { + br, ok := r.(*bufio.Reader) + if !ok { + br = bufio.NewReader(r) + } + return &chunkedReader{r: br} +} + +type chunkedReader struct { + r *bufio.Reader + n uint64 // unread bytes in chunk + err error + buf [2]byte + checkEnd bool // whether need to check for \r\n chunk footer +} + +func (cr *chunkedReader) beginChunk() { + // chunk-size CRLF + var line []byte + line, cr.err = readChunkLine(cr.r) + if cr.err != nil { + return + } + cr.n, cr.err = parseHexUint(line) + if cr.err != nil { + return + } + if cr.n == 0 { + cr.err = io.EOF + } +} + +func (cr *chunkedReader) chunkHeaderAvailable() bool { + n := cr.r.Buffered() + if n > 0 { + peek, _ := cr.r.Peek(n) + return bytes.IndexByte(peek, '\n') >= 0 + } + return false +} + +func (cr *chunkedReader) Read(b []uint8) (n int, err error) { + for cr.err == nil { + if cr.checkEnd { + if n > 0 && cr.r.Buffered() < 2 { + // We have some data. Return early (per the io.Reader + // contract) instead of potentially blocking while + // reading more. + break + } + if _, cr.err = io.ReadFull(cr.r, cr.buf[:2]); cr.err == nil { + if string(cr.buf[:]) != "\r\n" { + cr.err = errors.New("malformed chunked encoding") + break + } + } else { + if cr.err == io.EOF { + cr.err = io.ErrUnexpectedEOF + } + break + } + cr.checkEnd = false + } + if cr.n == 0 { + if n > 0 && !cr.chunkHeaderAvailable() { + // We've read enough. Don't potentially block + // reading a new chunk header. + break + } + cr.beginChunk() + continue + } + if len(b) == 0 { + break + } + rbuf := b + if uint64(len(rbuf)) > cr.n { + rbuf = rbuf[:cr.n] + } + var n0 int + n0, cr.err = cr.r.Read(rbuf) + n += n0 + b = b[n0:] + cr.n -= uint64(n0) + // If we're at the end of a chunk, read the next two + // bytes to verify they are "\r\n". + if cr.n == 0 && cr.err == nil { + cr.checkEnd = true + } else if cr.err == io.EOF { + cr.err = io.ErrUnexpectedEOF + } + } + return n, cr.err +} + +// Read a line of bytes (up to \n) from b. +// Give up if the line exceeds maxLineLength. +// The returned bytes are owned by the bufio.Reader +// so they are only valid until the next bufio read. +func readChunkLine(b *bufio.Reader) ([]byte, error) { + p, err := b.ReadSlice('\n') + if err != nil { + // We always know when EOF is coming. + // If the caller asked for a line, there should be a line. + if err == io.EOF { + err = io.ErrUnexpectedEOF + } else if err == bufio.ErrBufferFull { + err = ErrLineTooLong + } + return nil, err + } + if len(p) >= maxLineLength { + return nil, ErrLineTooLong + } + p = trimTrailingWhitespace(p) + p, err = removeChunkExtension(p) + if err != nil { + return nil, err + } + return p, nil +} + +func trimTrailingWhitespace(b []byte) []byte { + for len(b) > 0 && isASCIISpace(b[len(b)-1]) { + b = b[:len(b)-1] + } + return b +} + +func isASCIISpace(b byte) bool { + return b == ' ' || b == '\t' || b == '\n' || b == '\r' +} + +var semi = []byte(";") + +// removeChunkExtension removes any chunk-extension from p. +// For example, +// +// "0" => "0" +// "0;token" => "0" +// "0;token=val" => "0" +// `0;token="quoted string"` => "0" +func removeChunkExtension(p []byte) ([]byte, error) { + p, _, _ = bytes.Cut(p, semi) + // TODO: care about exact syntax of chunk extensions? We're + // ignoring and stripping them anyway. For now just never + // return an error. + return p, nil +} + +// NewChunkedWriter returns a new chunkedWriter that translates writes into HTTP +// "chunked" format before writing them to w. Closing the returned chunkedWriter +// sends the final 0-length chunk that marks the end of the stream but does +// not send the final CRLF that appears after trailers; trailers and the last +// CRLF must be written separately. +// +// NewChunkedWriter is not needed by normal applications. The http +// package adds chunking automatically if handlers don't set a +// Content-Length header. Using newChunkedWriter inside a handler +// would result in double chunking or chunking with a Content-Length +// length, both of which are wrong. +func NewChunkedWriter(w io.Writer) io.WriteCloser { + return &chunkedWriter{w} +} + +// Writing to chunkedWriter translates to writing in HTTP chunked Transfer +// Encoding wire format to the underlying Wire chunkedWriter. +type chunkedWriter struct { + Wire io.Writer +} + +// Write the contents of data as one chunk to Wire. +// NOTE: Note that the corresponding chunk-writing procedure in Conn.Write has +// a bug since it does not check for success of io.WriteString +func (cw *chunkedWriter) Write(data []byte) (n int, err error) { + + // Don't send 0-length data. It looks like EOF for chunked encoding. + if len(data) == 0 { + return 0, nil + } + + if _, err = fmt.Fprintf(cw.Wire, "%x\r\n", len(data)); err != nil { + return 0, err + } + if n, err = cw.Wire.Write(data); err != nil { + return + } + if n != len(data) { + err = io.ErrShortWrite + return + } + if _, err = io.WriteString(cw.Wire, "\r\n"); err != nil { + return + } + if bw, ok := cw.Wire.(*FlushAfterChunkWriter); ok { + err = bw.Flush() + } + return +} + +func (cw *chunkedWriter) Close() error { + _, err := io.WriteString(cw.Wire, "0\r\n") + return err +} + +// FlushAfterChunkWriter signals from the caller of NewChunkedWriter +// that each chunk should be followed by a flush. It is used by the +// http.Transport code to keep the buffering behavior for headers and +// trailers, but flush out chunks aggressively in the middle for +// request bodies which may be generated slowly. See Issue 6574. +type FlushAfterChunkWriter struct { + *bufio.Writer +} + +func parseHexUint(v []byte) (n uint64, err error) { + for i, b := range v { + switch { + case '0' <= b && b <= '9': + b = b - '0' + case 'a' <= b && b <= 'f': + b = b - 'a' + 10 + case 'A' <= b && b <= 'F': + b = b - 'A' + 10 + default: + return 0, errors.New("invalid byte in chunk length") + } + if i == 16 { + return 0, errors.New("http chunk length too large") + } + n <<= 4 + n |= uint64(b) + } + return +} diff --git a/src/net/http/internal/chunked_test.go b/src/net/http/internal/chunked_test.go new file mode 100644 index 0000000000..5e29a786dd --- /dev/null +++ b/src/net/http/internal/chunked_test.go @@ -0,0 +1,241 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package internal + +import ( + "bufio" + "bytes" + "fmt" + "io" + "strings" + "testing" + "testing/iotest" +) + +func TestChunk(t *testing.T) { + var b bytes.Buffer + + w := NewChunkedWriter(&b) + const chunk1 = "hello, " + const chunk2 = "world! 0123456789abcdef" + w.Write([]byte(chunk1)) + w.Write([]byte(chunk2)) + w.Close() + + if g, e := b.String(), "7\r\nhello, \r\n17\r\nworld! 0123456789abcdef\r\n0\r\n"; g != e { + t.Fatalf("chunk writer wrote %q; want %q", g, e) + } + + r := NewChunkedReader(&b) + data, err := io.ReadAll(r) + if err != nil { + t.Logf(`data: "%s"`, data) + t.Fatalf("ReadAll from reader: %v", err) + } + if g, e := string(data), chunk1+chunk2; g != e { + t.Errorf("chunk reader read %q; want %q", g, e) + } +} + +func TestChunkReadMultiple(t *testing.T) { + // Bunch of small chunks, all read together. + { + var b bytes.Buffer + w := NewChunkedWriter(&b) + w.Write([]byte("foo")) + w.Write([]byte("bar")) + w.Close() + + r := NewChunkedReader(&b) + buf := make([]byte, 10) + n, err := r.Read(buf) + if n != 6 || err != io.EOF { + t.Errorf("Read = %d, %v; want 6, EOF", n, err) + } + buf = buf[:n] + if string(buf) != "foobar" { + t.Errorf("Read = %q; want %q", buf, "foobar") + } + } + + // One big chunk followed by a little chunk, but the small bufio.Reader size + // should prevent the second chunk header from being read. + { + var b bytes.Buffer + w := NewChunkedWriter(&b) + // fillBufChunk is 11 bytes + 3 bytes header + 2 bytes footer = 16 bytes, + // the same as the bufio ReaderSize below (the minimum), so even + // though we're going to try to Read with a buffer larger enough to also + // receive "foo", the second chunk header won't be read yet. + const fillBufChunk = "0123456789a" + const shortChunk = "foo" + w.Write([]byte(fillBufChunk)) + w.Write([]byte(shortChunk)) + w.Close() + + r := NewChunkedReader(bufio.NewReaderSize(&b, 16)) + buf := make([]byte, len(fillBufChunk)+len(shortChunk)) + n, err := r.Read(buf) + if n != len(fillBufChunk) || err != nil { + t.Errorf("Read = %d, %v; want %d, nil", n, err, len(fillBufChunk)) + } + buf = buf[:n] + if string(buf) != fillBufChunk { + t.Errorf("Read = %q; want %q", buf, fillBufChunk) + } + + n, err = r.Read(buf) + if n != len(shortChunk) || err != io.EOF { + t.Errorf("Read = %d, %v; want %d, EOF", n, err, len(shortChunk)) + } + } + + // And test that we see an EOF chunk, even though our buffer is already full: + { + r := NewChunkedReader(bufio.NewReader(strings.NewReader("3\r\nfoo\r\n0\r\n"))) + buf := make([]byte, 3) + n, err := r.Read(buf) + if n != 3 || err != io.EOF { + t.Errorf("Read = %d, %v; want 3, EOF", n, err) + } + if string(buf) != "foo" { + t.Errorf("buf = %q; want foo", buf) + } + } +} + +func TestChunkReaderAllocs(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + var buf bytes.Buffer + w := NewChunkedWriter(&buf) + a, b, c := []byte("aaaaaa"), []byte("bbbbbbbbbbbb"), []byte("cccccccccccccccccccccccc") + w.Write(a) + w.Write(b) + w.Write(c) + w.Close() + + readBuf := make([]byte, len(a)+len(b)+len(c)+1) + byter := bytes.NewReader(buf.Bytes()) + bufr := bufio.NewReader(byter) + mallocs := testing.AllocsPerRun(100, func() { + byter.Seek(0, io.SeekStart) + bufr.Reset(byter) + r := NewChunkedReader(bufr) + n, err := io.ReadFull(r, readBuf) + if n != len(readBuf)-1 { + t.Fatalf("read %d bytes; want %d", n, len(readBuf)-1) + } + if err != io.ErrUnexpectedEOF { + t.Fatalf("read error = %v; want ErrUnexpectedEOF", err) + } + }) + if mallocs > 1.5 { + t.Errorf("mallocs = %v; want 1", mallocs) + } +} + +func TestParseHexUint(t *testing.T) { + type testCase struct { + in string + want uint64 + wantErr string + } + tests := []testCase{ + {"x", 0, "invalid byte in chunk length"}, + {"0000000000000000", 0, ""}, + {"0000000000000001", 1, ""}, + {"ffffffffffffffff", 1<<64 - 1, ""}, + {"000000000000bogus", 0, "invalid byte in chunk length"}, + {"00000000000000000", 0, "http chunk length too large"}, // could accept if we wanted + {"10000000000000000", 0, "http chunk length too large"}, + {"00000000000000001", 0, "http chunk length too large"}, // could accept if we wanted + } + for i := uint64(0); i <= 1234; i++ { + tests = append(tests, testCase{in: fmt.Sprintf("%x", i), want: i}) + } + for _, tt := range tests { + got, err := parseHexUint([]byte(tt.in)) + if tt.wantErr != "" { + if !strings.Contains(fmt.Sprint(err), tt.wantErr) { + t.Errorf("parseHexUint(%q) = %v, %v; want error %q", tt.in, got, err, tt.wantErr) + } + } else { + if err != nil || got != tt.want { + t.Errorf("parseHexUint(%q) = %v, %v; want %v", tt.in, got, err, tt.want) + } + } + } +} + +func TestChunkReadingIgnoresExtensions(t *testing.T) { + in := "7;ext=\"some quoted string\"\r\n" + // token=quoted string + "hello, \r\n" + + "17;someext\r\n" + // token without value + "world! 0123456789abcdef\r\n" + + "0;someextension=sometoken\r\n" // token=token + data, err := io.ReadAll(NewChunkedReader(strings.NewReader(in))) + if err != nil { + t.Fatalf("ReadAll = %q, %v", data, err) + } + if g, e := string(data), "hello, world! 0123456789abcdef"; g != e { + t.Errorf("read %q; want %q", g, e) + } +} + +// Issue 17355: ChunkedReader shouldn't block waiting for more data +// if it can return something. +func TestChunkReadPartial(t *testing.T) { + pr, pw := io.Pipe() + go func() { + pw.Write([]byte("7\r\n1234567")) + }() + cr := NewChunkedReader(pr) + readBuf := make([]byte, 7) + n, err := cr.Read(readBuf) + if err != nil { + t.Fatal(err) + } + want := "1234567" + if n != 7 || string(readBuf) != want { + t.Fatalf("Read: %v %q; want %d, %q", n, readBuf[:n], len(want), want) + } + go func() { + pw.Write([]byte("xx")) + }() + _, err = cr.Read(readBuf) + if got := fmt.Sprint(err); !strings.Contains(got, "malformed") { + t.Fatalf("second read = %v; want malformed error", err) + } + +} + +// Issue 48861: ChunkedReader should report incomplete chunks +func TestIncompleteChunk(t *testing.T) { + const valid = "4\r\nabcd\r\n" + "5\r\nabc\r\n\r\n" + "0\r\n" + + for i := 0; i < len(valid); i++ { + incomplete := valid[:i] + r := NewChunkedReader(strings.NewReader(incomplete)) + if _, err := io.ReadAll(r); err != io.ErrUnexpectedEOF { + t.Errorf("expected io.ErrUnexpectedEOF for %q, got %v", incomplete, err) + } + } + + r := NewChunkedReader(strings.NewReader(valid)) + if _, err := io.ReadAll(r); err != nil { + t.Errorf("unexpected error for %q: %v", valid, err) + } +} + +func TestChunkEndReadError(t *testing.T) { + readErr := fmt.Errorf("chunk end read error") + + r := NewChunkedReader(io.MultiReader(strings.NewReader("4\r\nabcd"), iotest.ErrReader(readErr))) + if _, err := io.ReadAll(r); err != readErr { + t.Errorf("expected %v, got %v", readErr, err) + } +} diff --git a/src/net/http/jar.go b/src/net/http/jar.go new file mode 100644 index 0000000000..3091c58aaa --- /dev/null +++ b/src/net/http/jar.go @@ -0,0 +1,29 @@ +// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. + +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "net/url" +) + +// A CookieJar manages storage and use of cookies in HTTP requests. +// +// Implementations of CookieJar must be safe for concurrent use by multiple +// goroutines. +// +// The net/http/cookiejar package provides a CookieJar implementation. +type CookieJar interface { + // SetCookies handles the receipt of the cookies in a reply for the + // given URL. It may or may not choose to save the cookies, depending + // on the jar's policy and implementation. + SetCookies(u *url.URL, cookies []*Cookie) + + // Cookies returns the cookies to send in a request for the given URL. + // It is up to the implementation to honor the standard cookie use + // restrictions such as in RFC 6265. + Cookies(u *url.URL) []*Cookie +} diff --git a/src/net/http/method.go b/src/net/http/method.go new file mode 100644 index 0000000000..b8a4c33beb --- /dev/null +++ b/src/net/http/method.go @@ -0,0 +1,22 @@ +// TINYGO: The following is copied from Go 1.19.3 official implementation. + +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +// Common HTTP methods. +// +// Unless otherwise noted, these are defined in RFC 7231 section 4.3. +const ( + MethodGet = "GET" + MethodHead = "HEAD" + MethodPost = "POST" + MethodPut = "PUT" + MethodPatch = "PATCH" // RFC 5789 + MethodDelete = "DELETE" + MethodConnect = "CONNECT" + MethodOptions = "OPTIONS" + MethodTrace = "TRACE" +) diff --git a/src/net/http/request.go b/src/net/http/request.go new file mode 100644 index 0000000000..1971ec4253 --- /dev/null +++ b/src/net/http/request.go @@ -0,0 +1,1447 @@ +// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. + +// TINYGO: Removed multipart stuff +// TINYGO: Removed trace stuff + +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// HTTP Request reading and parsing. + +package http + +import ( + "bufio" + "bytes" + "context" + "crypto/tls" + "encoding/base64" + "errors" + "fmt" + "io" + "mime" + "mime/multipart" + "net" + "net/http/internal/ascii" + "net/textproto" + "net/url" + urlpkg "net/url" + "strconv" + "strings" + "sync" +) + +const ( + defaultMaxMemory = 32 << 20 // 32 MB +) + +// ErrMissingFile is returned by FormFile when the provided file field name +// is either not present in the request or not a file field. +var ErrMissingFile = errors.New("http: no such file") + +// ProtocolError represents an HTTP protocol error. +// +// Deprecated: Not all errors in the http package related to protocol errors +// are of type ProtocolError. +type ProtocolError struct { + ErrorString string +} + +func (pe *ProtocolError) Error() string { return pe.ErrorString } + +var ( + // ErrNotSupported is returned by the Push method of Pusher + // implementations to indicate that HTTP/2 Push support is not + // available. + ErrNotSupported = &ProtocolError{"feature not supported"} + + // Deprecated: ErrUnexpectedTrailer is no longer returned by + // anything in the net/http package. Callers should not + // compare errors against this variable. + ErrUnexpectedTrailer = &ProtocolError{"trailer header without chunked transfer encoding"} + + // ErrMissingBoundary is returned by Request.MultipartReader when the + // request's Content-Type does not include a "boundary" parameter. + ErrMissingBoundary = &ProtocolError{"no multipart boundary param in Content-Type"} + + // ErrNotMultipart is returned by Request.MultipartReader when the + // request's Content-Type is not multipart/form-data. + ErrNotMultipart = &ProtocolError{"request Content-Type isn't multipart/form-data"} + + // Deprecated: ErrHeaderTooLong is no longer returned by + // anything in the net/http package. Callers should not + // compare errors against this variable. + ErrHeaderTooLong = &ProtocolError{"header too long"} + + // Deprecated: ErrShortBody is no longer returned by + // anything in the net/http package. Callers should not + // compare errors against this variable. + ErrShortBody = &ProtocolError{"entity body too short"} + + // Deprecated: ErrMissingContentLength is no longer returned by + // anything in the net/http package. Callers should not + // compare errors against this variable. + ErrMissingContentLength = &ProtocolError{"missing ContentLength in HEAD response"} +) + +func badStringError(what, val string) error { return fmt.Errorf("%s %q", what, val) } + +// Headers that Request.Write handles itself and should be skipped. +var reqWriteExcludeHeader = map[string]bool{ + "Host": true, // not in Header map anyway + "User-Agent": true, + "Content-Length": true, + "Transfer-Encoding": true, + "Trailer": true, +} + +// A Request represents an HTTP request received by a server +// or to be sent by a client. +// +// The field semantics differ slightly between client and server +// usage. In addition to the notes on the fields below, see the +// documentation for Request.Write and RoundTripper. +type Request struct { + // Method specifies the HTTP method (GET, POST, PUT, etc.). + // For client requests, an empty string means GET. + // + // Go's HTTP client does not support sending a request with + // the CONNECT method. See the documentation on Transport for + // details. + Method string + + // URL specifies either the URI being requested (for server + // requests) or the URL to access (for client requests). + // + // For server requests, the URL is parsed from the URI + // supplied on the Request-Line as stored in RequestURI. For + // most requests, fields other than Path and RawQuery will be + // empty. (See RFC 7230, Section 5.3) + // + // For client requests, the URL's Host specifies the server to + // connect to, while the Request's Host field optionally + // specifies the Host header value to send in the HTTP + // request. + URL *url.URL + + // The protocol version for incoming server requests. + // + // For client requests, these fields are ignored. The HTTP + // client code always uses either HTTP/1.1 or HTTP/2. + // See the docs on Transport for details. + Proto string // "HTTP/1.0" + ProtoMajor int // 1 + ProtoMinor int // 0 + + // Header contains the request header fields either received + // by the server or to be sent by the client. + // + // If a server received a request with header lines, + // + // Host: example.com + // accept-encoding: gzip, deflate + // Accept-Language: en-us + // fOO: Bar + // foo: two + // + // then + // + // Header = map[string][]string{ + // "Accept-Encoding": {"gzip, deflate"}, + // "Accept-Language": {"en-us"}, + // "Foo": {"Bar", "two"}, + // } + // + // For incoming requests, the Host header is promoted to the + // Request.Host field and removed from the Header map. + // + // HTTP defines that header names are case-insensitive. The + // request parser implements this by using CanonicalHeaderKey, + // making the first character and any characters following a + // hyphen uppercase and the rest lowercase. + // + // For client requests, certain headers such as Content-Length + // and Connection are automatically written when needed and + // values in Header may be ignored. See the documentation + // for the Request.Write method. + Header Header + + // Body is the request's body. + // + // For client requests, a nil body means the request has no + // body, such as a GET request. The HTTP Client's Transport + // is responsible for calling the Close method. + // + // For server requests, the Request Body is always non-nil + // but will return EOF immediately when no body is present. + // The Server will close the request body. The ServeHTTP + // Handler does not need to. + // + // Body must allow Read to be called concurrently with Close. + // In particular, calling Close should unblock a Read waiting + // for input. + Body io.ReadCloser + + // GetBody defines an optional func to return a new copy of + // Body. It is used for client requests when a redirect requires + // reading the body more than once. Use of GetBody still + // requires setting Body. + // + // For server requests, it is unused. + GetBody func() (io.ReadCloser, error) + + // ContentLength records the length of the associated content. + // The value -1 indicates that the length is unknown. + // Values >= 0 indicate that the given number of bytes may + // be read from Body. + // + // For client requests, a value of 0 with a non-nil Body is + // also treated as unknown. + ContentLength int64 + + // TransferEncoding lists the transfer encodings from outermost to + // innermost. An empty list denotes the "identity" encoding. + // TransferEncoding can usually be ignored; chunked encoding is + // automatically added and removed as necessary when sending and + // receiving requests. + TransferEncoding []string + + // Close indicates whether to close the connection after + // replying to this request (for servers) or after sending this + // request and reading its response (for clients). + // + // For server requests, the HTTP server handles this automatically + // and this field is not needed by Handlers. + // + // For client requests, setting this field prevents re-use of + // TCP connections between requests to the same hosts, as if + // Transport.DisableKeepAlives were set. + Close bool + + // For server requests, Host specifies the host on which the + // URL is sought. For HTTP/1 (per RFC 7230, section 5.4), this + // is either the value of the "Host" header or the host name + // given in the URL itself. For HTTP/2, it is the value of the + // ":authority" pseudo-header field. + // It may be of the form "host:port". For international domain + // names, Host may be in Punycode or Unicode form. Use + // golang.org/x/net/idna to convert it to either format if + // needed. + // To prevent DNS rebinding attacks, server Handlers should + // validate that the Host header has a value for which the + // Handler considers itself authoritative. The included + // ServeMux supports patterns registered to particular host + // names and thus protects its registered Handlers. + // + // For client requests, Host optionally overrides the Host + // header to send. If empty, the Request.Write method uses + // the value of URL.Host. Host may contain an international + // domain name. + Host string + + // Form contains the parsed form data, including both the URL + // field's query parameters and the PATCH, POST, or PUT form data. + // This field is only available after ParseForm is called. + // The HTTP client ignores Form and uses Body instead. + Form url.Values + + // PostForm contains the parsed form data from PATCH, POST + // or PUT body parameters. + // + // This field is only available after ParseForm is called. + // The HTTP client ignores PostForm and uses Body instead. + PostForm url.Values + + // MultipartForm is the parsed multipart form, including file uploads. + // This field is only available after ParseMultipartForm is called. + // The HTTP client ignores MultipartForm and uses Body instead. + MultipartForm *multipart.Form + + // Trailer specifies additional headers that are sent after the request + // body. + // + // For server requests, the Trailer map initially contains only the + // trailer keys, with nil values. (The client declares which trailers it + // will later send.) While the handler is reading from Body, it must + // not reference Trailer. After reading from Body returns EOF, Trailer + // can be read again and will contain non-nil values, if they were sent + // by the client. + // + // For client requests, Trailer must be initialized to a map containing + // the trailer keys to later send. The values may be nil or their final + // values. The ContentLength must be 0 or -1, to send a chunked request. + // After the HTTP request is sent the map values can be updated while + // the request body is read. Once the body returns EOF, the caller must + // not mutate Trailer. + // + // Few HTTP clients, servers, or proxies support HTTP trailers. + Trailer Header + + // RemoteAddr allows HTTP servers and other software to record + // the network address that sent the request, usually for + // logging. This field is not filled in by ReadRequest and + // has no defined format. The HTTP server in this package + // sets RemoteAddr to an "IP:port" address before invoking a + // handler. + // This field is ignored by the HTTP client. + RemoteAddr string + + // RequestURI is the unmodified request-target of the + // Request-Line (RFC 7230, Section 3.1.1) as sent by the client + // to a server. Usually the URL field should be used instead. + // It is an error to set this field in an HTTP client request. + RequestURI string + + // TLS allows HTTP servers and other software to record + // information about the TLS connection on which the request + // was received. This field is not filled in by ReadRequest. + // The HTTP server in this package sets the field for + // TLS-enabled connections before invoking a handler; + // otherwise it leaves the field nil. + // This field is ignored by the HTTP client. + TLS *tls.ConnectionState + + // Cancel is an optional channel whose closure indicates that the client + // request should be regarded as canceled. Not all implementations of + // RoundTripper may support Cancel. + // + // For server requests, this field is not applicable. + // + // Deprecated: Set the Request's context with NewRequestWithContext + // instead. If a Request's Cancel field and context are both + // set, it is undefined whether Cancel is respected. + Cancel <-chan struct{} + + // Response is the redirect response which caused this request + // to be created. This field is only populated during client + // redirects. + Response *Response + + // ctx is either the client or server context. It should only + // be modified via copying the whole Request using WithContext. + // It is unexported to prevent people from using Context wrong + // and mutating the contexts held by callers of the same request. + ctx context.Context + + // TINYGO: Add onEOF func for callback when response is fully read + // TINYGO: so we can close the connection. + onEOF func() +} + +// Context returns the request's context. To change the context, use +// WithContext. +// +// The returned context is always non-nil; it defaults to the +// background context. +// +// For outgoing client requests, the context controls cancellation. +// +// For incoming server requests, the context is canceled when the +// client's connection closes, the request is canceled (with HTTP/2), +// or when the ServeHTTP method returns. +func (r *Request) Context() context.Context { + if r.ctx != nil { + return r.ctx + } + return context.Background() +} + +// WithContext returns a shallow copy of r with its context changed +// to ctx. The provided ctx must be non-nil. +// +// For outgoing client request, the context controls the entire +// lifetime of a request and its response: obtaining a connection, +// sending the request, and reading the response headers and body. +// +// To create a new request with a context, use NewRequestWithContext. +// To change the context of a request, such as an incoming request you +// want to modify before sending back out, use Request.Clone. Between +// those two uses, it's rare to need WithContext. +func (r *Request) WithContext(ctx context.Context) *Request { + if ctx == nil { + panic("nil context") + } + r2 := new(Request) + *r2 = *r + r2.ctx = ctx + return r2 +} + +// Clone returns a deep copy of r with its context changed to ctx. +// The provided ctx must be non-nil. +// +// For an outgoing client request, the context controls the entire +// lifetime of a request and its response: obtaining a connection, +// sending the request, and reading the response headers and body. +func (r *Request) Clone(ctx context.Context) *Request { + if ctx == nil { + panic("nil context") + } + r2 := new(Request) + *r2 = *r + r2.ctx = ctx + r2.URL = cloneURL(r.URL) + if r.Header != nil { + r2.Header = r.Header.Clone() + } + if r.Trailer != nil { + r2.Trailer = r.Trailer.Clone() + } + if s := r.TransferEncoding; s != nil { + s2 := make([]string, len(s)) + copy(s2, s) + r2.TransferEncoding = s2 + } + r2.Form = cloneURLValues(r.Form) + r2.PostForm = cloneURLValues(r.PostForm) + r2.MultipartForm = cloneMultipartForm(r.MultipartForm) + return r2 +} + +// ProtoAtLeast reports whether the HTTP protocol used +// in the request is at least major.minor. +func (r *Request) ProtoAtLeast(major, minor int) bool { + return r.ProtoMajor > major || + r.ProtoMajor == major && r.ProtoMinor >= minor +} + +// UserAgent returns the client's User-Agent, if sent in the request. +func (r *Request) UserAgent() string { + return r.Header.Get("User-Agent") +} + +// Cookies parses and returns the HTTP cookies sent with the request. +func (r *Request) Cookies() []*Cookie { + return readCookies(r.Header, "") +} + +// ErrNoCookie is returned by Request's Cookie method when a cookie is not found. +var ErrNoCookie = errors.New("http: named cookie not present") + +// Cookie returns the named cookie provided in the request or +// ErrNoCookie if not found. +// If multiple cookies match the given name, only one cookie will +// be returned. +func (r *Request) Cookie(name string) (*Cookie, error) { + for _, c := range readCookies(r.Header, name) { + return c, nil + } + return nil, ErrNoCookie +} + +// AddCookie adds a cookie to the request. Per RFC 6265 section 5.4, +// AddCookie does not attach more than one Cookie header field. That +// means all cookies, if any, are written into the same line, +// separated by semicolon. +// AddCookie only sanitizes c's name and value, and does not sanitize +// a Cookie header already present in the request. +func (r *Request) AddCookie(c *Cookie) { + s := fmt.Sprintf("%s=%s", sanitizeCookieName(c.Name), sanitizeCookieValue(c.Value)) + if c := r.Header.Get("Cookie"); c != "" { + r.Header.Set("Cookie", c+"; "+s) + } else { + r.Header.Set("Cookie", s) + } +} + +// Referer returns the referring URL, if sent in the request. +// +// Referer is misspelled as in the request itself, a mistake from the +// earliest days of HTTP. This value can also be fetched from the +// Header map as Header["Referer"]; the benefit of making it available +// as a method is that the compiler can diagnose programs that use the +// alternate (correct English) spelling req.Referrer() but cannot +// diagnose programs that use Header["Referrer"]. +func (r *Request) Referer() string { + return r.Header.Get("Referer") +} + +// multipartByReader is a sentinel value. +// Its presence in Request.MultipartForm indicates that parsing of the request +// body has been handed off to a MultipartReader instead of ParseMultipartForm. +var multipartByReader = &multipart.Form{ + Value: make(map[string][]string), + File: make(map[string][]*multipart.FileHeader), +} + +// MultipartReader returns a MIME multipart reader if this is a +// multipart/form-data or a multipart/mixed POST request, else returns nil and an error. +// Use this function instead of ParseMultipartForm to +// process the request body as a stream. +func (r *Request) MultipartReader() (*multipart.Reader, error) { + if r.MultipartForm == multipartByReader { + return nil, errors.New("http: MultipartReader called twice") + } + if r.MultipartForm != nil { + return nil, errors.New("http: multipart handled by ParseMultipartForm") + } + r.MultipartForm = multipartByReader + return r.multipartReader(true) +} + +func (r *Request) multipartReader(allowMixed bool) (*multipart.Reader, error) { + v := r.Header.Get("Content-Type") + if v == "" { + return nil, ErrNotMultipart + } + if r.Body == nil { + return nil, errors.New("missing form body") + } + d, params, err := mime.ParseMediaType(v) + if err != nil || !(d == "multipart/form-data" || allowMixed && d == "multipart/mixed") { + return nil, ErrNotMultipart + } + boundary, ok := params["boundary"] + if !ok { + return nil, ErrMissingBoundary + } + return multipart.NewReader(r.Body, boundary), nil +} + +// isH2Upgrade reports whether r represents the http2 "client preface" +// magic string. +func (r *Request) isH2Upgrade() bool { + return r.Method == "PRI" && len(r.Header) == 0 && r.URL.Path == "*" && r.Proto == "HTTP/2.0" +} + +// Return value if nonempty, def otherwise. +func valueOrDefault(value, def string) string { + if value != "" { + return value + } + return def +} + +// NOTE: This is not intended to reflect the actual Go version being used. +// It was changed at the time of Go 1.1 release because the former User-Agent +// had ended up blocked by some intrusion detection systems. +// See https://codereview.appspot.com/7532043. +const defaultUserAgent = "Go-http-client/1.1" + +// Write writes an HTTP/1.1 request, which is the header and body, in wire format. +// This method consults the following fields of the request: +// +// Host +// URL +// Method (defaults to "GET") +// Header +// ContentLength +// TransferEncoding +// Body +// +// If Body is present, Content-Length is <= 0 and TransferEncoding +// hasn't been set to "identity", Write adds "Transfer-Encoding: +// chunked" to the header. Body is closed after it is sent. +func (r *Request) Write(w io.Writer) error { + return r.write(w, false, nil, nil) +} + +// WriteProxy is like Write but writes the request in the form +// expected by an HTTP proxy. In particular, WriteProxy writes the +// initial Request-URI line of the request with an absolute URI, per +// section 5.3 of RFC 7230, including the scheme and host. +// In either case, WriteProxy also writes a Host header, using +// either r.Host or r.URL.Host. +func (r *Request) WriteProxy(w io.Writer) error { + return r.write(w, true, nil, nil) +} + +// errMissingHost is returned by Write when there is no Host or URL present in +// the Request. +var errMissingHost = errors.New("http: Request.Write on Request with no Host or URL set") + +// extraHeaders may be nil +// waitForContinue may be nil +// always closes body +func (r *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, waitForContinue func() bool) (err error) { + closed := false + defer func() { + if closed { + return + } + if closeErr := r.closeBody(); closeErr != nil && err == nil { + err = closeErr + } + }() + + // Find the target host. Prefer the Host: header, but if that + // is not given, use the host from the request URL. + // + // Clean the host, in case it arrives with unexpected stuff in it. + host := cleanHost(r.Host) + if host == "" { + if r.URL == nil { + return errMissingHost + } + host = cleanHost(r.URL.Host) + } + + // According to RFC 6874, an HTTP client, proxy, or other + // intermediary must remove any IPv6 zone identifier attached + // to an outgoing URI. + host = removeZone(host) + + ruri := r.URL.RequestURI() + if usingProxy && r.URL.Scheme != "" && r.URL.Opaque == "" { + ruri = r.URL.Scheme + "://" + host + ruri + } else if r.Method == "CONNECT" && r.URL.Path == "" { + // CONNECT requests normally give just the host and port, not a full URL. + ruri = host + if r.URL.Opaque != "" { + ruri = r.URL.Opaque + } + } + if stringContainsCTLByte(ruri) { + return errors.New("net/http: can't write control character in Request.URL") + } + // TODO: validate r.Method too? At least it's less likely to + // come from an attacker (more likely to be a constant in + // code). + + // Wrap the writer in a bufio Writer if it's not already buffered. + // Don't always call NewWriter, as that forces a bytes.Buffer + // and other small bufio Writers to have a minimum 4k buffer + // size. + var bw *bufio.Writer + if _, ok := w.(io.ByteWriter); !ok { + bw = bufio.NewWriter(w) + w = bw + } + + _, err = fmt.Fprintf(w, "%s %s HTTP/1.1\r\n", valueOrDefault(r.Method, "GET"), ruri) + if err != nil { + return err + } + + // Header lines + _, err = fmt.Fprintf(w, "Host: %s\r\n", host) + if err != nil { + return err + } + + // Use the defaultUserAgent unless the Header contains one, which + // may be blank to not send the header. + userAgent := defaultUserAgent + if r.Header.has("User-Agent") { + userAgent = r.Header.Get("User-Agent") + } + if userAgent != "" { + _, err = fmt.Fprintf(w, "User-Agent: %s\r\n", userAgent) + if err != nil { + return err + } + } + + // Process Body,ContentLength,Close,Trailer + tw, err := newTransferWriter(r) + if err != nil { + return err + } + err = tw.writeHeader(w) + if err != nil { + return err + } + + err = r.Header.writeSubset(w, reqWriteExcludeHeader) + if err != nil { + return err + } + + if extraHeaders != nil { + err = extraHeaders.write(w) + if err != nil { + return err + } + } + + _, err = io.WriteString(w, "\r\n") + if err != nil { + return err + } + + // Flush and wait for 100-continue if expected. + if waitForContinue != nil { + if bw, ok := w.(*bufio.Writer); ok { + err = bw.Flush() + if err != nil { + return err + } + } + if !waitForContinue() { + closed = true + r.closeBody() + return nil + } + } + + if bw, ok := w.(*bufio.Writer); ok && tw.FlushHeaders { + if err := bw.Flush(); err != nil { + return err + } + } + + // Write body and trailer + closed = true + err = tw.writeBody(w) + if err != nil { + if tw.bodyReadError == err { + err = requestBodyReadError{err} + } + return err + } + + if bw != nil { + return bw.Flush() + } + return nil +} + +// requestBodyReadError wraps an error from (*Request).write to indicate +// that the error came from a Read call on the Request.Body. +// This error type should not escape the net/http package to users. +type requestBodyReadError struct{ error } + +// cleanHost cleans up the host sent in request's Host header. +// +// It both strips anything after '/' or ' ', and puts the value +// into Punycode form, if necessary. +// +// Ideally we'd clean the Host header according to the spec: +// +// https://tools.ietf.org/html/rfc7230#section-5.4 (Host = uri-host [ ":" port ]") +// https://tools.ietf.org/html/rfc7230#section-2.7 (uri-host -> rfc3986's host) +// https://tools.ietf.org/html/rfc3986#section-3.2.2 (definition of host) +// +// But practically, what we are trying to avoid is the situation in +// issue 11206, where a malformed Host header used in the proxy context +// would create a bad request. So it is enough to just truncate at the +// first offending character. + +// TINYGO: Removed IDNA checks...it doubled the binary size + +func cleanHost(in string) string { + if i := strings.IndexAny(in, " /"); i != -1 { + in = in[:i] + } + host, port, err := net.SplitHostPort(in) + if err != nil { // input was just a host + return in + } + return net.JoinHostPort(host, port) +} + +// removeZone removes IPv6 zone identifier from host. +// E.g., "[fe80::1%en0]:8080" to "[fe80::1]:8080" +func removeZone(host string) string { + if !strings.HasPrefix(host, "[") { + return host + } + i := strings.LastIndex(host, "]") + if i < 0 { + return host + } + j := strings.LastIndex(host[:i], "%") + if j < 0 { + return host + } + return host[:j] + host[i:] +} + +// ParseHTTPVersion parses an HTTP version string according to RFC 7230, section 2.6. +// "HTTP/1.0" returns (1, 0, true). Note that strings without +// a minor version, such as "HTTP/2", are not valid. +func ParseHTTPVersion(vers string) (major, minor int, ok bool) { + switch vers { + case "HTTP/1.1": + return 1, 1, true + case "HTTP/1.0": + return 1, 0, true + } + if !strings.HasPrefix(vers, "HTTP/") { + return 0, 0, false + } + if len(vers) != len("HTTP/X.Y") { + return 0, 0, false + } + if vers[6] != '.' { + return 0, 0, false + } + maj, err := strconv.ParseUint(vers[5:6], 10, 0) + if err != nil { + return 0, 0, false + } + min, err := strconv.ParseUint(vers[7:8], 10, 0) + if err != nil { + return 0, 0, false + } + return int(maj), int(min), true +} + +func validMethod(method string) bool { + /* + Method = "OPTIONS" ; Section 9.2 + | "GET" ; Section 9.3 + | "HEAD" ; Section 9.4 + | "POST" ; Section 9.5 + | "PUT" ; Section 9.6 + | "DELETE" ; Section 9.7 + | "TRACE" ; Section 9.8 + | "CONNECT" ; Section 9.9 + | extension-method + extension-method = token + token = 1* + */ + return len(method) > 0 && strings.IndexFunc(method, isNotToken) == -1 +} + +// NewRequest wraps NewRequestWithContext using context.Background. +func NewRequest(method, url string, body io.Reader) (*Request, error) { + return NewRequestWithContext(context.Background(), method, url, body) +} + +// NewRequestWithContext returns a new Request given a method, URL, and +// optional body. +// +// If the provided body is also an io.Closer, the returned +// Request.Body is set to body and will be closed by the Client +// methods Do, Post, and PostForm, and Transport.RoundTrip. +// +// NewRequestWithContext returns a Request suitable for use with +// Client.Do or Transport.RoundTrip. To create a request for use with +// testing a Server Handler, either use the NewRequest function in the +// net/http/httptest package, use ReadRequest, or manually update the +// Request fields. For an outgoing client request, the context +// controls the entire lifetime of a request and its response: +// obtaining a connection, sending the request, and reading the +// response headers and body. See the Request type's documentation for +// the difference between inbound and outbound request fields. +// +// If body is of type *bytes.Buffer, *bytes.Reader, or +// *strings.Reader, the returned request's ContentLength is set to its +// exact value (instead of -1), GetBody is populated (so 307 and 308 +// redirects can replay the body), and Body is set to NoBody if the +// ContentLength is 0. +func NewRequestWithContext(ctx context.Context, method, url string, body io.Reader) (*Request, error) { + if method == "" { + // We document that "" means "GET" for Request.Method, and people have + // relied on that from NewRequest, so keep that working. + // We still enforce validMethod for non-empty methods. + method = "GET" + } + if !validMethod(method) { + return nil, fmt.Errorf("net/http: invalid method %q", method) + } + if ctx == nil { + return nil, errors.New("net/http: nil Context") + } + u, err := urlpkg.Parse(url) + if err != nil { + return nil, err + } + rc, ok := body.(io.ReadCloser) + if !ok && body != nil { + rc = io.NopCloser(body) + } + // The host's colon:port should be normalized. See Issue 14836. + u.Host = removeEmptyPort(u.Host) + req := &Request{ + ctx: ctx, + Method: method, + URL: u, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: make(Header), + Body: rc, + Host: u.Host, + } + if body != nil { + switch v := body.(type) { + case *bytes.Buffer: + req.ContentLength = int64(v.Len()) + buf := v.Bytes() + req.GetBody = func() (io.ReadCloser, error) { + r := bytes.NewReader(buf) + return io.NopCloser(r), nil + } + case *bytes.Reader: + req.ContentLength = int64(v.Len()) + snapshot := *v + req.GetBody = func() (io.ReadCloser, error) { + r := snapshot + return io.NopCloser(&r), nil + } + case *strings.Reader: + req.ContentLength = int64(v.Len()) + snapshot := *v + req.GetBody = func() (io.ReadCloser, error) { + r := snapshot + return io.NopCloser(&r), nil + } + default: + // This is where we'd set it to -1 (at least + // if body != NoBody) to mean unknown, but + // that broke people during the Go 1.8 testing + // period. People depend on it being 0 I + // guess. Maybe retry later. See Issue 18117. + } + // For client requests, Request.ContentLength of 0 + // means either actually 0, or unknown. The only way + // to explicitly say that the ContentLength is zero is + // to set the Body to nil. But turns out too much code + // depends on NewRequest returning a non-nil Body, + // so we use a well-known ReadCloser variable instead + // and have the http package also treat that sentinel + // variable to mean explicitly zero. + if req.GetBody != nil && req.ContentLength == 0 { + req.Body = NoBody + req.GetBody = func() (io.ReadCloser, error) { return NoBody, nil } + } + } + + return req, nil +} + +// BasicAuth returns the username and password provided in the request's +// Authorization header, if the request uses HTTP Basic Authentication. +// See RFC 2617, Section 2. +func (r *Request) BasicAuth() (username, password string, ok bool) { + auth := r.Header.Get("Authorization") + if auth == "" { + return "", "", false + } + return parseBasicAuth(auth) +} + +// parseBasicAuth parses an HTTP Basic Authentication string. +// "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==" returns ("Aladdin", "open sesame", true). +func parseBasicAuth(auth string) (username, password string, ok bool) { + const prefix = "Basic " + // Case insensitive prefix match. See Issue 22736. + if len(auth) < len(prefix) || !ascii.EqualFold(auth[:len(prefix)], prefix) { + return "", "", false + } + c, err := base64.StdEncoding.DecodeString(auth[len(prefix):]) + if err != nil { + return "", "", false + } + cs := string(c) + username, password, ok = strings.Cut(cs, ":") + if !ok { + return "", "", false + } + return username, password, true +} + +// SetBasicAuth sets the request's Authorization header to use HTTP +// Basic Authentication with the provided username and password. +// +// With HTTP Basic Authentication the provided username and password +// are not encrypted. It should generally only be used in an HTTPS +// request. +// +// The username may not contain a colon. Some protocols may impose +// additional requirements on pre-escaping the username and +// password. For instance, when used with OAuth2, both arguments must +// be URL encoded first with url.QueryEscape. +func (r *Request) SetBasicAuth(username, password string) { + r.Header.Set("Authorization", "Basic "+basicAuth(username, password)) +} + +// parseRequestLine parses "GET /foo HTTP/1.1" into its three parts. +func parseRequestLine(line string) (method, requestURI, proto string, ok bool) { + method, rest, ok1 := strings.Cut(line, " ") + requestURI, proto, ok2 := strings.Cut(rest, " ") + if !ok1 || !ok2 { + return "", "", "", false + } + return method, requestURI, proto, true +} + +var textprotoReaderPool sync.Pool + +func newTextprotoReader(br *bufio.Reader) *textproto.Reader { + if v := textprotoReaderPool.Get(); v != nil { + tr := v.(*textproto.Reader) + tr.R = br + return tr + } + return textproto.NewReader(br) +} + +func putTextprotoReader(r *textproto.Reader) { + r.R = nil + textprotoReaderPool.Put(r) +} + +// ReadRequest reads and parses an incoming request from b. +// +// ReadRequest is a low-level function and should only be used for +// specialized applications; most code should use the Server to read +// requests and handle them via the Handler interface. ReadRequest +// only supports HTTP/1.x requests. For HTTP/2, use golang.org/x/net/http2. +func ReadRequest(b *bufio.Reader) (*Request, error) { + req, err := readRequest(b) + if err != nil { + return nil, err + } + + delete(req.Header, "Host") + return req, err +} + +func readRequest(b *bufio.Reader) (req *Request, err error) { + tp := newTextprotoReader(b) + req = new(Request) + + // First line: GET /index.html HTTP/1.0 + var s string + if s, err = tp.ReadLine(); err != nil { + return nil, err + } + defer func() { + putTextprotoReader(tp) + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + var ok bool + req.Method, req.RequestURI, req.Proto, ok = parseRequestLine(s) + if !ok { + return nil, badStringError("malformed HTTP request", s) + } + if !validMethod(req.Method) { + return nil, badStringError("invalid method", req.Method) + } + rawurl := req.RequestURI + if req.ProtoMajor, req.ProtoMinor, ok = ParseHTTPVersion(req.Proto); !ok { + return nil, badStringError("malformed HTTP version", req.Proto) + } + + // CONNECT requests are used two different ways, and neither uses a full URL: + // The standard use is to tunnel HTTPS through an HTTP proxy. + // It looks like "CONNECT www.google.com:443 HTTP/1.1", and the parameter is + // just the authority section of a URL. This information should go in req.URL.Host. + // + // The net/rpc package also uses CONNECT, but there the parameter is a path + // that starts with a slash. It can be parsed with the regular URL parser, + // and the path will end up in req.URL.Path, where it needs to be in order for + // RPC to work. + justAuthority := req.Method == "CONNECT" && !strings.HasPrefix(rawurl, "/") + if justAuthority { + rawurl = "http://" + rawurl + } + + if req.URL, err = url.ParseRequestURI(rawurl); err != nil { + return nil, err + } + + if justAuthority { + // Strip the bogus "http://" back off. + req.URL.Scheme = "" + } + + // Subsequent lines: Key: value. + mimeHeader, err := tp.ReadMIMEHeader() + if err != nil { + return nil, err + } + req.Header = Header(mimeHeader) + if len(req.Header["Host"]) > 1 { + return nil, fmt.Errorf("too many Host headers") + } + + // RFC 7230, section 5.3: Must treat + // GET /index.html HTTP/1.1 + // Host: www.google.com + // and + // GET http://www.google.com/index.html HTTP/1.1 + // Host: doesntmatter + // the same. In the second case, any Host line is ignored. + req.Host = req.URL.Host + if req.Host == "" { + req.Host = req.Header.get("Host") + } + + fixPragmaCacheControl(req.Header) + + req.Close = shouldClose(req.ProtoMajor, req.ProtoMinor, req.Header, false) + + err = readTransfer(req, b, nil) + if err != nil { + return nil, err + } + + if req.isH2Upgrade() { + // Because it's neither chunked, nor declared: + req.ContentLength = -1 + + // We want to give handlers a chance to hijack the + // connection, but we need to prevent the Server from + // dealing with the connection further if it's not + // hijacked. Set Close to ensure that: + req.Close = true + } + return req, nil +} + +// MaxBytesReader is similar to io.LimitReader but is intended for +// limiting the size of incoming request bodies. In contrast to +// io.LimitReader, MaxBytesReader's result is a ReadCloser, returns a +// non-nil error of type *MaxBytesError for a Read beyond the limit, +// and closes the underlying reader when its Close method is called. +// +// MaxBytesReader prevents clients from accidentally or maliciously +// sending a large request and wasting server resources. If possible, +// it tells the ResponseWriter to close the connection after the limit +// has been reached. +func MaxBytesReader(w ResponseWriter, r io.ReadCloser, n int64) io.ReadCloser { + if n < 0 { // Treat negative limits as equivalent to 0. + n = 0 + } + return &maxBytesReader{w: w, r: r, i: n, n: n} +} + +// MaxBytesError is returned by MaxBytesReader when its read limit is exceeded. +type MaxBytesError struct { + Limit int64 +} + +func (e *MaxBytesError) Error() string { + // Due to Hyrum's law, this text cannot be changed. + return "http: request body too large" +} + +type maxBytesReader struct { + w ResponseWriter + r io.ReadCloser // underlying reader + i int64 // max bytes initially, for MaxBytesError + n int64 // max bytes remaining + err error // sticky error +} + +func (l *maxBytesReader) Read(p []byte) (n int, err error) { + if l.err != nil { + return 0, l.err + } + if len(p) == 0 { + return 0, nil + } + // If they asked for a 32KB byte read but only 5 bytes are + // remaining, no need to read 32KB. 6 bytes will answer the + // question of the whether we hit the limit or go past it. + if int64(len(p)) > l.n+1 { + p = p[:l.n+1] + } + n, err = l.r.Read(p) + + if int64(n) <= l.n { + l.n -= int64(n) + l.err = err + return n, err + } + + n = int(l.n) + l.n = 0 + + // The server code and client code both use + // maxBytesReader. This "requestTooLarge" check is + // only used by the server code. To prevent binaries + // which only using the HTTP Client code (such as + // cmd/go) from also linking in the HTTP server, don't + // use a static type assertion to the server + // "*response" type. Check this interface instead: + type requestTooLarger interface { + requestTooLarge() + } + if res, ok := l.w.(requestTooLarger); ok { + res.requestTooLarge() + } + l.err = &MaxBytesError{l.i} + return n, l.err +} + +func (l *maxBytesReader) Close() error { + return l.r.Close() +} + +func copyValues(dst, src url.Values) { + for k, vs := range src { + dst[k] = append(dst[k], vs...) + } +} + +func parsePostForm(r *Request) (vs url.Values, err error) { + if r.Body == nil { + err = errors.New("missing form body") + return + } + ct := r.Header.Get("Content-Type") + // RFC 7231, section 3.1.1.5 - empty type + // MAY be treated as application/octet-stream + if ct == "" { + ct = "application/octet-stream" + } + ct, _, err = mime.ParseMediaType(ct) + switch { + case ct == "application/x-www-form-urlencoded": + var reader io.Reader = r.Body + maxFormSize := int64(1<<63 - 1) + if _, ok := r.Body.(*maxBytesReader); !ok { + maxFormSize = int64(10 << 20) // 10 MB is a lot of text. + reader = io.LimitReader(r.Body, maxFormSize+1) + } + b, e := io.ReadAll(reader) + if e != nil { + if err == nil { + err = e + } + break + } + if int64(len(b)) > maxFormSize { + err = errors.New("http: POST too large") + return + } + vs, e = url.ParseQuery(string(b)) + if err == nil { + err = e + } + case ct == "multipart/form-data": + // handled by ParseMultipartForm (which is calling us, or should be) + // TODO(bradfitz): there are too many possible + // orders to call too many functions here. + // Clean this up and write more tests. + // request_test.go contains the start of this, + // in TestParseMultipartFormOrder and others. + } + return +} + +// ParseForm populates r.Form and r.PostForm. +// +// For all requests, ParseForm parses the raw query from the URL and updates +// r.Form. +// +// For POST, PUT, and PATCH requests, it also reads the request body, parses it +// as a form and puts the results into both r.PostForm and r.Form. Request body +// parameters take precedence over URL query string values in r.Form. +// +// If the request Body's size has not already been limited by MaxBytesReader, +// the size is capped at 10MB. +// +// For other HTTP methods, or when the Content-Type is not +// application/x-www-form-urlencoded, the request Body is not read, and +// r.PostForm is initialized to a non-nil, empty value. +// +// ParseMultipartForm calls ParseForm automatically. +// ParseForm is idempotent. +func (r *Request) ParseForm() error { + var err error + if r.PostForm == nil { + if r.Method == "POST" || r.Method == "PUT" || r.Method == "PATCH" { + r.PostForm, err = parsePostForm(r) + } + if r.PostForm == nil { + r.PostForm = make(url.Values) + } + } + if r.Form == nil { + if len(r.PostForm) > 0 { + r.Form = make(url.Values) + copyValues(r.Form, r.PostForm) + } + var newValues url.Values + if r.URL != nil { + var e error + newValues, e = url.ParseQuery(r.URL.RawQuery) + if err == nil { + err = e + } + } + if newValues == nil { + newValues = make(url.Values) + } + if r.Form == nil { + r.Form = newValues + } else { + copyValues(r.Form, newValues) + } + } + return err +} + +// ParseMultipartForm parses a request body as multipart/form-data. +// The whole request body is parsed and up to a total of maxMemory bytes of +// its file parts are stored in memory, with the remainder stored on +// disk in temporary files. +// ParseMultipartForm calls ParseForm if necessary. +// If ParseForm returns an error, ParseMultipartForm returns it but also +// continues parsing the request body. +// After one call to ParseMultipartForm, subsequent calls have no effect. +func (r *Request) ParseMultipartForm(maxMemory int64) error { + if r.MultipartForm == multipartByReader { + return errors.New("http: multipart handled by MultipartReader") + } + var parseFormErr error + if r.Form == nil { + // Let errors in ParseForm fall through, and just + // return it at the end. + parseFormErr = r.ParseForm() + } + if r.MultipartForm != nil { + return nil + } + + mr, err := r.multipartReader(false) + if err != nil { + return err + } + + f, err := mr.ReadForm(maxMemory) + if err != nil { + return err + } + + if r.PostForm == nil { + r.PostForm = make(url.Values) + } + for k, v := range f.Value { + r.Form[k] = append(r.Form[k], v...) + // r.PostForm should also be populated. See Issue 9305. + r.PostForm[k] = append(r.PostForm[k], v...) + } + + r.MultipartForm = f + + return parseFormErr +} + +// FormValue returns the first value for the named component of the query. +// POST and PUT body parameters take precedence over URL query string values. +// FormValue calls ParseMultipartForm and ParseForm if necessary and ignores +// any errors returned by these functions. +// If key is not present, FormValue returns the empty string. +// To access multiple values of the same key, call ParseForm and +// then inspect Request.Form directly. +func (r *Request) FormValue(key string) string { + if r.Form == nil { + r.ParseMultipartForm(defaultMaxMemory) + } + if vs := r.Form[key]; len(vs) > 0 { + return vs[0] + } + return "" +} + +// PostFormValue returns the first value for the named component of the POST, +// PATCH, or PUT request body. URL query parameters are ignored. +// PostFormValue calls ParseMultipartForm and ParseForm if necessary and ignores +// any errors returned by these functions. +// If key is not present, PostFormValue returns the empty string. +func (r *Request) PostFormValue(key string) string { + if r.PostForm == nil { + r.ParseMultipartForm(defaultMaxMemory) + } + if vs := r.PostForm[key]; len(vs) > 0 { + return vs[0] + } + return "" +} + +// FormFile returns the first file for the provided form key. +// FormFile calls ParseMultipartForm and ParseForm if necessary. +func (r *Request) FormFile(key string) (multipart.File, *multipart.FileHeader, error) { + if r.MultipartForm == multipartByReader { + return nil, nil, errors.New("http: multipart handled by MultipartReader") + } + if r.MultipartForm == nil { + err := r.ParseMultipartForm(defaultMaxMemory) + if err != nil { + return nil, nil, err + } + } + if r.MultipartForm != nil && r.MultipartForm.File != nil { + if fhs := r.MultipartForm.File[key]; len(fhs) > 0 { + f, err := fhs[0].Open() + return f, fhs[0], err + } + } + return nil, nil, ErrMissingFile +} + +func (r *Request) expectsContinue() bool { + return hasToken(r.Header.get("Expect"), "100-continue") +} + +func (r *Request) wantsHttp10KeepAlive() bool { + if r.ProtoMajor != 1 || r.ProtoMinor != 0 { + return false + } + return hasToken(r.Header.get("Connection"), "keep-alive") +} + +func (r *Request) wantsClose() bool { + if r.Close { + return true + } + return hasToken(r.Header.get("Connection"), "close") +} + +func (r *Request) closeBody() error { + if r.Body == nil { + return nil + } + return r.Body.Close() +} + +func (r *Request) isReplayable() bool { + if r.Body == nil || r.Body == NoBody || r.GetBody != nil { + switch valueOrDefault(r.Method, "GET") { + case "GET", "HEAD", "OPTIONS", "TRACE": + return true + } + // The Idempotency-Key, while non-standard, is widely used to + // mean a POST or other request is idempotent. See + // https://golang.org/issue/19943#issuecomment-421092421 + if r.Header.has("Idempotency-Key") || r.Header.has("X-Idempotency-Key") { + return true + } + } + return false +} + +// outgoingLength reports the Content-Length of this outgoing (Client) request. +// It maps 0 into -1 (unknown) when the Body is non-nil. +func (r *Request) outgoingLength() int64 { + if r.Body == nil || r.Body == NoBody { + return 0 + } + if r.ContentLength != 0 { + return r.ContentLength + } + return -1 +} + +// requestMethodUsuallyLacksBody reports whether the given request +// method is one that typically does not involve a request body. +// This is used by the Transport (via +// transferWriter.shouldSendChunkedRequestBody) to determine whether +// we try to test-read a byte from a non-nil Request.Body when +// Request.outgoingLength() returns -1. See the comments in +// shouldSendChunkedRequestBody. +func requestMethodUsuallyLacksBody(method string) bool { + switch method { + case "GET", "HEAD", "DELETE", "OPTIONS", "PROPFIND", "SEARCH": + return true + } + return false +} + +// requiresHTTP1 reports whether this request requires being sent on +// an HTTP/1 connection. +func (r *Request) requiresHTTP1() bool { + return hasToken(r.Header.Get("Connection"), "upgrade") && + ascii.EqualFold(r.Header.Get("Upgrade"), "websocket") +} diff --git a/src/net/http/response.go b/src/net/http/response.go new file mode 100644 index 0000000000..980329f9c5 --- /dev/null +++ b/src/net/http/response.go @@ -0,0 +1,373 @@ +// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. + +// TINYGO: Removed TLS connection state +// TINYGO: Added onEOF hook to get callback when response has been read + +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// HTTP Response reading and parsing. + +package http + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "net/textproto" + "net/url" + "strconv" + "strings" + + "golang.org/x/net/http/httpguts" +) + +var respExcludeHeader = map[string]bool{ + "Content-Length": true, + "Transfer-Encoding": true, + "Trailer": true, +} + +// Response represents the response from an HTTP request. +// +// The Client and Transport return Responses from servers once +// the response headers have been received. The response body +// is streamed on demand as the Body field is read. +type Response struct { + Status string // e.g. "200 OK" + StatusCode int // e.g. 200 + Proto string // e.g. "HTTP/1.0" + ProtoMajor int // e.g. 1 + ProtoMinor int // e.g. 0 + + // Header maps header keys to values. If the response had multiple + // headers with the same key, they may be concatenated, with comma + // delimiters. (RFC 7230, section 3.2.2 requires that multiple headers + // be semantically equivalent to a comma-delimited sequence.) When + // Header values are duplicated by other fields in this struct (e.g., + // ContentLength, TransferEncoding, Trailer), the field values are + // authoritative. + // + // Keys in the map are canonicalized (see CanonicalHeaderKey). + Header Header + + // Body represents the response body. + // + // The response body is streamed on demand as the Body field + // is read. If the network connection fails or the server + // terminates the response, Body.Read calls return an error. + // + // The http Client and Transport guarantee that Body is always + // non-nil, even on responses without a body or responses with + // a zero-length body. It is the caller's responsibility to + // close Body. The default HTTP client's Transport may not + // reuse HTTP/1.x "keep-alive" TCP connections if the Body is + // not read to completion and closed. + // + // The Body is automatically dechunked if the server replied + // with a "chunked" Transfer-Encoding. + // + // As of Go 1.12, the Body will also implement io.Writer + // on a successful "101 Switching Protocols" response, + // as used by WebSockets and HTTP/2's "h2c" mode. + Body io.ReadCloser + + // ContentLength records the length of the associated content. The + // value -1 indicates that the length is unknown. Unless Request.Method + // is "HEAD", values >= 0 indicate that the given number of bytes may + // be read from Body. + ContentLength int64 + + // Contains transfer encodings from outer-most to inner-most. Value is + // nil, means that "identity" encoding is used. + TransferEncoding []string + + // Close records whether the header directed that the connection be + // closed after reading Body. The value is advice for clients: neither + // ReadResponse nor Response.Write ever closes a connection. + Close bool + + // Uncompressed reports whether the response was sent compressed but + // was decompressed by the http package. When true, reading from + // Body yields the uncompressed content instead of the compressed + // content actually set from the server, ContentLength is set to -1, + // and the "Content-Length" and "Content-Encoding" fields are deleted + // from the responseHeader. To get the original response from + // the server, set Transport.DisableCompression to true. + Uncompressed bool + + // Trailer maps trailer keys to values in the same + // format as Header. + // + // The Trailer initially contains only nil values, one for + // each key specified in the server's "Trailer" header + // value. Those values are not added to Header. + // + // Trailer must not be accessed concurrently with Read calls + // on the Body. + // + // After Body.Read has returned io.EOF, Trailer will contain + // any trailer values sent by the server. + Trailer Header + + // Request is the request that was sent to obtain this Response. + // Request's Body is nil (having already been consumed). + // This is only populated for Client requests. + Request *Request +} + +// Cookies parses and returns the cookies set in the Set-Cookie headers. +func (r *Response) Cookies() []*Cookie { + return readSetCookies(r.Header) +} + +// ErrNoLocation is returned by Response's Location method +// when no Location header is present. +var ErrNoLocation = errors.New("http: no Location header in response") + +// Location returns the URL of the response's "Location" header, +// if present. Relative redirects are resolved relative to +// the Response's Request. ErrNoLocation is returned if no +// Location header is present. +func (r *Response) Location() (*url.URL, error) { + lv := r.Header.Get("Location") + if lv == "" { + return nil, ErrNoLocation + } + if r.Request != nil && r.Request.URL != nil { + return r.Request.URL.Parse(lv) + } + return url.Parse(lv) +} + +// ReadResponse reads and returns an HTTP response from r. +// The req parameter optionally specifies the Request that corresponds +// to this Response. If nil, a GET request is assumed. +// Clients must call resp.Body.Close when finished reading resp.Body. +// After that call, clients can inspect resp.Trailer to find key/value +// pairs included in the response trailer. + +// TINYGO: Added onEOF func to be called when response body is closed +// TINYGO: so we can clean up the connection (r) + +func ReadResponse(r *bufio.Reader, req *Request) (*Response, error) { + tp := textproto.NewReader(r) + resp := &Response{ + Request: req, + } + + // Parse the first line of the response. + line, err := tp.ReadLine() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return nil, err + } + proto, status, ok := strings.Cut(line, " ") + if !ok { + return nil, badStringError("malformed HTTP response", line) + } + resp.Proto = proto + resp.Status = strings.TrimLeft(status, " ") + + statusCode, _, _ := strings.Cut(resp.Status, " ") + if len(statusCode) != 3 { + return nil, badStringError("malformed HTTP status code", statusCode) + } + resp.StatusCode, err = strconv.Atoi(statusCode) + if err != nil || resp.StatusCode < 0 { + return nil, badStringError("malformed HTTP status code", statusCode) + } + if resp.ProtoMajor, resp.ProtoMinor, ok = ParseHTTPVersion(resp.Proto); !ok { + return nil, badStringError("malformed HTTP version", resp.Proto) + } + + // Parse the response headers. + mimeHeader, err := tp.ReadMIMEHeader() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return nil, err + } + resp.Header = Header(mimeHeader) + + fixPragmaCacheControl(resp.Header) + + err = readTransfer(resp, r, req.onEOF) + if err != nil { + return nil, err + } + + return resp, nil +} + +// RFC 7234, section 5.4: Should treat +// +// Pragma: no-cache +// +// like +// +// Cache-Control: no-cache +func fixPragmaCacheControl(header Header) { + if hp, ok := header["Pragma"]; ok && len(hp) > 0 && hp[0] == "no-cache" { + if _, presentcc := header["Cache-Control"]; !presentcc { + header["Cache-Control"] = []string{"no-cache"} + } + } +} + +// ProtoAtLeast reports whether the HTTP protocol used +// in the response is at least major.minor. +func (r *Response) ProtoAtLeast(major, minor int) bool { + return r.ProtoMajor > major || + r.ProtoMajor == major && r.ProtoMinor >= minor +} + +// Write writes r to w in the HTTP/1.x server response format, +// including the status line, headers, body, and optional trailer. +// +// This method consults the following fields of the response r: +// +// StatusCode +// ProtoMajor +// ProtoMinor +// Request.Method +// TransferEncoding +// Trailer +// Body +// ContentLength +// Header, values for non-canonical keys will have unpredictable behavior +// +// The Response Body is closed after it is sent. +func (r *Response) Write(w io.Writer) error { + // Status line + text := r.Status + if text == "" { + text = StatusText(r.StatusCode) + if text == "" { + text = "status code " + strconv.Itoa(r.StatusCode) + } + } else { + // Just to reduce stutter, if user set r.Status to "200 OK" and StatusCode to 200. + // Not important. + text = strings.TrimPrefix(text, strconv.Itoa(r.StatusCode)+" ") + } + + if _, err := fmt.Fprintf(w, "HTTP/%d.%d %03d %s\r\n", r.ProtoMajor, r.ProtoMinor, r.StatusCode, text); err != nil { + return err + } + + // Clone it, so we can modify r1 as needed. + r1 := new(Response) + *r1 = *r + if r1.ContentLength == 0 && r1.Body != nil { + // Is it actually 0 length? Or just unknown? + var buf [1]byte + n, err := r1.Body.Read(buf[:]) + if err != nil && err != io.EOF { + return err + } + if n == 0 { + // Reset it to a known zero reader, in case underlying one + // is unhappy being read repeatedly. + r1.Body = NoBody + } else { + r1.ContentLength = -1 + r1.Body = struct { + io.Reader + io.Closer + }{ + io.MultiReader(bytes.NewReader(buf[:1]), r.Body), + r.Body, + } + } + } + // If we're sending a non-chunked HTTP/1.1 response without a + // content-length, the only way to do that is the old HTTP/1.0 + // way, by noting the EOF with a connection close, so we need + // to set Close. + if r1.ContentLength == -1 && !r1.Close && r1.ProtoAtLeast(1, 1) && !chunked(r1.TransferEncoding) && !r1.Uncompressed { + r1.Close = true + } + + // Process Body,ContentLength,Close,Trailer + tw, err := newTransferWriter(r1) + if err != nil { + return err + } + err = tw.writeHeader(w) + if err != nil { + return err + } + + // Rest of header + err = r.Header.WriteSubset(w, respExcludeHeader) + if err != nil { + return err + } + + // contentLengthAlreadySent may have been already sent for + // POST/PUT requests, even if zero length. See Issue 8180. + contentLengthAlreadySent := tw.shouldSendContentLength() + if r1.ContentLength == 0 && !chunked(r1.TransferEncoding) && !contentLengthAlreadySent && bodyAllowedForStatus(r.StatusCode) { + if _, err := io.WriteString(w, "Content-Length: 0\r\n"); err != nil { + return err + } + } + + // End-of-header + if _, err := io.WriteString(w, "\r\n"); err != nil { + return err + } + + // Write body and trailer + err = tw.writeBody(w) + if err != nil { + return err + } + + // Success + return nil +} + +func (r *Response) closeBody() { + if r.Body != nil { + r.Body.Close() + } +} + +// bodyIsWritable reports whether the Body supports writing. The +// Transport returns Writable bodies for 101 Switching Protocols +// responses. +// The Transport uses this method to determine whether a persistent +// connection is done being managed from its perspective. Once we +// return a writable response body to a user, the net/http package is +// done managing that connection. +func (r *Response) bodyIsWritable() bool { + _, ok := r.Body.(io.Writer) + return ok +} + +// isProtocolSwitch reports whether the response code and header +// indicate a successful protocol upgrade response. +func (r *Response) isProtocolSwitch() bool { + return isProtocolSwitchResponse(r.StatusCode, r.Header) +} + +// isProtocolSwitchResponse reports whether the response code and +// response header indicate a successful protocol upgrade response. +func isProtocolSwitchResponse(code int, h Header) bool { + return code == StatusSwitchingProtocols && isProtocolSwitchHeader(h) +} + +// isProtocolSwitchHeader reports whether the request or response header +// is for a protocol switch. +func isProtocolSwitchHeader(h Header) bool { + return h.Get("Upgrade") != "" && + httpguts.HeaderValuesContainsToken(h["Connection"], "Upgrade") +} diff --git a/src/net/http/server.go b/src/net/http/server.go new file mode 100644 index 0000000000..1a4264d17e --- /dev/null +++ b/src/net/http/server.go @@ -0,0 +1,3261 @@ +// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. + +// TINYGO: Removed ALPN protocol support +// TINYGO: Removed some HTTP/2 support +// TINYGO: Removed TimeoutHandler +// TINYGO: Removed ServeTLS and ListenAndServeTLS + +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// HTTP server. See RFC 7230 through 7235. + +package http + +import ( + "bufio" + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "log" + "math/rand" + "net" + "net/textproto" + "net/url" + urlpkg "net/url" + "path" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "golang.org/x/net/http/httpguts" +) + +// Errors used by the HTTP server. +var ( + // ErrBodyNotAllowed is returned by ResponseWriter.Write calls + // when the HTTP method or response code does not permit a + // body. + ErrBodyNotAllowed = errors.New("http: request method or response status code does not allow body") + + // ErrHijacked is returned by ResponseWriter.Write calls when + // the underlying connection has been hijacked using the + // Hijacker interface. A zero-byte write on a hijacked + // connection will return ErrHijacked without any other side + // effects. + ErrHijacked = errors.New("http: connection has been hijacked") + + // ErrContentLength is returned by ResponseWriter.Write calls + // when a Handler set a Content-Length response header with a + // declared size and then attempted to write more bytes than + // declared. + ErrContentLength = errors.New("http: wrote more than the declared Content-Length") + + // Deprecated: ErrWriteAfterFlush is no longer returned by + // anything in the net/http package. Callers should not + // compare errors against this variable. + ErrWriteAfterFlush = errors.New("unused") +) + +// A Handler responds to an HTTP request. +// +// ServeHTTP should write reply headers and data to the ResponseWriter +// and then return. Returning signals that the request is finished; it +// is not valid to use the ResponseWriter or read from the +// Request.Body after or concurrently with the completion of the +// ServeHTTP call. +// +// Depending on the HTTP client software, HTTP protocol version, and +// any intermediaries between the client and the Go server, it may not +// be possible to read from the Request.Body after writing to the +// ResponseWriter. Cautious handlers should read the Request.Body +// first, and then reply. +// +// Except for reading the body, handlers should not modify the +// provided Request. +// +// If ServeHTTP panics, the server (the caller of ServeHTTP) assumes +// that the effect of the panic was isolated to the active request. +// It recovers the panic, logs a stack trace to the server error log, +// and either closes the network connection or sends an HTTP/2 +// RST_STREAM, depending on the HTTP protocol. To abort a handler so +// the client sees an interrupted response but the server doesn't log +// an error, panic with the value ErrAbortHandler. +type Handler interface { + ServeHTTP(ResponseWriter, *Request) +} + +// A ResponseWriter interface is used by an HTTP handler to +// construct an HTTP response. +// +// A ResponseWriter may not be used after the Handler.ServeHTTP method +// has returned. +type ResponseWriter interface { + // Header returns the header map that will be sent by + // WriteHeader. The Header map also is the mechanism with which + // Handlers can set HTTP trailers. + // + // Changing the header map after a call to WriteHeader (or + // Write) has no effect unless the HTTP status code was of the + // 1xx class or the modified headers are trailers. + // + // There are two ways to set Trailers. The preferred way is to + // predeclare in the headers which trailers you will later + // send by setting the "Trailer" header to the names of the + // trailer keys which will come later. In this case, those + // keys of the Header map are treated as if they were + // trailers. See the example. The second way, for trailer + // keys not known to the Handler until after the first Write, + // is to prefix the Header map keys with the TrailerPrefix + // constant value. See TrailerPrefix. + // + // To suppress automatic response headers (such as "Date"), set + // their value to nil. + Header() Header + + // Write writes the data to the connection as part of an HTTP reply. + // + // If WriteHeader has not yet been called, Write calls + // WriteHeader(http.StatusOK) before writing the data. If the Header + // does not contain a Content-Type line, Write adds a Content-Type set + // to the result of passing the initial 512 bytes of written data to + // DetectContentType. Additionally, if the total size of all written + // data is under a few KB and there are no Flush calls, the + // Content-Length header is added automatically. + // + // Depending on the HTTP protocol version and the client, calling + // Write or WriteHeader may prevent future reads on the + // Request.Body. For HTTP/1.x requests, handlers should read any + // needed request body data before writing the response. Once the + // headers have been flushed (due to either an explicit Flusher.Flush + // call or writing enough data to trigger a flush), the request body + // may be unavailable. For HTTP/2 requests, the Go HTTP server permits + // handlers to continue to read the request body while concurrently + // writing the response. However, such behavior may not be supported + // by all HTTP/2 clients. Handlers should read before writing if + // possible to maximize compatibility. + Write([]byte) (int, error) + + // WriteHeader sends an HTTP response header with the provided + // status code. + // + // If WriteHeader is not called explicitly, the first call to Write + // will trigger an implicit WriteHeader(http.StatusOK). + // Thus explicit calls to WriteHeader are mainly used to + // send error codes or 1xx informational responses. + // + // The provided code must be a valid HTTP 1xx-5xx status code. + // Any number of 1xx headers may be written, followed by at most + // one 2xx-5xx header. 1xx headers are sent immediately, but 2xx-5xx + // headers may be buffered. Use the Flusher interface to send + // buffered data. The header map is cleared when 2xx-5xx headers are + // sent, but not with 1xx headers. + // + // The server will automatically send a 100 (Continue) header + // on the first read from the request body if the request has + // an "Expect: 100-continue" header. + WriteHeader(statusCode int) +} + +// The Flusher interface is implemented by ResponseWriters that allow +// an HTTP handler to flush buffered data to the client. +// +// The default HTTP/1.x and HTTP/2 ResponseWriter implementations +// support Flusher, but ResponseWriter wrappers may not. Handlers +// should always test for this ability at runtime. +// +// Note that even for ResponseWriters that support Flush, +// if the client is connected through an HTTP proxy, +// the buffered data may not reach the client until the response +// completes. +type Flusher interface { + // Flush sends any buffered data to the client. + Flush() +} + +// The Hijacker interface is implemented by ResponseWriters that allow +// an HTTP handler to take over the connection. +// +// The default ResponseWriter for HTTP/1.x connections supports +// Hijacker, but HTTP/2 connections intentionally do not. +// ResponseWriter wrappers may also not support Hijacker. Handlers +// should always test for this ability at runtime. +type Hijacker interface { + // Hijack lets the caller take over the connection. + // After a call to Hijack the HTTP server library + // will not do anything else with the connection. + // + // It becomes the caller's responsibility to manage + // and close the connection. + // + // The returned net.Conn may have read or write deadlines + // already set, depending on the configuration of the + // Server. It is the caller's responsibility to set + // or clear those deadlines as needed. + // + // The returned bufio.Reader may contain unprocessed buffered + // data from the client. + // + // After a call to Hijack, the original Request.Body must not + // be used. The original Request's Context remains valid and + // is not canceled until the Request's ServeHTTP method + // returns. + Hijack() (net.Conn, *bufio.ReadWriter, error) +} + +// The CloseNotifier interface is implemented by ResponseWriters which +// allow detecting when the underlying connection has gone away. +// +// This mechanism can be used to cancel long operations on the server +// if the client has disconnected before the response is ready. +// +// Deprecated: the CloseNotifier interface predates Go's context package. +// New code should use Request.Context instead. +type CloseNotifier interface { + // CloseNotify returns a channel that receives at most a + // single value (true) when the client connection has gone + // away. + // + // CloseNotify may wait to notify until Request.Body has been + // fully read. + // + // After the Handler has returned, there is no guarantee + // that the channel receives a value. + // + // If the protocol is HTTP/1.1 and CloseNotify is called while + // processing an idempotent request (such a GET) while + // HTTP/1.1 pipelining is in use, the arrival of a subsequent + // pipelined request may cause a value to be sent on the + // returned channel. In practice HTTP/1.1 pipelining is not + // enabled in browsers and not seen often in the wild. If this + // is a problem, use HTTP/2 or only use CloseNotify on methods + // such as POST. + CloseNotify() <-chan bool +} + +var ( + // ServerContextKey is a context key. It can be used in HTTP + // handlers with Context.Value to access the server that + // started the handler. The associated value will be of + // type *Server. + ServerContextKey = &contextKey{"http-server"} + + // LocalAddrContextKey is a context key. It can be used in + // HTTP handlers with Context.Value to access the local + // address the connection arrived on. + // The associated value will be of type net.Addr. + LocalAddrContextKey = &contextKey{"local-addr"} +) + +// A conn represents the server side of an HTTP connection. +type conn struct { + // server is the server on which the connection arrived. + // Immutable; never nil. + server *Server + + // cancelCtx cancels the connection-level context. + cancelCtx context.CancelFunc + + // rwc is the underlying network connection. + // This is never wrapped by other types and is the value given out + // to CloseNotifier callers. It is usually of type *net.TCPConn or + // *tls.Conn. + rwc net.Conn + + // remoteAddr is rwc.RemoteAddr().String(). It is not populated synchronously + // inside the Listener's Accept goroutine, as some implementations block. + // It is populated immediately inside the (*conn).serve goroutine. + // This is the value of a Handler's (*Request).RemoteAddr. + remoteAddr string + + // tlsState is the TLS connection state when using TLS. + // nil means not TLS. + tlsState *tls.ConnectionState + + // werr is set to the first write error to rwc. + // It is set via checkConnErrorWriter{w}, where bufw writes. + werr error + + // r is bufr's read source. It's a wrapper around rwc that provides + // io.LimitedReader-style limiting (while reading request headers) + // and functionality to support CloseNotifier. See *connReader docs. + r *connReader + + // bufr reads from r. + bufr *bufio.Reader + + // bufw writes to checkConnErrorWriter{c}, which populates werr on error. + bufw *bufio.Writer + + // lastMethod is the method of the most recent request + // on this connection, if any. + lastMethod string + + curReq atomic.Value // of *response (which has a Request in it) + + curState struct{ atomic uint64 } // packed (unixtime<<8|uint8(ConnState)) + + // mu guards hijackedv + mu sync.Mutex + + // hijackedv is whether this connection has been hijacked + // by a Handler with the Hijacker interface. + // It is guarded by mu. + hijackedv bool +} + +func (c *conn) hijacked() bool { + c.mu.Lock() + defer c.mu.Unlock() + return c.hijackedv +} + +// c.mu must be held. +func (c *conn) hijackLocked() (rwc net.Conn, buf *bufio.ReadWriter, err error) { + if c.hijackedv { + return nil, nil, ErrHijacked + } + c.r.abortPendingRead() + + c.hijackedv = true + rwc = c.rwc + rwc.SetDeadline(time.Time{}) + + buf = bufio.NewReadWriter(c.bufr, bufio.NewWriter(rwc)) + if c.r.hasByte { + if _, err := c.bufr.Peek(c.bufr.Buffered() + 1); err != nil { + return nil, nil, fmt.Errorf("unexpected Peek failure reading buffered byte: %v", err) + } + } + c.setState(rwc, StateHijacked, runHooks) + return +} + +// This should be >= 512 bytes for DetectContentType, +// but otherwise it's somewhat arbitrary. +const bufferBeforeChunkingSize = 2048 + +// chunkWriter writes to a response's conn buffer, and is the writer +// wrapped by the response.w buffered writer. +// +// chunkWriter also is responsible for finalizing the Header, including +// conditionally setting the Content-Type and setting a Content-Length +// in cases where the handler's final output is smaller than the buffer +// size. It also conditionally adds chunk headers, when in chunking mode. +// +// See the comment above (*response).Write for the entire write flow. +type chunkWriter struct { + res *response + + // header is either nil or a deep clone of res.handlerHeader + // at the time of res.writeHeader, if res.writeHeader is + // called and extra buffering is being done to calculate + // Content-Type and/or Content-Length. + header Header + + // wroteHeader tells whether the header's been written to "the + // wire" (or rather: w.conn.buf). this is unlike + // (*response).wroteHeader, which tells only whether it was + // logically written. + wroteHeader bool + + // set by the writeHeader method: + chunking bool // using chunked transfer encoding for reply body +} + +var ( + crlf = []byte("\r\n") + colonSpace = []byte(": ") +) + +func (cw *chunkWriter) Write(p []byte) (n int, err error) { + if !cw.wroteHeader { + cw.writeHeader(p) + } + if cw.res.req.Method == "HEAD" { + // Eat writes. + return len(p), nil + } + if cw.chunking { + _, err = fmt.Fprintf(cw.res.conn.bufw, "%x\r\n", len(p)) + if err != nil { + cw.res.conn.rwc.Close() + return + } + } + n, err = cw.res.conn.bufw.Write(p) + if cw.chunking && err == nil { + _, err = cw.res.conn.bufw.Write(crlf) + } + if err != nil { + cw.res.conn.rwc.Close() + } + return +} + +func (cw *chunkWriter) flush() { + if !cw.wroteHeader { + cw.writeHeader(nil) + } + cw.res.conn.bufw.Flush() +} + +func (cw *chunkWriter) close() { + if !cw.wroteHeader { + cw.writeHeader(nil) + } + if cw.chunking { + bw := cw.res.conn.bufw // conn's bufio writer + // zero chunk to mark EOF + bw.WriteString("0\r\n") + if trailers := cw.res.finalTrailers(); trailers != nil { + trailers.Write(bw) // the writer handles noting errors + } + // final blank line after the trailers (whether + // present or not) + bw.WriteString("\r\n") + } +} + +// A response represents the server side of an HTTP response. +type response struct { + conn *conn + req *Request // request for this response + reqBody io.ReadCloser + cancelCtx context.CancelFunc // when ServeHTTP exits + wroteHeader bool // a non-1xx header has been (logically) written + wroteContinue bool // 100 Continue response was written + wants10KeepAlive bool // HTTP/1.0 w/ Connection "keep-alive" + wantsClose bool // HTTP request has Connection "close" + + // canWriteContinue is a boolean value accessed as an atomic int32 + // that says whether or not a 100 Continue header can be written + // to the connection. + // writeContinueMu must be held while writing the header. + // These two fields together synchronize the body reader (the + // expectContinueReader, which wants to write 100 Continue) + // against the main writer. + canWriteContinue atomicBool + writeContinueMu sync.Mutex + + w *bufio.Writer // buffers output in chunks to chunkWriter + cw chunkWriter + + // handlerHeader is the Header that Handlers get access to, + // which may be retained and mutated even after WriteHeader. + // handlerHeader is copied into cw.header at WriteHeader + // time, and privately mutated thereafter. + handlerHeader Header + calledHeader bool // handler accessed handlerHeader via Header + + written int64 // number of bytes written in body + contentLength int64 // explicitly-declared Content-Length; or -1 + status int // status code passed to WriteHeader + + // close connection after this reply. set on request and + // updated after response from handler if there's a + // "Connection: keep-alive" response header and a + // Content-Length. + closeAfterReply bool + + // requestBodyLimitHit is set by requestTooLarge when + // maxBytesReader hits its max size. It is checked in + // WriteHeader, to make sure we don't consume the + // remaining request body to try to advance to the next HTTP + // request. Instead, when this is set, we stop reading + // subsequent requests on this connection and stop reading + // input from it. + requestBodyLimitHit bool + + // trailers are the headers to be sent after the handler + // finishes writing the body. This field is initialized from + // the Trailer response header when the response header is + // written. + trailers []string + + handlerDone atomicBool // set true when the handler exits + + // Buffers for Date, Content-Length, and status code + dateBuf [len(TimeFormat)]byte + clenBuf [10]byte + statusBuf [3]byte + + // closeNotifyCh is the channel returned by CloseNotify. + // TODO(bradfitz): this is currently (for Go 1.8) always + // non-nil. Make this lazily-created again as it used to be? + closeNotifyCh chan bool + didCloseNotify int32 // atomic (only 0->1 winner should send) +} + +// TrailerPrefix is a magic prefix for ResponseWriter.Header map keys +// that, if present, signals that the map entry is actually for +// the response trailers, and not the response headers. The prefix +// is stripped after the ServeHTTP call finishes and the values are +// sent in the trailers. +// +// This mechanism is intended only for trailers that are not known +// prior to the headers being written. If the set of trailers is fixed +// or known before the header is written, the normal Go trailers mechanism +// is preferred: +// +// https://pkg.go.dev/net/http#ResponseWriter +// https://pkg.go.dev/net/http#example-ResponseWriter-Trailers +const TrailerPrefix = "Trailer:" + +// finalTrailers is called after the Handler exits and returns a non-nil +// value if the Handler set any trailers. +func (w *response) finalTrailers() Header { + var t Header + for k, vv := range w.handlerHeader { + if strings.HasPrefix(k, TrailerPrefix) { + if t == nil { + t = make(Header) + } + t[strings.TrimPrefix(k, TrailerPrefix)] = vv + } + } + for _, k := range w.trailers { + if t == nil { + t = make(Header) + } + for _, v := range w.handlerHeader[k] { + t.Add(k, v) + } + } + return t +} + +type atomicBool int32 + +func (b *atomicBool) isSet() bool { return atomic.LoadInt32((*int32)(b)) != 0 } +func (b *atomicBool) setTrue() { atomic.StoreInt32((*int32)(b), 1) } +func (b *atomicBool) setFalse() { atomic.StoreInt32((*int32)(b), 0) } + +// declareTrailer is called for each Trailer header when the +// response header is written. It notes that a header will need to be +// written in the trailers at the end of the response. +func (w *response) declareTrailer(k string) { + k = CanonicalHeaderKey(k) + if !httpguts.ValidTrailerHeader(k) { + // Forbidden by RFC 7230, section 4.1.2 + return + } + w.trailers = append(w.trailers, k) +} + +// requestTooLarge is called by maxBytesReader when too much input has +// been read from the client. +func (w *response) requestTooLarge() { + w.closeAfterReply = true + w.requestBodyLimitHit = true + if !w.wroteHeader { + w.Header().Set("Connection", "close") + } +} + +// needsSniff reports whether a Content-Type still needs to be sniffed. +func (w *response) needsSniff() bool { + _, haveType := w.handlerHeader["Content-Type"] + return !w.cw.wroteHeader && !haveType && w.written < sniffLen +} + +// writerOnly hides an io.Writer value's optional ReadFrom method +// from io.Copy. +type writerOnly struct { + io.Writer +} + +// ReadFrom is here to optimize copying from an *os.File regular file +// to a *net.TCPConn with sendfile, or from a supported src type such +// as a *net.TCPConn on Linux with splice. +func (w *response) ReadFrom(src io.Reader) (n int64, err error) { + bufp := copyBufPool.Get().(*[]byte) + buf := *bufp + defer copyBufPool.Put(bufp) + + // Our underlying w.conn.rwc is usually a *TCPConn (with its + // own ReadFrom method). If not, just fall back to the normal + // copy method. + rf, ok := w.conn.rwc.(io.ReaderFrom) + if !ok { + return io.CopyBuffer(writerOnly{w}, src, buf) + } + + // Copy the first sniffLen bytes before switching to ReadFrom. + // This ensures we don't start writing the response before the + // source is available (see golang.org/issue/5660) and provides + // enough bytes to perform Content-Type sniffing when required. + if !w.cw.wroteHeader { + n0, err := io.CopyBuffer(writerOnly{w}, io.LimitReader(src, sniffLen), buf) + n += n0 + if err != nil || n0 < sniffLen { + return n, err + } + } + + w.w.Flush() // get rid of any previous writes + w.cw.flush() // make sure Header is written; flush data to rwc + + // Now that cw has been flushed, its chunking field is guaranteed initialized. + if !w.cw.chunking && w.bodyAllowed() { + n0, err := rf.ReadFrom(src) + n += n0 + w.written += n0 + return n, err + } + + n0, err := io.CopyBuffer(writerOnly{w}, src, buf) + n += n0 + return n, err +} + +// debugServerConnections controls whether all server connections are wrapped +// with a verbose logging wrapper. +const debugServerConnections = false + +// Create new connection from rwc. +func (srv *Server) newConn(rwc net.Conn) *conn { + c := &conn{ + server: srv, + rwc: rwc, + } + if debugServerConnections { + c.rwc = newLoggingConn("server", c.rwc) + } + return c +} + +type readResult struct { + _ incomparable + n int + err error + b byte // byte read, if n == 1 +} + +// connReader is the io.Reader wrapper used by *conn. It combines a +// selectively-activated io.LimitedReader (to bound request header +// read sizes) with support for selectively keeping an io.Reader.Read +// call blocked in a background goroutine to wait for activity and +// trigger a CloseNotifier channel. +type connReader struct { + conn *conn + + mu sync.Mutex // guards following + hasByte bool + byteBuf [1]byte + cond *sync.Cond + inRead bool + aborted bool // set true before conn.rwc deadline is set to past + remain int64 // bytes remaining +} + +func (cr *connReader) lock() { + cr.mu.Lock() + if cr.cond == nil { + cr.cond = sync.NewCond(&cr.mu) + } +} + +func (cr *connReader) unlock() { cr.mu.Unlock() } + +func (cr *connReader) startBackgroundRead() { + cr.lock() + defer cr.unlock() + if cr.inRead { + panic("invalid concurrent Body.Read call") + } + if cr.hasByte { + return + } + cr.inRead = true + cr.conn.rwc.SetReadDeadline(time.Time{}) + go cr.backgroundRead() +} + +func (cr *connReader) backgroundRead() { + n, err := cr.conn.rwc.Read(cr.byteBuf[:]) + cr.lock() + if n == 1 { + cr.hasByte = true + // We were past the end of the previous request's body already + // (since we wouldn't be in a background read otherwise), so + // this is a pipelined HTTP request. Prior to Go 1.11 we used to + // send on the CloseNotify channel and cancel the context here, + // but the behavior was documented as only "may", and we only + // did that because that's how CloseNotify accidentally behaved + // in very early Go releases prior to context support. Once we + // added context support, people used a Handler's + // Request.Context() and passed it along. Having that context + // cancel on pipelined HTTP requests caused problems. + // Fortunately, almost nothing uses HTTP/1.x pipelining. + // Unfortunately, apt-get does, or sometimes does. + // New Go 1.11 behavior: don't fire CloseNotify or cancel + // contexts on pipelined requests. Shouldn't affect people, but + // fixes cases like Issue 23921. This does mean that a client + // closing their TCP connection after sending a pipelined + // request won't cancel the context, but we'll catch that on any + // write failure (in checkConnErrorWriter.Write). + // If the server never writes, yes, there are still contrived + // server & client behaviors where this fails to ever cancel the + // context, but that's kinda why HTTP/1.x pipelining died + // anyway. + } + if ne, ok := err.(net.Error); ok && cr.aborted && ne.Timeout() { + // Ignore this error. It's the expected error from + // another goroutine calling abortPendingRead. + } else if err != nil { + cr.handleReadError(err) + } + cr.aborted = false + cr.inRead = false + cr.unlock() + cr.cond.Broadcast() +} + +func (cr *connReader) abortPendingRead() { + cr.lock() + defer cr.unlock() + if !cr.inRead { + return + } + cr.aborted = true + cr.conn.rwc.SetReadDeadline(aLongTimeAgo) + for cr.inRead { + cr.cond.Wait() + } + cr.conn.rwc.SetReadDeadline(time.Time{}) +} + +func (cr *connReader) setReadLimit(remain int64) { cr.remain = remain } +func (cr *connReader) setInfiniteReadLimit() { cr.remain = maxInt64 } +func (cr *connReader) hitReadLimit() bool { return cr.remain <= 0 } + +// handleReadError is called whenever a Read from the client returns a +// non-nil error. +// +// The provided non-nil err is almost always io.EOF or a "use of +// closed network connection". In any case, the error is not +// particularly interesting, except perhaps for debugging during +// development. Any error means the connection is dead and we should +// down its context. +// +// It may be called from multiple goroutines. +func (cr *connReader) handleReadError(_ error) { + cr.conn.cancelCtx() + cr.closeNotify() +} + +// may be called from multiple goroutines. +func (cr *connReader) closeNotify() { + res, _ := cr.conn.curReq.Load().(*response) + if res != nil && atomic.CompareAndSwapInt32(&res.didCloseNotify, 0, 1) { + res.closeNotifyCh <- true + } +} + +func (cr *connReader) Read(p []byte) (n int, err error) { + cr.lock() + if cr.inRead { + cr.unlock() + if cr.conn.hijacked() { + panic("invalid Body.Read call. After hijacked, the original Request must not be used") + } + panic("invalid concurrent Body.Read call") + } + if cr.hitReadLimit() { + cr.unlock() + return 0, io.EOF + } + if len(p) == 0 { + cr.unlock() + return 0, nil + } + if int64(len(p)) > cr.remain { + p = p[:cr.remain] + } + if cr.hasByte { + p[0] = cr.byteBuf[0] + cr.hasByte = false + cr.unlock() + return 1, nil + } + cr.inRead = true + cr.unlock() + n, err = cr.conn.rwc.Read(p) + + cr.lock() + cr.inRead = false + if err != nil { + cr.handleReadError(err) + } + cr.remain -= int64(n) + cr.unlock() + + cr.cond.Broadcast() + return n, err +} + +var ( + bufioReaderPool sync.Pool + bufioWriter2kPool sync.Pool + bufioWriter4kPool sync.Pool +) + +var copyBufPool = sync.Pool{ + New: func() any { + b := make([]byte, 32*1024) + return &b + }, +} + +func bufioWriterPool(size int) *sync.Pool { + switch size { + case 2 << 10: + return &bufioWriter2kPool + case 4 << 10: + return &bufioWriter4kPool + } + return nil +} + +func newBufioReader(r io.Reader) *bufio.Reader { + if v := bufioReaderPool.Get(); v != nil { + br := v.(*bufio.Reader) + br.Reset(r) + return br + } + // Note: if this reader size is ever changed, update + // TestHandlerBodyClose's assumptions. + return bufio.NewReader(r) +} + +func putBufioReader(br *bufio.Reader) { + br.Reset(nil) + bufioReaderPool.Put(br) +} + +func newBufioWriterSize(w io.Writer, size int) *bufio.Writer { + pool := bufioWriterPool(size) + if pool != nil { + if v := pool.Get(); v != nil { + bw := v.(*bufio.Writer) + bw.Reset(w) + return bw + } + } + return bufio.NewWriterSize(w, size) +} + +func putBufioWriter(bw *bufio.Writer) { + bw.Reset(nil) + if pool := bufioWriterPool(bw.Available()); pool != nil { + pool.Put(bw) + } +} + +// DefaultMaxHeaderBytes is the maximum permitted size of the headers +// in an HTTP request. +// This can be overridden by setting Server.MaxHeaderBytes. +// TINYGO: dropped default from 1 << 20 // 1 MB +const DefaultMaxHeaderBytes = 1 << 12 // 4 KB + +func (srv *Server) maxHeaderBytes() int { + if srv.MaxHeaderBytes > 0 { + return srv.MaxHeaderBytes + } + return DefaultMaxHeaderBytes +} + +func (srv *Server) initialReadLimitSize() int64 { + return int64(srv.maxHeaderBytes()) + 4096 // bufio slop +} + +// tlsHandshakeTimeout returns the time limit permitted for the TLS +// handshake, or zero for unlimited. +// +// It returns the minimum of any positive ReadHeaderTimeout, +// ReadTimeout, or WriteTimeout. +func (srv *Server) tlsHandshakeTimeout() time.Duration { + var ret time.Duration + for _, v := range [...]time.Duration{ + srv.ReadHeaderTimeout, + srv.ReadTimeout, + srv.WriteTimeout, + } { + if v <= 0 { + continue + } + if ret == 0 || v < ret { + ret = v + } + } + return ret +} + +// wrapper around io.ReadCloser which on first read, sends an +// HTTP/1.1 100 Continue header +type expectContinueReader struct { + resp *response + readCloser io.ReadCloser + closed atomicBool + sawEOF atomicBool +} + +func (ecr *expectContinueReader) Read(p []byte) (n int, err error) { + if ecr.closed.isSet() { + return 0, ErrBodyReadAfterClose + } + w := ecr.resp + if !w.wroteContinue && w.canWriteContinue.isSet() && !w.conn.hijacked() { + w.wroteContinue = true + w.writeContinueMu.Lock() + if w.canWriteContinue.isSet() { + w.conn.bufw.WriteString("HTTP/1.1 100 Continue\r\n\r\n") + w.conn.bufw.Flush() + w.canWriteContinue.setFalse() + } + w.writeContinueMu.Unlock() + } + n, err = ecr.readCloser.Read(p) + if err == io.EOF { + ecr.sawEOF.setTrue() + } + return +} + +func (ecr *expectContinueReader) Close() error { + ecr.closed.setTrue() + return ecr.readCloser.Close() +} + +// TimeFormat is the time format to use when generating times in HTTP +// headers. It is like time.RFC1123 but hard-codes GMT as the time +// zone. The time being formatted must be in UTC for Format to +// generate the correct format. +// +// For parsing this time format, see ParseTime. +const TimeFormat = "Mon, 02 Jan 2006 15:04:05 GMT" + +// appendTime is a non-allocating version of []byte(t.UTC().Format(TimeFormat)) +func appendTime(b []byte, t time.Time) []byte { + const days = "SunMonTueWedThuFriSat" + const months = "JanFebMarAprMayJunJulAugSepOctNovDec" + + t = t.UTC() + yy, mm, dd := t.Date() + hh, mn, ss := t.Clock() + day := days[3*t.Weekday():] + mon := months[3*(mm-1):] + + return append(b, + day[0], day[1], day[2], ',', ' ', + byte('0'+dd/10), byte('0'+dd%10), ' ', + mon[0], mon[1], mon[2], ' ', + byte('0'+yy/1000), byte('0'+(yy/100)%10), byte('0'+(yy/10)%10), byte('0'+yy%10), ' ', + byte('0'+hh/10), byte('0'+hh%10), ':', + byte('0'+mn/10), byte('0'+mn%10), ':', + byte('0'+ss/10), byte('0'+ss%10), ' ', + 'G', 'M', 'T') +} + +var errTooLarge = errors.New("http: request too large") + +// Read next request from connection. +func (c *conn) readRequest(ctx context.Context) (w *response, err error) { + if c.hijacked() { + return nil, ErrHijacked + } + + var ( + wholeReqDeadline time.Time // or zero if none + hdrDeadline time.Time // or zero if none + ) + t0 := time.Now() + if d := c.server.readHeaderTimeout(); d > 0 { + hdrDeadline = t0.Add(d) + } + if d := c.server.ReadTimeout; d > 0 { + wholeReqDeadline = t0.Add(d) + } + c.rwc.SetReadDeadline(hdrDeadline) + if d := c.server.WriteTimeout; d > 0 { + defer func() { + c.rwc.SetWriteDeadline(time.Now().Add(d)) + }() + } + + c.r.setReadLimit(c.server.initialReadLimitSize()) + if c.lastMethod == "POST" { + // RFC 7230 section 3 tolerance for old buggy clients. + peek, _ := c.bufr.Peek(4) // ReadRequest will get err below + c.bufr.Discard(numLeadingCRorLF(peek)) + } + req, err := readRequest(c.bufr) + if err != nil { + if c.r.hitReadLimit() { + return nil, errTooLarge + } + return nil, err + } + + if !http1ServerSupportsRequest(req) { + return nil, statusError{StatusHTTPVersionNotSupported, "unsupported protocol version"} + } + + c.lastMethod = req.Method + c.r.setInfiniteReadLimit() + + hosts, haveHost := req.Header["Host"] + isH2Upgrade := req.isH2Upgrade() + if req.ProtoAtLeast(1, 1) && (!haveHost || len(hosts) == 0) && !isH2Upgrade && req.Method != "CONNECT" { + return nil, badRequestError("missing required Host header") + } + if len(hosts) == 1 && !httpguts.ValidHostHeader(hosts[0]) { + return nil, badRequestError("malformed Host header") + } + for k, vv := range req.Header { + if !httpguts.ValidHeaderFieldName(k) { + return nil, badRequestError("invalid header name") + } + for _, v := range vv { + if !httpguts.ValidHeaderFieldValue(v) { + return nil, badRequestError("invalid header value") + } + } + } + delete(req.Header, "Host") + + ctx, cancelCtx := context.WithCancel(ctx) + req.ctx = ctx + req.RemoteAddr = c.remoteAddr + req.TLS = c.tlsState + if body, ok := req.Body.(*body); ok { + body.doEarlyClose = true + } + + // Adjust the read deadline if necessary. + if !hdrDeadline.Equal(wholeReqDeadline) { + c.rwc.SetReadDeadline(wholeReqDeadline) + } + + w = &response{ + conn: c, + cancelCtx: cancelCtx, + req: req, + reqBody: req.Body, + handlerHeader: make(Header), + contentLength: -1, + closeNotifyCh: make(chan bool, 1), + + // We populate these ahead of time so we're not + // reading from req.Header after their Handler starts + // and maybe mutates it (Issue 14940) + wants10KeepAlive: req.wantsHttp10KeepAlive(), + wantsClose: req.wantsClose(), + } + if isH2Upgrade { + w.closeAfterReply = true + } + w.cw.res = w + w.w = newBufioWriterSize(&w.cw, bufferBeforeChunkingSize) + return w, nil +} + +// http1ServerSupportsRequest reports whether Go's HTTP/1.x server +// supports the given request. +func http1ServerSupportsRequest(req *Request) bool { + if req.ProtoMajor == 1 { + return true + } + // Accept "PRI * HTTP/2.0" upgrade requests, so Handlers can + // wire up their own HTTP/2 upgrades. + if req.ProtoMajor == 2 && req.ProtoMinor == 0 && + req.Method == "PRI" && req.RequestURI == "*" { + return true + } + // Reject HTTP/0.x, and all other HTTP/2+ requests (which + // aren't encoded in ASCII anyway). + return false +} + +func (w *response) Header() Header { + if w.cw.header == nil && w.wroteHeader && !w.cw.wroteHeader { + // Accessing the header between logically writing it + // and physically writing it means we need to allocate + // a clone to snapshot the logically written state. + w.cw.header = w.handlerHeader.Clone() + } + w.calledHeader = true + return w.handlerHeader +} + +// maxPostHandlerReadBytes is the max number of Request.Body bytes not +// consumed by a handler that the server will read from the client +// in order to keep a connection alive. If there are more bytes than +// this then the server to be paranoid instead sends a "Connection: +// close" response. +// +// This number is approximately what a typical machine's TCP buffer +// size is anyway. (if we have the bytes on the machine, we might as +// well read them) +const maxPostHandlerReadBytes = 256 << 10 + +func checkWriteHeaderCode(code int) { + // Issue 22880: require valid WriteHeader status codes. + // For now we only enforce that it's three digits. + // In the future we might block things over 599 (600 and above aren't defined + // at https://httpwg.org/specs/rfc7231.html#status.codes). + // But for now any three digits. + // + // We used to send "HTTP/1.1 000 0" on the wire in responses but there's + // no equivalent bogus thing we can realistically send in HTTP/2, + // so we'll consistently panic instead and help people find their bugs + // early. (We can't return an error from WriteHeader even if we wanted to.) + if code < 100 || code > 999 { + panic(fmt.Sprintf("invalid WriteHeader code %v", code)) + } +} + +// relevantCaller searches the call stack for the first function outside of net/http. +// The purpose of this function is to provide more helpful error messages. +func relevantCaller() runtime.Frame { + pc := make([]uintptr, 16) + n := runtime.Callers(1, pc) + frames := runtime.CallersFrames(pc[:n]) + var frame runtime.Frame + for { + frame, more := frames.Next() + if !strings.HasPrefix(frame.Function, "net/http.") { + return frame + } + if !more { + break + } + } + return frame +} + +func (w *response) WriteHeader(code int) { + if w.conn.hijacked() { + caller := relevantCaller() + w.conn.server.logf("http: response.WriteHeader on hijacked connection from %s (%s:%d)", caller.Function, path.Base(caller.File), caller.Line) + return + } + if w.wroteHeader { + caller := relevantCaller() + w.conn.server.logf("http: superfluous response.WriteHeader call from %s (%s:%d)", caller.Function, path.Base(caller.File), caller.Line) + return + } + checkWriteHeaderCode(code) + + // Handle informational headers + if code >= 100 && code <= 199 { + // Prevent a potential race with an automatically-sent 100 Continue triggered by Request.Body.Read() + if code == 100 && w.canWriteContinue.isSet() { + w.writeContinueMu.Lock() + w.canWriteContinue.setFalse() + w.writeContinueMu.Unlock() + } + + writeStatusLine(w.conn.bufw, w.req.ProtoAtLeast(1, 1), code, w.statusBuf[:]) + + // Per RFC 8297 we must not clear the current header map + w.handlerHeader.WriteSubset(w.conn.bufw, excludedHeadersNoBody) + w.conn.bufw.Write(crlf) + w.conn.bufw.Flush() + + return + } + + w.wroteHeader = true + w.status = code + + if w.calledHeader && w.cw.header == nil { + w.cw.header = w.handlerHeader.Clone() + } + + if cl := w.handlerHeader.get("Content-Length"); cl != "" { + v, err := strconv.ParseInt(cl, 10, 64) + if err == nil && v >= 0 { + w.contentLength = v + } else { + w.conn.server.logf("http: invalid Content-Length of %q", cl) + w.handlerHeader.Del("Content-Length") + } + } +} + +// extraHeader is the set of headers sometimes added by chunkWriter.writeHeader. +// This type is used to avoid extra allocations from cloning and/or populating +// the response Header map and all its 1-element slices. +type extraHeader struct { + contentType string + connection string + transferEncoding string + date []byte // written if not nil + contentLength []byte // written if not nil +} + +// Sorted the same as extraHeader.Write's loop. +var extraHeaderKeys = [][]byte{ + []byte("Content-Type"), + []byte("Connection"), + []byte("Transfer-Encoding"), +} + +var ( + headerContentLength = []byte("Content-Length: ") + headerDate = []byte("Date: ") +) + +// Write writes the headers described in h to w. +// +// This method has a value receiver, despite the somewhat large size +// of h, because it prevents an allocation. The escape analysis isn't +// smart enough to realize this function doesn't mutate h. +func (h extraHeader) Write(w *bufio.Writer) { + if h.date != nil { + w.Write(headerDate) + w.Write(h.date) + w.Write(crlf) + } + if h.contentLength != nil { + w.Write(headerContentLength) + w.Write(h.contentLength) + w.Write(crlf) + } + for i, v := range []string{h.contentType, h.connection, h.transferEncoding} { + if v != "" { + w.Write(extraHeaderKeys[i]) + w.Write(colonSpace) + w.WriteString(v) + w.Write(crlf) + } + } +} + +// writeHeader finalizes the header sent to the client and writes it +// to cw.res.conn.bufw. +// +// p is not written by writeHeader, but is the first chunk of the body +// that will be written. It is sniffed for a Content-Type if none is +// set explicitly. It's also used to set the Content-Length, if the +// total body size was small and the handler has already finished +// running. +func (cw *chunkWriter) writeHeader(p []byte) { + if cw.wroteHeader { + return + } + cw.wroteHeader = true + + w := cw.res + keepAlivesEnabled := w.conn.server.doKeepAlives() + isHEAD := w.req.Method == "HEAD" + + // header is written out to w.conn.buf below. Depending on the + // state of the handler, we either own the map or not. If we + // don't own it, the exclude map is created lazily for + // WriteSubset to remove headers. The setHeader struct holds + // headers we need to add. + header := cw.header + owned := header != nil + if !owned { + header = w.handlerHeader + } + var excludeHeader map[string]bool + delHeader := func(key string) { + if owned { + header.Del(key) + return + } + if _, ok := header[key]; !ok { + return + } + if excludeHeader == nil { + excludeHeader = make(map[string]bool) + } + excludeHeader[key] = true + } + var setHeader extraHeader + + // Don't write out the fake "Trailer:foo" keys. See TrailerPrefix. + trailers := false + for k := range cw.header { + if strings.HasPrefix(k, TrailerPrefix) { + if excludeHeader == nil { + excludeHeader = make(map[string]bool) + } + excludeHeader[k] = true + trailers = true + } + } + for _, v := range cw.header["Trailer"] { + trailers = true + foreachHeaderElement(v, cw.res.declareTrailer) + } + + te := header.get("Transfer-Encoding") + hasTE := te != "" + + // If the handler is done but never sent a Content-Length + // response header and this is our first (and last) write, set + // it, even to zero. This helps HTTP/1.0 clients keep their + // "keep-alive" connections alive. + // Exceptions: 304/204/1xx responses never get Content-Length, and if + // it was a HEAD request, we don't know the difference between + // 0 actual bytes and 0 bytes because the handler noticed it + // was a HEAD request and chose not to write anything. So for + // HEAD, the handler should either write the Content-Length or + // write non-zero bytes. If it's actually 0 bytes and the + // handler never looked at the Request.Method, we just don't + // send a Content-Length header. + // Further, we don't send an automatic Content-Length if they + // set a Transfer-Encoding, because they're generally incompatible. + if w.handlerDone.isSet() && !trailers && !hasTE && bodyAllowedForStatus(w.status) && header.get("Content-Length") == "" && (!isHEAD || len(p) > 0) { + w.contentLength = int64(len(p)) + setHeader.contentLength = strconv.AppendInt(cw.res.clenBuf[:0], int64(len(p)), 10) + } + + // If this was an HTTP/1.0 request with keep-alive and we sent a + // Content-Length back, we can make this a keep-alive response ... + if w.wants10KeepAlive && keepAlivesEnabled { + sentLength := header.get("Content-Length") != "" + if sentLength && header.get("Connection") == "keep-alive" { + w.closeAfterReply = false + } + } + + // Check for an explicit (and valid) Content-Length header. + hasCL := w.contentLength != -1 + + if w.wants10KeepAlive && (isHEAD || hasCL || !bodyAllowedForStatus(w.status)) { + _, connectionHeaderSet := header["Connection"] + if !connectionHeaderSet { + setHeader.connection = "keep-alive" + } + } else if !w.req.ProtoAtLeast(1, 1) || w.wantsClose { + w.closeAfterReply = true + } + + if header.get("Connection") == "close" || !keepAlivesEnabled { + w.closeAfterReply = true + } + + // If the client wanted a 100-continue but we never sent it to + // them (or, more strictly: we never finished reading their + // request body), don't reuse this connection because it's now + // in an unknown state: we might be sending this response at + // the same time the client is now sending its request body + // after a timeout. (Some HTTP clients send Expect: + // 100-continue but knowing that some servers don't support + // it, the clients set a timer and send the body later anyway) + // If we haven't seen EOF, we can't skip over the unread body + // because we don't know if the next bytes on the wire will be + // the body-following-the-timer or the subsequent request. + // See Issue 11549. + if ecr, ok := w.req.Body.(*expectContinueReader); ok && !ecr.sawEOF.isSet() { + w.closeAfterReply = true + } + + // Per RFC 2616, we should consume the request body before + // replying, if the handler hasn't already done so. But we + // don't want to do an unbounded amount of reading here for + // DoS reasons, so we only try up to a threshold. + // TODO(bradfitz): where does RFC 2616 say that? See Issue 15527 + // about HTTP/1.x Handlers concurrently reading and writing, like + // HTTP/2 handlers can do. Maybe this code should be relaxed? + if w.req.ContentLength != 0 && !w.closeAfterReply { + var discard, tooBig bool + + switch bdy := w.req.Body.(type) { + case *expectContinueReader: + if bdy.resp.wroteContinue { + discard = true + } + case *body: + bdy.mu.Lock() + switch { + case bdy.closed: + if !bdy.sawEOF { + // Body was closed in handler with non-EOF error. + w.closeAfterReply = true + } + case bdy.unreadDataSizeLocked() >= maxPostHandlerReadBytes: + tooBig = true + default: + discard = true + } + bdy.mu.Unlock() + default: + discard = true + } + + if discard { + _, err := io.CopyN(io.Discard, w.reqBody, maxPostHandlerReadBytes+1) + switch err { + case nil: + // There must be even more data left over. + tooBig = true + case ErrBodyReadAfterClose: + // Body was already consumed and closed. + case io.EOF: + // The remaining body was just consumed, close it. + err = w.reqBody.Close() + if err != nil { + w.closeAfterReply = true + } + default: + // Some other kind of error occurred, like a read timeout, or + // corrupt chunked encoding. In any case, whatever remains + // on the wire must not be parsed as another HTTP request. + w.closeAfterReply = true + } + } + + if tooBig { + w.requestTooLarge() + delHeader("Connection") + setHeader.connection = "close" + } + } + + code := w.status + if bodyAllowedForStatus(code) { + // If no content type, apply sniffing algorithm to body. + _, haveType := header["Content-Type"] + + // If the Content-Encoding was set and is non-blank, + // we shouldn't sniff the body. See Issue 31753. + ce := header.Get("Content-Encoding") + hasCE := len(ce) > 0 + if !hasCE && !haveType && !hasTE && len(p) > 0 { + setHeader.contentType = DetectContentType(p) + } + } else { + for _, k := range suppressedHeaders(code) { + delHeader(k) + } + } + + if !header.has("Date") { + setHeader.date = appendTime(cw.res.dateBuf[:0], time.Now()) + } + + if hasCL && hasTE && te != "identity" { + // TODO: return an error if WriteHeader gets a return parameter + // For now just ignore the Content-Length. + w.conn.server.logf("http: WriteHeader called with both Transfer-Encoding of %q and a Content-Length of %d", + te, w.contentLength) + delHeader("Content-Length") + hasCL = false + } + + if w.req.Method == "HEAD" || !bodyAllowedForStatus(code) || code == StatusNoContent { + // Response has no body. + delHeader("Transfer-Encoding") + } else if hasCL { + // Content-Length has been provided, so no chunking is to be done. + delHeader("Transfer-Encoding") + } else if w.req.ProtoAtLeast(1, 1) { + // HTTP/1.1 or greater: Transfer-Encoding has been set to identity, and no + // content-length has been provided. The connection must be closed after the + // reply is written, and no chunking is to be done. This is the setup + // recommended in the Server-Sent Events candidate recommendation 11, + // section 8. + if hasTE && te == "identity" { + cw.chunking = false + w.closeAfterReply = true + delHeader("Transfer-Encoding") + } else { + // HTTP/1.1 or greater: use chunked transfer encoding + // to avoid closing the connection at EOF. + cw.chunking = true + setHeader.transferEncoding = "chunked" + if hasTE && te == "chunked" { + // We will send the chunked Transfer-Encoding header later. + delHeader("Transfer-Encoding") + } + } + } else { + // HTTP version < 1.1: cannot do chunked transfer + // encoding and we don't know the Content-Length so + // signal EOF by closing connection. + w.closeAfterReply = true + delHeader("Transfer-Encoding") // in case already set + } + + // Cannot use Content-Length with non-identity Transfer-Encoding. + if cw.chunking { + delHeader("Content-Length") + } + if !w.req.ProtoAtLeast(1, 0) { + return + } + + // Only override the Connection header if it is not a successful + // protocol switch response and if KeepAlives are not enabled. + // See https://golang.org/issue/36381. + delConnectionHeader := w.closeAfterReply && + (!keepAlivesEnabled || !hasToken(cw.header.get("Connection"), "close")) && + !isProtocolSwitchResponse(w.status, header) + if delConnectionHeader { + delHeader("Connection") + if w.req.ProtoAtLeast(1, 1) { + setHeader.connection = "close" + } + } + + writeStatusLine(w.conn.bufw, w.req.ProtoAtLeast(1, 1), code, w.statusBuf[:]) + cw.header.WriteSubset(w.conn.bufw, excludeHeader) + setHeader.Write(w.conn.bufw) + w.conn.bufw.Write(crlf) +} + +// foreachHeaderElement splits v according to the "#rule" construction +// in RFC 7230 section 7 and calls fn for each non-empty element. +func foreachHeaderElement(v string, fn func(string)) { + v = textproto.TrimString(v) + if v == "" { + return + } + if !strings.Contains(v, ",") { + fn(v) + return + } + for _, f := range strings.Split(v, ",") { + if f = textproto.TrimString(f); f != "" { + fn(f) + } + } +} + +// writeStatusLine writes an HTTP/1.x Status-Line (RFC 7230 Section 3.1.2) +// to bw. is11 is whether the HTTP request is HTTP/1.1. false means HTTP/1.0. +// code is the response status code. +// scratch is an optional scratch buffer. If it has at least capacity 3, it's used. +func writeStatusLine(bw *bufio.Writer, is11 bool, code int, scratch []byte) { + if is11 { + bw.WriteString("HTTP/1.1 ") + } else { + bw.WriteString("HTTP/1.0 ") + } + if text := StatusText(code); text != "" { + bw.Write(strconv.AppendInt(scratch[:0], int64(code), 10)) + bw.WriteByte(' ') + bw.WriteString(text) + bw.WriteString("\r\n") + } else { + // don't worry about performance + fmt.Fprintf(bw, "%03d status code %d\r\n", code, code) + } +} + +// bodyAllowed reports whether a Write is allowed for this response type. +// It's illegal to call this before the header has been flushed. +func (w *response) bodyAllowed() bool { + if !w.wroteHeader { + panic("") + } + return bodyAllowedForStatus(w.status) +} + +// The Life Of A Write is like this: +// +// Handler starts. No header has been sent. The handler can either +// write a header, or just start writing. Writing before sending a header +// sends an implicitly empty 200 OK header. +// +// If the handler didn't declare a Content-Length up front, we either +// go into chunking mode or, if the handler finishes running before +// the chunking buffer size, we compute a Content-Length and send that +// in the header instead. +// +// Likewise, if the handler didn't set a Content-Type, we sniff that +// from the initial chunk of output. +// +// The Writers are wired together like: +// +// 1. *response (the ResponseWriter) -> +// 2. (*response).w, a *bufio.Writer of bufferBeforeChunkingSize bytes -> +// 3. chunkWriter.Writer (whose writeHeader finalizes Content-Length/Type) +// and which writes the chunk headers, if needed -> +// 4. conn.bufw, a *bufio.Writer of default (4kB) bytes, writing to -> +// 5. checkConnErrorWriter{c}, which notes any non-nil error on Write +// and populates c.werr with it if so, but otherwise writes to -> +// 6. the rwc, the net.Conn. +// +// TODO(bradfitz): short-circuit some of the buffering when the +// initial header contains both a Content-Type and Content-Length. +// Also short-circuit in (1) when the header's been sent and not in +// chunking mode, writing directly to (4) instead, if (2) has no +// buffered data. More generally, we could short-circuit from (1) to +// (3) even in chunking mode if the write size from (1) is over some +// threshold and nothing is in (2). The answer might be mostly making +// bufferBeforeChunkingSize smaller and having bufio's fast-paths deal +// with this instead. +func (w *response) Write(data []byte) (n int, err error) { + return w.write(len(data), data, "") +} + +func (w *response) WriteString(data string) (n int, err error) { + return w.write(len(data), nil, data) +} + +// either dataB or dataS is non-zero. +func (w *response) write(lenData int, dataB []byte, dataS string) (n int, err error) { + if w.conn.hijacked() { + if lenData > 0 { + caller := relevantCaller() + w.conn.server.logf("http: response.Write on hijacked connection from %s (%s:%d)", caller.Function, path.Base(caller.File), caller.Line) + } + return 0, ErrHijacked + } + + if w.canWriteContinue.isSet() { + // Body reader wants to write 100 Continue but hasn't yet. + // Tell it not to. The store must be done while holding the lock + // because the lock makes sure that there is not an active write + // this very moment. + w.writeContinueMu.Lock() + w.canWriteContinue.setFalse() + w.writeContinueMu.Unlock() + } + + if !w.wroteHeader { + w.WriteHeader(StatusOK) + } + if lenData == 0 { + return 0, nil + } + if !w.bodyAllowed() { + return 0, ErrBodyNotAllowed + } + + w.written += int64(lenData) // ignoring errors, for errorKludge + if w.contentLength != -1 && w.written > w.contentLength { + return 0, ErrContentLength + } + if dataB != nil { + return w.w.Write(dataB) + } else { + return w.w.WriteString(dataS) + } +} + +func (w *response) finishRequest() { + w.handlerDone.setTrue() + + if !w.wroteHeader { + w.WriteHeader(StatusOK) + } + + w.w.Flush() + putBufioWriter(w.w) + w.cw.close() + w.conn.bufw.Flush() + + w.conn.r.abortPendingRead() + + // Close the body (regardless of w.closeAfterReply) so we can + // re-use its bufio.Reader later safely. + w.reqBody.Close() + + if w.req.MultipartForm != nil { + w.req.MultipartForm.RemoveAll() + } +} + +// shouldReuseConnection reports whether the underlying TCP connection can be reused. +// It must only be called after the handler is done executing. +func (w *response) shouldReuseConnection() bool { + if w.closeAfterReply { + // The request or something set while executing the + // handler indicated we shouldn't reuse this + // connection. + return false + } + + if w.req.Method != "HEAD" && w.contentLength != -1 && w.bodyAllowed() && w.contentLength != w.written { + // Did not write enough. Avoid getting out of sync. + return false + } + + // There was some error writing to the underlying connection + // during the request, so don't re-use this conn. + if w.conn.werr != nil { + return false + } + + if w.closedRequestBodyEarly() { + return false + } + + return true +} + +func (w *response) closedRequestBodyEarly() bool { + body, ok := w.req.Body.(*body) + return ok && body.didEarlyClose() +} + +func (w *response) Flush() { + if !w.wroteHeader { + w.WriteHeader(StatusOK) + } + w.w.Flush() + w.cw.flush() +} + +func (c *conn) finalFlush() { + if c.bufr != nil { + // Steal the bufio.Reader (~4KB worth of memory) and its associated + // reader for a future connection. + putBufioReader(c.bufr) + c.bufr = nil + } + + if c.bufw != nil { + c.bufw.Flush() + // Steal the bufio.Writer (~4KB worth of memory) and its associated + // writer for a future connection. + putBufioWriter(c.bufw) + c.bufw = nil + } +} + +// Close the connection. +func (c *conn) close() { + c.finalFlush() + c.rwc.Close() +} + +// rstAvoidanceDelay is the amount of time we sleep after closing the +// write side of a TCP connection before closing the entire socket. +// By sleeping, we increase the chances that the client sees our FIN +// and processes its final data before they process the subsequent RST +// from closing a connection with known unread data. +// This RST seems to occur mostly on BSD systems. (And Windows?) +// This timeout is somewhat arbitrary (~latency around the planet). +const rstAvoidanceDelay = 500 * time.Millisecond + +type closeWriter interface { + CloseWrite() error +} + +var _ closeWriter = (*net.TCPConn)(nil) + +// closeWrite flushes any outstanding data and sends a FIN packet (if +// client is connected via TCP), signaling that we're done. We then +// pause for a bit, hoping the client processes it before any +// subsequent RST. +// +// See https://golang.org/issue/3595 +func (c *conn) closeWriteAndWait() { + c.finalFlush() + if tcp, ok := c.rwc.(closeWriter); ok { + tcp.CloseWrite() + } + time.Sleep(rstAvoidanceDelay) +} + +// validNextProto reports whether the proto is a valid ALPN protocol name. +// Everything is valid except the empty string and built-in protocol types, +// so that those can't be overridden with alternate implementations. +func validNextProto(proto string) bool { + switch proto { + case "", "http/1.1", "http/1.0": + return false + } + return true +} + +const ( + runHooks = true + skipHooks = false +) + +func (c *conn) setState(nc net.Conn, state ConnState, runHook bool) { + srv := c.server + switch state { + case StateNew: + srv.trackConn(c, true) + case StateHijacked, StateClosed: + srv.trackConn(c, false) + } + if state > 0xff || state < 0 { + panic("internal error") + } + packedState := uint64(time.Now().Unix()<<8) | uint64(state) + atomic.StoreUint64(&c.curState.atomic, packedState) + if !runHook { + return + } + if hook := srv.ConnState; hook != nil { + hook(nc, state) + } +} + +func (c *conn) getState() (state ConnState, unixSec int64) { + packedState := atomic.LoadUint64(&c.curState.atomic) + return ConnState(packedState & 0xff), int64(packedState >> 8) +} + +// badRequestError is a literal string (used by in the server in HTML, +// unescaped) to tell the user why their request was bad. It should +// be plain text without user info or other embedded errors. +func badRequestError(e string) error { return statusError{StatusBadRequest, e} } + +// statusError is an error used to respond to a request with an HTTP status. +// The text should be plain text without user info or other embedded errors. +type statusError struct { + code int + text string +} + +func (e statusError) Error() string { return StatusText(e.code) + ": " + e.text } + +// ErrAbortHandler is a sentinel panic value to abort a handler. +// While any panic from ServeHTTP aborts the response to the client, +// panicking with ErrAbortHandler also suppresses logging of a stack +// trace to the server's error log. +var ErrAbortHandler = errors.New("net/http: abort Handler") + +// isCommonNetReadError reports whether err is a common error +// encountered during reading a request off the network when the +// client has gone away or had its read fail somehow. This is used to +// determine which logs are interesting enough to log about. +func isCommonNetReadError(err error) bool { + if err == io.EOF { + return true + } + if neterr, ok := err.(net.Error); ok && neterr.Timeout() { + return true + } + if oe, ok := err.(*net.OpError); ok && oe.Op == "read" { + return true + } + return false +} + +// Serve a new connection. +func (c *conn) serve(ctx context.Context) { + c.remoteAddr = c.rwc.RemoteAddr().String() + ctx = context.WithValue(ctx, LocalAddrContextKey, c.rwc.LocalAddr()) + var inFlightResponse *response + defer func() { + if err := recover(); err != nil && err != ErrAbortHandler { + const size = 64 << 10 + buf := make([]byte, size) + buf = buf[:runtime.Stack(buf, false)] + c.server.logf("http: panic serving %v: %v\n%s", c.remoteAddr, err, buf) + } + if inFlightResponse != nil { + inFlightResponse.cancelCtx() + } + if !c.hijacked() { + if inFlightResponse != nil { + inFlightResponse.conn.r.abortPendingRead() + inFlightResponse.reqBody.Close() + } + c.close() + c.setState(c.rwc, StateClosed, runHooks) + } + }() + + // TINYGO: Removed TLS conn check + + // HTTP/1.x from here on. + + ctx, cancelCtx := context.WithCancel(ctx) + c.cancelCtx = cancelCtx + defer cancelCtx() + + c.r = &connReader{conn: c} + c.bufr = newBufioReader(c.r) + c.bufw = newBufioWriterSize(checkConnErrorWriter{c}, 4<<10) + + for { + w, err := c.readRequest(ctx) + if c.r.remain != c.server.initialReadLimitSize() { + // If we read any bytes off the wire, we're active. + c.setState(c.rwc, StateActive, runHooks) + } + if err != nil { + const errorHeaders = "\r\nContent-Type: text/plain; charset=utf-8\r\nConnection: close\r\n\r\n" + + switch { + case err == errTooLarge: + // Their HTTP client may or may not be + // able to read this if we're + // responding to them and hanging up + // while they're still writing their + // request. Undefined behavior. + const publicErr = "431 Request Header Fields Too Large" + fmt.Fprintf(c.rwc, "HTTP/1.1 "+publicErr+errorHeaders+publicErr) + c.closeWriteAndWait() + return + + case isUnsupportedTEError(err): + // Respond as per RFC 7230 Section 3.3.1 which says, + // A server that receives a request message with a + // transfer coding it does not understand SHOULD + // respond with 501 (Unimplemented). + code := StatusNotImplemented + + // We purposefully aren't echoing back the transfer-encoding's value, + // so as to mitigate the risk of cross side scripting by an attacker. + fmt.Fprintf(c.rwc, "HTTP/1.1 %d %s%sUnsupported transfer encoding", code, StatusText(code), errorHeaders) + return + + case isCommonNetReadError(err): + return // don't reply + + default: + if v, ok := err.(statusError); ok { + fmt.Fprintf(c.rwc, "HTTP/1.1 %d %s: %s%s%d %s: %s", v.code, StatusText(v.code), v.text, errorHeaders, v.code, StatusText(v.code), v.text) + return + } + publicErr := "400 Bad Request" + fmt.Fprintf(c.rwc, "HTTP/1.1 "+publicErr+errorHeaders+publicErr) + return + } + } + + // Expect 100 Continue support + req := w.req + if req.expectsContinue() { + if req.ProtoAtLeast(1, 1) && req.ContentLength != 0 { + // Wrap the Body reader with one that replies on the connection + req.Body = &expectContinueReader{readCloser: req.Body, resp: w} + w.canWriteContinue.setTrue() + } + } else if req.Header.get("Expect") != "" { + w.sendExpectationFailed() + return + } + + c.curReq.Store(w) + + if requestBodyRemains(req.Body) { + registerOnHitEOF(req.Body, w.conn.r.startBackgroundRead) + } else { + w.conn.r.startBackgroundRead() + } + + // HTTP cannot have multiple simultaneous active requests.[*] + // Until the server replies to this request, it can't read another, + // so we might as well run the handler in this goroutine. + // [*] Not strictly true: HTTP pipelining. We could let them all process + // in parallel even if their responses need to be serialized. + // But we're not going to implement HTTP pipelining because it + // was never deployed in the wild and the answer is HTTP/2. + inFlightResponse = w + serverHandler{c.server}.ServeHTTP(w, w.req) + inFlightResponse = nil + w.cancelCtx() + if c.hijacked() { + return + } + w.finishRequest() + if !w.shouldReuseConnection() { + if w.requestBodyLimitHit || w.closedRequestBodyEarly() { + c.closeWriteAndWait() + } + return + } + c.setState(c.rwc, StateIdle, runHooks) + c.curReq.Store((*response)(nil)) + + if !w.conn.server.doKeepAlives() { + // We're in shutdown mode. We might've replied + // to the user without "Connection: close" and + // they might think they can send another + // request, but such is life with HTTP/1.1. + return + } + + if d := c.server.idleTimeout(); d != 0 { + c.rwc.SetReadDeadline(time.Now().Add(d)) + if _, err := c.bufr.Peek(4); err != nil { + return + } + } + c.rwc.SetReadDeadline(time.Time{}) + } +} + +func (w *response) sendExpectationFailed() { + // TODO(bradfitz): let ServeHTTP handlers handle + // requests with non-standard expectation[s]? Seems + // theoretical at best, and doesn't fit into the + // current ServeHTTP model anyway. We'd need to + // make the ResponseWriter an optional + // "ExpectReplier" interface or something. + // + // For now we'll just obey RFC 7231 5.1.1 which says + // "A server that receives an Expect field-value other + // than 100-continue MAY respond with a 417 (Expectation + // Failed) status code to indicate that the unexpected + // expectation cannot be met." + w.Header().Set("Connection", "close") + w.WriteHeader(StatusExpectationFailed) + w.finishRequest() +} + +// Hijack implements the Hijacker.Hijack method. Our response is both a ResponseWriter +// and a Hijacker. +func (w *response) Hijack() (rwc net.Conn, buf *bufio.ReadWriter, err error) { + if w.handlerDone.isSet() { + panic("net/http: Hijack called after ServeHTTP finished") + } + if w.wroteHeader { + w.cw.flush() + } + + c := w.conn + c.mu.Lock() + defer c.mu.Unlock() + + // Release the bufioWriter that writes to the chunk writer, it is not + // used after a connection has been hijacked. + rwc, buf, err = c.hijackLocked() + if err == nil { + putBufioWriter(w.w) + w.w = nil + } + return rwc, buf, err +} + +func (w *response) CloseNotify() <-chan bool { + if w.handlerDone.isSet() { + panic("net/http: CloseNotify called after ServeHTTP finished") + } + return w.closeNotifyCh +} + +func registerOnHitEOF(rc io.ReadCloser, fn func()) { + switch v := rc.(type) { + case *expectContinueReader: + registerOnHitEOF(v.readCloser, fn) + case *body: + v.registerOnHitEOF(fn) + default: + panic("unexpected type " + fmt.Sprintf("%T", rc)) + } +} + +// requestBodyRemains reports whether future calls to Read +// on rc might yield more data. +func requestBodyRemains(rc io.ReadCloser) bool { + if rc == NoBody { + return false + } + switch v := rc.(type) { + case *expectContinueReader: + return requestBodyRemains(v.readCloser) + case *body: + return v.bodyRemains() + default: + panic("unexpected type " + fmt.Sprintf("%T", rc)) + } +} + +// The HandlerFunc type is an adapter to allow the use of +// ordinary functions as HTTP handlers. If f is a function +// with the appropriate signature, HandlerFunc(f) is a +// Handler that calls f. +type HandlerFunc func(ResponseWriter, *Request) + +// ServeHTTP calls f(w, r). +func (f HandlerFunc) ServeHTTP(w ResponseWriter, r *Request) { + f(w, r) +} + +// Helper handlers + +// Error replies to the request with the specified error message and HTTP code. +// It does not otherwise end the request; the caller should ensure no further +// writes are done to w. +// The error message should be plain text. +func Error(w ResponseWriter, error string, code int) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(code) + fmt.Fprintln(w, error) +} + +// NotFound replies to the request with an HTTP 404 not found error. +func NotFound(w ResponseWriter, r *Request) { Error(w, "404 page not found", StatusNotFound) } + +// NotFoundHandler returns a simple request handler +// that replies to each request with a “404 page not found” reply. +func NotFoundHandler() Handler { return HandlerFunc(NotFound) } + +// StripPrefix returns a handler that serves HTTP requests by removing the +// given prefix from the request URL's Path (and RawPath if set) and invoking +// the handler h. StripPrefix handles a request for a path that doesn't begin +// with prefix by replying with an HTTP 404 not found error. The prefix must +// match exactly: if the prefix in the request contains escaped characters +// the reply is also an HTTP 404 not found error. +func StripPrefix(prefix string, h Handler) Handler { + if prefix == "" { + return h + } + return HandlerFunc(func(w ResponseWriter, r *Request) { + p := strings.TrimPrefix(r.URL.Path, prefix) + rp := strings.TrimPrefix(r.URL.RawPath, prefix) + if len(p) < len(r.URL.Path) && (r.URL.RawPath == "" || len(rp) < len(r.URL.RawPath)) { + r2 := new(Request) + *r2 = *r + r2.URL = new(url.URL) + *r2.URL = *r.URL + r2.URL.Path = p + r2.URL.RawPath = rp + h.ServeHTTP(w, r2) + } else { + NotFound(w, r) + } + }) +} + +// Redirect replies to the request with a redirect to url, +// which may be a path relative to the request path. +// +// The provided code should be in the 3xx range and is usually +// StatusMovedPermanently, StatusFound or StatusSeeOther. +// +// If the Content-Type header has not been set, Redirect sets it +// to "text/html; charset=utf-8" and writes a small HTML body. +// Setting the Content-Type header to any value, including nil, +// disables that behavior. +func Redirect(w ResponseWriter, r *Request, url string, code int) { + if u, err := urlpkg.Parse(url); err == nil { + // If url was relative, make its path absolute by + // combining with request path. + // The client would probably do this for us, + // but doing it ourselves is more reliable. + // See RFC 7231, section 7.1.2 + if u.Scheme == "" && u.Host == "" { + oldpath := r.URL.Path + if oldpath == "" { // should not happen, but avoid a crash if it does + oldpath = "/" + } + + // no leading http://server + if url == "" || url[0] != '/' { + // make relative path absolute + olddir, _ := path.Split(oldpath) + url = olddir + url + } + + var query string + if i := strings.Index(url, "?"); i != -1 { + url, query = url[:i], url[i:] + } + + // clean up but preserve trailing slash + trailing := strings.HasSuffix(url, "/") + url = path.Clean(url) + if trailing && !strings.HasSuffix(url, "/") { + url += "/" + } + url += query + } + } + + h := w.Header() + + // RFC 7231 notes that a short HTML body is usually included in + // the response because older user agents may not understand 301/307. + // Do it only if the request didn't already have a Content-Type header. + _, hadCT := h["Content-Type"] + + h.Set("Location", hexEscapeNonASCII(url)) + if !hadCT && (r.Method == "GET" || r.Method == "HEAD") { + h.Set("Content-Type", "text/html; charset=utf-8") + } + w.WriteHeader(code) + + // Shouldn't send the body for POST or HEAD; that leaves GET. + if !hadCT && r.Method == "GET" { + body := "" + StatusText(code) + ".\n" + fmt.Fprintln(w, body) + } +} + +var htmlReplacer = strings.NewReplacer( + "&", "&", + "<", "<", + ">", ">", + // """ is shorter than """. + `"`, """, + // "'" is shorter than "'" and apos was not in HTML until HTML5. + "'", "'", +) + +func htmlEscape(s string) string { + return htmlReplacer.Replace(s) +} + +// Redirect to a fixed URL +type redirectHandler struct { + url string + code int +} + +func (rh *redirectHandler) ServeHTTP(w ResponseWriter, r *Request) { + Redirect(w, r, rh.url, rh.code) +} + +// RedirectHandler returns a request handler that redirects +// each request it receives to the given url using the given +// status code. +// +// The provided code should be in the 3xx range and is usually +// StatusMovedPermanently, StatusFound or StatusSeeOther. +func RedirectHandler(url string, code int) Handler { + return &redirectHandler{url, code} +} + +// ServeMux is an HTTP request multiplexer. +// It matches the URL of each incoming request against a list of registered +// patterns and calls the handler for the pattern that +// most closely matches the URL. +// +// Patterns name fixed, rooted paths, like "/favicon.ico", +// or rooted subtrees, like "/images/" (note the trailing slash). +// Longer patterns take precedence over shorter ones, so that +// if there are handlers registered for both "/images/" +// and "/images/thumbnails/", the latter handler will be +// called for paths beginning "/images/thumbnails/" and the +// former will receive requests for any other paths in the +// "/images/" subtree. +// +// Note that since a pattern ending in a slash names a rooted subtree, +// the pattern "/" matches all paths not matched by other registered +// patterns, not just the URL with Path == "/". +// +// If a subtree has been registered and a request is received naming the +// subtree root without its trailing slash, ServeMux redirects that +// request to the subtree root (adding the trailing slash). This behavior can +// be overridden with a separate registration for the path without +// the trailing slash. For example, registering "/images/" causes ServeMux +// to redirect a request for "/images" to "/images/", unless "/images" has +// been registered separately. +// +// Patterns may optionally begin with a host name, restricting matches to +// URLs on that host only. Host-specific patterns take precedence over +// general patterns, so that a handler might register for the two patterns +// "/codesearch" and "codesearch.google.com/" without also taking over +// requests for "http://www.google.com/". +// +// ServeMux also takes care of sanitizing the URL request path and the Host +// header, stripping the port number and redirecting any request containing . or +// .. elements or repeated slashes to an equivalent, cleaner URL. +type ServeMux struct { + mu sync.RWMutex + m map[string]muxEntry + es []muxEntry // slice of entries sorted from longest to shortest. + hosts bool // whether any patterns contain hostnames +} + +type muxEntry struct { + h Handler + pattern string +} + +// NewServeMux allocates and returns a new ServeMux. +func NewServeMux() *ServeMux { return new(ServeMux) } + +// DefaultServeMux is the default ServeMux used by Serve. +var DefaultServeMux = &defaultServeMux + +var defaultServeMux ServeMux + +// cleanPath returns the canonical path for p, eliminating . and .. elements. +func cleanPath(p string) string { + if p == "" { + return "/" + } + if p[0] != '/' { + p = "/" + p + } + np := path.Clean(p) + // path.Clean removes trailing slash except for root; + // put the trailing slash back if necessary. + if p[len(p)-1] == '/' && np != "/" { + // Fast path for common case of p being the string we want: + if len(p) == len(np)+1 && strings.HasPrefix(p, np) { + np = p + } else { + np += "/" + } + } + return np +} + +// stripHostPort returns h without any trailing ":". +func stripHostPort(h string) string { + // If no port on host, return unchanged + if !strings.Contains(h, ":") { + return h + } + host, _, err := net.SplitHostPort(h) + if err != nil { + return h // on error, return unchanged + } + return host +} + +// Find a handler on a handler map given a path string. +// Most-specific (longest) pattern wins. +func (mux *ServeMux) match(path string) (h Handler, pattern string) { + // Check for exact match first. + v, ok := mux.m[path] + if ok { + return v.h, v.pattern + } + + // Check for longest valid match. mux.es contains all patterns + // that end in / sorted from longest to shortest. + for _, e := range mux.es { + if strings.HasPrefix(path, e.pattern) { + return e.h, e.pattern + } + } + return nil, "" +} + +// redirectToPathSlash determines if the given path needs appending "/" to it. +// This occurs when a handler for path + "/" was already registered, but +// not for path itself. If the path needs appending to, it creates a new +// URL, setting the path to u.Path + "/" and returning true to indicate so. +func (mux *ServeMux) redirectToPathSlash(host, path string, u *url.URL) (*url.URL, bool) { + mux.mu.RLock() + shouldRedirect := mux.shouldRedirectRLocked(host, path) + mux.mu.RUnlock() + if !shouldRedirect { + return u, false + } + path = path + "/" + u = &url.URL{Path: path, RawQuery: u.RawQuery} + return u, true +} + +// shouldRedirectRLocked reports whether the given path and host should be redirected to +// path+"/". This should happen if a handler is registered for path+"/" but +// not path -- see comments at ServeMux. +func (mux *ServeMux) shouldRedirectRLocked(host, path string) bool { + p := []string{path, host + path} + + for _, c := range p { + if _, exist := mux.m[c]; exist { + return false + } + } + + n := len(path) + if n == 0 { + return false + } + for _, c := range p { + if _, exist := mux.m[c+"/"]; exist { + return path[n-1] != '/' + } + } + + return false +} + +// Handler returns the handler to use for the given request, +// consulting r.Method, r.Host, and r.URL.Path. It always returns +// a non-nil handler. If the path is not in its canonical form, the +// handler will be an internally-generated handler that redirects +// to the canonical path. If the host contains a port, it is ignored +// when matching handlers. +// +// The path and host are used unchanged for CONNECT requests. +// +// Handler also returns the registered pattern that matches the +// request or, in the case of internally-generated redirects, +// the pattern that will match after following the redirect. +// +// If there is no registered handler that applies to the request, +// Handler returns a “page not found” handler and an empty pattern. +func (mux *ServeMux) Handler(r *Request) (h Handler, pattern string) { + + // CONNECT requests are not canonicalized. + if r.Method == "CONNECT" { + // If r.URL.Path is /tree and its handler is not registered, + // the /tree -> /tree/ redirect applies to CONNECT requests + // but the path canonicalization does not. + if u, ok := mux.redirectToPathSlash(r.URL.Host, r.URL.Path, r.URL); ok { + return RedirectHandler(u.String(), StatusMovedPermanently), u.Path + } + + return mux.handler(r.Host, r.URL.Path) + } + + // All other requests have any port stripped and path cleaned + // before passing to mux.handler. + host := stripHostPort(r.Host) + path := cleanPath(r.URL.Path) + + // If the given path is /tree and its handler is not registered, + // redirect for /tree/. + if u, ok := mux.redirectToPathSlash(host, path, r.URL); ok { + return RedirectHandler(u.String(), StatusMovedPermanently), u.Path + } + + if path != r.URL.Path { + _, pattern = mux.handler(host, path) + u := &url.URL{Path: path, RawQuery: r.URL.RawQuery} + return RedirectHandler(u.String(), StatusMovedPermanently), pattern + } + + return mux.handler(host, r.URL.Path) +} + +// handler is the main implementation of Handler. +// The path is known to be in canonical form, except for CONNECT methods. +func (mux *ServeMux) handler(host, path string) (h Handler, pattern string) { + mux.mu.RLock() + defer mux.mu.RUnlock() + + // Host-specific pattern takes precedence over generic ones + if mux.hosts { + h, pattern = mux.match(host + path) + } + if h == nil { + h, pattern = mux.match(path) + } + if h == nil { + h, pattern = NotFoundHandler(), "" + } + return +} + +// ServeHTTP dispatches the request to the handler whose +// pattern most closely matches the request URL. +func (mux *ServeMux) ServeHTTP(w ResponseWriter, r *Request) { + if r.RequestURI == "*" { + if r.ProtoAtLeast(1, 1) { + w.Header().Set("Connection", "close") + } + w.WriteHeader(StatusBadRequest) + return + } + h, _ := mux.Handler(r) + h.ServeHTTP(w, r) +} + +// Handle registers the handler for the given pattern. +// If a handler already exists for pattern, Handle panics. +func (mux *ServeMux) Handle(pattern string, handler Handler) { + mux.mu.Lock() + defer mux.mu.Unlock() + + if pattern == "" { + panic("http: invalid pattern") + } + if handler == nil { + panic("http: nil handler") + } + if _, exist := mux.m[pattern]; exist { + panic("http: multiple registrations for " + pattern) + } + + if mux.m == nil { + mux.m = make(map[string]muxEntry) + } + e := muxEntry{h: handler, pattern: pattern} + mux.m[pattern] = e + if pattern[len(pattern)-1] == '/' { + mux.es = appendSorted(mux.es, e) + } + + if pattern[0] != '/' { + mux.hosts = true + } +} + +func appendSorted(es []muxEntry, e muxEntry) []muxEntry { + n := len(es) + i := sort.Search(n, func(i int) bool { + return len(es[i].pattern) < len(e.pattern) + }) + if i == n { + return append(es, e) + } + // we now know that i points at where we want to insert + es = append(es, muxEntry{}) // try to grow the slice in place, any entry works. + copy(es[i+1:], es[i:]) // Move shorter entries down + es[i] = e + return es +} + +// HandleFunc registers the handler function for the given pattern. +func (mux *ServeMux) HandleFunc(pattern string, handler func(ResponseWriter, *Request)) { + if handler == nil { + panic("http: nil handler") + } + mux.Handle(pattern, HandlerFunc(handler)) +} + +// Handle registers the handler for the given pattern +// in the DefaultServeMux. +// The documentation for ServeMux explains how patterns are matched. +func Handle(pattern string, handler Handler) { DefaultServeMux.Handle(pattern, handler) } + +// HandleFunc registers the handler function for the given pattern +// in the DefaultServeMux. +// The documentation for ServeMux explains how patterns are matched. +func HandleFunc(pattern string, handler func(ResponseWriter, *Request)) { + DefaultServeMux.HandleFunc(pattern, handler) +} + +// Serve accepts incoming HTTP connections on the listener l, +// creating a new service goroutine for each. The service goroutines +// read requests and then call handler to reply to them. +// +// The handler is typically nil, in which case the DefaultServeMux is used. +// +// HTTP/2 support is only enabled if the Listener returns *tls.Conn +// connections and they were configured with "h2" in the TLS +// Config.NextProtos. +// +// Serve always returns a non-nil error. +func Serve(l net.Listener, handler Handler) error { + srv := &Server{Handler: handler} + return srv.Serve(l) +} + +// A Server defines parameters for running an HTTP server. +// The zero value for Server is a valid configuration. +type Server struct { + // Addr optionally specifies the TCP address for the server to listen on, + // in the form "host:port". If empty, ":http" (port 80) is used. + // The service names are defined in RFC 6335 and assigned by IANA. + // See net.Dial for details of the address format. + Addr string + + Handler Handler // handler to invoke, http.DefaultServeMux if nil + + // TLSConfig optionally provides a TLS configuration for use + // by ServeTLS and ListenAndServeTLS. Note that this value is + // cloned by ServeTLS and ListenAndServeTLS, so it's not + // possible to modify the configuration with methods like + // tls.Config.SetSessionTicketKeys. To use + // SetSessionTicketKeys, use Server.Serve with a TLS Listener + // instead. + TLSConfig *tls.Config + + // ReadTimeout is the maximum duration for reading the entire + // request, including the body. A zero or negative value means + // there will be no timeout. + // + // Because ReadTimeout does not let Handlers make per-request + // decisions on each request body's acceptable deadline or + // upload rate, most users will prefer to use + // ReadHeaderTimeout. It is valid to use them both. + ReadTimeout time.Duration + + // ReadHeaderTimeout is the amount of time allowed to read + // request headers. The connection's read deadline is reset + // after reading the headers and the Handler can decide what + // is considered too slow for the body. If ReadHeaderTimeout + // is zero, the value of ReadTimeout is used. If both are + // zero, there is no timeout. + ReadHeaderTimeout time.Duration + + // WriteTimeout is the maximum duration before timing out + // writes of the response. It is reset whenever a new + // request's header is read. Like ReadTimeout, it does not + // let Handlers make decisions on a per-request basis. + // A zero or negative value means there will be no timeout. + WriteTimeout time.Duration + + // IdleTimeout is the maximum amount of time to wait for the + // next request when keep-alives are enabled. If IdleTimeout + // is zero, the value of ReadTimeout is used. If both are + // zero, there is no timeout. + IdleTimeout time.Duration + + // MaxHeaderBytes controls the maximum number of bytes the + // server will read parsing the request header's keys and + // values, including the request line. It does not limit the + // size of the request body. + // If zero, DefaultMaxHeaderBytes is used. + MaxHeaderBytes int + + // ConnState specifies an optional callback function that is + // called when a client connection changes state. See the + // ConnState type and associated constants for details. + ConnState func(net.Conn, ConnState) + + // ErrorLog specifies an optional logger for errors accepting + // connections, unexpected behavior from handlers, and + // underlying FileSystem errors. + // If nil, logging is done via the log package's standard logger. + ErrorLog *log.Logger + + // BaseContext optionally specifies a function that returns + // the base context for incoming requests on this server. + // The provided Listener is the specific Listener that's + // about to start accepting requests. + // If BaseContext is nil, the default is context.Background(). + // If non-nil, it must return a non-nil context. + BaseContext func(net.Listener) context.Context + + // ConnContext optionally specifies a function that modifies + // the context used for a new connection c. The provided ctx + // is derived from the base context and has a ServerContextKey + // value. + ConnContext func(ctx context.Context, c net.Conn) context.Context + + inShutdown atomicBool // true when server is in shutdown + + disableKeepAlives int32 // accessed atomically. + nextProtoOnce sync.Once // guards setupHTTP2_* init + nextProtoErr error // result of http2.ConfigureServer if used + + mu sync.Mutex + listeners map[*net.Listener]struct{} + activeConn map[*conn]struct{} + doneChan chan struct{} + onShutdown []func() + + listenerGroup sync.WaitGroup +} + +func (s *Server) getDoneChan() <-chan struct{} { + s.mu.Lock() + defer s.mu.Unlock() + return s.getDoneChanLocked() +} + +func (s *Server) getDoneChanLocked() chan struct{} { + if s.doneChan == nil { + s.doneChan = make(chan struct{}) + } + return s.doneChan +} + +func (s *Server) closeDoneChanLocked() { + ch := s.getDoneChanLocked() + select { + case <-ch: + // Already closed. Don't close again. + default: + // Safe to close here. We're the only closer, guarded + // by s.mu. + close(ch) + } +} + +// Close immediately closes all active net.Listeners and any +// connections in state StateNew, StateActive, or StateIdle. For a +// graceful shutdown, use Shutdown. +// +// Close does not attempt to close (and does not even know about) +// any hijacked connections, such as WebSockets. +// +// Close returns any error returned from closing the Server's +// underlying Listener(s). +func (srv *Server) Close() error { + srv.inShutdown.setTrue() + srv.mu.Lock() + defer srv.mu.Unlock() + srv.closeDoneChanLocked() + err := srv.closeListenersLocked() + + // Unlock srv.mu while waiting for listenerGroup. + // The group Add and Done calls are made with srv.mu held, + // to avoid adding a new listener in the window between + // us setting inShutdown above and waiting here. + srv.mu.Unlock() + srv.listenerGroup.Wait() + srv.mu.Lock() + + for c := range srv.activeConn { + c.rwc.Close() + delete(srv.activeConn, c) + } + return err +} + +// shutdownPollIntervalMax is the max polling interval when checking +// quiescence during Server.Shutdown. Polling starts with a small +// interval and backs off to the max. +// Ideally we could find a solution that doesn't involve polling, +// but which also doesn't have a high runtime cost (and doesn't +// involve any contentious mutexes), but that is left as an +// exercise for the reader. +const shutdownPollIntervalMax = 500 * time.Millisecond + +// Shutdown gracefully shuts down the server without interrupting any +// active connections. Shutdown works by first closing all open +// listeners, then closing all idle connections, and then waiting +// indefinitely for connections to return to idle and then shut down. +// If the provided context expires before the shutdown is complete, +// Shutdown returns the context's error, otherwise it returns any +// error returned from closing the Server's underlying Listener(s). +// +// When Shutdown is called, Serve, ListenAndServe, and +// ListenAndServeTLS immediately return ErrServerClosed. Make sure the +// program doesn't exit and waits instead for Shutdown to return. +// +// Shutdown does not attempt to close nor wait for hijacked +// connections such as WebSockets. The caller of Shutdown should +// separately notify such long-lived connections of shutdown and wait +// for them to close, if desired. See RegisterOnShutdown for a way to +// register shutdown notification functions. +// +// Once Shutdown has been called on a server, it may not be reused; +// future calls to methods such as Serve will return ErrServerClosed. +func (srv *Server) Shutdown(ctx context.Context) error { + srv.inShutdown.setTrue() + + srv.mu.Lock() + lnerr := srv.closeListenersLocked() + srv.closeDoneChanLocked() + for _, f := range srv.onShutdown { + go f() + } + srv.mu.Unlock() + srv.listenerGroup.Wait() + + pollIntervalBase := time.Millisecond + nextPollInterval := func() time.Duration { + // Add 10% jitter. + interval := pollIntervalBase + time.Duration(rand.Intn(int(pollIntervalBase/10))) + // Double and clamp for next time. + pollIntervalBase *= 2 + if pollIntervalBase > shutdownPollIntervalMax { + pollIntervalBase = shutdownPollIntervalMax + } + return interval + } + + timer := time.NewTimer(nextPollInterval()) + defer timer.Stop() + for { + if srv.closeIdleConns() { + return lnerr + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + timer.Reset(nextPollInterval()) + } + } +} + +// RegisterOnShutdown registers a function to call on Shutdown. +// This can be used to gracefully shutdown connections that have +// undergone ALPN protocol upgrade or that have been hijacked. +// This function should start protocol-specific graceful shutdown, +// but should not wait for shutdown to complete. +func (srv *Server) RegisterOnShutdown(f func()) { + srv.mu.Lock() + srv.onShutdown = append(srv.onShutdown, f) + srv.mu.Unlock() +} + +// closeIdleConns closes all idle connections and reports whether the +// server is quiescent. +func (s *Server) closeIdleConns() bool { + s.mu.Lock() + defer s.mu.Unlock() + quiescent := true + for c := range s.activeConn { + st, unixSec := c.getState() + // Issue 22682: treat StateNew connections as if + // they're idle if we haven't read the first request's + // header in over 5 seconds. + if st == StateNew && unixSec < time.Now().Unix()-5 { + st = StateIdle + } + if st != StateIdle || unixSec == 0 { + // Assume unixSec == 0 means it's a very new + // connection, without state set yet. + quiescent = false + continue + } + c.rwc.Close() + delete(s.activeConn, c) + } + return quiescent +} + +func (s *Server) closeListenersLocked() error { + var err error + for ln := range s.listeners { + if cerr := (*ln).Close(); cerr != nil && err == nil { + err = cerr + } + } + return err +} + +// A ConnState represents the state of a client connection to a server. +// It's used by the optional Server.ConnState hook. +type ConnState int + +const ( + // StateNew represents a new connection that is expected to + // send a request immediately. Connections begin at this + // state and then transition to either StateActive or + // StateClosed. + StateNew ConnState = iota + + // StateActive represents a connection that has read 1 or more + // bytes of a request. The Server.ConnState hook for + // StateActive fires before the request has entered a handler + // and doesn't fire again until the request has been + // handled. After the request is handled, the state + // transitions to StateClosed, StateHijacked, or StateIdle. + // For HTTP/2, StateActive fires on the transition from zero + // to one active request, and only transitions away once all + // active requests are complete. That means that ConnState + // cannot be used to do per-request work; ConnState only notes + // the overall state of the connection. + StateActive + + // StateIdle represents a connection that has finished + // handling a request and is in the keep-alive state, waiting + // for a new request. Connections transition from StateIdle + // to either StateActive or StateClosed. + StateIdle + + // StateHijacked represents a hijacked connection. + // This is a terminal state. It does not transition to StateClosed. + StateHijacked + + // StateClosed represents a closed connection. + // This is a terminal state. Hijacked connections do not + // transition to StateClosed. + StateClosed +) + +var stateName = map[ConnState]string{ + StateNew: "new", + StateActive: "active", + StateIdle: "idle", + StateHijacked: "hijacked", + StateClosed: "closed", +} + +func (c ConnState) String() string { + return stateName[c] +} + +// serverHandler delegates to either the server's Handler or +// DefaultServeMux and also handles "OPTIONS *" requests. +type serverHandler struct { + srv *Server +} + +func (sh serverHandler) ServeHTTP(rw ResponseWriter, req *Request) { + handler := sh.srv.Handler + if handler == nil { + handler = DefaultServeMux + } + if req.RequestURI == "*" && req.Method == "OPTIONS" { + handler = globalOptionsHandler{} + } + + if req.URL != nil && strings.Contains(req.URL.RawQuery, ";") { + var allowQuerySemicolonsInUse int32 + req = req.WithContext(context.WithValue(req.Context(), silenceSemWarnContextKey, func() { + atomic.StoreInt32(&allowQuerySemicolonsInUse, 1) + })) + defer func() { + if atomic.LoadInt32(&allowQuerySemicolonsInUse) == 0 { + sh.srv.logf("http: URL query contains semicolon, which is no longer a supported separator; parts of the query may be stripped when parsed; see golang.org/issue/25192") + } + }() + } + + handler.ServeHTTP(rw, req) +} + +var silenceSemWarnContextKey = &contextKey{"silence-semicolons"} + +// AllowQuerySemicolons returns a handler that serves requests by converting any +// unescaped semicolons in the URL query to ampersands, and invoking the handler h. +// +// This restores the pre-Go 1.17 behavior of splitting query parameters on both +// semicolons and ampersands. (See golang.org/issue/25192). Note that this +// behavior doesn't match that of many proxies, and the mismatch can lead to +// security issues. +// +// AllowQuerySemicolons should be invoked before Request.ParseForm is called. +func AllowQuerySemicolons(h Handler) Handler { + return HandlerFunc(func(w ResponseWriter, r *Request) { + if silenceSemicolonsWarning, ok := r.Context().Value(silenceSemWarnContextKey).(func()); ok { + silenceSemicolonsWarning() + } + if strings.Contains(r.URL.RawQuery, ";") { + r2 := new(Request) + *r2 = *r + r2.URL = new(url.URL) + *r2.URL = *r.URL + r2.URL.RawQuery = strings.ReplaceAll(r.URL.RawQuery, ";", "&") + h.ServeHTTP(w, r2) + } else { + h.ServeHTTP(w, r) + } + }) +} + +// ListenAndServe listens on the TCP network address srv.Addr and then +// calls Serve to handle requests on incoming connections. +// Accepted connections are configured to enable TCP keep-alives. +// +// If srv.Addr is blank, ":http" is used. +// +// ListenAndServe always returns a non-nil error. After Shutdown or Close, +// the returned error is ErrServerClosed. +func (srv *Server) ListenAndServe() error { + if srv.shuttingDown() { + return ErrServerClosed + } + addr := srv.Addr + if addr == "" { + addr = ":http" + } + ln, err := net.Listen("tcp", addr) + if err != nil { + return err + } + return srv.Serve(ln) +} + +var testHookServerServe func(*Server, net.Listener) // used if non-nil + +// ErrServerClosed is returned by the Server's Serve, ServeTLS, ListenAndServe, +// and ListenAndServeTLS methods after a call to Shutdown or Close. +var ErrServerClosed = errors.New("http: Server closed") + +// Serve accepts incoming connections on the Listener l, creating a +// new service goroutine for each. The service goroutines read requests and +// then call srv.Handler to reply to them. +// +// HTTP/2 support is only enabled if the Listener returns *tls.Conn +// connections and they were configured with "h2" in the TLS +// Config.NextProtos. +// +// Serve always returns a non-nil error and closes l. +// After Shutdown or Close, the returned error is ErrServerClosed. +func (srv *Server) Serve(l net.Listener) error { + if fn := testHookServerServe; fn != nil { + fn(srv, l) // call hook with unwrapped listener + } + + origListener := l + l = &onceCloseListener{Listener: l} + defer l.Close() + + if !srv.trackListener(&l, true) { + return ErrServerClosed + } + defer srv.trackListener(&l, false) + + baseCtx := context.Background() + if srv.BaseContext != nil { + baseCtx = srv.BaseContext(origListener) + if baseCtx == nil { + panic("BaseContext returned a nil context") + } + } + + var tempDelay time.Duration // how long to sleep on accept failure + + ctx := context.WithValue(baseCtx, ServerContextKey, srv) + for { + rw, err := l.Accept() + if err != nil { + select { + case <-srv.getDoneChan(): + return ErrServerClosed + default: + } + if ne, ok := err.(net.Error); ok && ne.Temporary() { + if tempDelay == 0 { + tempDelay = 5 * time.Millisecond + } else { + tempDelay *= 2 + } + if max := 1 * time.Second; tempDelay > max { + tempDelay = max + } + srv.logf("http: Accept error: %v; retrying in %v", err, tempDelay) + time.Sleep(tempDelay) + continue + } + return err + } + connCtx := ctx + if cc := srv.ConnContext; cc != nil { + connCtx = cc(connCtx, rw) + if connCtx == nil { + panic("ConnContext returned nil") + } + } + tempDelay = 0 + c := srv.newConn(rw) + c.setState(c.rwc, StateNew, runHooks) // before Serve can return + go c.serve(connCtx) + } +} + +// trackListener adds or removes a net.Listener to the set of tracked +// listeners. +// +// We store a pointer to interface in the map set, in case the +// net.Listener is not comparable. This is safe because we only call +// trackListener via Serve and can track+defer untrack the same +// pointer to local variable there. We never need to compare a +// Listener from another caller. +// +// It reports whether the server is still up (not Shutdown or Closed). +func (s *Server) trackListener(ln *net.Listener, add bool) bool { + s.mu.Lock() + defer s.mu.Unlock() + if s.listeners == nil { + s.listeners = make(map[*net.Listener]struct{}) + } + if add { + if s.shuttingDown() { + return false + } + s.listeners[ln] = struct{}{} + s.listenerGroup.Add(1) + } else { + delete(s.listeners, ln) + s.listenerGroup.Done() + } + return true +} + +func (s *Server) trackConn(c *conn, add bool) { + s.mu.Lock() + defer s.mu.Unlock() + if s.activeConn == nil { + s.activeConn = make(map[*conn]struct{}) + } + if add { + s.activeConn[c] = struct{}{} + } else { + delete(s.activeConn, c) + } +} + +func (s *Server) idleTimeout() time.Duration { + if s.IdleTimeout != 0 { + return s.IdleTimeout + } + return s.ReadTimeout +} + +func (s *Server) readHeaderTimeout() time.Duration { + if s.ReadHeaderTimeout != 0 { + return s.ReadHeaderTimeout + } + return s.ReadTimeout +} + +func (s *Server) doKeepAlives() bool { + return atomic.LoadInt32(&s.disableKeepAlives) == 0 && !s.shuttingDown() +} + +func (s *Server) shuttingDown() bool { + return s.inShutdown.isSet() +} + +// SetKeepAlivesEnabled controls whether HTTP keep-alives are enabled. +// By default, keep-alives are always enabled. Only very +// resource-constrained environments or servers in the process of +// shutting down should disable them. +func (srv *Server) SetKeepAlivesEnabled(v bool) { + if v { + atomic.StoreInt32(&srv.disableKeepAlives, 0) + return + } + atomic.StoreInt32(&srv.disableKeepAlives, 1) + + // Close idle HTTP/1 conns: + srv.closeIdleConns() + + // TODO: Issue 26303: close HTTP/2 conns as soon as they become idle. +} + +func (s *Server) logf(format string, args ...any) { + if s.ErrorLog != nil { + s.ErrorLog.Printf(format, args...) + } else { + log.Printf(format, args...) + } +} + +// logf prints to the ErrorLog of the *Server associated with request r +// via ServerContextKey. If there's no associated server, or if ErrorLog +// is nil, logging is done via the log package's standard logger. +func logf(r *Request, format string, args ...any) { + s, _ := r.Context().Value(ServerContextKey).(*Server) + if s != nil && s.ErrorLog != nil { + s.ErrorLog.Printf(format, args...) + } else { + log.Printf(format, args...) + } +} + +// ListenAndServe listens on the TCP network address addr and then calls +// Serve with handler to handle requests on incoming connections. +// Accepted connections are configured to enable TCP keep-alives. +// +// The handler is typically nil, in which case the DefaultServeMux is used. +// +// ListenAndServe always returns a non-nil error. +func ListenAndServe(addr string, handler Handler) error { + server := &Server{Addr: addr, Handler: handler} + return server.ListenAndServe() +} + +// onceCloseListener wraps a net.Listener, protecting it from +// multiple Close calls. +type onceCloseListener struct { + net.Listener + once sync.Once + closeErr error +} + +func (oc *onceCloseListener) Close() error { + oc.once.Do(oc.close) + return oc.closeErr +} + +func (oc *onceCloseListener) close() { oc.closeErr = oc.Listener.Close() } + +// globalOptionsHandler responds to "OPTIONS *" requests. +type globalOptionsHandler struct{} + +func (globalOptionsHandler) ServeHTTP(w ResponseWriter, r *Request) { + w.Header().Set("Content-Length", "0") + if r.ContentLength != 0 { + // Read up to 4KB of OPTIONS body (as mentioned in the + // spec as being reserved for future use), but anything + // over that is considered a waste of server resources + // (or an attack) and we abort and close the connection, + // courtesy of MaxBytesReader's EOF behavior. + mb := MaxBytesReader(w, r.Body, 4<<10) + io.Copy(io.Discard, mb) + } +} + +// loggingConn is used for debugging. +type loggingConn struct { + name string + net.Conn +} + +var ( + uniqNameMu sync.Mutex + uniqNameNext = make(map[string]int) +) + +func newLoggingConn(baseName string, c net.Conn) net.Conn { + uniqNameMu.Lock() + defer uniqNameMu.Unlock() + uniqNameNext[baseName]++ + return &loggingConn{ + name: fmt.Sprintf("%s-%d", baseName, uniqNameNext[baseName]), + Conn: c, + } +} + +func (c *loggingConn) Write(p []byte) (n int, err error) { + log.Printf("%s.Write(%d) = ....", c.name, len(p)) + n, err = c.Conn.Write(p) + log.Printf("%s.Write(%d) = %d, %v", c.name, len(p), n, err) + return +} + +func (c *loggingConn) Read(p []byte) (n int, err error) { + log.Printf("%s.Read(%d) = ....", c.name, len(p)) + n, err = c.Conn.Read(p) + log.Printf("%s.Read(%d) = %d, %v", c.name, len(p), n, err) + return +} + +func (c *loggingConn) Close() (err error) { + log.Printf("%s.Close() = ...", c.name) + err = c.Conn.Close() + log.Printf("%s.Close() = %v", c.name, err) + return +} + +// checkConnErrorWriter writes to c.rwc and records any write errors to c.werr. +// It only contains one field (and a pointer field at that), so it +// fits in an interface value without an extra allocation. +type checkConnErrorWriter struct { + c *conn +} + +func (w checkConnErrorWriter) Write(p []byte) (n int, err error) { + n, err = w.c.rwc.Write(p) + if err != nil && w.c.werr == nil { + w.c.werr = err + w.c.cancelCtx() + } + return +} + +func numLeadingCRorLF(v []byte) (n int) { + for _, b := range v { + if b == '\r' || b == '\n' { + n++ + continue + } + break + } + return + +} + +func strSliceContains(ss []string, s string) bool { + for _, v := range ss { + if v == s { + return true + } + } + return false +} + +// tlsRecordHeaderLooksLikeHTTP reports whether a TLS record header +// looks like it might've been a misdirected plaintext HTTP request. +func tlsRecordHeaderLooksLikeHTTP(hdr [5]byte) bool { + switch string(hdr[:]) { + case "GET /", "HEAD ", "POST ", "PUT /", "OPTIO": + return true + } + return false +} + +// MaxBytesHandler returns a Handler that runs h with its ResponseWriter and Request.Body wrapped by a MaxBytesReader. +func MaxBytesHandler(h Handler, n int64) Handler { + return HandlerFunc(func(w ResponseWriter, r *Request) { + r2 := *r + r2.Body = MaxBytesReader(w, r.Body, n) + h.ServeHTTP(w, &r2) + }) +} diff --git a/src/net/http/sniff.go b/src/net/http/sniff.go new file mode 100644 index 0000000000..3fee91284b --- /dev/null +++ b/src/net/http/sniff.go @@ -0,0 +1,306 @@ +// TINYGO: The following is copied from Go 1.19.3 official implementation. + +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "bytes" + "encoding/binary" +) + +// The algorithm uses at most sniffLen bytes to make its decision. +const sniffLen = 512 + +// DetectContentType implements the algorithm described +// at https://mimesniff.spec.whatwg.org/ to determine the +// Content-Type of the given data. It considers at most the +// first 512 bytes of data. DetectContentType always returns +// a valid MIME type: if it cannot determine a more specific one, it +// returns "application/octet-stream". +func DetectContentType(data []byte) string { + if len(data) > sniffLen { + data = data[:sniffLen] + } + + // Index of the first non-whitespace byte in data. + firstNonWS := 0 + for ; firstNonWS < len(data) && isWS(data[firstNonWS]); firstNonWS++ { + } + + for _, sig := range sniffSignatures { + if ct := sig.match(data, firstNonWS); ct != "" { + return ct + } + } + + return "application/octet-stream" // fallback +} + +// isWS reports whether the provided byte is a whitespace byte (0xWS) +// as defined in https://mimesniff.spec.whatwg.org/#terminology. +func isWS(b byte) bool { + switch b { + case '\t', '\n', '\x0c', '\r', ' ': + return true + } + return false +} + +// isTT reports whether the provided byte is a tag-terminating byte (0xTT) +// as defined in https://mimesniff.spec.whatwg.org/#terminology. +func isTT(b byte) bool { + switch b { + case ' ', '>': + return true + } + return false +} + +type sniffSig interface { + // match returns the MIME type of the data, or "" if unknown. + match(data []byte, firstNonWS int) string +} + +// Data matching the table in section 6. +var sniffSignatures = []sniffSig{ + htmlSig("= 0 || t.Body == nil { // redundant checks; caller did them + return false + } + if t.Method == "CONNECT" { + return false + } + if requestMethodUsuallyLacksBody(t.Method) { + // Only probe the Request.Body for GET/HEAD/DELETE/etc + // requests, because it's only those types of requests + // that confuse servers. + t.probeRequestBody() // adjusts t.Body, t.ContentLength + return t.Body != nil + } + // For all other request types (PUT, POST, PATCH, or anything + // made-up we've never heard of), assume it's normal and the server + // can deal with a chunked request body. Maybe we'll adjust this + // later. + return true +} + +// probeRequestBody reads a byte from t.Body to see whether it's empty +// (returns io.EOF right away). +// +// But because we've had problems with this blocking users in the past +// (issue 17480) when the body is a pipe (perhaps waiting on the response +// headers before the pipe is fed data), we need to be careful and bound how +// long we wait for it. This delay will only affect users if all the following +// are true: +// - the request body blocks +// - the content length is not set (or set to -1) +// - the method doesn't usually have a body (GET, HEAD, DELETE, ...) +// - there is no transfer-encoding=chunked already set. +// +// In other words, this delay will not normally affect anybody, and there +// are workarounds if it does. +func (t *transferWriter) probeRequestBody() { + t.ByteReadCh = make(chan readResult, 1) + go func(body io.Reader) { + var buf [1]byte + var rres readResult + rres.n, rres.err = body.Read(buf[:]) + if rres.n == 1 { + rres.b = buf[0] + } + t.ByteReadCh <- rres + close(t.ByteReadCh) + }(t.Body) + timer := time.NewTimer(200 * time.Millisecond) + select { + case rres := <-t.ByteReadCh: + timer.Stop() + if rres.n == 0 && rres.err == io.EOF { + // It was empty. + t.Body = nil + t.ContentLength = 0 + } else if rres.n == 1 { + if rres.err != nil { + t.Body = io.MultiReader(&byteReader{b: rres.b}, errorReader{rres.err}) + } else { + t.Body = io.MultiReader(&byteReader{b: rres.b}, t.Body) + } + } else if rres.err != nil { + t.Body = errorReader{rres.err} + } + case <-timer.C: + // Too slow. Don't wait. Read it later, and keep + // assuming that this is ContentLength == -1 + // (unknown), which means we'll send a + // "Transfer-Encoding: chunked" header. + t.Body = io.MultiReader(finishAsyncByteRead{t}, t.Body) + // Request that Request.Write flush the headers to the + // network before writing the body, since our body may not + // become readable until it's seen the response headers. + t.FlushHeaders = true + } +} + +func noResponseBodyExpected(requestMethod string) bool { + return requestMethod == "HEAD" +} + +func (t *transferWriter) shouldSendContentLength() bool { + if chunked(t.TransferEncoding) { + return false + } + if t.ContentLength > 0 { + return true + } + if t.ContentLength < 0 { + return false + } + // Many servers expect a Content-Length for these methods + if t.Method == "POST" || t.Method == "PUT" || t.Method == "PATCH" { + return true + } + if t.ContentLength == 0 && isIdentity(t.TransferEncoding) { + if t.Method == "GET" || t.Method == "HEAD" { + return false + } + return true + } + + return false +} + +func (t *transferWriter) writeHeader(w io.Writer) error { + if t.Close && !hasToken(t.Header.get("Connection"), "close") { + if _, err := io.WriteString(w, "Connection: close\r\n"); err != nil { + return err + } + } + + // Write Content-Length and/or Transfer-Encoding whose values are a + // function of the sanitized field triple (Body, ContentLength, + // TransferEncoding) + if t.shouldSendContentLength() { + if _, err := io.WriteString(w, "Content-Length: "); err != nil { + return err + } + if _, err := io.WriteString(w, strconv.FormatInt(t.ContentLength, 10)+"\r\n"); err != nil { + return err + } + } else if chunked(t.TransferEncoding) { + if _, err := io.WriteString(w, "Transfer-Encoding: chunked\r\n"); err != nil { + return err + } + } + + // Write Trailer header + if t.Trailer != nil { + keys := make([]string, 0, len(t.Trailer)) + for k := range t.Trailer { + k = CanonicalHeaderKey(k) + switch k { + case "Transfer-Encoding", "Trailer", "Content-Length": + return badStringError("invalid Trailer key", k) + } + keys = append(keys, k) + } + if len(keys) > 0 { + sort.Strings(keys) + // TODO: could do better allocation-wise here, but trailers are rare, + // so being lazy for now. + if _, err := io.WriteString(w, "Trailer: "+strings.Join(keys, ",")+"\r\n"); err != nil { + return err + } + } + } + + return nil +} + +// always closes t.BodyCloser +func (t *transferWriter) writeBody(w io.Writer) (err error) { + var ncopy int64 + closed := false + defer func() { + if closed || t.BodyCloser == nil { + return + } + if closeErr := t.BodyCloser.Close(); closeErr != nil && err == nil { + err = closeErr + } + }() + + // Write body. We "unwrap" the body first if it was wrapped in a + // nopCloser or readTrackingBody. This is to ensure that we can take advantage of + // OS-level optimizations in the event that the body is an + // *os.File. + if t.Body != nil { + var body = t.unwrapBody() + if chunked(t.TransferEncoding) { + if bw, ok := w.(*bufio.Writer); ok && !t.IsResponse { + w = &internal.FlushAfterChunkWriter{Writer: bw} + } + cw := internal.NewChunkedWriter(w) + _, err = t.doBodyCopy(cw, body) + if err == nil { + err = cw.Close() + } + } else if t.ContentLength == -1 { + dst := w + if t.Method == "CONNECT" { + dst = bufioFlushWriter{dst} + } + ncopy, err = t.doBodyCopy(dst, body) + } else { + ncopy, err = t.doBodyCopy(w, io.LimitReader(body, t.ContentLength)) + if err != nil { + return err + } + var nextra int64 + nextra, err = t.doBodyCopy(io.Discard, body) + ncopy += nextra + } + if err != nil { + return err + } + } + if t.BodyCloser != nil { + closed = true + if err := t.BodyCloser.Close(); err != nil { + return err + } + } + + if !t.ResponseToHEAD && t.ContentLength != -1 && t.ContentLength != ncopy { + return fmt.Errorf("http: ContentLength=%d with Body length %d", + t.ContentLength, ncopy) + } + + if chunked(t.TransferEncoding) { + // Write Trailer header + if t.Trailer != nil { + if err := t.Trailer.Write(w); err != nil { + return err + } + } + // Last chunk, empty trailer + _, err = io.WriteString(w, "\r\n") + } + return err +} + +// doBodyCopy wraps a copy operation, with any resulting error also +// being saved in bodyReadError. +// +// This function is only intended for use in writeBody. +func (t *transferWriter) doBodyCopy(dst io.Writer, src io.Reader) (n int64, err error) { + n, err = io.Copy(dst, src) + if err != nil && err != io.EOF { + t.bodyReadError = err + } + return +} + +// unwrapBodyReader unwraps the body's inner reader if it's a +// nopCloser. This is to ensure that body writes sourced from local +// files (*os.File types) are properly optimized. +// +// This function is only intended for use in writeBody. +func (t *transferWriter) unwrapBody() io.Reader { + if r, ok := unwrapNopCloser(t.Body); ok { + return r + } + if r, ok := t.Body.(*readTrackingBody); ok { + r.didRead = true + return r.ReadCloser + } + return t.Body +} + +type transferReader struct { + // Input + Header Header + StatusCode int + RequestMethod string + ProtoMajor int + ProtoMinor int + // Output + Body io.ReadCloser + ContentLength int64 + Chunked bool + Close bool + Trailer Header +} + +func (t *transferReader) protoAtLeast(m, n int) bool { + return t.ProtoMajor > m || (t.ProtoMajor == m && t.ProtoMinor >= n) +} + +// bodyAllowedForStatus reports whether a given response status code +// permits a body. See RFC 7230, section 3.3. +func bodyAllowedForStatus(status int) bool { + switch { + case status >= 100 && status <= 199: + return false + case status == 204: + return false + case status == 304: + return false + } + return true +} + +var ( + suppressedHeaders304 = []string{"Content-Type", "Content-Length", "Transfer-Encoding"} + suppressedHeadersNoBody = []string{"Content-Length", "Transfer-Encoding"} + excludedHeadersNoBody = map[string]bool{"Content-Length": true, "Transfer-Encoding": true} +) + +func suppressedHeaders(status int) []string { + switch { + case status == 304: + // RFC 7232 section 4.1 + return suppressedHeaders304 + case !bodyAllowedForStatus(status): + return suppressedHeadersNoBody + } + return nil +} + +// msg is *Request or *Response. +func readTransfer(msg any, r *bufio.Reader, onEOF func()) (err error) { + t := &transferReader{RequestMethod: "GET"} + + // TINYGO: Added onEOF func to be called when response body is closed + // TINYGO: so we can clean up the connection (r) + + // Unify input + isResponse := false + switch rr := msg.(type) { + case *Response: + t.Header = rr.Header + t.StatusCode = rr.StatusCode + t.ProtoMajor = rr.ProtoMajor + t.ProtoMinor = rr.ProtoMinor + t.Close = shouldClose(t.ProtoMajor, t.ProtoMinor, t.Header, true) + isResponse = true + if rr.Request != nil { + t.RequestMethod = rr.Request.Method + } + case *Request: + t.Header = rr.Header + t.RequestMethod = rr.Method + t.ProtoMajor = rr.ProtoMajor + t.ProtoMinor = rr.ProtoMinor + // Transfer semantics for Requests are exactly like those for + // Responses with status code 200, responding to a GET method + t.StatusCode = 200 + t.Close = rr.Close + default: + panic("unexpected type") + } + + // Default to HTTP/1.1 + if t.ProtoMajor == 0 && t.ProtoMinor == 0 { + t.ProtoMajor, t.ProtoMinor = 1, 1 + } + + // Transfer-Encoding: chunked, and overriding Content-Length. + if err := t.parseTransferEncoding(); err != nil { + return err + } + + realLength, err := fixLength(isResponse, t.StatusCode, t.RequestMethod, t.Header, t.Chunked) + if err != nil { + return err + } + if isResponse && t.RequestMethod == "HEAD" { + if n, err := parseContentLength(t.Header.get("Content-Length")); err != nil { + return err + } else { + t.ContentLength = n + } + } else { + t.ContentLength = realLength + } + + // Trailer + t.Trailer, err = fixTrailer(t.Header, t.Chunked) + if err != nil { + return err + } + + // If there is no Content-Length or chunked Transfer-Encoding on a *Response + // and the status is not 1xx, 204 or 304, then the body is unbounded. + // See RFC 7230, section 3.3. + switch msg.(type) { + case *Response: + if realLength == -1 && !t.Chunked && bodyAllowedForStatus(t.StatusCode) { + // Unbounded body. + t.Close = true + } + } + + // Prepare body reader. ContentLength < 0 means chunked encoding + // or close connection when finished, since multipart is not supported yet + switch { + case t.Chunked: + if noResponseBodyExpected(t.RequestMethod) || !bodyAllowedForStatus(t.StatusCode) { + t.Body = NoBody + } else { + t.Body = &body{src: internal.NewChunkedReader(r), hdr: msg, r: r, closing: t.Close, onHitEOF: onEOF} + } + case realLength == 0: + t.Body = NoBody + case realLength > 0: + t.Body = &body{src: io.LimitReader(r, realLength), closing: t.Close, onHitEOF: onEOF} + default: + // realLength < 0, i.e. "Content-Length" not mentioned in header + if t.Close { + // Close semantics (i.e. HTTP/1.0) + t.Body = &body{src: r, closing: t.Close, onHitEOF: onEOF} + } else { + // Persistent connection (i.e. HTTP/1.1) + t.Body = NoBody + } + } + + // Unify output + switch rr := msg.(type) { + case *Request: + rr.Body = t.Body + rr.ContentLength = t.ContentLength + if t.Chunked { + rr.TransferEncoding = []string{"chunked"} + } + rr.Close = t.Close + rr.Trailer = t.Trailer + case *Response: + rr.Body = t.Body + rr.ContentLength = t.ContentLength + if t.Chunked { + rr.TransferEncoding = []string{"chunked"} + } + rr.Close = t.Close + rr.Trailer = t.Trailer + } + + return nil +} + +// Checks whether chunked is part of the encodings stack +func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" } + +// Checks whether the encoding is explicitly "identity". +func isIdentity(te []string) bool { return len(te) == 1 && te[0] == "identity" } + +// unsupportedTEError reports unsupported transfer-encodings. +type unsupportedTEError struct { + err string +} + +func (uste *unsupportedTEError) Error() string { + return uste.err +} + +// isUnsupportedTEError checks if the error is of type +// unsupportedTEError. It is usually invoked with a non-nil err. +func isUnsupportedTEError(err error) bool { + _, ok := err.(*unsupportedTEError) + return ok +} + +// parseTransferEncoding sets t.Chunked based on the Transfer-Encoding header. +func (t *transferReader) parseTransferEncoding() error { + raw, present := t.Header["Transfer-Encoding"] + if !present { + return nil + } + delete(t.Header, "Transfer-Encoding") + + // Issue 12785; ignore Transfer-Encoding on HTTP/1.0 requests. + if !t.protoAtLeast(1, 1) { + return nil + } + + // Like nginx, we only support a single Transfer-Encoding header field, and + // only if set to "chunked". This is one of the most security sensitive + // surfaces in HTTP/1.1 due to the risk of request smuggling, so we keep it + // strict and simple. + if len(raw) != 1 { + return &unsupportedTEError{fmt.Sprintf("too many transfer encodings: %q", raw)} + } + if !ascii.EqualFold(raw[0], "chunked") { + return &unsupportedTEError{fmt.Sprintf("unsupported transfer encoding: %q", raw[0])} + } + + // RFC 7230 3.3.2 says "A sender MUST NOT send a Content-Length header field + // in any message that contains a Transfer-Encoding header field." + // + // but also: "If a message is received with both a Transfer-Encoding and a + // Content-Length header field, the Transfer-Encoding overrides the + // Content-Length. Such a message might indicate an attempt to perform + // request smuggling (Section 9.5) or response splitting (Section 9.4) and + // ought to be handled as an error. A sender MUST remove the received + // Content-Length field prior to forwarding such a message downstream." + // + // Reportedly, these appear in the wild. + delete(t.Header, "Content-Length") + + t.Chunked = true + return nil +} + +// Determine the expected body length, using RFC 7230 Section 3.3. This +// function is not a method, because ultimately it should be shared by +// ReadResponse and ReadRequest. +func fixLength(isResponse bool, status int, requestMethod string, header Header, chunked bool) (int64, error) { + isRequest := !isResponse + contentLens := header["Content-Length"] + + // Hardening against HTTP request smuggling + if len(contentLens) > 1 { + // Per RFC 7230 Section 3.3.2, prevent multiple + // Content-Length headers if they differ in value. + // If there are dups of the value, remove the dups. + // See Issue 16490. + first := textproto.TrimString(contentLens[0]) + for _, ct := range contentLens[1:] { + if first != textproto.TrimString(ct) { + return 0, fmt.Errorf("http: message cannot contain multiple Content-Length headers; got %q", contentLens) + } + } + + // deduplicate Content-Length + header.Del("Content-Length") + header.Add("Content-Length", first) + + contentLens = header["Content-Length"] + } + + // Logic based on response type or status + if noResponseBodyExpected(requestMethod) { + // For HTTP requests, as part of hardening against request + // smuggling (RFC 7230), don't allow a Content-Length header for + // methods which don't permit bodies. As an exception, allow + // exactly one Content-Length header if its value is "0". + if isRequest && len(contentLens) > 0 && !(len(contentLens) == 1 && contentLens[0] == "0") { + return 0, fmt.Errorf("http: method cannot contain a Content-Length; got %q", contentLens) + } + return 0, nil + } + if status/100 == 1 { + return 0, nil + } + switch status { + case 204, 304: + return 0, nil + } + + // Logic based on Transfer-Encoding + if chunked { + return -1, nil + } + + // Logic based on Content-Length + var cl string + if len(contentLens) == 1 { + cl = textproto.TrimString(contentLens[0]) + } + if cl != "" { + n, err := parseContentLength(cl) + if err != nil { + return -1, err + } + return n, nil + } + header.Del("Content-Length") + + if isRequest { + // RFC 7230 neither explicitly permits nor forbids an + // entity-body on a GET request so we permit one if + // declared, but we default to 0 here (not -1 below) + // if there's no mention of a body. + // Likewise, all other request methods are assumed to have + // no body if neither Transfer-Encoding chunked nor a + // Content-Length are set. + return 0, nil + } + + // Body-EOF logic based on other methods (like closing, or chunked coding) + return -1, nil +} + +// Determine whether to hang up after sending a request and body, or +// receiving a response and body +// 'header' is the request headers. +func shouldClose(major, minor int, header Header, removeCloseHeader bool) bool { + if major < 1 { + return true + } + + conv := header["Connection"] + hasClose := httpguts.HeaderValuesContainsToken(conv, "close") + if major == 1 && minor == 0 { + return hasClose || !httpguts.HeaderValuesContainsToken(conv, "keep-alive") + } + + if hasClose && removeCloseHeader { + header.Del("Connection") + } + + return hasClose +} + +// Parse the trailer header. +func fixTrailer(header Header, chunked bool) (Header, error) { + vv, ok := header["Trailer"] + if !ok { + return nil, nil + } + if !chunked { + // Trailer and no chunking: + // this is an invalid use case for trailer header. + // Nevertheless, no error will be returned and we + // let users decide if this is a valid HTTP message. + // The Trailer header will be kept in Response.Header + // but not populate Response.Trailer. + // See issue #27197. + return nil, nil + } + header.Del("Trailer") + + trailer := make(Header) + var err error + for _, v := range vv { + foreachHeaderElement(v, func(key string) { + key = CanonicalHeaderKey(key) + switch key { + case "Transfer-Encoding", "Trailer", "Content-Length": + if err == nil { + err = badStringError("bad trailer key", key) + return + } + } + trailer[key] = nil + }) + } + if err != nil { + return nil, err + } + if len(trailer) == 0 { + return nil, nil + } + return trailer, nil +} + +// body turns a Reader into a ReadCloser. +// Close ensures that the body has been fully read +// and then reads the trailer if necessary. +type body struct { + src io.Reader + hdr any // non-nil (Response or Request) value means read trailer + r *bufio.Reader // underlying wire-format reader for the trailer + closing bool // is the connection to be closed after reading body? + doEarlyClose bool // whether Close should stop early + + mu sync.Mutex // guards following, and calls to Read and Close + sawEOF bool + closed bool + earlyClose bool // Close called and we didn't read to the end of src + onHitEOF func() // if non-nil, func to call when EOF is Read +} + +// ErrBodyReadAfterClose is returned when reading a Request or Response +// Body after the body has been closed. This typically happens when the body is +// read after an HTTP Handler calls WriteHeader or Write on its +// ResponseWriter. +var ErrBodyReadAfterClose = errors.New("http: invalid Read on closed Body") + +func (b *body) Read(p []byte) (n int, err error) { + b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + return 0, ErrBodyReadAfterClose + } + return b.readLocked(p) +} + +// Must hold b.mu. +func (b *body) readLocked(p []byte) (n int, err error) { + if b.sawEOF { + return 0, io.EOF + } + n, err = b.src.Read(p) + + if err == io.EOF { + b.sawEOF = true + // Chunked case. Read the trailer. + if b.hdr != nil { + if e := b.readTrailer(); e != nil { + err = e + // Something went wrong in the trailer, we must not allow any + // further reads of any kind to succeed from body, nor any + // subsequent requests on the server connection. See + // golang.org/issue/12027 + b.sawEOF = false + b.closed = true + } + b.hdr = nil + } else { + // If the server declared the Content-Length, our body is a LimitedReader + // and we need to check whether this EOF arrived early. + if lr, ok := b.src.(*io.LimitedReader); ok && lr.N > 0 { + err = io.ErrUnexpectedEOF + } + } + } + + // If we can return an EOF here along with the read data, do + // so. This is optional per the io.Reader contract, but doing + // so helps the HTTP transport code recycle its connection + // earlier (since it will see this EOF itself), even if the + // client doesn't do future reads or Close. + if err == nil && n > 0 { + if lr, ok := b.src.(*io.LimitedReader); ok && lr.N == 0 { + err = io.EOF + b.sawEOF = true + } + } + + if b.sawEOF && b.onHitEOF != nil { + b.onHitEOF() + } + + return n, err +} + +var ( + singleCRLF = []byte("\r\n") + doubleCRLF = []byte("\r\n\r\n") +) + +func seeUpcomingDoubleCRLF(r *bufio.Reader) bool { + for peekSize := 4; ; peekSize++ { + // This loop stops when Peek returns an error, + // which it does when r's buffer has been filled. + buf, err := r.Peek(peekSize) + if bytes.HasSuffix(buf, doubleCRLF) { + return true + } + if err != nil { + break + } + } + return false +} + +var errTrailerEOF = errors.New("http: unexpected EOF reading trailer") + +func (b *body) readTrailer() error { + // The common case, since nobody uses trailers. + buf, err := b.r.Peek(2) + if bytes.Equal(buf, singleCRLF) { + b.r.Discard(2) + return nil + } + if len(buf) < 2 { + return errTrailerEOF + } + if err != nil { + return err + } + + // Make sure there's a header terminator coming up, to prevent + // a DoS with an unbounded size Trailer. It's not easy to + // slip in a LimitReader here, as textproto.NewReader requires + // a concrete *bufio.Reader. Also, we can't get all the way + // back up to our conn's LimitedReader that *might* be backing + // this bufio.Reader. Instead, a hack: we iteratively Peek up + // to the bufio.Reader's max size, looking for a double CRLF. + // This limits the trailer to the underlying buffer size, typically 4kB. + if !seeUpcomingDoubleCRLF(b.r) { + return errors.New("http: suspiciously long trailer after chunked body") + } + + hdr, err := textproto.NewReader(b.r).ReadMIMEHeader() + if err != nil { + if err == io.EOF { + return errTrailerEOF + } + return err + } + switch rr := b.hdr.(type) { + case *Request: + mergeSetHeader(&rr.Trailer, Header(hdr)) + case *Response: + mergeSetHeader(&rr.Trailer, Header(hdr)) + } + return nil +} + +func mergeSetHeader(dst *Header, src Header) { + if *dst == nil { + *dst = src + return + } + for k, vv := range src { + (*dst)[k] = vv + } +} + +// unreadDataSizeLocked returns the number of bytes of unread input. +// It returns -1 if unknown. +// b.mu must be held. +func (b *body) unreadDataSizeLocked() int64 { + if lr, ok := b.src.(*io.LimitedReader); ok { + return lr.N + } + return -1 +} + +func (b *body) Close() error { + b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + return nil + } + var err error + switch { + case b.sawEOF: + // Already saw EOF, so no need going to look for it. + case b.hdr == nil && b.closing: + // no trailer and closing the connection next. + // no point in reading to EOF. + case b.doEarlyClose: + // Read up to maxPostHandlerReadBytes bytes of the body, looking + // for EOF (and trailers), so we can re-use this connection. + if lr, ok := b.src.(*io.LimitedReader); ok && lr.N > maxPostHandlerReadBytes { + // There was a declared Content-Length, and we have more bytes remaining + // than our maxPostHandlerReadBytes tolerance. So, give up. + b.earlyClose = true + } else { + var n int64 + // Consume the body, or, which will also lead to us reading + // the trailer headers after the body, if present. + n, err = io.CopyN(io.Discard, bodyLocked{b}, maxPostHandlerReadBytes) + if err == io.EOF { + err = nil + } + if n == maxPostHandlerReadBytes { + b.earlyClose = true + } + } + default: + // Fully consume the body, which will also lead to us reading + // the trailer headers after the body, if present. + _, err = io.Copy(io.Discard, bodyLocked{b}) + } + b.closed = true + return err +} + +func (b *body) didEarlyClose() bool { + b.mu.Lock() + defer b.mu.Unlock() + return b.earlyClose +} + +// bodyRemains reports whether future Read calls might +// yield data. +func (b *body) bodyRemains() bool { + b.mu.Lock() + defer b.mu.Unlock() + return !b.sawEOF +} + +func (b *body) registerOnHitEOF(fn func()) { + b.mu.Lock() + defer b.mu.Unlock() + b.onHitEOF = fn +} + +// bodyLocked is an io.Reader reading from a *body when its mutex is +// already held. +type bodyLocked struct { + b *body +} + +func (bl bodyLocked) Read(p []byte) (n int, err error) { + if bl.b.closed { + return 0, ErrBodyReadAfterClose + } + return bl.b.readLocked(p) +} + +// parseContentLength trims whitespace from s and returns -1 if no value +// is set, or the value if it's >= 0. +func parseContentLength(cl string) (int64, error) { + cl = textproto.TrimString(cl) + if cl == "" { + return -1, nil + } + n, err := strconv.ParseUint(cl, 10, 63) + if err != nil { + return 0, badStringError("bad Content-Length", cl) + } + return int64(n), nil + +} + +// finishAsyncByteRead finishes reading the 1-byte sniff +// from the ContentLength==0, Body!=nil case. +type finishAsyncByteRead struct { + tw *transferWriter +} + +func (fr finishAsyncByteRead) Read(p []byte) (n int, err error) { + if len(p) == 0 { + return + } + rres := <-fr.tw.ByteReadCh + n, err = rres.n, rres.err + if n == 1 { + p[0] = rres.b + } + if err == nil { + err = io.EOF + } + return +} + +var nopCloserType = reflect.TypeOf(io.NopCloser(nil)) +var nopCloserWriterToType = reflect.TypeOf(io.NopCloser(struct { + io.Reader + io.WriterTo +}{})) + +// unwrapNopCloser return the underlying reader and true if r is a NopCloser +// else it return false. +func unwrapNopCloser(r io.Reader) (underlyingReader io.Reader, isNopCloser bool) { + switch reflect.TypeOf(r) { + case nopCloserType, nopCloserWriterToType: + return reflect.ValueOf(r).Field(0).Interface().(io.Reader), true + default: + return nil, false + } +} + +// isKnownInMemoryReader reports whether r is a type known to not +// block on Read. Its caller uses this as an optional optimization to +// send fewer TCP packets. +func isKnownInMemoryReader(r io.Reader) bool { + switch r.(type) { + case *bytes.Reader, *bytes.Buffer, *strings.Reader: + return true + } + if r, ok := unwrapNopCloser(r); ok { + return isKnownInMemoryReader(r) + } + if r, ok := r.(*readTrackingBody); ok { + return isKnownInMemoryReader(r.ReadCloser) + } + return false +} + +// bufioFlushWriter is an io.Writer wrapper that flushes all writes +// on its wrapped writer if it's a *bufio.Writer. +type bufioFlushWriter struct{ w io.Writer } + +func (fw bufioFlushWriter) Write(p []byte) (n int, err error) { + n, err = fw.w.Write(p) + if bw, ok := fw.w.(*bufio.Writer); n > 0 && ok { + ferr := bw.Flush() + if ferr != nil && err == nil { + err = ferr + } + } + return +} diff --git a/src/net/http/transport.go b/src/net/http/transport.go new file mode 100644 index 0000000000..a5f0f49346 --- /dev/null +++ b/src/net/http/transport.go @@ -0,0 +1,22 @@ +// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. + +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// HTTP client implementation. See RFC 7230 through 7235. +// +// This is the low-level Transport implementation of RoundTripper. +// The high-level interface is in client.go. + +package http + +import ( + "io" +) + +type readTrackingBody struct { + io.ReadCloser + didRead bool + didClose bool +} diff --git a/src/net/interface.go b/src/net/interface.go deleted file mode 100644 index 32206f78fc..0000000000 --- a/src/net/interface.go +++ /dev/null @@ -1,253 +0,0 @@ -// The following is copied from Go 1.16 official implementation. - -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package net - -import ( - "errors" - "internal/itoa" - "sync" - "time" -) - -var ( - errInvalidInterface = errors.New("invalid network interface") - errInvalidInterfaceIndex = errors.New("invalid network interface index") - errInvalidInterfaceName = errors.New("invalid network interface name") - errNoSuchInterface = errors.New("no such network interface") - errNoSuchMulticastInterface = errors.New("no such multicast network interface") -) - -// Interface represents a mapping between network interface name -// and index. It also represents network interface facility -// information. -type Interface struct { - Index int // positive integer that starts at one, zero is never used - MTU int // maximum transmission unit - Name string // e.g., "en0", "lo0", "eth0.100" - HardwareAddr HardwareAddr // IEEE MAC-48, EUI-48 and EUI-64 form - Flags Flags // e.g., FlagUp, FlagLoopback, FlagMulticast -} - -type Flags uint - -const ( - FlagUp Flags = 1 << iota // interface is up - FlagBroadcast // interface supports broadcast access capability - FlagLoopback // interface is a loopback interface - FlagPointToPoint // interface belongs to a point-to-point link - FlagMulticast // interface supports multicast access capability -) - -var flagNames = []string{ - "up", - "broadcast", - "loopback", - "pointtopoint", - "multicast", -} - -func (f Flags) String() string { - s := "" - for i, name := range flagNames { - if f&(1<", if ip has length 0 // - dotted decimal ("192.0.2.1"), if ip is an IPv4 or IP4-mapped IPv6 address -// - IPv6 ("2001:db8::1"), if ip is a valid IPv6 address +// - IPv6 conforming to RFC 5952 ("2001:db8::1"), if ip is a valid IPv6 address // - the hexadecimal form of ip, without punctuation, if no other cases apply func (ip IP) String() string { p := ip @@ -528,6 +547,9 @@ func (n *IPNet) Network() string { return "ip+net" } // character and a mask expressed as hexadecimal form with no // punctuation like "198.51.100.0/c000ff00". func (n *IPNet) String() string { + if n == nil { + return "" + } nn, m := networkNumberAndMask(n) if nn == nil || m == nil { return "" @@ -557,6 +579,10 @@ func parseIPv4(s string) IP { if !ok || n > 0xFF { return nil } + if c > 1 && s[0] == '0' { + // Reject non-zero components with leading zeroes. + return nil + } s = s[c:] p[i] = byte(n) } diff --git a/src/net/iprawsock.go b/src/net/iprawsock.go index d16d5eac1d..8f82ec8e34 100644 --- a/src/net/iprawsock.go +++ b/src/net/iprawsock.go @@ -1,4 +1,4 @@ -// The following is copied from Go 1.16 official implementation. +// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. // Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style @@ -6,6 +6,24 @@ package net +// BUG(mikio): On every POSIX platform, reads from the "ip4" network +// using the ReadFrom or ReadFromIP method might not return a complete +// IPv4 packet, including its header, even if there is space +// available. This can occur even in cases where Read or ReadMsgIP +// could return a complete packet. For this reason, it is recommended +// that you do not use these methods if it is important to receive a +// full packet. +// +// The Go 1 compatibility guidelines make it impossible for us to +// change the behavior of these methods; use Read or ReadMsgIP +// instead. + +// BUG(mikio): On JS and Plan 9, methods and functions related +// to IPConn are not implemented. + +// BUG(mikio): On Windows, the File method of IPConn is not +// implemented. + // IPAddr represents the address of an IP end point. type IPAddr struct { IP IP diff --git a/src/net/ipsock.go b/src/net/ipsock.go index 57ceaebf09..52d1f7dc2a 100644 --- a/src/net/ipsock.go +++ b/src/net/ipsock.go @@ -1,4 +1,4 @@ -// The following is copied from Go 1.16 official implementation. +// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style @@ -6,7 +6,9 @@ package net -import "internal/bytealg" +import ( + "internal/bytealg" +) // SplitHostPort splits a network address of the form "host:port", // "host%zone:port", "[host]:port" or "[host%zone]:port" into host or diff --git a/src/net/mac.go b/src/net/mac.go index 2bad98c462..320b209d64 100644 --- a/src/net/mac.go +++ b/src/net/mac.go @@ -1,4 +1,4 @@ -// The following is copied from Go 1.16 official implementation. +// TINYGO: The following is copied from Go 1.19.3 official implementation. // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style @@ -40,49 +40,49 @@ func (a HardwareAddr) String() string { // 0000.0000.fe80.0000.0000.0000.0200.5e10.0000.0001 func ParseMAC(s string) (hw HardwareAddr, err error) { if len(s) < 14 { - goto err + goto error } if s[2] == ':' || s[2] == '-' { if (len(s)+1)%3 != 0 { - goto err + goto error } n := (len(s) + 1) / 3 if n != 6 && n != 8 && n != 20 { - goto err + goto error } hw = make(HardwareAddr, n) for x, i := 0, 0; i < n; i++ { var ok bool if hw[i], ok = xtoi2(s[x:], s[2]); !ok { - goto err + goto error } x += 3 } } else if s[4] == '.' { if (len(s)+1)%5 != 0 { - goto err + goto error } n := 2 * (len(s) + 1) / 5 if n != 6 && n != 8 && n != 20 { - goto err + goto error } hw = make(HardwareAddr, n) for x, i := 0, 0; i < n; i += 2 { var ok bool if hw[i], ok = xtoi2(s[x:x+2], 0); !ok { - goto err + goto error } if hw[i+1], ok = xtoi2(s[x+2:], s[4]); !ok { - goto err + goto error } x += 5 } } else { - goto err + goto error } return hw, nil -err: +error: return nil, &AddrError{Err: "invalid MAC address", Addr: s} } diff --git a/src/net/mac_test.go b/src/net/mac_test.go new file mode 100644 index 0000000000..cad884fcf5 --- /dev/null +++ b/src/net/mac_test.go @@ -0,0 +1,109 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "reflect" + "strings" + "testing" +) + +var parseMACTests = []struct { + in string + out HardwareAddr + err string +}{ + // See RFC 7042, Section 2.1.1. + {"00:00:5e:00:53:01", HardwareAddr{0x00, 0x00, 0x5e, 0x00, 0x53, 0x01}, ""}, + {"00-00-5e-00-53-01", HardwareAddr{0x00, 0x00, 0x5e, 0x00, 0x53, 0x01}, ""}, + {"0000.5e00.5301", HardwareAddr{0x00, 0x00, 0x5e, 0x00, 0x53, 0x01}, ""}, + + // See RFC 7042, Section 2.2.2. + {"02:00:5e:10:00:00:00:01", HardwareAddr{0x02, 0x00, 0x5e, 0x10, 0x00, 0x00, 0x00, 0x01}, ""}, + {"02-00-5e-10-00-00-00-01", HardwareAddr{0x02, 0x00, 0x5e, 0x10, 0x00, 0x00, 0x00, 0x01}, ""}, + {"0200.5e10.0000.0001", HardwareAddr{0x02, 0x00, 0x5e, 0x10, 0x00, 0x00, 0x00, 0x01}, ""}, + + // See RFC 4391, Section 9.1.1. + { + "00:00:00:00:fe:80:00:00:00:00:00:00:02:00:5e:10:00:00:00:01", + HardwareAddr{ + 0x00, 0x00, 0x00, 0x00, + 0xfe, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x5e, 0x10, 0x00, 0x00, 0x00, 0x01, + }, + "", + }, + { + "00-00-00-00-fe-80-00-00-00-00-00-00-02-00-5e-10-00-00-00-01", + HardwareAddr{ + 0x00, 0x00, 0x00, 0x00, + 0xfe, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x5e, 0x10, 0x00, 0x00, 0x00, 0x01, + }, + "", + }, + { + "0000.0000.fe80.0000.0000.0000.0200.5e10.0000.0001", + HardwareAddr{ + 0x00, 0x00, 0x00, 0x00, + 0xfe, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x5e, 0x10, 0x00, 0x00, 0x00, 0x01, + }, + "", + }, + + {"ab:cd:ef:AB:CD:EF", HardwareAddr{0xab, 0xcd, 0xef, 0xab, 0xcd, 0xef}, ""}, + {"ab:cd:ef:AB:CD:EF:ab:cd", HardwareAddr{0xab, 0xcd, 0xef, 0xab, 0xcd, 0xef, 0xab, 0xcd}, ""}, + { + "ab:cd:ef:AB:CD:EF:ab:cd:ef:AB:CD:EF:ab:cd:ef:AB:CD:EF:ab:cd", + HardwareAddr{ + 0xab, 0xcd, 0xef, 0xab, + 0xcd, 0xef, 0xab, 0xcd, 0xef, 0xab, 0xcd, 0xef, + 0xab, 0xcd, 0xef, 0xab, 0xcd, 0xef, 0xab, 0xcd, + }, + "", + }, + + {"01.02.03.04.05.06", nil, "invalid MAC address"}, + {"01:02:03:04:05:06:", nil, "invalid MAC address"}, + {"x1:02:03:04:05:06", nil, "invalid MAC address"}, + {"01002:03:04:05:06", nil, "invalid MAC address"}, + {"01:02003:04:05:06", nil, "invalid MAC address"}, + {"01:02:03004:05:06", nil, "invalid MAC address"}, + {"01:02:03:04005:06", nil, "invalid MAC address"}, + {"01:02:03:04:05006", nil, "invalid MAC address"}, + {"01-02:03:04:05:06", nil, "invalid MAC address"}, + {"01:02-03-04-05-06", nil, "invalid MAC address"}, + {"0123:4567:89AF", nil, "invalid MAC address"}, + {"0123-4567-89AF", nil, "invalid MAC address"}, +} + +func TestParseMAC(t *testing.T) { + match := func(err error, s string) bool { + if s == "" { + return err == nil + } + return err != nil && strings.Contains(err.Error(), s) + } + + for i, tt := range parseMACTests { + out, err := ParseMAC(tt.in) + if !reflect.DeepEqual(out, tt.out) || !match(err, tt.err) { + t.Errorf("ParseMAC(%q) = %v, %v, want %v, %v", tt.in, out, err, tt.out, tt.err) + } + if tt.err == "" { + // Verify that serialization works too, and that it round-trips. + s := out.String() + out2, err := ParseMAC(s) + if err != nil { + t.Errorf("%d. ParseMAC(%q) = %v", i, s, err) + continue + } + if !reflect.DeepEqual(out2, out) { + t.Errorf("%d. ParseMAC(%q) = %v, want %v", i, s, out2, out) + } + } + } +} diff --git a/src/net/net.go b/src/net/net.go index db4d8f117f..2e7f9054c9 100644 --- a/src/net/net.go +++ b/src/net/net.go @@ -1,4 +1,4 @@ -// The following is copied from Go 1.18 official implementation. +// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style @@ -7,7 +7,6 @@ package net import ( - "io" "time" ) @@ -81,10 +80,6 @@ type Conn interface { SetWriteDeadline(t time.Time) error } -type conn struct { - // -} - // A Listener is a generic network listener for stream-oriented protocols. // // Multiple goroutines may invoke methods on a Listener simultaneously. @@ -193,87 +188,3 @@ func (e *AddrError) Error() string { } return s } - -func (e *AddrError) Timeout() bool { return false } -func (e *AddrError) Temporary() bool { return false } - -// ErrClosed is the error returned by an I/O call on a network -// connection that has already been closed, or that is closed by -// another goroutine before the I/O is completed. This may be wrapped -// in another error, and should normally be tested using -// errors.Is(err, net.ErrClosed). -var ErrClosed = errClosed - -// buffersWriter is the interface implemented by Conns that support a -// "writev"-like batch write optimization. -// writeBuffers should fully consume and write all chunks from the -// provided Buffers, else it should report a non-nil error. -type buffersWriter interface { - writeBuffers(*Buffers) (int64, error) -} - -// Buffers contains zero or more runs of bytes to write. -// -// On certain machines, for certain types of connections, this is -// optimized into an OS-specific batch write operation (such as -// "writev"). -type Buffers [][]byte - -var ( - _ io.WriterTo = (*Buffers)(nil) - _ io.Reader = (*Buffers)(nil) -) - -// WriteTo writes contents of the buffers to w. -// -// WriteTo implements io.WriterTo for Buffers. -// -// WriteTo modifies the slice v as well as v[i] for 0 <= i < len(v), -// but does not modify v[i][j] for any i, j. -func (v *Buffers) WriteTo(w io.Writer) (n int64, err error) { - if wv, ok := w.(buffersWriter); ok { - return wv.writeBuffers(v) - } - for _, b := range *v { - nb, err := w.Write(b) - n += int64(nb) - if err != nil { - v.consume(n) - return n, err - } - } - v.consume(n) - return n, nil -} - -// Read from the buffers. -// -// Read implements io.Reader for Buffers. -// -// Read modifies the slice v as well as v[i] for 0 <= i < len(v), -// but does not modify v[i][j] for any i, j. -func (v *Buffers) Read(p []byte) (n int, err error) { - for len(p) > 0 && len(*v) > 0 { - n0 := copy(p, (*v)[0]) - v.consume(int64(n0)) - p = p[n0:] - n += n0 - } - if len(*v) == 0 { - err = io.EOF - } - return -} - -func (v *Buffers) consume(n int64) { - for len(*v) > 0 { - ln0 := int64(len((*v)[0])) - if ln0 > n { - (*v)[0] = (*v)[0][n:] - return - } - n -= ln0 - (*v)[0] = nil - *v = (*v)[1:] - } -} diff --git a/src/net/netdev.go b/src/net/netdev.go new file mode 100644 index 0000000000..2d294c15b3 --- /dev/null +++ b/src/net/netdev.go @@ -0,0 +1,46 @@ +package net + +import ( + "time" +) + +// netdev is the current netdev, set by the application with useNetdev() +var netdev netdever + +// (useNetdev is go:linkname'd from tinygo/drivers package) +func useNetdev(dev netdever) { + netdev = dev +} + +// Netdev is TinyGo's network device driver model. Network drivers implement +// the netdever interface, providing a common network I/O interface to TinyGo's +// "net" package. The interface is modeled after the BSD socket interface. +// net.Conn implementations (TCPConn, UDPConn, and TLSConn) use the netdev +// interface for device I/O access. +// +// A netdever is passed to the "net" package using net.useNetdev(). +// +// Just like a net.Conn, multiple goroutines may invoke methods on a netdever +// simultaneously. +// +// NOTE: The netdever interface is mirrored in drivers/netdev.go. +// NOTE: If making changes to this interface, mirror the changes in +// NOTE: drivers/netdev.go, and visa-versa. + +type netdever interface { + + // GetHostByName returns the IP address of either a hostname or IPv4 + // address in standard dot notation + GetHostByName(name string) (IP, error) + + // Berkely Sockets-like interface, Go-ified. See man page for socket(2), etc. + Socket(domain int, stype int, protocol int) (int, error) + Bind(sockfd int, ip IP, port int) error + Connect(sockfd int, host string, ip IP, port int) error + Listen(sockfd int, backlog int) error + Accept(sockfd int, ip IP, port int) (int, error) + Send(sockfd int, buf []byte, flags int, timeout time.Duration) (int, error) + Recv(sockfd int, buf []byte, flags int, timeout time.Duration) (int, error) + Close(sockfd int) error + SetSockOpt(sockfd int, level int, opt int, value interface{}) error +} diff --git a/src/net/parse.go b/src/net/parse.go index f1f2ccb5c9..b263271fc7 100644 --- a/src/net/parse.go +++ b/src/net/parse.go @@ -1,11 +1,121 @@ -// The following is copied from Go 1.16 official implementation. +// TINYGO: The following is copied from Go 1.19.3 official implementation. // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// Simple file i/o and string manipulation, to avoid +// depending on strconv and bufio and strings. + package net +import ( + "internal/bytealg" + "io" + "os" + "time" +) + +type file struct { + file *os.File + data []byte + atEOF bool +} + +func (f *file) close() { f.file.Close() } + +func (f *file) getLineFromData() (s string, ok bool) { + data := f.data + i := 0 + for i = 0; i < len(data); i++ { + if data[i] == '\n' { + s = string(data[0:i]) + ok = true + // move data + i++ + n := len(data) - i + copy(data[0:], data[i:]) + f.data = data[0:n] + return + } + } + if f.atEOF && len(f.data) > 0 { + // EOF, return all we have + s = string(data) + f.data = f.data[0:0] + ok = true + } + return +} + +func (f *file) readLine() (s string, ok bool) { + if s, ok = f.getLineFromData(); ok { + return + } + if len(f.data) < cap(f.data) { + ln := len(f.data) + n, err := io.ReadFull(f.file, f.data[ln:cap(f.data)]) + if n >= 0 { + f.data = f.data[0 : ln+n] + } + if err == io.EOF || err == io.ErrUnexpectedEOF { + f.atEOF = true + } + } + s, ok = f.getLineFromData() + return +} + +func open(name string) (*file, error) { + fd, err := os.Open(name) + if err != nil { + return nil, err + } + return &file{fd, make([]byte, 0, 64*1024), false}, nil +} + +func stat(name string) (mtime time.Time, size int64, err error) { + st, err := os.Stat(name) + if err != nil { + return time.Time{}, 0, err + } + return st.ModTime(), st.Size(), nil +} + +// Count occurrences in s of any bytes in t. +func countAnyByte(s string, t string) int { + n := 0 + for i := 0; i < len(s); i++ { + if bytealg.IndexByteString(t, s[i]) >= 0 { + n++ + } + } + return n +} + +// Split s at any bytes in t. +func splitAtBytes(s string, t string) []string { + a := make([]string, 1+countAnyByte(s, t)) + n := 0 + last := 0 + for i := 0; i < len(s); i++ { + if bytealg.IndexByteString(t, s[i]) >= 0 { + if last < i { + a[n] = s[last:i] + n++ + } + last = i + 1 + } + } + if last < len(s) { + a[n] = s[last:] + n++ + } + return a[0:n] +} + +func getFields(s string) []string { return splitAtBytes(s, " \r\t\n") } + // Bigger than we need, not too big to worry about overflow const big = 0xFFFFFF @@ -78,6 +188,17 @@ func appendHex(dst []byte, i uint32) []byte { return dst } +// Number of occurrences of b in s. +func count(s string, b byte) int { + n := 0 + for i := 0; i < len(s); i++ { + if s[i] == b { + n++ + } + } + return n +} + // Index of rightmost occurrence of b in s. func last(s string, b byte) int { i := len(s) @@ -88,3 +209,137 @@ func last(s string, b byte) int { } return i } + +// hasUpperCase tells whether the given string contains at least one upper-case. +func hasUpperCase(s string) bool { + for i := range s { + if 'A' <= s[i] && s[i] <= 'Z' { + return true + } + } + return false +} + +// lowerASCIIBytes makes x ASCII lowercase in-place. +func lowerASCIIBytes(x []byte) { + for i, b := range x { + if 'A' <= b && b <= 'Z' { + x[i] += 'a' - 'A' + } + } +} + +// lowerASCII returns the ASCII lowercase version of b. +func lowerASCII(b byte) byte { + if 'A' <= b && b <= 'Z' { + return b + ('a' - 'A') + } + return b +} + +// trimSpace returns x without any leading or trailing ASCII whitespace. +func trimSpace(x []byte) []byte { + for len(x) > 0 && isSpace(x[0]) { + x = x[1:] + } + for len(x) > 0 && isSpace(x[len(x)-1]) { + x = x[:len(x)-1] + } + return x +} + +// isSpace reports whether b is an ASCII space character. +func isSpace(b byte) bool { + return b == ' ' || b == '\t' || b == '\n' || b == '\r' +} + +// removeComment returns line, removing any '#' byte and any following +// bytes. +func removeComment(line []byte) []byte { + if i := bytealg.IndexByte(line, '#'); i != -1 { + return line[:i] + } + return line +} + +// foreachLine runs fn on each line of x. +// Each line (except for possibly the last) ends in '\n'. +// It returns the first non-nil error returned by fn. +func foreachLine(x []byte, fn func(line []byte) error) error { + for len(x) > 0 { + nl := bytealg.IndexByte(x, '\n') + if nl == -1 { + return fn(x) + } + line := x[:nl+1] + x = x[nl+1:] + if err := fn(line); err != nil { + return err + } + } + return nil +} + +// foreachField runs fn on each non-empty run of non-space bytes in x. +// It returns the first non-nil error returned by fn. +func foreachField(x []byte, fn func(field []byte) error) error { + x = trimSpace(x) + for len(x) > 0 { + sp := bytealg.IndexByte(x, ' ') + if sp == -1 { + return fn(x) + } + if field := trimSpace(x[:sp]); len(field) > 0 { + if err := fn(field); err != nil { + return err + } + } + x = trimSpace(x[sp+1:]) + } + return nil +} + +// stringsHasSuffix is strings.HasSuffix. It reports whether s ends in +// suffix. +func stringsHasSuffix(s, suffix string) bool { + return len(s) >= len(suffix) && s[len(s)-len(suffix):] == suffix +} + +// stringsHasSuffixFold reports whether s ends in suffix, +// ASCII-case-insensitively. +func stringsHasSuffixFold(s, suffix string) bool { + return len(s) >= len(suffix) && stringsEqualFold(s[len(s)-len(suffix):], suffix) +} + +// stringsHasPrefix is strings.HasPrefix. It reports whether s begins with prefix. +func stringsHasPrefix(s, prefix string) bool { + return len(s) >= len(prefix) && s[:len(prefix)] == prefix +} + +// stringsEqualFold is strings.EqualFold, ASCII only. It reports whether s and t +// are equal, ASCII-case-insensitively. +func stringsEqualFold(s, t string) bool { + if len(s) != len(t) { + return false + } + for i := 0; i < len(s); i++ { + if lowerASCII(s[i]) != lowerASCII(t[i]) { + return false + } + } + return true +} + +func readFull(r io.Reader) (all []byte, err error) { + buf := make([]byte, 1024) + for { + n, err := r.Read(buf) + all = append(all, buf[:n]...) + if err == io.EOF { + return all, nil + } + if err != nil { + return nil, err + } + } +} diff --git a/src/net/pipe.go b/src/net/pipe.go index 02dd07cf9a..238da0c727 100644 --- a/src/net/pipe.go +++ b/src/net/pipe.go @@ -1,4 +1,4 @@ -// The following is copied from Go 1.19.2 official implementation. +// The following is copied from Go 1.19.3 official implementation. // Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/src/net/pipe_test.go b/src/net/pipe_test.go deleted file mode 100644 index 7978fc6aa0..0000000000 --- a/src/net/pipe_test.go +++ /dev/null @@ -1,48 +0,0 @@ -// The following is copied from Go 1.19.2 official implementation. - -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package net - -import ( - "io" - "testing" - "time" -) - -func TestPipe(t *testing.T) { - testConn(t, func() (c1, c2 Conn, stop func(), err error) { - c1, c2 = Pipe() - stop = func() { - c1.Close() - c2.Close() - } - return - }) -} - -func TestPipeCloseError(t *testing.T) { - c1, c2 := Pipe() - c1.Close() - - if _, err := c1.Read(nil); err != io.ErrClosedPipe { - t.Errorf("c1.Read() = %v, want io.ErrClosedPipe", err) - } - if _, err := c1.Write(nil); err != io.ErrClosedPipe { - t.Errorf("c1.Write() = %v, want io.ErrClosedPipe", err) - } - if err := c1.SetDeadline(time.Time{}); err != io.ErrClosedPipe { - t.Errorf("c1.SetDeadline() = %v, want io.ErrClosedPipe", err) - } - if _, err := c2.Read(nil); err != io.EOF { - t.Errorf("c2.Read() = %v, want io.EOF", err) - } - if _, err := c2.Write(nil); err != io.ErrClosedPipe { - t.Errorf("c2.Write() = %v, want io.ErrClosedPipe", err) - } - if err := c2.SetDeadline(time.Time{}); err != io.ErrClosedPipe { - t.Errorf("c2.SetDeadline() = %v, want io.ErrClosedPipe", err) - } -} diff --git a/src/net/tcpsock.go b/src/net/tcpsock.go index d6aa602a9a..f5b1b65eff 100644 --- a/src/net/tcpsock.go +++ b/src/net/tcpsock.go @@ -1,8 +1,18 @@ +// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. + +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package net import ( + "fmt" "internal/itoa" "net/netip" + "strconv" + "syscall" + "time" ) // TCPAddr represents the address of a TCP end point. @@ -54,12 +64,231 @@ func (a *TCPAddr) opAddr() Addr { return a } +// ResolveTCPAddr returns an address of TCP end point. +// +// The network must be a TCP network name. +// +// If the host in the address parameter is not a literal IP address or +// the port is not a literal port number, ResolveTCPAddr resolves the +// address to an address of TCP end point. +// Otherwise, it parses the address as a pair of literal IP address +// and port number. +// The address parameter can use a host name, but this is not +// recommended, because it will return at most one of the host name's +// IP addresses. +// +// See func Dial for a description of the network and address +// parameters. +func ResolveTCPAddr(network, address string) (*TCPAddr, error) { + + switch network { + case "tcp", "tcp4": + default: + return nil, fmt.Errorf("Network '%s' not supported", network) + } + + // TINYGO: Use netdev resolver + + host, sport, err := SplitHostPort(address) + if err != nil { + return nil, err + } + + port, err := strconv.Atoi(sport) + if err != nil { + return nil, fmt.Errorf("Error parsing port '%s' in address: %s", + sport, err) + } + + if host == "" { + return &TCPAddr{Port: port}, nil + } + + ip, err := netdev.GetHostByName(host) + if err != nil { + return nil, fmt.Errorf("Lookup of host name '%s' failed: %s", host, err) + } + + return &TCPAddr{IP: ip, Port: port}, nil +} + // TCPConn is an implementation of the Conn interface for TCP network // connections. type TCPConn struct { - conn + fd int + laddr *TCPAddr + raddr *TCPAddr + readDeadline time.Time + writeDeadline time.Time +} + +// DialTCP acts like Dial for TCP networks. +// +// The network must be a TCP network name; see func Dial for details. +// +// If laddr is nil, a local address is automatically chosen. +// If the IP field of raddr is nil or an unspecified IP address, the +// local system is assumed. +func DialTCP(network string, laddr, raddr *TCPAddr) (*TCPConn, error) { + + switch network { + case "tcp", "tcp4": + default: + return nil, fmt.Errorf("Network '%s' not supported", network) + } + + // TINYGO: Use netdev to create TCP socket and connect + + if raddr == nil { + raddr = &TCPAddr{} + } + + if raddr.IP.IsUnspecified() { + return nil, fmt.Errorf("Sorry, localhost isn't available on Tinygo") + } + + fd, err := netdev.Socket(syscall.AF_INET, syscall.SOCK_STREAM, syscall.IPPROTO_TCP) + if err != nil { + return nil, err + } + + if err = netdev.Connect(fd, "", raddr.IP, raddr.Port); err != nil { + netdev.Close(fd) + return nil, err + } + + return &TCPConn{ + fd: fd, + laddr: laddr, + raddr: raddr, + }, nil +} + +// TINYGO: Use netdev for Conn methods: Read = Recv, Write = Send, etc. + +func (c *TCPConn) Read(b []byte) (int, error) { + var timeout time.Duration + + now := time.Now() + + if !c.readDeadline.IsZero() { + if c.readDeadline.Before(now) { + return 0, fmt.Errorf("Read deadline expired") + } else { + timeout = c.readDeadline.Sub(now) + } + } + + n, err := netdev.Recv(c.fd, b, 0, timeout) + // Turn the -1 socket error into 0 and let err speak for error + if n < 0 { + n = 0 + } + return n, err +} + +func (c *TCPConn) Write(b []byte) (int, error) { + var timeout time.Duration + + now := time.Now() + + if !c.writeDeadline.IsZero() { + if c.writeDeadline.Before(now) { + return 0, fmt.Errorf("Write deadline expired") + } else { + timeout = c.writeDeadline.Sub(now) + } + } + + n, err := netdev.Send(c.fd, b, 0, timeout) + // Turn the -1 socket error into 0 and let err speak for error + if n < 0 { + n = 0 + } + return n, err +} + +func (c *TCPConn) Close() error { + return netdev.Close(c.fd) +} + +func (c *TCPConn) LocalAddr() Addr { + return c.laddr +} + +func (c *TCPConn) RemoteAddr() Addr { + return c.raddr +} + +func (c *TCPConn) SetDeadline(t time.Time) error { + c.readDeadline = t + c.writeDeadline = t + return nil +} + +func (c *TCPConn) SetKeepAlive(keepalive bool) error { + return netdev.SetSockOpt(c.fd, syscall.SOL_SOCKET, syscall.SO_KEEPALIVE, keepalive) +} + +func (c *TCPConn) SetKeepAlivePeriod(d time.Duration) error { + // Units are 1/2 seconds + return netdev.SetSockOpt(c.fd, syscall.SOL_TCP, syscall.TCP_KEEPINTVL, 2*d.Seconds()) +} + +func (c *TCPConn) SetReadDeadline(t time.Time) error { + c.readDeadline = t + return nil +} + +func (c *TCPConn) SetWriteDeadline(t time.Time) error { + c.writeDeadline = t + return nil } func (c *TCPConn) CloseWrite() error { - return &OpError{"close", "", nil, nil, ErrNotImplemented} + return fmt.Errorf("CloseWrite not implemented") +} + +type listener struct { + fd int + laddr *TCPAddr +} + +func (l *listener) Accept() (Conn, error) { + fd, err := netdev.Accept(l.fd, IP{}, 0) + if err != nil { + return nil, err + } + + return &TCPConn{ + fd: fd, + laddr: l.laddr, + }, nil +} + +func (l *listener) Close() error { + return netdev.Close(l.fd) +} + +func (l *listener) Addr() Addr { + return l.laddr +} + +func listenTCP(laddr *TCPAddr) (Listener, error) { + fd, err := netdev.Socket(syscall.AF_INET, syscall.SOCK_STREAM, syscall.IPPROTO_TCP) + if err != nil { + return nil, err + } + + err = netdev.Bind(fd, laddr.IP, laddr.Port) + if err != nil { + return nil, err + } + + err = netdev.Listen(fd, 5) + if err != nil { + return nil, err + } + + return &listener{fd: fd, laddr: laddr}, nil } diff --git a/src/net/tlssock.go b/src/net/tlssock.go new file mode 100644 index 0000000000..b5653edd6a --- /dev/null +++ b/src/net/tlssock.go @@ -0,0 +1,154 @@ +// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. + +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// TLS low level connection and record layer + +package net + +import ( + "fmt" + "strconv" + "syscall" + "time" +) + +func DialTLS(addr string) (*TLSConn, error) { + + host, sport, err := SplitHostPort(addr) + if err != nil { + return nil, err + } + + port, err := strconv.Atoi(sport) + if err != nil { + return nil, err + } + + if port == 0 { + port = 443 + } + + fd, err := netdev.Socket(syscall.AF_INET, syscall.SOCK_STREAM, syscall.IPPROTO_TLS) + if err != nil { + return nil, err + } + + if err = netdev.Connect(fd, host, IP{}, port); err != nil { + netdev.Close(fd) + return nil, err + } + + return &TLSConn{ + fd: fd, + }, nil +} + +// A TLSConn represents a secured connection. +// It implements the net.Conn interface. +type TLSConn struct { + fd int + readDeadline time.Time + writeDeadline time.Time +} + +// Access to net.Conn methods. +// Cannot just embed net.Conn because that would +// export the struct field too. + +// LocalAddr returns the local network address. +func (c *TLSConn) LocalAddr() Addr { + // TODO + return nil +} + +// RemoteAddr returns the remote network address. +func (c *TLSConn) RemoteAddr() Addr { + // TODO + return nil +} + +// SetDeadline sets the read and write deadlines associated with the connection. +// A zero value for t means Read and Write will not time out. +// After a Write has timed out, the TLS state is corrupt and all future writes will return the same error. +func (c *TLSConn) SetDeadline(t time.Time) error { + c.readDeadline = t + c.writeDeadline = t + return nil +} + +// SetReadDeadline sets the read deadline on the underlying connection. +// A zero value for t means Read will not time out. +func (c *TLSConn) SetReadDeadline(t time.Time) error { + c.readDeadline = t + return nil +} + +// SetWriteDeadline sets the write deadline on the underlying connection. +// A zero value for t means Write will not time out. +// After a Write has timed out, the TLS state is corrupt and all future writes will return the same error. +func (c *TLSConn) SetWriteDeadline(t time.Time) error { + c.writeDeadline = t + return nil +} + +func (c *TLSConn) Read(b []byte) (int, error) { + var timeout time.Duration + + now := time.Now() + + if !c.readDeadline.IsZero() { + if c.readDeadline.Before(now) { + return 0, fmt.Errorf("Read deadline expired") + } else { + timeout = c.readDeadline.Sub(now) + } + } + + n, err := netdev.Recv(c.fd, b, 0, timeout) + // Turn the -1 socket error into 0 and let err speak for error + if n < 0 { + n = 0 + } + return n, err +} + +func (c *TLSConn) Write(b []byte) (int, error) { + var timeout time.Duration + + now := time.Now() + + if !c.writeDeadline.IsZero() { + if c.writeDeadline.Before(now) { + return 0, fmt.Errorf("Write deadline expired") + } else { + timeout = c.writeDeadline.Sub(now) + } + } + + n, err := netdev.Send(c.fd, b, 0, timeout) + // Turn the -1 socket error into 0 and let err speak for error + if n < 0 { + n = 0 + } + return n, err +} + +func (c *TLSConn) Close() error { + return netdev.Close(c.fd) +} + +// Handshake runs the client or server handshake +// protocol if it has not yet been run. +// +// Most uses of this package need not call Handshake explicitly: the +// first Read or Write will call it automatically. +// +// For control over canceling or setting a timeout on a handshake, use +// HandshakeContext or the Dialer's DialContext method instead. +func (c *TLSConn) Handshake() error { + panic("TLSConn.Handshake() not implemented") + return nil +} diff --git a/src/net/udpsock.go b/src/net/udpsock.go index f10800220c..5ffe697e7f 100644 --- a/src/net/udpsock.go +++ b/src/net/udpsock.go @@ -1,8 +1,18 @@ +// TINYGO: The following is copied and modified from Go 1.19.3 official implementation. + +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package net import ( + "fmt" "internal/itoa" "net/netip" + "strconv" + "syscall" + "time" ) // UDPAddr represents the address of a UDP end point. @@ -53,3 +63,204 @@ func (a *UDPAddr) opAddr() Addr { } return a } + +// ResolveUDPAddr returns an address of UDP end point. +// +// The network must be a UDP network name. +// +// If the host in the address parameter is not a literal IP address or +// the port is not a literal port number, ResolveUDPAddr resolves the +// address to an address of UDP end point. +// Otherwise, it parses the address as a pair of literal IP address +// and port number. +// The address parameter can use a host name, but this is not +// recommended, because it will return at most one of the host name's +// IP addresses. +// +// See func Dial for a description of the network and address +// parameters. +func ResolveUDPAddr(network, address string) (*UDPAddr, error) { + + switch network { + case "udp", "udp4": + default: + return nil, fmt.Errorf("Network '%s' not supported", network) + } + + // TINYGO: Use netdev resolver + + host, sport, err := SplitHostPort(address) + if err != nil { + return nil, err + } + + port, err := strconv.Atoi(sport) + if err != nil { + return nil, fmt.Errorf("Error parsing port '%s' in address: %s", + sport, err) + } + + if host == "" { + return &UDPAddr{Port: port}, nil + } + + ip, err := netdev.GetHostByName(host) + if err != nil { + return nil, fmt.Errorf("Lookup of host name '%s' failed: %s", host, err) + } + + return &UDPAddr{IP: ip, Port: port}, nil +} + +// UDPConn is the implementation of the Conn and PacketConn interfaces +// for UDP network connections. +type UDPConn struct { + fd int + laddr *UDPAddr + raddr *UDPAddr + readDeadline time.Time + writeDeadline time.Time +} + +// Use IANA RFC 6335 port range 49152–65535 for ephemeral (dynamic) ports +var eport = int32(49151) + +func ephemeralPort() int { + // TODO: this is racy, if concurrent DialUDPs; use atomic? + if eport == int32(65535) { + eport = int32(49151) + } else { + eport++ + } + return int(eport) +} + +// DialUDP acts like Dial for UDP networks. +// +// The network must be a UDP network name; see func Dial for details. +// +// If laddr is nil, a local address is automatically chosen. +// If the IP field of raddr is nil or an unspecified IP address, the +// local system is assumed. +func DialUDP(network string, laddr, raddr *UDPAddr) (*UDPConn, error) { + switch network { + case "udp", "udp4": + default: + return nil, fmt.Errorf("Network '%s' not supported", network) + } + + // TINYGO: Use netdev to create UDP socket and connect + + if laddr == nil { + laddr = &UDPAddr{} + } + + if raddr == nil { + raddr = &UDPAddr{} + } + + if raddr.IP.IsUnspecified() { + return nil, fmt.Errorf("Sorry, localhost isn't available on Tinygo") + } + + // If no port was given, grab an ephemeral port + if laddr.Port == 0 { + laddr.Port = ephemeralPort() + } + + fd, err := netdev.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_UDP) + if err != nil { + return nil, err + } + + // Local bind + err = netdev.Bind(fd, laddr.IP, laddr.Port) + if err != nil { + netdev.Close(fd) + return nil, err + } + + // Remote connect + if err = netdev.Connect(fd, "", raddr.IP, raddr.Port); err != nil { + netdev.Close(fd) + return nil, err + } + + return &UDPConn{ + fd: fd, + laddr: laddr, + raddr: raddr, + }, nil +} + +// TINYGO: Use netdev for Conn methods: Read = Recv, Write = Send, etc. + +func (c *UDPConn) Read(b []byte) (int, error) { + var timeout time.Duration + + now := time.Now() + + if !c.readDeadline.IsZero() { + if c.readDeadline.Before(now) { + return 0, fmt.Errorf("Read deadline expired") + } else { + timeout = c.readDeadline.Sub(now) + } + } + + n, err := netdev.Recv(c.fd, b, 0, timeout) + // Turn the -1 socket error into 0 and let err speak for error + if n < 0 { + n = 0 + } + return n, err +} + +func (c *UDPConn) Write(b []byte) (int, error) { + var timeout time.Duration + + now := time.Now() + + if !c.writeDeadline.IsZero() { + if c.writeDeadline.Before(now) { + return 0, fmt.Errorf("Write deadline expired") + } else { + timeout = c.writeDeadline.Sub(now) + } + } + + n, err := netdev.Send(c.fd, b, 0, timeout) + // Turn the -1 socket error into 0 and let err speak for error + if n < 0 { + n = 0 + } + return n, err +} + +func (c *UDPConn) Close() error { + return netdev.Close(c.fd) +} + +func (c *UDPConn) LocalAddr() Addr { + return c.laddr +} + +func (c *UDPConn) RemoteAddr() Addr { + return c.raddr +} + +func (c *UDPConn) SetDeadline(t time.Time) error { + c.readDeadline = t + c.writeDeadline = t + return nil +} + +func (c *UDPConn) SetReadDeadline(t time.Time) error { + c.readDeadline = t + return nil +} + +func (c *UDPConn) SetWriteDeadline(t time.Time) error { + c.writeDeadline = t + return nil +} diff --git a/src/net/writev_test.go b/src/net/writev_test.go deleted file mode 100644 index 3a2c3efa3c..0000000000 --- a/src/net/writev_test.go +++ /dev/null @@ -1,132 +0,0 @@ -// The following is copied from Go 1.17 official implementation and -// modified to accommodate TinyGo. - -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package net - -import ( - "bytes" - "fmt" - "io" - "reflect" - "testing" -) - -func TestBuffers_read(t *testing.T) { - const story = "once upon a time in Gopherland ... " - buffers := Buffers{ - []byte("once "), - []byte("upon "), - []byte("a "), - []byte("time "), - []byte("in "), - []byte("Gopherland ... "), - } - got, err := io.ReadAll(&buffers) - if err != nil { - t.Fatal(err) - } - if string(got) != story { - t.Errorf("read %q; want %q", got, story) - } - if len(buffers) != 0 { - t.Errorf("len(buffers) = %d; want 0", len(buffers)) - } -} - -func TestBuffers_consume(t *testing.T) { - tests := []struct { - in Buffers - consume int64 - want Buffers - }{ - { - in: Buffers{[]byte("foo"), []byte("bar")}, - consume: 0, - want: Buffers{[]byte("foo"), []byte("bar")}, - }, - { - in: Buffers{[]byte("foo"), []byte("bar")}, - consume: 2, - want: Buffers{[]byte("o"), []byte("bar")}, - }, - { - in: Buffers{[]byte("foo"), []byte("bar")}, - consume: 3, - want: Buffers{[]byte("bar")}, - }, - { - in: Buffers{[]byte("foo"), []byte("bar")}, - consume: 4, - want: Buffers{[]byte("ar")}, - }, - { - in: Buffers{nil, nil, nil, []byte("bar")}, - consume: 1, - want: Buffers{[]byte("ar")}, - }, - { - in: Buffers{nil, nil, nil, []byte("foo")}, - consume: 0, - want: Buffers{[]byte("foo")}, - }, - { - in: Buffers{nil, nil, nil}, - consume: 0, - want: Buffers{}, - }, - } - for i, tt := range tests { - in := tt.in - in.consume(tt.consume) - if !reflect.DeepEqual(in, tt.want) { - t.Errorf("%d. after consume(%d) = %+v, want %+v", i, tt.consume, in, tt.want) - } - } -} - -func TestBuffers_WriteTo(t *testing.T) { - for _, name := range []string{"WriteTo", "Copy"} { - for _, size := range []int{0, 10, 1023, 1024, 1025} { - t.Run(fmt.Sprintf("%s/%d", name, size), func(t *testing.T) { - testBuffer_writeTo(t, size, name == "Copy") - }) - } - } -} - -func testBuffer_writeTo(t *testing.T, chunks int, useCopy bool) { - var want bytes.Buffer - for i := 0; i < chunks; i++ { - want.WriteByte(byte(i)) - } - - var b bytes.Buffer - buffers := make(Buffers, chunks) - for i := range buffers { - buffers[i] = want.Bytes()[i : i+1] - } - var n int64 - var err error - if useCopy { - n, err = io.Copy(&b, &buffers) - } else { - n, err = buffers.WriteTo(&b) - } - if err != nil { - t.Fatal(err) - } - if len(buffers) != 0 { - t.Fatal(fmt.Errorf("len(buffers) = %d; want 0", len(buffers))) - } - if n != int64(want.Len()) { - t.Fatal(fmt.Errorf("Buffers.WriteTo returned %d; want %d", n, want.Len())) - } - all, err := io.ReadAll(&b) - if !bytes.Equal(all, want.Bytes()) || err != nil { - t.Fatal(fmt.Errorf("read %q, %v; want %q, nil", all, err, want.Bytes())) - } -} diff --git a/src/os/file_other.go b/src/os/file_other.go index 7e4642c0bf..c95cf546ba 100644 --- a/src/os/file_other.go +++ b/src/os/file_other.go @@ -41,6 +41,14 @@ func NewFile(fd uintptr, name string) *File { return &File{&file{stdioFileHandle(fd), name}} } +// Rename renames (moves) oldpath to newpath. +// If newpath already exists and is not a directory, Rename replaces it. +// OS-specific restrictions may apply when oldpath and newpath are in different directories. +// If there is an error, it will be of type *LinkError. +func Rename(oldpath, newpath string) error { + return ErrNotImplemented +} + // Read reads up to len(b) bytes from machine.Serial. // It returns the number of bytes read and any error encountered. func (f stdioFileHandle) Read(b []byte) (n int, err error) { diff --git a/src/syscall/net.go b/src/syscall/net.go index 531fa80d8f..5f8c50da9a 100644 --- a/src/syscall/net.go +++ b/src/syscall/net.go @@ -32,3 +32,22 @@ type Conn interface { // SyscallConn returns a raw network connection. SyscallConn() (RawConn, error) } + +const ( + AF_INET = 0x2 + SOCK_STREAM = 0x1 + SOCK_DGRAM = 0x2 + SOL_SOCKET = 0x1 + SO_KEEPALIVE = 0x9 + SOL_TCP = 0x6 + TCP_KEEPINTVL = 0x5 + IPPROTO_TCP = 0x6 + IPPROTO_UDP = 0x11 + F_SETFL = 0x4 + + // TINYGO: Made up, not a real IP protocol number. This is used to + // create a TLS socket on the device, assuming the device supports mbed + // TLS. + + IPPROTO_TLS = 0xFE +) diff --git a/src/syscall/syscall_libc_darwin.go b/src/syscall/syscall_libc_darwin.go index 0b908255ae..066f12275b 100644 --- a/src/syscall/syscall_libc_darwin.go +++ b/src/syscall/syscall_libc_darwin.go @@ -53,7 +53,6 @@ const ( DT_UNKNOWN = 0x0 DT_WHT = 0xe F_GETFL = 0x3 - F_SETFL = 0x4 O_NONBLOCK = 0x4 ) diff --git a/src/syscall/syscall_libc_wasi.go b/src/syscall/syscall_libc_wasi.go index 29043b4a36..d80986a704 100644 --- a/src/syscall/syscall_libc_wasi.go +++ b/src/syscall/syscall_libc_wasi.go @@ -90,7 +90,6 @@ const ( // ../../lib/wasi-libc/expected/wasm32-wasi/predefined-macros.txt F_GETFL = 3 - F_SETFL = 4 ) // These values are needed as a stub until Go supports WASI as a full target. From e1de929dc942d5bb719602cdf9b2aa1bac59a2d1 Mon Sep 17 00:00:00 2001 From: Scott Feldman Date: Tue, 28 Mar 2023 11:19:17 -0700 Subject: [PATCH 02/11] move IPPROTO_TLS to netdev to avoid src/syscall dependency --- src/net/netdev.go | 6 ++++++ src/net/tlssock.go | 2 +- src/syscall/net.go | 6 ------ 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/net/netdev.go b/src/net/netdev.go index 2d294c15b3..833804f236 100644 --- a/src/net/netdev.go +++ b/src/net/netdev.go @@ -4,6 +4,12 @@ import ( "time" ) +const ( + // Made up, not a real IP protocol number. This is used to create a + // TLS socket on the device, assuming the device supports mbed TLS. + IPPROTO_TLS = 0xFE +) + // netdev is the current netdev, set by the application with useNetdev() var netdev netdever diff --git a/src/net/tlssock.go b/src/net/tlssock.go index b5653edd6a..6612e95fe7 100644 --- a/src/net/tlssock.go +++ b/src/net/tlssock.go @@ -31,7 +31,7 @@ func DialTLS(addr string) (*TLSConn, error) { port = 443 } - fd, err := netdev.Socket(syscall.AF_INET, syscall.SOCK_STREAM, syscall.IPPROTO_TLS) + fd, err := netdev.Socket(syscall.AF_INET, syscall.SOCK_STREAM, IPPROTO_TLS) if err != nil { return nil, err } diff --git a/src/syscall/net.go b/src/syscall/net.go index 5f8c50da9a..c1dd3e150b 100644 --- a/src/syscall/net.go +++ b/src/syscall/net.go @@ -44,10 +44,4 @@ const ( IPPROTO_TCP = 0x6 IPPROTO_UDP = 0x11 F_SETFL = 0x4 - - // TINYGO: Made up, not a real IP protocol number. This is used to - // create a TLS socket on the device, assuming the device supports mbed - // TLS. - - IPPROTO_TLS = 0xFE ) From 63b78b2f938775617be6e57246eb84e0679836a9 Mon Sep 17 00:00:00 2001 From: Scott Feldman Date: Sat, 1 Apr 2023 14:14:53 -0700 Subject: [PATCH 03/11] udpsock: remove TODO comment on ephemeral ports being racy --- src/net/udpsock.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/net/udpsock.go b/src/net/udpsock.go index 5ffe697e7f..f6a57136af 100644 --- a/src/net/udpsock.go +++ b/src/net/udpsock.go @@ -126,7 +126,6 @@ type UDPConn struct { var eport = int32(49151) func ephemeralPort() int { - // TODO: this is racy, if concurrent DialUDPs; use atomic? if eport == int32(65535) { eport = int32(49151) } else { From 1d2106fd7b802ec7b2d42038fd477ce5541777ac Mon Sep 17 00:00:00 2001 From: Scott Feldman Date: Sat, 1 Apr 2023 14:56:54 -0700 Subject: [PATCH 04/11] Makefile: remove net package from tests to workaround Windows failure --- Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Makefile b/Makefile index d575f5d87d..dea1ab6cbb 100644 --- a/Makefile +++ b/Makefile @@ -308,8 +308,6 @@ TEST_PACKAGES_FAST = \ internal/profile \ math \ math/cmplx \ - net/http/internal/ascii \ - net/mail \ os \ path \ reflect \ From 155ada5c3c001cba3e0e6a0aedc2f56affadeb4d Mon Sep 17 00:00:00 2001 From: Scott Feldman Date: Sat, 1 Apr 2023 16:46:36 -0700 Subject: [PATCH 05/11] Makefile: revert removing net package from test for Windows --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index dea1ab6cbb..d575f5d87d 100644 --- a/Makefile +++ b/Makefile @@ -308,6 +308,8 @@ TEST_PACKAGES_FAST = \ internal/profile \ math \ math/cmplx \ + net/http/internal/ascii \ + net/mail \ os \ path \ reflect \ From c71f2083cce090962344b46eb7dd7c759681c5dd Mon Sep 17 00:00:00 2001 From: Scott Feldman Date: Sat, 1 Apr 2023 17:15:49 -0700 Subject: [PATCH 06/11] loader: add windows to targets to merge syscall package --- loader/goroot.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loader/goroot.go b/loader/goroot.go index d1d8e044dd..d32c3e09d6 100644 --- a/loader/goroot.go +++ b/loader/goroot.go @@ -214,7 +214,7 @@ func listGorootMergeLinks(goroot, tinygoroot string, overrides map[string]bool) // with the TinyGo version. This is the case on some targets. func needsSyscallPackage(buildTags []string) bool { for _, tag := range buildTags { - if tag == "baremetal" || tag == "darwin" || tag == "nintendoswitch" || tag == "tinygo.wasm" { + if tag == "baremetal" || tag == "darwin" || tag == "nintendoswitch" || tag == "tinygo.wasm" || tag == "windows" { return true } } From d89f431a9957704b62d2c3e4e90aa7513f8314bd Mon Sep 17 00:00:00 2001 From: Scott Feldman Date: Mon, 3 Apr 2023 10:14:50 -0700 Subject: [PATCH 07/11] loader: revert adding windows to syscall package merge --- loader/goroot.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loader/goroot.go b/loader/goroot.go index d32c3e09d6..d1d8e044dd 100644 --- a/loader/goroot.go +++ b/loader/goroot.go @@ -214,7 +214,7 @@ func listGorootMergeLinks(goroot, tinygoroot string, overrides map[string]bool) // with the TinyGo version. This is the case on some targets. func needsSyscallPackage(buildTags []string) bool { for _, tag := range buildTags { - if tag == "baremetal" || tag == "darwin" || tag == "nintendoswitch" || tag == "tinygo.wasm" || tag == "windows" { + if tag == "baremetal" || tag == "darwin" || tag == "nintendoswitch" || tag == "tinygo.wasm" { return true } } From 42c2d3ca20586c88c812ba91049775252e56bb71 Mon Sep 17 00:00:00 2001 From: Scott Feldman Date: Sun, 16 Apr 2023 09:48:09 -0700 Subject: [PATCH 08/11] BUG Fix: return proper net.OpError's for Read/Write operations Need to return the same error structure/content as regular Go for net.Conn Read/Write operations. Found/fixed when testing deadlines on Read/Write operations. --- src/net/net.go | 9 ++++ src/net/netdev.go | 4 +- src/net/tcpsock.go | 38 +++++-------- src/net/tlssock.go | 129 ++++++++++++++++++++------------------------- src/net/udpsock.go | 37 ++++--------- 5 files changed, 92 insertions(+), 125 deletions(-) diff --git a/src/net/net.go b/src/net/net.go index 2e7f9054c9..cf88c6bfbd 100644 --- a/src/net/net.go +++ b/src/net/net.go @@ -161,6 +161,15 @@ func (e *OpError) Error() string { return s } +type timeout interface { + Timeout() bool +} + +func (e *OpError) Timeout() bool { + t, ok := e.Err.(timeout) + return ok && t.Timeout() +} + // A ParseError is the error type of literal network address parsers. type ParseError struct { // Type is the type of string that was expected, such as diff --git a/src/net/netdev.go b/src/net/netdev.go index 833804f236..f7add3bb72 100644 --- a/src/net/netdev.go +++ b/src/net/netdev.go @@ -45,8 +45,8 @@ type netdever interface { Connect(sockfd int, host string, ip IP, port int) error Listen(sockfd int, backlog int) error Accept(sockfd int, ip IP, port int) (int, error) - Send(sockfd int, buf []byte, flags int, timeout time.Duration) (int, error) - Recv(sockfd int, buf []byte, flags int, timeout time.Duration) (int, error) + Send(sockfd int, buf []byte, flags int, deadline time.Time) (int, error) + Recv(sockfd int, buf []byte, flags int, deadline time.Time) (int, error) Close(sockfd int) error SetSockOpt(sockfd int, level int, opt int, value interface{}) error } diff --git a/src/net/tcpsock.go b/src/net/tcpsock.go index f5b1b65eff..abcbc2cbe6 100644 --- a/src/net/tcpsock.go +++ b/src/net/tcpsock.go @@ -9,6 +9,7 @@ package net import ( "fmt" "internal/itoa" + "io" "net/netip" "strconv" "syscall" @@ -116,6 +117,7 @@ func ResolveTCPAddr(network, address string) (*TCPAddr, error) { // connections. type TCPConn struct { fd int + net string laddr *TCPAddr raddr *TCPAddr readDeadline time.Time @@ -159,6 +161,7 @@ func DialTCP(network string, laddr, raddr *TCPAddr) (*TCPConn, error) { return &TCPConn{ fd: fd, + net: network, laddr: laddr, raddr: raddr, }, nil @@ -167,44 +170,26 @@ func DialTCP(network string, laddr, raddr *TCPAddr) (*TCPConn, error) { // TINYGO: Use netdev for Conn methods: Read = Recv, Write = Send, etc. func (c *TCPConn) Read(b []byte) (int, error) { - var timeout time.Duration - - now := time.Now() - - if !c.readDeadline.IsZero() { - if c.readDeadline.Before(now) { - return 0, fmt.Errorf("Read deadline expired") - } else { - timeout = c.readDeadline.Sub(now) - } - } - - n, err := netdev.Recv(c.fd, b, 0, timeout) + n, err := netdev.Recv(c.fd, b, 0, c.readDeadline) // Turn the -1 socket error into 0 and let err speak for error if n < 0 { n = 0 } + if err != nil && err != io.EOF { + err = &OpError{Op: "read", Net: c.net, Source: c.laddr, Addr: c.raddr, Err: err} + } return n, err } func (c *TCPConn) Write(b []byte) (int, error) { - var timeout time.Duration - - now := time.Now() - - if !c.writeDeadline.IsZero() { - if c.writeDeadline.Before(now) { - return 0, fmt.Errorf("Write deadline expired") - } else { - timeout = c.writeDeadline.Sub(now) - } - } - - n, err := netdev.Send(c.fd, b, 0, timeout) + n, err := netdev.Send(c.fd, b, 0, c.writeDeadline) // Turn the -1 socket error into 0 and let err speak for error if n < 0 { n = 0 } + if err != nil { + err = &OpError{Op: "write", Net: c.net, Source: c.laddr, Addr: c.raddr, Err: err} + } return n, err } @@ -262,6 +247,7 @@ func (l *listener) Accept() (Conn, error) { return &TCPConn{ fd: fd, + net: "tcp", laddr: l.laddr, }, nil } diff --git a/src/net/tlssock.go b/src/net/tlssock.go index 6612e95fe7..81c426a059 100644 --- a/src/net/tlssock.go +++ b/src/net/tlssock.go @@ -9,12 +9,39 @@ package net import ( - "fmt" + "internal/itoa" + "io" "strconv" "syscall" "time" ) +// TLSAddr represents the address of a TLS end point. +type TLSAddr struct { + Host string + Port int +} + +func (a *TLSAddr) Network() string { return "tls" } + +func (a *TLSAddr) String() string { + if a == nil { + return "" + } + return JoinHostPort(a.Host, itoa.Itoa(a.Port)) +} + +// A TLSConn represents a secured connection. +// It implements the net.Conn interface. +type TLSConn struct { + fd int + net string + laddr *TLSAddr + raddr *TLSAddr + readDeadline time.Time + writeDeadline time.Time +} + func DialTLS(addr string) (*TLSConn, error) { host, sport, err := SplitHostPort(addr) @@ -42,104 +69,64 @@ func DialTLS(addr string) (*TLSConn, error) { } return &TLSConn{ - fd: fd, + fd: fd, + net: "tls", + raddr: &TLSAddr{host, port}, }, nil } -// A TLSConn represents a secured connection. -// It implements the net.Conn interface. -type TLSConn struct { - fd int - readDeadline time.Time - writeDeadline time.Time +func (c *TLSConn) Read(b []byte) (int, error) { + n, err := netdev.Recv(c.fd, b, 0, c.readDeadline) + // Turn the -1 socket error into 0 and let err speak for error + if n < 0 { + n = 0 + } + if err != nil && err != io.EOF { + err = &OpError{Op: "read", Net: c.net, Source: c.laddr, Addr: c.raddr, Err: err} + } + return n, err +} + +func (c *TLSConn) Write(b []byte) (int, error) { + n, err := netdev.Send(c.fd, b, 0, c.writeDeadline) + // Turn the -1 socket error into 0 and let err speak for error + if n < 0 { + n = 0 + } + if err != nil { + err = &OpError{Op: "write", Net: c.net, Source: c.laddr, Addr: c.raddr, Err: err} + } + return n, err } -// Access to net.Conn methods. -// Cannot just embed net.Conn because that would -// export the struct field too. +func (c *TLSConn) Close() error { + return netdev.Close(c.fd) +} -// LocalAddr returns the local network address. func (c *TLSConn) LocalAddr() Addr { - // TODO - return nil + return c.laddr } -// RemoteAddr returns the remote network address. func (c *TLSConn) RemoteAddr() Addr { - // TODO - return nil + return c.raddr } -// SetDeadline sets the read and write deadlines associated with the connection. -// A zero value for t means Read and Write will not time out. -// After a Write has timed out, the TLS state is corrupt and all future writes will return the same error. func (c *TLSConn) SetDeadline(t time.Time) error { c.readDeadline = t c.writeDeadline = t return nil } -// SetReadDeadline sets the read deadline on the underlying connection. -// A zero value for t means Read will not time out. func (c *TLSConn) SetReadDeadline(t time.Time) error { c.readDeadline = t return nil } -// SetWriteDeadline sets the write deadline on the underlying connection. -// A zero value for t means Write will not time out. -// After a Write has timed out, the TLS state is corrupt and all future writes will return the same error. func (c *TLSConn) SetWriteDeadline(t time.Time) error { c.writeDeadline = t return nil } -func (c *TLSConn) Read(b []byte) (int, error) { - var timeout time.Duration - - now := time.Now() - - if !c.readDeadline.IsZero() { - if c.readDeadline.Before(now) { - return 0, fmt.Errorf("Read deadline expired") - } else { - timeout = c.readDeadline.Sub(now) - } - } - - n, err := netdev.Recv(c.fd, b, 0, timeout) - // Turn the -1 socket error into 0 and let err speak for error - if n < 0 { - n = 0 - } - return n, err -} - -func (c *TLSConn) Write(b []byte) (int, error) { - var timeout time.Duration - - now := time.Now() - - if !c.writeDeadline.IsZero() { - if c.writeDeadline.Before(now) { - return 0, fmt.Errorf("Write deadline expired") - } else { - timeout = c.writeDeadline.Sub(now) - } - } - - n, err := netdev.Send(c.fd, b, 0, timeout) - // Turn the -1 socket error into 0 and let err speak for error - if n < 0 { - n = 0 - } - return n, err -} - -func (c *TLSConn) Close() error { - return netdev.Close(c.fd) -} - // Handshake runs the client or server handshake // protocol if it has not yet been run. // diff --git a/src/net/udpsock.go b/src/net/udpsock.go index f6a57136af..55bc4c325a 100644 --- a/src/net/udpsock.go +++ b/src/net/udpsock.go @@ -9,6 +9,7 @@ package net import ( "fmt" "internal/itoa" + "io" "net/netip" "strconv" "syscall" @@ -116,6 +117,7 @@ func ResolveUDPAddr(network, address string) (*UDPAddr, error) { // for UDP network connections. type UDPConn struct { fd int + net string laddr *UDPAddr raddr *UDPAddr readDeadline time.Time @@ -187,6 +189,7 @@ func DialUDP(network string, laddr, raddr *UDPAddr) (*UDPConn, error) { return &UDPConn{ fd: fd, + net: network, laddr: laddr, raddr: raddr, }, nil @@ -195,44 +198,26 @@ func DialUDP(network string, laddr, raddr *UDPAddr) (*UDPConn, error) { // TINYGO: Use netdev for Conn methods: Read = Recv, Write = Send, etc. func (c *UDPConn) Read(b []byte) (int, error) { - var timeout time.Duration - - now := time.Now() - - if !c.readDeadline.IsZero() { - if c.readDeadline.Before(now) { - return 0, fmt.Errorf("Read deadline expired") - } else { - timeout = c.readDeadline.Sub(now) - } - } - - n, err := netdev.Recv(c.fd, b, 0, timeout) + n, err := netdev.Recv(c.fd, b, 0, c.readDeadline) // Turn the -1 socket error into 0 and let err speak for error if n < 0 { n = 0 } + if err != nil && err != io.EOF { + err = &OpError{Op: "read", Net: c.net, Source: c.laddr, Addr: c.raddr, Err: err} + } return n, err } func (c *UDPConn) Write(b []byte) (int, error) { - var timeout time.Duration - - now := time.Now() - - if !c.writeDeadline.IsZero() { - if c.writeDeadline.Before(now) { - return 0, fmt.Errorf("Write deadline expired") - } else { - timeout = c.writeDeadline.Sub(now) - } - } - - n, err := netdev.Send(c.fd, b, 0, timeout) + n, err := netdev.Send(c.fd, b, 0, c.writeDeadline) // Turn the -1 socket error into 0 and let err speak for error if n < 0 { n = 0 } + if err != nil { + err = &OpError{Op: "write", Net: c.net, Source: c.laddr, Addr: c.raddr, Err: err} + } return n, err } From b7589678aa14f30453419220aaa67bac7e84d230 Mon Sep 17 00:00:00 2001 From: Scott Feldman Date: Wed, 26 Apr 2023 01:38:05 -0700 Subject: [PATCH 09/11] BUG Fix: use port :80 if server.Addr is empty --- src/net/tcpsock.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/net/tcpsock.go b/src/net/tcpsock.go index abcbc2cbe6..6222c9788f 100644 --- a/src/net/tcpsock.go +++ b/src/net/tcpsock.go @@ -88,6 +88,11 @@ func ResolveTCPAddr(network, address string) (*TCPAddr, error) { return nil, fmt.Errorf("Network '%s' not supported", network) } + switch address { + case ":http": + address = ":80" + } + // TINYGO: Use netdev resolver host, sport, err := SplitHostPort(address) From a48baf2fc511a584c2dd42de9b1abd40e66d4dc7 Mon Sep 17 00:00:00 2001 From: Scott Feldman Date: Tue, 2 May 2023 16:39:11 -0700 Subject: [PATCH 10/11] move syscall constants for networking into net space to avoid windows build issue --- src/net/netdev.go | 12 +++++++++++- src/net/tcpsock.go | 9 ++++----- src/net/tlssock.go | 3 +-- src/net/udpsock.go | 3 +-- src/syscall/net.go | 13 ------------- src/syscall/syscall_libc_darwin.go | 1 + src/syscall/syscall_libc_wasi.go | 1 + 7 files changed, 19 insertions(+), 23 deletions(-) diff --git a/src/net/netdev.go b/src/net/netdev.go index f7add3bb72..05150e270e 100644 --- a/src/net/netdev.go +++ b/src/net/netdev.go @@ -5,9 +5,19 @@ import ( ) const ( + AF_INET = 0x2 + SOCK_STREAM = 0x1 + SOCK_DGRAM = 0x2 + SOL_SOCKET = 0x1 + SO_KEEPALIVE = 0x9 + SOL_TCP = 0x6 + TCP_KEEPINTVL = 0x5 + IPPROTO_TCP = 0x6 + IPPROTO_UDP = 0x11 // Made up, not a real IP protocol number. This is used to create a // TLS socket on the device, assuming the device supports mbed TLS. IPPROTO_TLS = 0xFE + F_SETFL = 0x4 ) // netdev is the current netdev, set by the application with useNetdev() @@ -31,7 +41,7 @@ func useNetdev(dev netdever) { // // NOTE: The netdever interface is mirrored in drivers/netdev.go. // NOTE: If making changes to this interface, mirror the changes in -// NOTE: drivers/netdev.go, and visa-versa. +// NOTE: drivers/netdev.go, and vice-versa. type netdever interface { diff --git a/src/net/tcpsock.go b/src/net/tcpsock.go index 6222c9788f..25a5475063 100644 --- a/src/net/tcpsock.go +++ b/src/net/tcpsock.go @@ -12,7 +12,6 @@ import ( "io" "net/netip" "strconv" - "syscall" "time" ) @@ -154,7 +153,7 @@ func DialTCP(network string, laddr, raddr *TCPAddr) (*TCPConn, error) { return nil, fmt.Errorf("Sorry, localhost isn't available on Tinygo") } - fd, err := netdev.Socket(syscall.AF_INET, syscall.SOCK_STREAM, syscall.IPPROTO_TCP) + fd, err := netdev.Socket(AF_INET, SOCK_STREAM, IPPROTO_TCP) if err != nil { return nil, err } @@ -217,12 +216,12 @@ func (c *TCPConn) SetDeadline(t time.Time) error { } func (c *TCPConn) SetKeepAlive(keepalive bool) error { - return netdev.SetSockOpt(c.fd, syscall.SOL_SOCKET, syscall.SO_KEEPALIVE, keepalive) + return netdev.SetSockOpt(c.fd, SOL_SOCKET, SO_KEEPALIVE, keepalive) } func (c *TCPConn) SetKeepAlivePeriod(d time.Duration) error { // Units are 1/2 seconds - return netdev.SetSockOpt(c.fd, syscall.SOL_TCP, syscall.TCP_KEEPINTVL, 2*d.Seconds()) + return netdev.SetSockOpt(c.fd, SOL_TCP, TCP_KEEPINTVL, 2*d.Seconds()) } func (c *TCPConn) SetReadDeadline(t time.Time) error { @@ -266,7 +265,7 @@ func (l *listener) Addr() Addr { } func listenTCP(laddr *TCPAddr) (Listener, error) { - fd, err := netdev.Socket(syscall.AF_INET, syscall.SOCK_STREAM, syscall.IPPROTO_TCP) + fd, err := netdev.Socket(AF_INET, SOCK_STREAM, IPPROTO_TCP) if err != nil { return nil, err } diff --git a/src/net/tlssock.go b/src/net/tlssock.go index 81c426a059..77d0245e15 100644 --- a/src/net/tlssock.go +++ b/src/net/tlssock.go @@ -12,7 +12,6 @@ import ( "internal/itoa" "io" "strconv" - "syscall" "time" ) @@ -58,7 +57,7 @@ func DialTLS(addr string) (*TLSConn, error) { port = 443 } - fd, err := netdev.Socket(syscall.AF_INET, syscall.SOCK_STREAM, IPPROTO_TLS) + fd, err := netdev.Socket(AF_INET, SOCK_STREAM, IPPROTO_TLS) if err != nil { return nil, err } diff --git a/src/net/udpsock.go b/src/net/udpsock.go index 55bc4c325a..9fa2054224 100644 --- a/src/net/udpsock.go +++ b/src/net/udpsock.go @@ -12,7 +12,6 @@ import ( "io" "net/netip" "strconv" - "syscall" "time" ) @@ -169,7 +168,7 @@ func DialUDP(network string, laddr, raddr *UDPAddr) (*UDPConn, error) { laddr.Port = ephemeralPort() } - fd, err := netdev.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_UDP) + fd, err := netdev.Socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) if err != nil { return nil, err } diff --git a/src/syscall/net.go b/src/syscall/net.go index c1dd3e150b..531fa80d8f 100644 --- a/src/syscall/net.go +++ b/src/syscall/net.go @@ -32,16 +32,3 @@ type Conn interface { // SyscallConn returns a raw network connection. SyscallConn() (RawConn, error) } - -const ( - AF_INET = 0x2 - SOCK_STREAM = 0x1 - SOCK_DGRAM = 0x2 - SOL_SOCKET = 0x1 - SO_KEEPALIVE = 0x9 - SOL_TCP = 0x6 - TCP_KEEPINTVL = 0x5 - IPPROTO_TCP = 0x6 - IPPROTO_UDP = 0x11 - F_SETFL = 0x4 -) diff --git a/src/syscall/syscall_libc_darwin.go b/src/syscall/syscall_libc_darwin.go index 066f12275b..0b908255ae 100644 --- a/src/syscall/syscall_libc_darwin.go +++ b/src/syscall/syscall_libc_darwin.go @@ -53,6 +53,7 @@ const ( DT_UNKNOWN = 0x0 DT_WHT = 0xe F_GETFL = 0x3 + F_SETFL = 0x4 O_NONBLOCK = 0x4 ) diff --git a/src/syscall/syscall_libc_wasi.go b/src/syscall/syscall_libc_wasi.go index d80986a704..29043b4a36 100644 --- a/src/syscall/syscall_libc_wasi.go +++ b/src/syscall/syscall_libc_wasi.go @@ -90,6 +90,7 @@ const ( // ../../lib/wasi-libc/expected/wasm32-wasi/predefined-macros.txt F_GETFL = 3 + F_SETFL = 4 ) // These values are needed as a stub until Go supports WASI as a full target. From 0bc2aa62474cf86bd401c49f9e2bbdf21cbd4dd5 Mon Sep 17 00:00:00 2001 From: Scott Feldman Date: Tue, 2 May 2023 18:05:11 -0700 Subject: [PATCH 11/11] Unexport net constants copied over from syscall --- src/net/netdev.go | 22 +++++++++++----------- src/net/tcpsock.go | 8 ++++---- src/net/tlssock.go | 2 +- src/net/udpsock.go | 2 +- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/net/netdev.go b/src/net/netdev.go index 05150e270e..95d42db1c4 100644 --- a/src/net/netdev.go +++ b/src/net/netdev.go @@ -5,19 +5,19 @@ import ( ) const ( - AF_INET = 0x2 - SOCK_STREAM = 0x1 - SOCK_DGRAM = 0x2 - SOL_SOCKET = 0x1 - SO_KEEPALIVE = 0x9 - SOL_TCP = 0x6 - TCP_KEEPINTVL = 0x5 - IPPROTO_TCP = 0x6 - IPPROTO_UDP = 0x11 + _AF_INET = 0x2 + _SOCK_STREAM = 0x1 + _SOCK_DGRAM = 0x2 + _SOL_SOCKET = 0x1 + _SO_KEEPALIVE = 0x9 + _SOL_TCP = 0x6 + _TCP_KEEPINTVL = 0x5 + _IPPROTO_TCP = 0x6 + _IPPROTO_UDP = 0x11 // Made up, not a real IP protocol number. This is used to create a // TLS socket on the device, assuming the device supports mbed TLS. - IPPROTO_TLS = 0xFE - F_SETFL = 0x4 + _IPPROTO_TLS = 0xFE + _F_SETFL = 0x4 ) // netdev is the current netdev, set by the application with useNetdev() diff --git a/src/net/tcpsock.go b/src/net/tcpsock.go index 25a5475063..42af8552dc 100644 --- a/src/net/tcpsock.go +++ b/src/net/tcpsock.go @@ -153,7 +153,7 @@ func DialTCP(network string, laddr, raddr *TCPAddr) (*TCPConn, error) { return nil, fmt.Errorf("Sorry, localhost isn't available on Tinygo") } - fd, err := netdev.Socket(AF_INET, SOCK_STREAM, IPPROTO_TCP) + fd, err := netdev.Socket(_AF_INET, _SOCK_STREAM, _IPPROTO_TCP) if err != nil { return nil, err } @@ -216,12 +216,12 @@ func (c *TCPConn) SetDeadline(t time.Time) error { } func (c *TCPConn) SetKeepAlive(keepalive bool) error { - return netdev.SetSockOpt(c.fd, SOL_SOCKET, SO_KEEPALIVE, keepalive) + return netdev.SetSockOpt(c.fd, _SOL_SOCKET, _SO_KEEPALIVE, keepalive) } func (c *TCPConn) SetKeepAlivePeriod(d time.Duration) error { // Units are 1/2 seconds - return netdev.SetSockOpt(c.fd, SOL_TCP, TCP_KEEPINTVL, 2*d.Seconds()) + return netdev.SetSockOpt(c.fd, _SOL_TCP, _TCP_KEEPINTVL, 2*d.Seconds()) } func (c *TCPConn) SetReadDeadline(t time.Time) error { @@ -265,7 +265,7 @@ func (l *listener) Addr() Addr { } func listenTCP(laddr *TCPAddr) (Listener, error) { - fd, err := netdev.Socket(AF_INET, SOCK_STREAM, IPPROTO_TCP) + fd, err := netdev.Socket(_AF_INET, _SOCK_STREAM, _IPPROTO_TCP) if err != nil { return nil, err } diff --git a/src/net/tlssock.go b/src/net/tlssock.go index 77d0245e15..f4f228b8dc 100644 --- a/src/net/tlssock.go +++ b/src/net/tlssock.go @@ -57,7 +57,7 @@ func DialTLS(addr string) (*TLSConn, error) { port = 443 } - fd, err := netdev.Socket(AF_INET, SOCK_STREAM, IPPROTO_TLS) + fd, err := netdev.Socket(_AF_INET, _SOCK_STREAM, _IPPROTO_TLS) if err != nil { return nil, err } diff --git a/src/net/udpsock.go b/src/net/udpsock.go index 9fa2054224..f782c5f55d 100644 --- a/src/net/udpsock.go +++ b/src/net/udpsock.go @@ -168,7 +168,7 @@ func DialUDP(network string, laddr, raddr *UDPAddr) (*UDPConn, error) { laddr.Port = ephemeralPort() } - fd, err := netdev.Socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) + fd, err := netdev.Socket(_AF_INET, _SOCK_DGRAM, _IPPROTO_UDP) if err != nil { return nil, err }