3 files changed, 513 insertions, 0 deletions
diff --git a/src/net/http/internal/chunked.go b/src/net/http/internal/chunked.go
new file mode 100644
index 0000000..f06e572
--- /dev/null
+++ b/src/net/http/internal/chunked.go
@@ -0,0 +1,255 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// The wire protocol for HTTP's "chunked" Transfer-Encoding.
+
+// Package internal contains HTTP internals shared by net/http and
+// net/http/httputil.
+package internal
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+)
+
+const maxLineLength = 4096 // assumed <= bufio.defaultBufSize
+
+var ErrLineTooLong = errors.New("header line too long")
+
+// NewChunkedReader returns a new chunkedReader that translates the data read from r
+// out of HTTP "chunked" format before returning it.
+// The chunkedReader returns io.EOF when the final 0-length chunk is read.
+//
+// NewChunkedReader is not needed by normal applications. The http package
+// automatically decodes chunking when reading response bodies.
+func NewChunkedReader(r io.Reader) io.Reader {
+	br, ok := r.(*bufio.Reader)
+	if !ok {
+		br = bufio.NewReader(r)
+	}
+	return &chunkedReader{r: br}
+}
+
+type chunkedReader struct {
+	r        *bufio.Reader
+	n        uint64 // unread bytes in chunk
+	err      error
+	buf      [2]byte
+	checkEnd bool // whether need to check for \r\n chunk footer
+}
+
+func (cr *chunkedReader) beginChunk() {
+	// chunk-size CRLF
+	var line []byte
+	line, cr.err = readChunkLine(cr.r)
+	if cr.err != nil {
+		return
+	}
+	cr.n, cr.err = parseHexUint(line)
+	if cr.err != nil {
+		return
+	}
+	if cr.n == 0 {
+		cr.err = io.EOF
+	}
+}
+
+func (cr *chunkedReader) chunkHeaderAvailable() bool {
+	n := cr.r.Buffered()
+	if n > 0 {
+		peek, _ := cr.r.Peek(n)
+		return bytes.IndexByte(peek, '\n') >= 0
+	}
+	return false
+}
+
+func (cr *chunkedReader) Read(b []uint8) (n int, err error) {
+	for cr.err == nil {
+		if cr.checkEnd {
+			if n > 0 && cr.r.Buffered() < 2 {
+				// We have some data. Return early (per the io.Reader
+				// contract) instead of potentially blocking while
+				// reading more.
+				break
+			}
+			if _, cr.err = io.ReadFull(cr.r, cr.buf[:2]); cr.err == nil {
+				if string(cr.buf[:]) != "\r\n" {
+					cr.err = errors.New("malformed chunked encoding")
+					break
+				}
+			}
+			cr.checkEnd = false
+		}
+		if cr.n == 0 {
+			if n > 0 && !cr.chunkHeaderAvailable() {
+				// We've read enough. Don't potentially block
+				// reading a new chunk header.
+				break
+			}
+			cr.beginChunk()
+			continue
+		}
+		if len(b) == 0 {
+			break
+		}
+		rbuf := b
+		if uint64(len(rbuf)) > cr.n {
+			rbuf = rbuf[:cr.n]
+		}
+		var n0 int
+		n0, cr.err = cr.r.Read(rbuf)
+		n += n0
+		b = b[n0:]
+		cr.n -= uint64(n0)
+		// If we're at the end of a chunk, read the next two
+		// bytes to verify they are "\r\n".
+		if cr.n == 0 && cr.err == nil {
+			cr.checkEnd = true
+		}
+	}
+	return n, cr.err
+}
+
+// Read a line of bytes (up to \n) from b.
+// Give up if the line exceeds maxLineLength.
+// The returned bytes are owned by the bufio.Reader
+// so they are only valid until the next bufio read.
+func readChunkLine(b *bufio.Reader) ([]byte, error) {
+	p, err := b.ReadSlice('\n')
+	if err != nil {
+		// We always know when EOF is coming.
+		// If the caller asked for a line, there should be a line.
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		} else if err == bufio.ErrBufferFull {
+			err = ErrLineTooLong
+		}
+		return nil, err
+	}
+	if len(p) >= maxLineLength {
+		return nil, ErrLineTooLong
+	}
+	p = trimTrailingWhitespace(p)
+	p, err = removeChunkExtension(p)
+	if err != nil {
+		return nil, err
+	}
+	return p, nil
+}
+
+func trimTrailingWhitespace(b []byte) []byte {
+	for len(b) > 0 && isASCIISpace(b[len(b)-1]) {
+		b = b[:len(b)-1]
+	}
+	return b
+}
+
+func isASCIISpace(b byte) bool {
+	return b == ' ' || b == '\t' || b == '\n' || b == '\r'
+}
+
+// removeChunkExtension removes any chunk-extension from p.
+// For example,
+//     "0" => "0"
+//     "0;token" => "0"
+//     "0;token=val" => "0"
+//     `0;token="quoted string"` => "0"
+func removeChunkExtension(p []byte) ([]byte, error) {
+	semi := bytes.IndexByte(p, ';')
+	if semi == -1 {
+		return p, nil
+	}
+	// TODO: care about exact syntax of chunk extensions? We're
+	// ignoring and stripping them anyway. For now just never
+	// return an error.
+	return p[:semi], nil
+}
+
+// NewChunkedWriter returns a new chunkedWriter that translates writes into HTTP
+// "chunked" format before writing them to w. Closing the returned chunkedWriter
+// sends the final 0-length chunk that marks the end of the stream but does
+// not send the final CRLF that appears after trailers; trailers and the last
+// CRLF must be written separately.
+//
+// NewChunkedWriter is not needed by normal applications. The http
+// package adds chunking automatically if handlers don't set a
+// Content-Length header. Using newChunkedWriter inside a handler
+// would result in double chunking or chunking with a Content-Length
+// length, both of which are wrong.
+func NewChunkedWriter(w io.Writer) io.WriteCloser {
+	return &chunkedWriter{w}
+}
+
+// Writing to chunkedWriter translates to writing in HTTP chunked Transfer
+// Encoding wire format to the underlying Wire chunkedWriter.
+type chunkedWriter struct {
+	Wire io.Writer
+}
+
+// Write the contents of data as one chunk to Wire.
+// NOTE: Note that the corresponding chunk-writing procedure in Conn.Write has
+// a bug since it does not check for success of io.WriteString
+func (cw *chunkedWriter) Write(data []byte) (n int, err error) {
+
+	// Don't send 0-length data. It looks like EOF for chunked encoding.
+	if len(data) == 0 {
+		return 0, nil
+	}
+
+	if _, err = fmt.Fprintf(cw.Wire, "%x\r\n", len(data)); err != nil {
+		return 0, err
+	}
+	if n, err = cw.Wire.Write(data); err != nil {
+		return
+	}
+	if n != len(data) {
+		err = io.ErrShortWrite
+		return
+	}
+	if _, err = io.WriteString(cw.Wire, "\r\n"); err != nil {
+		return
+	}
+	if bw, ok := cw.Wire.(*FlushAfterChunkWriter); ok {
+		err = bw.Flush()
+	}
+	return
+}
+
+func (cw *chunkedWriter) Close() error {
+	_, err := io.WriteString(cw.Wire, "0\r\n")
+	return err
+}
+
+// FlushAfterChunkWriter signals from the caller of NewChunkedWriter
+// that each chunk should be followed by a flush. It is used by the
+// http.Transport code to keep the buffering behavior for headers and
+// trailers, but flush out chunks aggressively in the middle for
+// request bodies which may be generated slowly. See Issue 6574.
+type FlushAfterChunkWriter struct {
+	*bufio.Writer
+}
+
+func parseHexUint(v []byte) (n uint64, err error) {
+	for i, b := range v {
+		switch {
+		case '0' <= b && b <= '9':
+			b = b - '0'
+		case 'a' <= b && b <= 'f':
+			b = b - 'a' + 10
+		case 'A' <= b && b <= 'F':
+			b = b - 'A' + 10
+		default:
+			return 0, errors.New("invalid byte in chunk length")
+		}
+		if i == 16 {
+			return 0, errors.New("http chunk length too large")
+		}
+		n <<= 4
+		n |= uint64(b)
+	}
+	return
+}
diff --git a/src/net/http/internal/chunked_test.go b/src/net/http/internal/chunked_test.go
new file mode 100644
index 0000000..08152ed
--- /dev/null
+++ b/src/net/http/internal/chunked_test.go
@@ -0,0 +1,213 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package internal
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"strings"
+	"testing"
+)
+
+func TestChunk(t *testing.T) {
+	var b bytes.Buffer
+
+	w := NewChunkedWriter(&b)
+	const chunk1 = "hello, "
+	const chunk2 = "world! 0123456789abcdef"
+	w.Write([]byte(chunk1))
+	w.Write([]byte(chunk2))
+	w.Close()
+
+	if g, e := b.String(), "7\r\nhello, \r\n17\r\nworld! 0123456789abcdef\r\n0\r\n"; g != e {
+		t.Fatalf("chunk writer wrote %q; want %q", g, e)
+	}
+
+	r := NewChunkedReader(&b)
+	data, err := io.ReadAll(r)
+	if err != nil {
+		t.Logf(`data: "%s"`, data)
+		t.Fatalf("ReadAll from reader: %v", err)
+	}
+	if g, e := string(data), chunk1+chunk2; g != e {
+		t.Errorf("chunk reader read %q; want %q", g, e)
+	}
+}
+
+func TestChunkReadMultiple(t *testing.T) {
+	// Bunch of small chunks, all read together.
+	{
+		var b bytes.Buffer
+		w := NewChunkedWriter(&b)
+		w.Write([]byte("foo"))
+		w.Write([]byte("bar"))
+		w.Close()
+
+		r := NewChunkedReader(&b)
+		buf := make([]byte, 10)
+		n, err := r.Read(buf)
+		if n != 6 || err != io.EOF {
+			t.Errorf("Read = %d, %v; want 6, EOF", n, err)
+		}
+		buf = buf[:n]
+		if string(buf) != "foobar" {
+			t.Errorf("Read = %q; want %q", buf, "foobar")
+		}
+	}
+
+	// One big chunk followed by a little chunk, but the small bufio.Reader size
+	// should prevent the second chunk header from being read.
+	{
+		var b bytes.Buffer
+		w := NewChunkedWriter(&b)
+		// fillBufChunk is 11 bytes + 3 bytes header + 2 bytes footer = 16 bytes,
+		// the same as the bufio ReaderSize below (the minimum), so even
+		// though we're going to try to Read with a buffer larger enough to also
+		// receive "foo", the second chunk header won't be read yet.
+		const fillBufChunk = "0123456789a"
+		const shortChunk = "foo"
+		w.Write([]byte(fillBufChunk))
+		w.Write([]byte(shortChunk))
+		w.Close()
+
+		r := NewChunkedReader(bufio.NewReaderSize(&b, 16))
+		buf := make([]byte, len(fillBufChunk)+len(shortChunk))
+		n, err := r.Read(buf)
+		if n != len(fillBufChunk) || err != nil {
+			t.Errorf("Read = %d, %v; want %d, nil", n, err, len(fillBufChunk))
+		}
+		buf = buf[:n]
+		if string(buf) != fillBufChunk {
+			t.Errorf("Read = %q; want %q", buf, fillBufChunk)
+		}
+
+		n, err = r.Read(buf)
+		if n != len(shortChunk) || err != io.EOF {
+			t.Errorf("Read = %d, %v; want %d, EOF", n, err, len(shortChunk))
+		}
+	}
+
+	// And test that we see an EOF chunk, even though our buffer is already full:
+	{
+		r := NewChunkedReader(bufio.NewReader(strings.NewReader("3\r\nfoo\r\n0\r\n")))
+		buf := make([]byte, 3)
+		n, err := r.Read(buf)
+		if n != 3 || err != io.EOF {
+			t.Errorf("Read = %d, %v; want 3, EOF", n, err)
+		}
+		if string(buf) != "foo" {
+			t.Errorf("buf = %q; want foo", buf)
+		}
+	}
+}
+
+func TestChunkReaderAllocs(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping in short mode")
+	}
+	var buf bytes.Buffer
+	w := NewChunkedWriter(&buf)
+	a, b, c := []byte("aaaaaa"), []byte("bbbbbbbbbbbb"), []byte("cccccccccccccccccccccccc")
+	w.Write(a)
+	w.Write(b)
+	w.Write(c)
+	w.Close()
+
+	readBuf := make([]byte, len(a)+len(b)+len(c)+1)
+	byter := bytes.NewReader(buf.Bytes())
+	bufr := bufio.NewReader(byter)
+	mallocs := testing.AllocsPerRun(100, func() {
+		byter.Seek(0, io.SeekStart)
+		bufr.Reset(byter)
+		r := NewChunkedReader(bufr)
+		n, err := io.ReadFull(r, readBuf)
+		if n != len(readBuf)-1 {
+			t.Fatalf("read %d bytes; want %d", n, len(readBuf)-1)
+		}
+		if err != io.ErrUnexpectedEOF {
+			t.Fatalf("read error = %v; want ErrUnexpectedEOF", err)
+		}
+	})
+	if mallocs > 1.5 {
+		t.Errorf("mallocs = %v; want 1", mallocs)
+	}
+}
+
+func TestParseHexUint(t *testing.T) {
+	type testCase struct {
+		in      string
+		want    uint64
+		wantErr string
+	}
+	tests := []testCase{
+		{"x", 0, "invalid byte in chunk length"},
+		{"0000000000000000", 0, ""},
+		{"0000000000000001", 1, ""},
+		{"ffffffffffffffff", 1<<64 - 1, ""},
+		{"000000000000bogus", 0, "invalid byte in chunk length"},
+		{"00000000000000000", 0, "http chunk length too large"}, // could accept if we wanted
+		{"10000000000000000", 0, "http chunk length too large"},
+		{"00000000000000001", 0, "http chunk length too large"}, // could accept if we wanted
+	}
+	for i := uint64(0); i <= 1234; i++ {
+		tests = append(tests, testCase{in: fmt.Sprintf("%x", i), want: i})
+	}
+	for _, tt := range tests {
+		got, err := parseHexUint([]byte(tt.in))
+		if tt.wantErr != "" {
+			if !strings.Contains(fmt.Sprint(err), tt.wantErr) {
+				t.Errorf("parseHexUint(%q) = %v, %v; want error %q", tt.in, got, err, tt.wantErr)
+			}
+		} else {
+			if err != nil || got != tt.want {
+				t.Errorf("parseHexUint(%q) = %v, %v; want %v", tt.in, got, err, tt.want)
+			}
+		}
+	}
+}
+
+func TestChunkReadingIgnoresExtensions(t *testing.T) {
+	in := "7;ext=\"some quoted string\"\r\n" + // token=quoted string
+		"hello, \r\n" +
+		"17;someext\r\n" + // token without value
+		"world! 0123456789abcdef\r\n" +
+		"0;someextension=sometoken\r\n" // token=token
+	data, err := io.ReadAll(NewChunkedReader(strings.NewReader(in)))
+	if err != nil {
+		t.Fatalf("ReadAll = %q, %v", data, err)
+	}
+	if g, e := string(data), "hello, world! 0123456789abcdef"; g != e {
+		t.Errorf("read %q; want %q", g, e)
+	}
+}
+
+// Issue 17355: ChunkedReader shouldn't block waiting for more data
+// if it can return something.
+func TestChunkReadPartial(t *testing.T) {
+	pr, pw := io.Pipe()
+	go func() {
+		pw.Write([]byte("7\r\n1234567"))
+	}()
+	cr := NewChunkedReader(pr)
+	readBuf := make([]byte, 7)
+	n, err := cr.Read(readBuf)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := "1234567"
+	if n != 7 || string(readBuf) != want {
+		t.Fatalf("Read: %v %q; want %d, %q", n, readBuf[:n], len(want), want)
+	}
+	go func() {
+		pw.Write([]byte("xx"))
+	}()
+	_, err = cr.Read(readBuf)
+	if got := fmt.Sprint(err); !strings.Contains(got, "malformed") {
+		t.Fatalf("second read = %v; want malformed error", err)
+	}
+
+}
diff --git a/src/net/http/internal/testcert.go b/src/net/http/internal/testcert.go
new file mode 100644
index 0000000..2284a83
--- /dev/null
+++ b/src/net/http/internal/testcert.go
@@ -0,0 +1,45 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package internal
+
+import "strings"
+
+// LocalhostCert is a PEM-encoded TLS cert with SAN IPs
+// "127.0.0.1" and "[::1]", expiring at Jan 29 16:00:00 2084 GMT.
+// generated from src/crypto/tls:
+// go run generate_cert.go  --rsa-bits 1024 --host 127.0.0.1,::1,example.com --ca --start-date "Jan 1 00:00:00 1970" --duration=1000000h
+var LocalhostCert = []byte(`-----BEGIN CERTIFICATE-----
+MIICEzCCAXygAwIBAgIQMIMChMLGrR+QvmQvpwAU6zANBgkqhkiG9w0BAQsFADAS
+MRAwDgYDVQQKEwdBY21lIENvMCAXDTcwMDEwMTAwMDAwMFoYDzIwODQwMTI5MTYw
+MDAwWjASMRAwDgYDVQQKEwdBY21lIENvMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCB
+iQKBgQDuLnQAI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9SjY1bIw4
+iA5sBBZzHi3z0h1YV8QPuxEbi4nW91IJm2gsvvZhIrCHS3l6afab4pZBl2+XsDul
+rKBxKKtD1rGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTvqJQIDAQABo2gwZjAO
+BgNVHQ8BAf8EBAMCAqQwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDwYDVR0TAQH/BAUw
+AwEB/zAuBgNVHREEJzAlggtleGFtcGxlLmNvbYcEfwAAAYcQAAAAAAAAAAAAAAAA
+AAAAATANBgkqhkiG9w0BAQsFAAOBgQCEcetwO59EWk7WiJsG4x8SY+UIAA+flUI9
+tyC4lNhbcF2Idq9greZwbYCqTTTr2XiRNSMLCOjKyI7ukPoPjo16ocHj+P3vZGfs
+h1fIw3cSS2OolhloGw/XM6RWPWtPAlGykKLciQrBru5NAPvCMsb/I1DAceTiotQM
+fblo6RBxUQ==
+-----END CERTIFICATE-----`)
+
+// LocalhostKey is the private key for localhostCert.
+var LocalhostKey = []byte(testingKey(`-----BEGIN RSA TESTING KEY-----
+MIICXgIBAAKBgQDuLnQAI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9
+SjY1bIw4iA5sBBZzHi3z0h1YV8QPuxEbi4nW91IJm2gsvvZhIrCHS3l6afab4pZB
+l2+XsDulrKBxKKtD1rGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTvqJQIDAQAB
+AoGAGRzwwir7XvBOAy5tM/uV6e+Zf6anZzus1s1Y1ClbjbE6HXbnWWF/wbZGOpet
+3Zm4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKwZatTXtdZrhu+Jk7hx0nTPy8Jcb
+uJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+hgN4H
+qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz8vp
+jy4SHEg1AkEA/v13/5M47K9vCxmb8QeD/asydfsgS5TeuNi8DoUBEmiSJwma7FXY
+fFUtxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su4b3sjXNueLKH85Q+phy2U
+fQtuUE9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xlp/DoCzjA0CQQDU
+y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6PYj013sovGKUFfYAqVXVlxtIX
+qyUBnu3X9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYZHOde3JEMhNRcVFMO8dJDaFeo
+f9Oeos0UUothgiDktdQHxdNEwLjQf7lJJBzV+5OtwswCWA==
+-----END RSA TESTING KEY-----`))
+
+func testingKey(s string) string { return strings.ReplaceAll(s, "TESTING KEY", "PRIVATE KEY") }