Subversion Repositories openrisc

// Copyright 2009 The Go Authors. All rights reserved.

// Use of this source code is governed by a BSD-style

// license that can be found in the LICENSE file.

// HTTP Request reading and parsing.

package http

import (

        "bufio"

        "bytes"

        "crypto/tls"

        "encoding/base64"

        "errors"

        "fmt"

        "io"

        "io/ioutil"

        "mime"

        "mime/multipart"

        "net/textproto"

        "net/url"

        "strings"

)

const (

        maxValueLength   = 4096

        maxHeaderLines   = 1024

        chunkSize        = 4 << 10  // 4 KB chunks

        defaultMaxMemory = 32 << 20 // 32 MB

)

// ErrMissingFile is returned by FormFile when the provided file field name

// is either not present in the request or not a file field.

var ErrMissingFile = errors.New("http: no such file")

// HTTP request parsing errors.

type ProtocolError struct {

        ErrorString string

}

func (err *ProtocolError) Error() string { return err.ErrorString }

var (

        ErrHeaderTooLong        = &ProtocolError{"header too long"}

        ErrShortBody            = &ProtocolError{"entity body too short"}

        ErrNotSupported         = &ProtocolError{"feature not supported"}

        ErrUnexpectedTrailer    = &ProtocolError{"trailer header without chunked transfer encoding"}

        ErrMissingContentLength = &ProtocolError{"missing ContentLength in HEAD response"}

        ErrNotMultipart         = &ProtocolError{"request Content-Type isn't multipart/form-data"}

        ErrMissingBoundary      = &ProtocolError{"no multipart boundary param Content-Type"}

)

type badStringError struct {

        what string

        str  string

}

func (e *badStringError) Error() string { return fmt.Sprintf("%s %q", e.what, e.str) }

// Headers that Request.Write handles itself and should be skipped.

var reqWriteExcludeHeader = map[string]bool{

        "Host":              true, // not in Header map anyway

        "User-Agent":        true,

        "Content-Length":    true,

        "Transfer-Encoding": true,

        "Trailer":           true,

}

// A Request represents an HTTP request received by a server

// or to be sent by a client.

type Request struct {

        Method string // GET, POST, PUT, etc.

        URL    *url.URL

        // The protocol version for incoming requests.

        // Outgoing requests always use HTTP/1.1.

        Proto      string // "HTTP/1.0"

        ProtoMajor int    // 1

        ProtoMinor int    // 0

        // A header maps request lines to their values.

        // If the header says

        //

        //      accept-encoding: gzip, deflate

        //      Accept-Language: en-us

        //      Connection: keep-alive

        //

        // then

        //

        //      Header = map[string][]string{

        //              "Accept-Encoding": {"gzip, deflate"},

        //              "Accept-Language": {"en-us"},

        //              "Connection": {"keep-alive"},

        //      }

        //

        // HTTP defines that header names are case-insensitive.

        // The request parser implements this by canonicalizing the

        // name, making the first character and any characters

        // following a hyphen uppercase and the rest lowercase.

        Header Header

        // The message body.

        Body io.ReadCloser

        // ContentLength records the length of the associated content.

        // The value -1 indicates that the length is unknown.

        // Values >= 0 indicate that the given number of bytes may

        // be read from Body.

        // For outgoing requests, a value of 0 means unknown if Body is not nil.

        ContentLength int64

        // TransferEncoding lists the transfer encodings from outermost to

        // innermost. An empty list denotes the "identity" encoding.

        // TransferEncoding can usually be ignored; chunked encoding is

        // automatically added and removed as necessary when sending and

        // receiving requests.

        TransferEncoding []string

        // Close indicates whether to close the connection after

        // replying to this request.

        Close bool

        // The host on which the URL is sought.

        // Per RFC 2616, this is either the value of the Host: header

        // or the host name given in the URL itself.

        Host string

        // Form contains the parsed form data, including both the URL

        // field's query parameters and the POST or PUT form data.

        // This field is only available after ParseForm is called.

        // The HTTP client ignores Form and uses Body instead.

        Form url.Values

        // MultipartForm is the parsed multipart form, including file uploads.

        // This field is only available after ParseMultipartForm is called.

        // The HTTP client ignores MultipartForm and uses Body instead.

        MultipartForm *multipart.Form

        // Trailer maps trailer keys to values.  Like for Header, if the

        // response has multiple trailer lines with the same key, they will be

        // concatenated, delimited by commas.

        // For server requests, Trailer is only populated after Body has been

        // closed or fully consumed.

        // Trailer support is only partially complete.

        Trailer Header

        // RemoteAddr allows HTTP servers and other software to record

        // the network address that sent the request, usually for

        // logging. This field is not filled in by ReadRequest and

        // has no defined format. The HTTP server in this package

        // sets RemoteAddr to an "IP:port" address before invoking a

        // handler.

        // This field is ignored by the HTTP client.

        RemoteAddr string

        // RequestURI is the unmodified Request-URI of the

        // Request-Line (RFC 2616, Section 5.1) as sent by the client

        // to a server. Usually the URL field should be used instead.

        // It is an error to set this field in an HTTP client request.

        RequestURI string

        // TLS allows HTTP servers and other software to record

        // information about the TLS connection on which the request

        // was received. This field is not filled in by ReadRequest.

        // The HTTP server in this package sets the field for

        // TLS-enabled connections before invoking a handler;

        // otherwise it leaves the field nil.

        // This field is ignored by the HTTP client.

        TLS *tls.ConnectionState

}

// ProtoAtLeast returns whether the HTTP protocol used

// in the request is at least major.minor.

func (r *Request) ProtoAtLeast(major, minor int) bool {

        return r.ProtoMajor > major ||

                r.ProtoMajor == major && r.ProtoMinor >= minor

}

// UserAgent returns the client's User-Agent, if sent in the request.

func (r *Request) UserAgent() string {

        return r.Header.Get("User-Agent")

}

// Cookies parses and returns the HTTP cookies sent with the request.

func (r *Request) Cookies() []*Cookie {

        return readCookies(r.Header, "")

}

var ErrNoCookie = errors.New("http: named cookied not present")

// Cookie returns the named cookie provided in the request or

// ErrNoCookie if not found.

func (r *Request) Cookie(name string) (*Cookie, error) {

        for _, c := range readCookies(r.Header, name) {

                return c, nil

        }

        return nil, ErrNoCookie

}

// AddCookie adds a cookie to the request.  Per RFC 6265 section 5.4,

// AddCookie does not attach more than one Cookie header field.  That

// means all cookies, if any, are written into the same line,

// separated by semicolon.

func (r *Request) AddCookie(c *Cookie) {

        s := fmt.Sprintf("%s=%s", sanitizeName(c.Name), sanitizeValue(c.Value))

        if c := r.Header.Get("Cookie"); c != "" {

                r.Header.Set("Cookie", c+"; "+s)

        } else {

                r.Header.Set("Cookie", s)

        }

}

// Referer returns the referring URL, if sent in the request.

//

// Referer is misspelled as in the request itself, a mistake from the

// earliest days of HTTP.  This value can also be fetched from the

// Header map as Header["Referer"]; the benefit of making it available

// as a method is that the compiler can diagnose programs that use the

// alternate (correct English) spelling req.Referrer() but cannot

// diagnose programs that use Header["Referrer"].

func (r *Request) Referer() string {

        return r.Header.Get("Referer")

}

// multipartByReader is a sentinel value.

// Its presence in Request.MultipartForm indicates that parsing of the request

// body has been handed off to a MultipartReader instead of ParseMultipartFrom.

var multipartByReader = &multipart.Form{

        Value: make(map[string][]string),

        File:  make(map[string][]*multipart.FileHeader),

}

// MultipartReader returns a MIME multipart reader if this is a

// multipart/form-data POST request, else returns nil and an error.

// Use this function instead of ParseMultipartForm to

// process the request body as a stream.

func (r *Request) MultipartReader() (*multipart.Reader, error) {

        if r.MultipartForm == multipartByReader {

                return nil, errors.New("http: MultipartReader called twice")

        }

        if r.MultipartForm != nil {

                return nil, errors.New("http: multipart handled by ParseMultipartForm")

        }

        r.MultipartForm = multipartByReader

        return r.multipartReader()

}

func (r *Request) multipartReader() (*multipart.Reader, error) {

        v := r.Header.Get("Content-Type")

        if v == "" {

                return nil, ErrNotMultipart

        }

        d, params, err := mime.ParseMediaType(v)

        if err != nil || d != "multipart/form-data" {

                return nil, ErrNotMultipart

        }

        boundary, ok := params["boundary"]

        if !ok {

                return nil, ErrMissingBoundary

        }

        return multipart.NewReader(r.Body, boundary), nil

}

// Return value if nonempty, def otherwise.

func valueOrDefault(value, def string) string {

        if value != "" {

                return value

        }

        return def

}

const defaultUserAgent = "Go http package"

// Write writes an HTTP/1.1 request -- header and body -- in wire format.

// This method consults the following fields of the request:

//      Host

//      URL

//      Method (defaults to "GET")

//      Header

//      ContentLength

//      TransferEncoding

//      Body

//

// If Body is present, Content-Length is <= 0 and TransferEncoding

// hasn't been set to "identity", Write adds "Transfer-Encoding:

// chunked" to the header. Body is closed after it is sent.

func (r *Request) Write(w io.Writer) error {

        return r.write(w, false, nil)

}

// WriteProxy is like Write but writes the request in the form

// expected by an HTTP proxy.  In particular, WriteProxy writes the

// initial Request-URI line of the request with an absolute URI, per

// section 5.1.2 of RFC 2616, including the scheme and host.

// In either case, WriteProxy also writes a Host header, using

// either r.Host or r.URL.Host.

func (r *Request) WriteProxy(w io.Writer) error {

        return r.write(w, true, nil)

}

// extraHeaders may be nil

func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header) error {

        host := req.Host

        if host == "" {

                if req.URL == nil {

                        return errors.New("http: Request.Write on Request with no Host or URL set")

                }

                host = req.URL.Host

        }

        ruri := req.URL.RequestURI()

        if usingProxy && req.URL.Scheme != "" && req.URL.Opaque == "" {

                ruri = req.URL.Scheme + "://" + host + ruri

        } else if req.Method == "CONNECT" && req.URL.Path == "" {

                // CONNECT requests normally give just the host and port, not a full URL.

                ruri = host

        }

        // TODO(bradfitz): escape at least newlines in ruri?

        bw := bufio.NewWriter(w)

        fmt.Fprintf(bw, "%s %s HTTP/1.1\r\n", valueOrDefault(req.Method, "GET"), ruri)

        // Header lines

        fmt.Fprintf(bw, "Host: %s\r\n", host)

        // Use the defaultUserAgent unless the Header contains one, which

        // may be blank to not send the header.

        userAgent := defaultUserAgent

        if req.Header != nil {

                if ua := req.Header["User-Agent"]; len(ua) > 0 {

                        userAgent = ua[0]

                }

        }

        if userAgent != "" {

                fmt.Fprintf(bw, "User-Agent: %s\r\n", userAgent)

        }

        // Process Body,ContentLength,Close,Trailer

        tw, err := newTransferWriter(req)

        if err != nil {

                return err

        }

        err = tw.WriteHeader(bw)

        if err != nil {

                return err

        }

        // TODO: split long values?  (If so, should share code with Conn.Write)

        err = req.Header.WriteSubset(bw, reqWriteExcludeHeader)

        if err != nil {

                return err

        }

        if extraHeaders != nil {

                err = extraHeaders.Write(bw)

                if err != nil {

                        return err

                }

        }

        io.WriteString(bw, "\r\n")

        // Write body and trailer

        err = tw.WriteBody(bw)

        if err != nil {

                return err

        }

        return bw.Flush()

}

// Convert decimal at s[i:len(s)] to integer,

// returning value, string position where the digits stopped,

// and whether there was a valid number (digits, not too big).

func atoi(s string, i int) (n, i1 int, ok bool) {

        const Big = 1000000

        if i >= len(s) || s[i] < '0' || s[i] > '9' {

                return 0, 0, false

        }

        n = 0

        for ; i < len(s) && '0' <= s[i] && s[i] <= '9'; i++ {

                n = n*10 + int(s[i]-'0')

                if n > Big {

                        return 0, 0, false

                }

        }

        return n, i, true

}

// ParseHTTPVersion parses a HTTP version string.

// "HTTP/1.0" returns (1, 0, true).

func ParseHTTPVersion(vers string) (major, minor int, ok bool) {

        if len(vers) < 5 || vers[0:5] != "HTTP/" {

                return 0, 0, false

        }

        major, i, ok := atoi(vers, 5)

        if !ok || i >= len(vers) || vers[i] != '.' {

                return 0, 0, false

        }

        minor, i, ok = atoi(vers, i+1)

        if !ok || i != len(vers) {

                return 0, 0, false

        }

        return major, minor, true

}

// NewRequest returns a new Request given a method, URL, and optional body.

func NewRequest(method, urlStr string, body io.Reader) (*Request, error) {

        u, err := url.Parse(urlStr)

        if err != nil {

                return nil, err

        }

        rc, ok := body.(io.ReadCloser)

        if !ok && body != nil {

                rc = ioutil.NopCloser(body)

        }

        req := &Request{

                Method:     method,

                URL:        u,

                Proto:      "HTTP/1.1",

                ProtoMajor: 1,

                ProtoMinor: 1,

                Header:     make(Header),

                Body:       rc,

                Host:       u.Host,

        }

        if body != nil {

                switch v := body.(type) {

                case *strings.Reader:

                        req.ContentLength = int64(v.Len())

                case *bytes.Buffer:

                        req.ContentLength = int64(v.Len())

                }

        }

        return req, nil

}

// SetBasicAuth sets the request's Authorization header to use HTTP

// Basic Authentication with the provided username and password.

//

// With HTTP Basic Authentication the provided username and password

// are not encrypted.

func (r *Request) SetBasicAuth(username, password string) {

        s := username + ":" + password

        r.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(s)))

}

// ReadRequest reads and parses a request from b.

func ReadRequest(b *bufio.Reader) (req *Request, err error) {

        tp := textproto.NewReader(b)

        req = new(Request)

        // First line: GET /index.html HTTP/1.0

        var s string

        if s, err = tp.ReadLine(); err != nil {

                if err == io.EOF {

                        err = io.ErrUnexpectedEOF

                }

                return nil, err

        }

        var f []string

        if f = strings.SplitN(s, " ", 3); len(f) < 3 {

                return nil, &badStringError{"malformed HTTP request", s}

        }

        req.Method, req.RequestURI, req.Proto = f[0], f[1], f[2]

        rawurl := req.RequestURI

        var ok bool

        if req.ProtoMajor, req.ProtoMinor, ok = ParseHTTPVersion(req.Proto); !ok {

                return nil, &badStringError{"malformed HTTP version", req.Proto}

        }

        // CONNECT requests are used two different ways, and neither uses a full URL:

        // The standard use is to tunnel HTTPS through an HTTP proxy.

        // It looks like "CONNECT www.google.com:443 HTTP/1.1", and the parameter is

        // just the authority section of a URL. This information should go in req.URL.Host.

        //

        // The net/rpc package also uses CONNECT, but there the parameter is a path

        // that starts with a slash. It can be parsed with the regular URL parser,

        // and the path will end up in req.URL.Path, where it needs to be in order for

        // RPC to work.

        justAuthority := req.Method == "CONNECT" && !strings.HasPrefix(rawurl, "/")

        if justAuthority {

                rawurl = "http://" + rawurl

        }

        if req.URL, err = url.ParseRequest(rawurl); err != nil {

                return nil, err

        }

        if justAuthority {

                // Strip the bogus "http://" back off.

                req.URL.Scheme = ""

        }

        // Subsequent lines: Key: value.

        mimeHeader, err := tp.ReadMIMEHeader()

        if err != nil {

                return nil, err

        }

        req.Header = Header(mimeHeader)

        // RFC2616: Must treat

        //      GET /index.html HTTP/1.1

        //      Host: www.google.com

        // and

        //      GET http://www.google.com/index.html HTTP/1.1

        //      Host: doesntmatter

        // the same.  In the second case, any Host line is ignored.

        req.Host = req.URL.Host

        if req.Host == "" {

                req.Host = req.Header.Get("Host")

        }

        req.Header.Del("Host")

        fixPragmaCacheControl(req.Header)

        // TODO: Parse specific header values:

        //      Accept

        //      Accept-Encoding

        //      Accept-Language

        //      Authorization

        //      Cache-Control

        //      Connection

        //      Date

        //      Expect

        //      From

        //      If-Match

        //      If-Modified-Since

        //      If-None-Match

        //      If-Range

        //      If-Unmodified-Since

        //      Max-Forwards

        //      Proxy-Authorization

        //      Referer [sic]

        //      TE (transfer-codings)

        //      Trailer

        //      Transfer-Encoding

        //      Upgrade

        //      User-Agent

        //      Via

        //      Warning

        err = readTransfer(req, b)

        if err != nil {

                return nil, err

        }

        return req, nil

}

// MaxBytesReader is similar to io.LimitReader but is intended for

// limiting the size of incoming request bodies. In contrast to

// io.LimitReader, MaxBytesReader's result is a ReadCloser, returns a

// non-EOF error for a Read beyond the limit, and Closes the

// underlying reader when its Close method is called.

//

// MaxBytesReader prevents clients from accidentally or maliciously

// sending a large request and wasting server resources.

func MaxBytesReader(w ResponseWriter, r io.ReadCloser, n int64) io.ReadCloser {

        return &maxBytesReader{w: w, r: r, n: n}

}

type maxBytesReader struct {

        w       ResponseWriter

        r       io.ReadCloser // underlying reader

        n       int64         // max bytes remaining

        stopped bool

}

func (l *maxBytesReader) Read(p []byte) (n int, err error) {

        if l.n <= 0 {

                if !l.stopped {

                        l.stopped = true

                        if res, ok := l.w.(*response); ok {

                                res.requestTooLarge()

                        }

                }

                return 0, errors.New("http: request body too large")

        }

        if int64(len(p)) > l.n {

                p = p[:l.n]

        }

        n, err = l.r.Read(p)

        l.n -= int64(n)

        return

}

func (l *maxBytesReader) Close() error {

        return l.r.Close()

}

// ParseForm parses the raw query from the URL.

//

// For POST or PUT requests, it also parses the request body as a form.

// If the request Body's size has not already been limited by MaxBytesReader,

// the size is capped at 10MB.

//

// ParseMultipartForm calls ParseForm automatically.

// It is idempotent.

func (r *Request) ParseForm() (err error) {

        if r.Form != nil {

                return

        }

        if r.URL != nil {

                r.Form, err = url.ParseQuery(r.URL.RawQuery)

        }

        if r.Method == "POST" || r.Method == "PUT" {

                if r.Body == nil {

                        return errors.New("missing form body")

                }

                ct := r.Header.Get("Content-Type")

                ct, _, err = mime.ParseMediaType(ct)

                switch {

                case ct == "application/x-www-form-urlencoded":

                        var reader io.Reader = r.Body

                        maxFormSize := int64(1<<63 - 1)

                        if _, ok := r.Body.(*maxBytesReader); !ok {

                                maxFormSize = int64(10 << 20) // 10 MB is a lot of text.

                                reader = io.LimitReader(r.Body, maxFormSize+1)

                        }

                        b, e := ioutil.ReadAll(reader)

                        if e != nil {

                                if err == nil {

                                        err = e

                                }

                                break

                        }

                        if int64(len(b)) > maxFormSize {

                                return errors.New("http: POST too large")

                        }

                        var newValues url.Values

                        newValues, e = url.ParseQuery(string(b))

                        if err == nil {

                                err = e

                        }

                        if r.Form == nil {

                                r.Form = make(url.Values)

                        }

                        // Copy values into r.Form. TODO: make this smoother.

                        for k, vs := range newValues {

                                for _, value := range vs {

                                        r.Form.Add(k, value)

                                }

                        }

                case ct == "multipart/form-data":

                        // handled by ParseMultipartForm (which is calling us, or should be)

                        // TODO(bradfitz): there are too many possible

                        // orders to call too many functions here.

                        // Clean this up and write more tests.

                        // request_test.go contains the start of this,

                        // in TestRequestMultipartCallOrder.

                }

        }

        return err

}

// ParseMultipartForm parses a request body as multipart/form-data.

// The whole request body is parsed and up to a total of maxMemory bytes of

// its file parts are stored in memory, with the remainder stored on

// disk in temporary files.

// ParseMultipartForm calls ParseForm if necessary.

// After one call to ParseMultipartForm, subsequent calls have no effect.

func (r *Request) ParseMultipartForm(maxMemory int64) error {

        if r.MultipartForm == multipartByReader {

                return errors.New("http: multipart handled by MultipartReader")

        }

        if r.Form == nil {

                err := r.ParseForm()

                if err != nil {

                        return err

                }

        }

        if r.MultipartForm != nil {

                return nil

        }

        mr, err := r.multipartReader()

        if err == ErrNotMultipart {

                return nil

        } else if err != nil {

                return err

        }

        f, err := mr.ReadForm(maxMemory)

        if err != nil {

                return err

        }

        for k, v := range f.Value {

                r.Form[k] = append(r.Form[k], v...)

        }

        r.MultipartForm = f

        return nil

}

// FormValue returns the first value for the named component of the query.

// FormValue calls ParseMultipartForm and ParseForm if necessary.

func (r *Request) FormValue(key string) string {

        if r.Form == nil {

                r.ParseMultipartForm(defaultMaxMemory)

        }

        if vs := r.Form[key]; len(vs) > 0 {

                return vs[0]

        }

        return ""

}

// FormFile returns the first file for the provided form key.

// FormFile calls ParseMultipartForm and ParseForm if necessary.

func (r *Request) FormFile(key string) (multipart.File, *multipart.FileHeader, error) {

        if r.MultipartForm == multipartByReader {

                return nil, nil, errors.New("http: multipart handled by MultipartReader")

        }

        if r.MultipartForm == nil {

                err := r.ParseMultipartForm(defaultMaxMemory)

                if err != nil {

                        return nil, nil, err

                }

        }

        if r.MultipartForm != nil && r.MultipartForm.File != nil {

                if fhs := r.MultipartForm.File[key]; len(fhs) > 0 {

                        f, err := fhs[0].Open()

                        return f, fhs[0], err

                }

        }

        return nil, nil, ErrMissingFile

}

func (r *Request) expectsContinue() bool {

        return strings.ToLower(r.Header.Get("Expect")) == "100-continue"

}

func (r *Request) wantsHttp10KeepAlive() bool {

        if r.ProtoMajor != 1 || r.ProtoMinor != 0 {

                return false

        }

        return strings.Contains(strings.ToLower(r.Header.Get("Connection")), "keep-alive")

}