text: // Copyright 2013 The Go Authors. All rights reserved.
  // Use of this source code is governed by a BSD-style
  // license that can be found in the LICENSE file.
  
  // Package charset provides common text encodings for HTML documents.
  //
  // The mapping from encoding labels to encodings is defined at
  // https://encoding.spec.whatwg.org/.
  package charset // import "golang.org/x/net/html/charset"
  
  import (
  	"bytes"
  	"fmt"
  	"io"
  	"mime"
  	"strings"
  	"unicode/utf8"
  
  	"golang.org/x/net/html"
  	"golang.org/x/text/encoding"
  	"golang.org/x/text/encoding/charmap"
  	"golang.org/x/text/encoding/htmlindex"
  	"golang.org/x/text/transform"
  )
  
  // Lookup returns the encoding with the specified label, and its canonical
  // name. It returns nil and the empty string if label is not one of the
  // standard encodings for HTML. Matching is case-insensitive and ignores
  // leading and trailing whitespace. Encoders will use HTML escape sequences for
  // runes that are not supported by the character set.
  func Lookup(label string) (e encoding.Encoding, name string) {
  	e, err := htmlindex.Get(label)
  	if err != nil {
  		return nil, ""
  	}
  	name, _ = htmlindex.Name(e)
  	return &htmlEncoding{e}, name
  }
  
  type htmlEncoding struct{ encoding.Encoding }
  
  func (h *htmlEncoding) NewEncoder() *encoding.Encoder {
  	// HTML requires a non-terminating legacy encoder. We use HTML escapes to
  	// substitute unsupported code points.
  	return encoding.HTMLEscapeUnsupported(h.Encoding.NewEncoder())
  }
  
  // DetermineEncoding determines the encoding of an HTML document by examining
  // up to the first 1024 bytes of content and the declared Content-Type.
  //
  // See http://www.whatwg.org/specs/web-apps/current-work/multipage/parsing.html#determining-the-character-encoding
  func DetermineEncoding(content []byte, contentType string) (e encoding.Encoding, name string, certain bool) {
  	if len(content) > 1024 {
  		content = content[:1024]
  	}
  
  	for _, b := range boms {
  		if bytes.HasPrefix(content, b.bom) {
  			e, name = Lookup(b.enc)
  			return e, name, true
  		}
  	}
  
  	if _, params, err := mime.ParseMediaType(contentType); err == nil {
  		if cs, ok := params["charset"]; ok {
  			if e, name = Lookup(cs); e != nil {
  				return e, name, true
  			}
  		}
  	}
  
  	if len(content) > 0 {
  		e, name = prescan(content)
  		if e != nil {
  			return e, name, false
  		}
  	}
  
  	// Try to detect UTF-8.
  	// First eliminate any partial rune at the end.
  	for i := len(content) - 1; i >= 0 && i > len(content)-4; i-- {
  		b := content[i]
  		if b < 0x80 {
  			break
  		}
  		if utf8.RuneStart(b) {
  			content = content[:i]
  			break
  		}
  	}
  	hasHighBit := false
  	for _, c := range content {
  		if c >= 0x80 {
  			hasHighBit = true
  			break
  		}
  	}
  	if hasHighBit && utf8.Valid(content) {
  		return encoding.Nop, "utf-8", false
  	}
  
  	// TODO: change default depending on user's locale?
  	return charmap.Windows1252, "windows-1252", false
  }
  
  // NewReader returns an io.Reader that converts the content of r to UTF-8.
  // It calls DetermineEncoding to find out what r's encoding is.
  func NewReader(r io.Reader, contentType string) (io.Reader, error) {
  	preview := make([]byte, 1024)
  	n, err := io.ReadFull(r, preview)
  	switch {
  	case err == io.ErrUnexpectedEOF:
  		preview = preview[:n]
  		r = bytes.NewReader(preview)
  	case err != nil:
  		return nil, err
  	default:
  		r = io.MultiReader(bytes.NewReader(preview), r)
  	}
  
  	if e, _, _ := DetermineEncoding(preview, contentType); e != encoding.Nop {
  		r = transform.NewReader(r, e.NewDecoder())
  	}
  	return r, nil
  }
  
  // NewReaderLabel returns a reader that converts from the specified charset to
  // UTF-8. It uses Lookup to find the encoding that corresponds to label, and
  // returns an error if Lookup returns nil. It is suitable for use as
  // encoding/xml.Decoder's CharsetReader function.
  func NewReaderLabel(label string, input io.Reader) (io.Reader, error) {
  	e, _ := Lookup(label)
  	if e == nil {
  		return nil, fmt.Errorf("unsupported charset: %q", label)
  	}
  	return transform.NewReader(input, e.NewDecoder()), nil
  }
  
  func prescan(content []byte) (e encoding.Encoding, name string) {
  	z := html.NewTokenizer(bytes.NewReader(content))
  	for {
  		switch z.Next() {
  		case html.ErrorToken:
  			return nil, ""
  
  		case html.StartTagToken, html.SelfClosingTagToken:
  			tagName, hasAttr := z.TagName()
  			if !bytes.Equal(tagName, []byte("meta")) {
  				continue
  			}
  			attrList := make(map[string]bool)
  			gotPragma := false
  
  			const (
  				dontKnow = iota
  				doNeedPragma
  				doNotNeedPragma
  			)
  			needPragma := dontKnow
  
  			name = ""
  			e = nil
  			for hasAttr {
  				var key, val []byte
  				key, val, hasAttr = z.TagAttr()
  				ks := string(key)
  				if attrList[ks] {
  					continue
  				}
  				attrList[ks] = true
  				for i, c := range val {
  					if 'A' <= c && c <= 'Z' {
  						val[i] = c + 0x20
  					}
  				}
  
  				switch ks {
  				case "http-equiv":
  					if bytes.Equal(val, []byte("content-type")) {
  						gotPragma = true
  					}
  
  				case "content":
  					if e == nil {
  						name = fromMetaElement(string(val))
  						if name != "" {
  							e, name = Lookup(name)
  							if e != nil {
  								needPragma = doNeedPragma
  							}
  						}
  					}
  
  				case "charset":
  					e, name = Lookup(string(val))
  					needPragma = doNotNeedPragma
  				}
  			}
  
  			if needPragma == dontKnow || needPragma == doNeedPragma && !gotPragma {
  				continue
  			}
  
  			if strings.HasPrefix(name, "utf-16") {
  				name = "utf-8"
  				e = encoding.Nop
  			}
  
  			if e != nil {
  				return e, name
  			}
  		}
  	}
  }
  
  func fromMetaElement(s string) string {
  	for s != "" {
  		csLoc := strings.Index(s, "charset")
  		if csLoc == -1 {
  			return ""
  		}
  		s = s[csLoc+len("charset"):]
  		s = strings.TrimLeft(s, " \t\n\f\r")
  		if !strings.HasPrefix(s, "=") {
  			continue
  		}
  		s = s[1:]
  		s = strings.TrimLeft(s, " \t\n\f\r")
  		if s == "" {
  			return ""
  		}
  		if q := s[0]; q == '"' || q == '\'' {
  			s = s[1:]
  			closeQuote := strings.IndexRune(s, rune(q))
  			if closeQuote == -1 {
  				return ""
  			}
  			return s[:closeQuote]
  		}
  
  		end := strings.IndexAny(s, "; \t\n\f\r")
  		if end == -1 {
  			end = len(s)
  		}
  		return s[:end]
  	}
  	return ""
  }
  
  var boms = []struct {
  	bom []byte
  	enc string
  }{
  	{[]byte{0xfe, 0xff}, "utf-16be"},
  	{[]byte{0xff, 0xfe}, "utf-16le"},
  	{[]byte{0xef, 0xbb, 0xbf}, "utf-8"},
  }
  