func nextRecord(reader *bufio.Reader) (*record, error) {
  rec := record{}
  var failSafe int = 100
  var line string
  var err error
  for {
    line, err = reader.ReadString('\n')
    if err != nil { return nil, err }
    line = strings.TrimSpace(line)
    if line == "WARC/1.0" {
      break
    } else if line == "" {
      continue
    } else {
      panic(fmt.Sprintf("Malformed first line: '%s'\n", line))
    }
  }
  rec.header = append(rec.header, line)
  for {
    line, err = reader.ReadString('\n')
    if err != nil {
      if err == io.EOF { break }
      return nil, err
    }
    line = strings.TrimSpace(line)
    if line == "" { break }
    if strings.HasPrefix(line, "WARC-Type: ") {
      rec.warcType = line[11:]
    } else if strings.HasPrefix(line, "Content-Length: ") {
      rec.length, err = strconv.Atoi(line[16:])
      if err != nil { return nil, err }
    } else if strings.HasPrefix(line, "WARC-Refers-To: ") {
      rec.refersTo = line[16:]
    }
    rec.header = append(rec.header, line)
    failSafe--
    if failSafe == 0 { panic("Hit failsafe when reading header") }
  }
  if rec.length == 0 { panic("No record length") }
  rec.data = make([]byte, rec.length)
  var bytes int
  for bytes < rec.length {
    n, err := reader.Read(rec.data[bytes:])
    if err != nil { return nil, err }
    bytes += n
  }
  if bytes != rec.length {
    panic(fmt.Sprintf("Only read %d bytes, expecting %d\n", bytes, rec.length))
  }
  return &rec, err
}
Returns nil on EOF
package main
import (
  "bufio"
  "encoding/json"
  "fmt"
  "log"
  "io"
  "os"
  "strings"
  "strconv"
)
type GzipMetaData struct {
  FooterLength int `json:"Footer-Length,string"`
  DeflateLength int `json:"Deflate-Length,string"`
  HeaderLength int `json:"Header-Length,string"`
  InflatedLength int `json:"Inflated-Length,string"`
}
type WarcContainer struct {
  Compressed bool
  Offset int `json:",string"`
  Filename string
  GzipMeta GzipMetaData `json:"Gzip-Metadata"`
}
type WarcEnvelope struct {
  Format string
  HeaderLength int `json:"WARC-Header-Length,string"`
  Digest string `json:"Block-Digest"`
  ContentLength int `json:"Actual-Content-Length,string"`
  HeaderMetaData WarcMetaData `json:"WARC-Header-Metadata"`
  PayloadMetaData PayloadMetaData `json:"Payload-Metadata"`
}
type WarcMetaData struct {
  Type string `json:"WARC-Type"`
  Length int `json:"Content-Length,string"`
  RecordId string `json:"WARC-Record-ID"`
  Uri string `json:"WARC-Target-URI"`
  ContentType string `json:"Content-Type"`
}
type PayloadMetaData struct {
  ResponseMeta ResponseMetaData `json:"HTTP-Response-Metadata"`
}
type HTTPHeaders struct {
  ContentType string `json:"Content-Type"`
}
type ResponseMetaData struct {
  ResponseInfo ResponseMessage `json:"Response-Message"`
  Headers HTTPHeaders
}
type ResponseMessage struct {
  Status int `json:",string"`
}
type WarcMeta struct {
  Envelope WarcEnvelope
  Container WarcContainer
}
type record struct {
  length int
  refersTo string
  header []string
  warcType string
  data []byte
}