Multistream WARCs and better Zstandard support

[tofuproxy.git] / warc / record.go

diff --git a/warc/record.go b/warc/record.go
index 2dd123d3c972a515fb78a5705b507e7097110178..a2fa6bc5a2370838cdb4c4d5d867003a2a037ee9 100644 (file)
--- a/warc/record.go
+++ b/warc/record.go
@@ -25,9 +25,11 @@ import (
 type Record struct {
        WARCPath string
        Offset   int64
+       Size     int64
+
        Hdr      Header
        HdrLen   int
-       Size     int64
+       HdrLines []string
 
        Continuations []*Record
 }
@@ -45,12 +47,12 @@ func (rec *Record) TotalSize() int64 {
 }
 
 type SelfRecordReader struct {
-       r   *io.LimitedReader
-       rsc io.ReadSeekCloser
+       lr  *io.LimitedReader
+       rrr io.ReadCloser
 }
 
 func (srr *SelfRecordReader) Read(p []byte) (n int, err error) {
-       n, err = srr.r.Read(p)
+       n, err = srr.rrr.Read(p)
        if err != nil {
                srr.Close()
        }
@@ -58,23 +60,19 @@ func (srr *SelfRecordReader) Read(p []byte) (n int, err error) {
 }
 
 func (srr *SelfRecordReader) Close() error {
-       return srr.rsc.Close()
+       return srr.rrr.Close()
 }
 
-func (rec *Record) selfReader(noHdr bool) (*SelfRecordReader, error) {
-       rsc, err := Open(rec.WARCPath)
-       if err != nil {
-               return nil, err
-       }
+func (rec *Record) selfReader(noHdr bool, offsets []Offset) (*SelfRecordReader, error) {
        offset := rec.Offset
        if noHdr {
                offset += int64(rec.HdrLen)
        }
-       if _, err = rsc.Seek(offset, io.SeekStart); err != nil {
-               rsc.Close()
+       rrr, err := Open(rec.WARCPath, offsets, offset)
+       if err != nil {
                return nil, err
        }
-       return &SelfRecordReader{r: &io.LimitedReader{R: rsc, N: rec.Size}, rsc: rsc}, nil
+       return &SelfRecordReader{lr: &io.LimitedReader{R: rrr, N: rec.Size}, rrr: rrr}, nil
 }
 
 type RecordReader struct {
@@ -82,14 +80,17 @@ type RecordReader struct {
        srrs []*SelfRecordReader
 }
 
-func (rec *Record) Reader(noHdr bool) (*RecordReader, error) {
+func (rec *Record) Reader(
+       noHdr bool,
+       warcOffsets map[string][]Offset,
+) (*RecordReader, error) {
        srrs := make([]*SelfRecordReader, 0, 1+len(rec.Continuations))
        rs := make([]io.Reader, 0, 1+len(rec.Continuations))
        for i, r := range append([]*Record{rec}, rec.Continuations...) {
                if i > 0 {
                        noHdr = true
                }
-               srr, err := r.selfReader(noHdr)
+               srr, err := r.selfReader(noHdr, warcOffsets[rec.WARCPath])
                if err != nil {
                        for _, srr := range srrs {
                                srr.Close()