// tofuproxy -- flexible HTTP/HTTPS proxy, TLS terminator, X.509 TOFU // manager, WARC/geminispace browser // Copyright (C) 2021-2024 Sergey Matveev // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation, version 3 of the License. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with this program. If not, see . package warc import ( "io" "strings" ) type Record struct { WARCPath string Offset int64 Size int64 Hdr Header HdrLen int HdrLines []string Continuations []*Record } func (rec *Record) URI() string { return strings.Trim(rec.Hdr.Get("WARC-Target-URI"), "<>") } func (rec *Record) TotalSize() int64 { s := rec.Size for _, r := range rec.Continuations { s += r.Size } return s } type SelfRecordReader struct { lr *io.LimitedReader rrr io.ReadCloser } func (srr *SelfRecordReader) Read(p []byte) (n int, err error) { n, err = srr.lr.Read(p) if err != nil { srr.Close() } return } func (srr *SelfRecordReader) Close() error { return srr.rrr.Close() } func (rec *Record) selfReader(noHdr bool, offsets []Offset) (*SelfRecordReader, error) { offset := rec.Offset if noHdr { offset += int64(rec.HdrLen) } rrr, err := Open(rec.WARCPath, offsets, offset) if err != nil { return nil, err } size := rec.Size if !noHdr { size += int64(rec.HdrLen) } return &SelfRecordReader{lr: &io.LimitedReader{R: rrr, N: size}, rrr: rrr}, nil } type RecordReader struct { r io.Reader srrs []*SelfRecordReader } func (rec *Record) Reader( noHdr bool, warcOffsets map[string][]Offset, ) (*RecordReader, error) { srrs := make([]*SelfRecordReader, 0, 1+len(rec.Continuations)) rs := make([]io.Reader, 0, 1+len(rec.Continuations)) for i, r := range append([]*Record{rec}, rec.Continuations...) { if i > 0 { noHdr = true } srr, err := r.selfReader(noHdr, warcOffsets[rec.WARCPath]) if err != nil { for _, srr := range srrs { srr.Close() } return nil, err } srrs = append(srrs, srr) rs = append(rs, srr) } return &RecordReader{r: io.MultiReader(rs...), srrs: srrs}, nil } func (rr *RecordReader) Read(p []byte) (n int, err error) { n, err = rr.r.Read(p) if err != nil { rr.Close() } return } func (rr *RecordReader) Close() error { for _, srr := range rr.srrs { srr.Close() } return nil }