Multistream WARCs and better Zstandard support

[tofuproxy.git] / warc / open.go

diff --git a/warc/open.go b/warc/open.go
new file mode 100644 (file)
index 0000000..22792d5
--- /dev/null
+++ b/warc/open.go
@@ -0,0 +1,49 @@
+/*
+tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management
+Copyright (C) 2021 Sergey Matveev <stargrave@stargrave.org>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, version 3 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package warc
+
+import (
+       "fmt"
+       "io"
+       "path"
+)
+
+var UnZSTDPath = "/home/stargrave/work/tofuproxy/cmd/unzstd/unzstd"
+
+type Offset struct {
+       Z int64 // Compressed frame size
+       U int64 // Its uncompressed size
+}
+
+type RawRecordReader interface {
+       io.ReadCloser
+       Offsets() []Offset
+}
+
+func Open(warcPath string, offsets []Offset, offset int64) (RawRecordReader, error) {
+       ext := path.Ext(warcPath)
+       switch ext {
+       case ".warc":
+               return NewUncompressedReader(warcPath, offset)
+       case ".gz":
+               return NewGZIPReader(warcPath, offsets, offset)
+       case ".zst":
+               return NewCompressedReader(warcPath, UnZSTDPath, offsets, offset)
+       }
+       return nil, fmt.Errorf("unknown extension: %s", ext)
+}