2 tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management
3 Copyright (C) 2021 Sergey Matveev <stargrave@stargrave.org>
5 This program is free software: you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation, version 3 of the License.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>.
27 "github.com/dustin/go-humanize"
28 "go.stargrave.org/tofuproxy/warc"
32 uri := flag.String("uri", "", "URI to extract, if specified")
33 hdr := flag.Bool("hdr", false, "Also extract WARC's header")
34 idx := flag.Bool("idx", false, "Save WARC indexes")
36 log.SetFlags(log.Lshortfile)
38 for _, p := range flag.Args() {
39 log.Println("adding", p)
40 if err := warc.Add(p); err != nil {
43 log.Println("added", p, len(warc.WARCs[p]), "URIs")
45 if err := warc.SaveIndexes(); err != nil {
51 for warcPath, uris := range warc.WARCs {
52 for uri, rec := range uris {
56 humanize.IBytes(uint64(rec.TotalSize())),
61 for _, uris := range warc.WARCs {
66 r, err := rec.Reader(!*hdr)