2 tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management
3 Copyright (C) 2021 Sergey Matveev <stargrave@stargrave.org>
5 This program is free software: you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation, version 3 of the License.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>.
30 const IndexExt = ".idx.gob"
33 WARCs = map[string]map[string]*Record{}
36 Incomplete = map[string]*Record{}
39 func Add(warcPath string) error {
40 fd, err := os.Open(warcPath + IndexExt)
43 var uris map[string]*Record
44 if err := gob.NewDecoder(fd).Decode(&uris); err != nil {
48 WARCs[warcPath] = uris
52 if err != nil && !os.IsNotExist(err) {
55 r, err := NewReader(warcPath)
60 uris := map[string]*Record{}
62 rec, err := r.ReadRecord()
69 segNum := rec.Hdr.Get("WARC-Segment-Number")
70 switch rec.Hdr.Get("WARC-Type") {
77 Incomplete[rec.Hdr.Get("WARC-Record-ID")] = rec
82 originID := rec.Hdr.Get("WARC-Segment-Origin-ID")
83 incomplete := Incomplete[originID]
84 if incomplete == nil {
85 return fmt.Errorf("can not find WARC-Segment-Origin-ID: %q", originID)
87 segNumExpected := strconv.Itoa(len(incomplete.Continuations) + 1 + 1)
88 if segNum != segNumExpected {
90 "unexpected WARC-Segment-Number %s != %s",
91 segNum, segNumExpected,
94 incomplete.Continuations = append(incomplete.Continuations, rec)
95 if rec.Hdr.Get("WARC-Segment-Total-Length") != "" {
97 WARCs[incomplete.WARCPath][incomplete.URI()] = incomplete
99 delete(Incomplete, originID)
104 WARCs[warcPath] = uris
109 func SaveIndexes() error {
111 defer WARCsM.RUnlock()
112 for warcPath, uris := range WARCs {
113 p := warcPath + IndexExt
114 if _, err := os.Stat(p); err == nil {
117 fd, err := os.OpenFile(
119 os.O_CREATE|os.O_WRONLY|os.O_EXCL,
125 if err = gob.NewEncoder(fd).Encode(&uris); err != nil {
130 if err = os.Rename(p+".tmp", p); err != nil {
133 log.Println("saved:", p)