/* tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 3 of the License. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . */ package fifos import ( "bufio" "fmt" "log" "os" "go.stargrave.org/tofuproxy/warc" ) func listWARCs(p string) { for { fd, err := os.OpenFile(p, os.O_WRONLY|os.O_APPEND, os.FileMode(0666)) if err != nil { log.Fatalln(err) } warc.WARCsM.RLock() for warcPath, uris := range warc.WARCs { fmt.Fprintf( fd, "%s\t%d\t%d\n", warcPath, len(uris), len(warc.WARCs[warcPath]), ) } warc.WARCsM.RUnlock() fd.Close() } } func addWARC(p string) { for { fd, err := os.OpenFile(p, os.O_RDONLY, os.FileMode(0666)) if err != nil { log.Fatalln(err) } var warcPaths []string scanner := bufio.NewScanner(fd) for scanner.Scan() { t := scanner.Text() if len(t) > 0 { warcPaths = append(warcPaths, t) } } fd.Close() for _, warcPath := range warcPaths { if warcPath == "SAVE" { if err = warc.SaveIndexes(); err != nil { log.Printf("%s: can not save index %s: %+v\n", p, warcPath, err) } continue } if _, exists := warc.WARCs[warcPath]; exists { continue } log.Printf("%s: adding WARC %s\n", p, warcPath) err = warc.Add(warcPath) if err != nil { log.Printf("%s: can not open %s: %+v\n", p, warcPath, err) break } log.Printf( "%s: %s: added %d URIs %d segments\n", p, warcPath, len(warc.WARCs[warcPath]), len(warc.WARCsOffsets[warcPath]), ) } } }