/*
-tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management
-Copyright (C) 2021 Sergey Matveev <stargrave@stargrave.org>
+tofuproxy -- flexible HTTP/HTTPS proxy, TLS terminator, X.509 TOFU
+ manager, WARC/geminispace browser
+Copyright (C) 2021-2023 Sergey Matveev <stargrave@stargrave.org>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
package fifos
import (
- "bufio"
"fmt"
"log"
"os"
}
warc.WARCsM.RLock()
for warcPath, uris := range warc.WARCs {
- fmt.Fprintf(fd, "%s\t%d\n", warcPath, len(uris))
+ fmt.Fprintf(
+ fd, "%s\t%d\t%d\n",
+ warcPath, len(uris), len(warc.WARCs[warcPath]),
+ )
}
warc.WARCsM.RUnlock()
fd.Close()
func addWARC(p string) {
for {
- fd, err := os.OpenFile(p, os.O_RDONLY, os.FileMode(0666))
- if err != nil {
- log.Fatalln(err)
- }
- var warcPaths []string
- scanner := bufio.NewScanner(fd)
- for scanner.Scan() {
- t := scanner.Text()
- if len(t) > 0 {
- warcPaths = append(warcPaths, t)
- }
- }
- fd.Close()
- for _, warcPath := range warcPaths {
+ for _, warcPath := range readLinesFromFIFO(p) {
if warcPath == "SAVE" {
- if err = warc.SaveIndexes(); err != nil {
+ if err := warc.SaveIndices(); err != nil {
log.Printf("%s: can not save index %s: %+v\n", p, warcPath, err)
}
continue
continue
}
log.Printf("%s: adding WARC %s\n", p, warcPath)
- err = warc.Add(warcPath)
- if err != nil {
+ if err := warc.Add(warcPath); err != nil {
log.Printf("%s: can not open %s: %+v\n", p, warcPath, err)
break
}
- log.Printf("%s: %s: added %d URIs\n", p, warcPath, len(warc.WARCs[warcPath]))
+ log.Printf(
+ "%s: %s: added %d URIs %d segments\n",
+ p, warcPath,
+ len(warc.WARCs[warcPath]),
+ len(warc.WARCsOffsets[warcPath]),
+ )
}
}
}