fifos/warcs.go

   1 // tofuproxy -- flexible HTTP/HTTPS proxy, TLS terminator, X.509 TOFU
   2 //              manager, WARC/geminispace browser
   3 // Copyright (C) 2021-2024 Sergey Matveev <stargrave@stargrave.org>
   4 //
   5 // This program is free software: you can redistribute it and/or modify
   6 // it under the terms of the GNU General Public License as published by
   7 // the Free Software Foundation, version 3 of the License.
   8 //
   9 // This program is distributed in the hope that it will be useful,
  10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 // GNU General Public License for more details.
  13 //
  14 // You should have received a copy of the GNU General Public License
  15 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 package fifos
  18
  19 import (
  20         "fmt"
  21         "log"
  22         "os"
  23
  24         "go.stargrave.org/tofuproxy/warc"
  25 )
  26
  27 func listWARCs(p string) {
  28         for {
  29                 fd, err := os.OpenFile(p, os.O_WRONLY|os.O_APPEND, os.FileMode(0666))
  30                 if err != nil {
  31                         log.Fatalln(err)
  32                 }
  33                 warc.WARCsM.RLock()
  34                 for warcPath, uris := range warc.WARCs {
  35                         fmt.Fprintf(
  36                                 fd, "%s\t%d\t%d\n",
  37                                 warcPath, len(uris), len(warc.WARCs[warcPath]),
  38                         )
  39                 }
  40                 warc.WARCsM.RUnlock()
  41                 fd.Close()
  42         }
  43 }
  44
  45 func addWARC(p string) {
  46         for {
  47                 for _, warcPath := range readLinesFromFIFO(p) {
  48                         if warcPath == "SAVE" {
  49                                 if err := warc.SaveIndices(); err != nil {
  50                                         log.Printf("%s: can not save index %s: %+v\n", p, warcPath, err)
  51                                 }
  52                                 continue
  53                         }
  54                         if _, exists := warc.WARCs[warcPath]; exists {
  55                                 continue
  56                         }
  57                         log.Printf("%s: adding WARC %s\n", p, warcPath)
  58                         if err := warc.Add(warcPath); err != nil {
  59                                 log.Printf("%s: can not open %s: %+v\n", p, warcPath, err)
  60                                 break
  61                         }
  62                         log.Printf(
  63                                 "%s: %s: added %d URIs %d segments\n",
  64                                 p, warcPath,
  65                                 len(warc.WARCs[warcPath]),
  66                                 len(warc.WARCsOffsets[warcPath]),
  67                         )
  68                 }
  69         }
  70 }