fifos/warcs.go

   1 /*
   2 tofuproxy -- flexible HTTP/HTTPS proxy, TLS terminator, X.509 TOFU
   3              manager, WARC/geminispace browser
   4 Copyright (C) 2021-2023 Sergey Matveev <stargrave@stargrave.org>
   5
   6 This program is free software: you can redistribute it and/or modify
   7 it under the terms of the GNU General Public License as published by
   8 the Free Software Foundation, version 3 of the License.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU General Public License for more details.
  14
  15 You should have received a copy of the GNU General Public License
  16 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17 */
  18
  19 package fifos
  20
  21 import (
  22         "fmt"
  23         "log"
  24         "os"
  25
  26         "go.stargrave.org/tofuproxy/warc"
  27 )
  28
  29 func listWARCs(p string) {
  30         for {
  31                 fd, err := os.OpenFile(p, os.O_WRONLY|os.O_APPEND, os.FileMode(0666))
  32                 if err != nil {
  33                         log.Fatalln(err)
  34                 }
  35                 warc.WARCsM.RLock()
  36                 for warcPath, uris := range warc.WARCs {
  37                         fmt.Fprintf(
  38                                 fd, "%s\t%d\t%d\n",
  39                                 warcPath, len(uris), len(warc.WARCs[warcPath]),
  40                         )
  41                 }
  42                 warc.WARCsM.RUnlock()
  43                 fd.Close()
  44         }
  45 }
  46
  47 func addWARC(p string) {
  48         for {
  49                 for _, warcPath := range readLinesFromFIFO(p) {
  50                         if warcPath == "SAVE" {
  51                                 if err := warc.SaveIndices(); err != nil {
  52                                         log.Printf("%s: can not save index %s: %+v\n", p, warcPath, err)
  53                                 }
  54                                 continue
  55                         }
  56                         if _, exists := warc.WARCs[warcPath]; exists {
  57                                 continue
  58                         }
  59                         log.Printf("%s: adding WARC %s\n", p, warcPath)
  60                         if err := warc.Add(warcPath); err != nil {
  61                                 log.Printf("%s: can not open %s: %+v\n", p, warcPath, err)
  62                                 break
  63                         }
  64                         log.Printf(
  65                                 "%s: %s: added %d URIs %d segments\n",
  66                                 p, warcPath,
  67                                 len(warc.WARCs[warcPath]),
  68                                 len(warc.WARCsOffsets[warcPath]),
  69                         )
  70                 }
  71         }
  72 }