fifos/warcs.go

   1 /*
   2 tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates
   3              manager, WARC/Gemini browser
   4 Copyright (C) 2021 Sergey Matveev <stargrave@stargrave.org>
   5
   6 This program is free software: you can redistribute it and/or modify
   7 it under the terms of the GNU General Public License as published by
   8 the Free Software Foundation, version 3 of the License.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU General Public License for more details.
  14
  15 You should have received a copy of the GNU General Public License
  16 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17 */
  18
  19 package fifos
  20
  21 import (
  22         "bufio"
  23         "fmt"
  24         "log"
  25         "os"
  26
  27         "go.stargrave.org/tofuproxy/warc"
  28 )
  29
  30 func listWARCs(p string) {
  31         for {
  32                 fd, err := os.OpenFile(p, os.O_WRONLY|os.O_APPEND, os.FileMode(0666))
  33                 if err != nil {
  34                         log.Fatalln(err)
  35                 }
  36                 warc.WARCsM.RLock()
  37                 for warcPath, uris := range warc.WARCs {
  38                         fmt.Fprintf(
  39                                 fd, "%s\t%d\t%d\n",
  40                                 warcPath, len(uris), len(warc.WARCs[warcPath]),
  41                         )
  42                 }
  43                 warc.WARCsM.RUnlock()
  44                 fd.Close()
  45         }
  46 }
  47
  48 func addWARC(p string) {
  49         for {
  50                 fd, err := os.OpenFile(p, os.O_RDONLY, os.FileMode(0666))
  51                 if err != nil {
  52                         log.Fatalln(err)
  53                 }
  54                 var warcPaths []string
  55                 scanner := bufio.NewScanner(fd)
  56                 for scanner.Scan() {
  57                         t := scanner.Text()
  58                         if len(t) > 0 {
  59                                 warcPaths = append(warcPaths, t)
  60                         }
  61                 }
  62                 fd.Close()
  63                 for _, warcPath := range warcPaths {
  64                         if warcPath == "SAVE" {
  65                                 if err = warc.SaveIndexes(); err != nil {
  66                                         log.Printf("%s: can not save index %s: %+v\n", p, warcPath, err)
  67                                 }
  68                                 continue
  69                         }
  70                         if _, exists := warc.WARCs[warcPath]; exists {
  71                                 continue
  72                         }
  73                         log.Printf("%s: adding WARC %s\n", p, warcPath)
  74                         err = warc.Add(warcPath)
  75                         if err != nil {
  76                                 log.Printf("%s: can not open %s: %+v\n", p, warcPath, err)
  77                                 break
  78                         }
  79                         log.Printf(
  80                                 "%s: %s: added %d URIs %d segments\n",
  81                                 p, warcPath,
  82                                 len(warc.WARCs[warcPath]),
  83                                 len(warc.WARCsOffsets[warcPath]),
  84                         )
  85                 }
  86         }
  87 }