fifos/warcs.go

   1 /*
   2 tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management
   3 Copyright (C) 2021 Sergey Matveev <stargrave@stargrave.org>
   4
   5 This program is free software: you can redistribute it and/or modify
   6 it under the terms of the GNU General Public License as published by
   7 the Free Software Foundation, version 3 of the License.
   8
   9 This program is distributed in the hope that it will be useful,
  10 but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 GNU General Public License for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 */
  17
  18 package fifos
  19
  20 import (
  21         "bufio"
  22         "fmt"
  23         "log"
  24         "os"
  25
  26         "go.stargrave.org/tofuproxy/warc"
  27 )
  28
  29 func listWARCs(p string) {
  30         for {
  31                 fd, err := os.OpenFile(p, os.O_WRONLY|os.O_APPEND, os.FileMode(0666))
  32                 if err != nil {
  33                         log.Fatalln(err)
  34                 }
  35                 warc.WARCsM.RLock()
  36                 for warcPath, uris := range warc.WARCs {
  37                         fmt.Fprintf(
  38                                 fd, "%s\t%d\t%d\n",
  39                                 warcPath, len(uris), len(warc.WARCs[warcPath]),
  40                         )
  41                 }
  42                 warc.WARCsM.RUnlock()
  43                 fd.Close()
  44         }
  45 }
  46
  47 func addWARC(p string) {
  48         for {
  49                 fd, err := os.OpenFile(p, os.O_RDONLY, os.FileMode(0666))
  50                 if err != nil {
  51                         log.Fatalln(err)
  52                 }
  53                 var warcPaths []string
  54                 scanner := bufio.NewScanner(fd)
  55                 for scanner.Scan() {
  56                         t := scanner.Text()
  57                         if len(t) > 0 {
  58                                 warcPaths = append(warcPaths, t)
  59                         }
  60                 }
  61                 fd.Close()
  62                 for _, warcPath := range warcPaths {
  63                         if warcPath == "SAVE" {
  64                                 if err = warc.SaveIndexes(); err != nil {
  65                                         log.Printf("%s: can not save index %s: %+v\n", p, warcPath, err)
  66                                 }
  67                                 continue
  68                         }
  69                         if _, exists := warc.WARCs[warcPath]; exists {
  70                                 continue
  71                         }
  72                         log.Printf("%s: adding WARC %s\n", p, warcPath)
  73                         err = warc.Add(warcPath)
  74                         if err != nil {
  75                                 log.Printf("%s: can not open %s: %+v\n", p, warcPath, err)
  76                                 break
  77                         }
  78                         log.Printf(
  79                                 "%s: %s: added %d URIs %d segments\n",
  80                                 p, warcPath,
  81                                 len(warc.WARCs[warcPath]),
  82                                 len(warc.WARCsOffsets[warcPath]),
  83                         )
  84                 }
  85         }
  86 }