fifos/warcs.go

   1 /*
   2 tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management
   3 Copyright (C) 2021 Sergey Matveev <stargrave@stargrave.org>
   4
   5 This program is free software: you can redistribute it and/or modify
   6 it under the terms of the GNU General Public License as published by
   7 the Free Software Foundation, version 3 of the License.
   8
   9 This program is distributed in the hope that it will be useful,
  10 but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 GNU General Public License for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 */
  17
  18 package fifos
  19
  20 import (
  21         "bufio"
  22         "fmt"
  23         "log"
  24         "os"
  25
  26         "go.stargrave.org/tofuproxy/warc"
  27 )
  28
  29 func listWARCs(p string) {
  30         for {
  31                 fd, err := os.OpenFile(p, os.O_WRONLY|os.O_APPEND, os.FileMode(0666))
  32                 if err != nil {
  33                         log.Fatalln(err)
  34                 }
  35                 warc.WARCsM.RLock()
  36                 for warcPath, uris := range warc.WARCs {
  37                         fmt.Fprintf(fd, "%s\t%d\n", warcPath, len(uris))
  38                 }
  39                 warc.WARCsM.RUnlock()
  40                 fd.Close()
  41         }
  42 }
  43
  44 func addWARC(p string) {
  45         for {
  46                 fd, err := os.OpenFile(p, os.O_RDONLY, os.FileMode(0666))
  47                 if err != nil {
  48                         log.Fatalln(err)
  49                 }
  50                 var warcPaths []string
  51                 scanner := bufio.NewScanner(fd)
  52                 for scanner.Scan() {
  53                         t := scanner.Text()
  54                         if len(t) > 0 {
  55                                 warcPaths = append(warcPaths, t)
  56                         }
  57                 }
  58                 fd.Close()
  59                 for _, warcPath := range warcPaths {
  60                         if warcPath == "SAVE" {
  61                                 if err = warc.SaveIndexes(); err != nil {
  62                                         log.Printf("%s: can not save index %s: %+v\n", p, warcPath, err)
  63                                 }
  64                                 continue
  65                         }
  66                         if _, exists := warc.WARCs[warcPath]; exists {
  67                                 continue
  68                         }
  69                         log.Printf("%s: adding WARC %s\n", p, warcPath)
  70                         err = warc.Add(warcPath)
  71                         if err != nil {
  72                                 log.Printf("%s: can not open %s: %+v\n", p, warcPath, err)
  73                                 break
  74                         }
  75                         log.Printf("%s: %s: added %d URIs\n", p, warcPath, len(warc.WARCs[warcPath]))
  76                 }
  77         }
  78 }