]> Sergey Matveev's repositories - tofuproxy.git/blob - fifos/warcs.go
90e17a0ccceb3848ad947c9aed87ed1bf56435dd
[tofuproxy.git] / fifos / warcs.go
1 /*
2 tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management
3 Copyright (C) 2021 Sergey Matveev <stargrave@stargrave.org>
4
5 This program is free software: you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation, version 3 of the License.
8
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12 GNU General Public License for more details.
13
14 You should have received a copy of the GNU General Public License
15 along with this program.  If not, see <http://www.gnu.org/licenses/>.
16 */
17
18 package fifos
19
20 import (
21         "bufio"
22         "fmt"
23         "log"
24         "os"
25
26         "go.stargrave.org/tofuproxy/warc"
27 )
28
29 func listWARCs(p string) {
30         for {
31                 fd, err := os.OpenFile(p, os.O_WRONLY|os.O_APPEND, os.FileMode(0666))
32                 if err != nil {
33                         log.Fatalln(err)
34                 }
35                 warc.WARCsM.RLock()
36                 for warcPath, uris := range warc.WARCs {
37                         fmt.Fprintf(
38                                 fd, "%s\t%d\t%d\n",
39                                 warcPath, len(uris), len(warc.WARCs[warcPath]),
40                         )
41                 }
42                 warc.WARCsM.RUnlock()
43                 fd.Close()
44         }
45 }
46
47 func addWARC(p string) {
48         for {
49                 fd, err := os.OpenFile(p, os.O_RDONLY, os.FileMode(0666))
50                 if err != nil {
51                         log.Fatalln(err)
52                 }
53                 var warcPaths []string
54                 scanner := bufio.NewScanner(fd)
55                 for scanner.Scan() {
56                         t := scanner.Text()
57                         if len(t) > 0 {
58                                 warcPaths = append(warcPaths, t)
59                         }
60                 }
61                 fd.Close()
62                 for _, warcPath := range warcPaths {
63                         if warcPath == "SAVE" {
64                                 if err = warc.SaveIndexes(); err != nil {
65                                         log.Printf("%s: can not save index %s: %+v\n", p, warcPath, err)
66                                 }
67                                 continue
68                         }
69                         if _, exists := warc.WARCs[warcPath]; exists {
70                                 continue
71                         }
72                         log.Printf("%s: adding WARC %s\n", p, warcPath)
73                         err = warc.Add(warcPath)
74                         if err != nil {
75                                 log.Printf("%s: can not open %s: %+v\n", p, warcPath, err)
76                                 break
77                         }
78                         log.Printf(
79                                 "%s: %s: added %d URIs %d segments\n",
80                                 p, warcPath,
81                                 len(warc.WARCs[warcPath]),
82                                 len(warc.WARCsOffsets[warcPath]),
83                         )
84                 }
85         }
86 }