From 6eee3c6c83cc535855e254426a90f7a2abba04ce Mon Sep 17 00:00:00 2001 From: Sergey Matveev Date: Thu, 28 Oct 2021 11:43:08 +0300 Subject: [PATCH] gemini:// support --- README | 3 +- cmd/certgen/main.go | 3 +- cmd/enzstd/enzstd.c | 3 +- cmd/tofuproxy/main.go | 10 +- cmd/ungzip/main.go | 3 +- cmd/unzstd/unzstd.c | 3 +- cmd/warc-extract/main.go | 3 +- conn.go | 3 +- doc/gemini.texi | 14 ++ doc/index.texi | 11 +- doc/usage.texi | 1 + fifos/del.go | 3 +- fifos/list.go | 15 +- fifos/log.go | 3 +- fifos/spies.go | 3 +- fifos/start.go | 3 +- fifos/warcs.go | 3 +- httpauth.go | 7 +- rounds/denyFonts.go | 3 +- rounds/gemini.go | 265 +++++++++++++++++++++++++++++++++++ rounds/habrImage.go | 3 +- rounds/noHead.go | 3 +- rounds/reddit.go | 3 +- rounds/redirectHTML.go | 6 +- rounds/spy.go | 3 +- rounds/transcodeAVIF.go | 3 +- rounds/transcodeJXL.go | 3 +- rounds/transcodeWebP.go | 3 +- rounds/warc.go | 3 +- tls.go | 56 +------- dane.go => tls/dane.go | 3 +- tls/dial.go | 81 +++++++++++ tlsauth.go => tls/tlsauth.go | 16 ++- verify.go => tls/verify.go | 3 +- trip.go | 7 +- warc/compressed.go | 3 +- warc/gzip.go | 3 +- warc/header.go | 3 +- warc/open.go | 3 +- warc/reader.go | 3 +- warc/record.go | 3 +- warc/uncompressed.go | 3 +- warc/uris.go | 3 +- x509.go | 3 +- 44 files changed, 479 insertions(+), 106 deletions(-) create mode 100644 doc/gemini.texi create mode 100644 rounds/gemini.go rename dane.go => tls/dane.go (94%) create mode 100644 tls/dial.go rename tlsauth.go => tls/tlsauth.go (86%) rename verify.go => tls/verify.go (98%) diff --git a/README b/README index 1f14d6e..e767a42 100644 --- a/README +++ b/README @@ -1,4 +1,5 @@ -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates +manager, WARC/Gemini browser. Home page: http://www.tofuproxy.stargrave.org/ diff --git a/cmd/certgen/main.go b/cmd/certgen/main.go index ab2b9f6..93b5e7f 100644 --- a/cmd/certgen/main.go +++ b/cmd/certgen/main.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/cmd/enzstd/enzstd.c b/cmd/enzstd/enzstd.c index cedb084..ee5e35a 100644 --- a/cmd/enzstd/enzstd.c +++ b/cmd/enzstd/enzstd.c @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/cmd/tofuproxy/main.go b/cmd/tofuproxy/main.go index d552339..941438d 100644 --- a/cmd/tofuproxy/main.go +++ b/cmd/tofuproxy/main.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify @@ -27,6 +28,7 @@ import ( "go.stargrave.org/tofuproxy" "go.stargrave.org/tofuproxy/fifos" "go.stargrave.org/tofuproxy/rounds" + ttls "go.stargrave.org/tofuproxy/tls" ) func main() { @@ -54,9 +56,9 @@ func main() { fifos.NoTAI = *notai fifos.Start(*fifosDir) - tofuproxy.Certs = *certs - tofuproxy.CCerts = *ccerts - tofuproxy.DNSSrv = *dnsSrv + ttls.Certs = *certs + ttls.CCerts = *ccerts + ttls.DNSSrv = *dnsSrv tofuproxy.CACert = caCert tofuproxy.CAPrv = caPrv rounds.WARCOnly = *warcOnly diff --git a/cmd/ungzip/main.go b/cmd/ungzip/main.go index 0895aa5..64ec0f7 100644 --- a/cmd/ungzip/main.go +++ b/cmd/ungzip/main.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/cmd/unzstd/unzstd.c b/cmd/unzstd/unzstd.c index 4864db6..4815251 100644 --- a/cmd/unzstd/unzstd.c +++ b/cmd/unzstd/unzstd.c @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/cmd/warc-extract/main.go b/cmd/warc-extract/main.go index 177d251..85bc273 100644 --- a/cmd/warc-extract/main.go +++ b/cmd/warc-extract/main.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/conn.go b/conn.go index 4b88f13..d2a05a2 100644 --- a/conn.go +++ b/conn.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/doc/gemini.texi b/doc/gemini.texi new file mode 100644 index 0000000..73acece --- /dev/null +++ b/doc/gemini.texi @@ -0,0 +1,14 @@ +@node Gemini +@section Geminispace + +You can browse @url{https://en.wikipedia.org/wiki/Gemini_(protocol), +geminispace} by going to @url{https://gemini/HOST/PATH}, where +@var{HOST} and @var{PATH} are taken from ordinary gemini URL. For +example to browse @url{gemini://gemini.circumlunar.space/docs/}, you must go +through @command{tofuproxy} to @url{https://gemini/gemini.circumlunar.space/docs/}. + +If you request @url{https://gemini/} URL with @code{Accept: text/gemini} +HTTP header, then gemfiles (@file{.gmi} will be passed as-is without +conversion to HTML. + +No @code{INPUT} is supported currently. diff --git a/doc/index.texi b/doc/index.texi index b97500d..7d552be 100644 --- a/doc/index.texi +++ b/doc/index.texi @@ -54,6 +54,10 @@ compression ratio for lossless screenshots, is not supported everywhere. WARC-related software is written on Python, that nowadays is close to be impossible to install and use with all its broken dependencies system. +@item And yet another piece of software is needed for browsing the +@url{https://en.wikipedia.org/wiki/Gemini_(protocol), geminispace}? +Too many bicycles already! + @end itemize That is why I wrote @command{tofuproxy} -- pure Go HTTP proxy, MitMing @@ -129,11 +133,12 @@ And Go itself tries also to act as a @url{https://http2.github.io/, HTTP/2} client too. @item -Ability to load @url{https://en.wikipedia.org/wiki/Web_ARChive, WARC} -files, possibly compressed, possibly continued and replace responses. +Ability to load and browse @url{https://en.wikipedia.org/wiki/Web_ARChive, +WARC} archives, possibly @command{gzip}/@command{zstd} compressed. @item - +Ability to browse geminispace, by transparent conversion to HTMLs with +URL rewriting. @end itemize diff --git a/doc/usage.texi b/doc/usage.texi index c6f2611..6ff4cd3 100644 --- a/doc/usage.texi +++ b/doc/usage.texi @@ -65,3 +65,4 @@ $ ( cd fifos ; ./multitail.sh ) @include spies.texi @include certs.texi @include warcs.texi +@include gemini.texi diff --git a/fifos/del.go b/fifos/del.go index 8e5a184..7700baf 100644 --- a/fifos/del.go +++ b/fifos/del.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/fifos/list.go b/fifos/list.go index 8625a81..d8f77ad 100644 --- a/fifos/list.go +++ b/fifos/list.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify @@ -77,11 +78,15 @@ func listTLSAuth(p string) { } caches.TLSAuthCacheM.RLock() for host, tlsCert := range caches.TLSAuthCache { - cert, err := x509.ParseCertificate(tlsCert.Certificate[0]) - if err != nil { - log.Fatalln(err) + subj := "NONE" + if len(tlsCert.Certificate) != 0 { + cert, err := x509.ParseCertificate(tlsCert.Certificate[0]) + if err != nil { + log.Fatalln(err) + } + subj = cert.Subject.String() } - if _, err = fd.WriteString(fmt.Sprintf("%s\t%s\n", host, cert.Subject)); err != nil { + if _, err = fd.WriteString(fmt.Sprintf("%s\t%s\n", host, subj)); err != nil { break } } diff --git a/fifos/log.go b/fifos/log.go index 0b717a1..b5308c8 100644 --- a/fifos/log.go +++ b/fifos/log.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/fifos/spies.go b/fifos/spies.go index 4edc151..2428ba8 100644 --- a/fifos/spies.go +++ b/fifos/spies.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/fifos/start.go b/fifos/start.go index 2f535e2..570f4bc 100644 --- a/fifos/start.go +++ b/fifos/start.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/fifos/warcs.go b/fifos/warcs.go index 90e17a0..ca8da6d 100644 --- a/fifos/warcs.go +++ b/fifos/warcs.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/httpauth.go b/httpauth.go index ade7801..025c42d 100644 --- a/httpauth.go +++ b/httpauth.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify @@ -27,6 +28,8 @@ import ( "os/exec" "path/filepath" "strings" + + ttls "go.stargrave.org/tofuproxy/tls" ) func findInNetrc(host string) (string, string) { @@ -90,7 +93,7 @@ bind . {switch -exact %%K { l login }} `, realm, userInit, passInit)) - cmd := exec.Command(CmdWish) + cmd := exec.Command(ttls.CmdWish) cmd.Stdin = &b out, err := cmd.Output() if err != nil { diff --git a/rounds/denyFonts.go b/rounds/denyFonts.go index efc3d4f..ea29361 100644 --- a/rounds/denyFonts.go +++ b/rounds/denyFonts.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/rounds/gemini.go b/rounds/gemini.go new file mode 100644 index 0000000..4876ff7 --- /dev/null +++ b/rounds/gemini.go @@ -0,0 +1,265 @@ +/* +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser +Copyright (C) 2021 Sergey Matveev + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package rounds + +import ( + "bufio" + "bytes" + "context" + "fmt" + "html" + "io" + "log" + "net/http" + "strconv" + "strings" + + "go.stargrave.org/tofuproxy/fifos" + ttls "go.stargrave.org/tofuproxy/tls" +) + +const ( + ContentTypeGemini = "text/gemini" + SchemeGemini = "gemini://" + GeminiEntrypoint = "https://gemini" + GeminiPort = ":1965" +) + +var GemCodeName = map[int]string{ + 10: "INPUT", + 11: "SENSITIVE INPUT", + 20: "SUCCESS", + 30: "REDIRECT - TEMPORARY", + 31: "REDIRECT - PERMANENT", + 40: "TEMPORARY FAILURE", + 41: "SERVER UNAVAILABLE", + 42: "CGI ERROR", + 43: "PROXY ERROR", + 44: "SLOW DOWN", + 50: "PERMANENT FAILURE", + 51: "NOT FOUND", + 52: "GONE", + 53: "PROXY REQUEST REFUSED", + 59: "BAD REQUEST", + 60: "CLIENT CERTIFICATE REQUIRED", + 61: "CERTIFICATE NOT AUTHORISED", + 62: "CERTIFICATE NOT VALID", +} + +func absolutizeURL(host, u string, paths ...string) string { + host = strings.TrimSuffix(host, GeminiPort) + if strings.Contains(u, "://") { + return u + } + if strings.HasPrefix(u, "/") { + return GeminiEntrypoint + "/" + host + u + } + paths = append([]string{GeminiEntrypoint, host}, paths...) + paths = append(paths, u) + return strings.Join(paths, "/") +} + +func geminifyURL(host, u string, paths ...string) string { + u = absolutizeURL(host, u, paths...) + if !strings.HasPrefix(u, SchemeGemini) { + return u + } + return GeminiEntrypoint + "/" + strings.TrimPrefix(u, SchemeGemini) +} + +func RoundGemini( + host string, + resp *http.Response, + w http.ResponseWriter, + req *http.Request, +) (bool, error) { + if host != "gemini" { + return true, nil + } + paths := strings.Split(strings.TrimPrefix(req.URL.Path, "/"), "/") + host, paths = paths[0], paths[1:] + hostWithPort := host + if !strings.Contains(hostWithPort, ":") { + hostWithPort += GeminiPort + } + conn, err := ttls.DialTLS(context.TODO(), "tcp", hostWithPort) + if err != nil { + log.Printf("%s: can not dial: %+v\n", req.URL, err) + return false, err + } + _, err = fmt.Fprintf( + conn, "%s%s/%s\r\n", + SchemeGemini, host, strings.Join(paths, "/"), + ) + if err != nil { + log.Printf("%s: can not send request: %+v\n", req.URL, err) + return false, err + } + if len(paths) > 0 && paths[len(paths)-1] == "" { + paths = paths[:len(paths)-1] + } + br := bufio.NewReader(conn) + rawResp, err := br.ReadString('\n') + if err != nil { + log.Printf("%s: can not read response: %+v\n", req.URL, err) + return false, err + } + cols := strings.SplitN(rawResp, " ", 2) + if len(cols) < 2 { + err = fmt.Errorf("invalid response format: %s", rawResp) + log.Printf("%s: %s\n", req.URL, err) + return false, err + } + code, err := strconv.Atoi(cols[0]) + if err != nil { + log.Printf("%s: can not parse response code: %+v\n", req.URL, err) + return false, err + } + codeName := GemCodeName[code] + if codeName == "" { + codeName = "UNKNOWN" + } + if 10 <= code && code <= 19 { + w.Header().Add("Content-Type", "text/plain") + w.WriteHeader(http.StatusBadRequest) + fmt.Fprintf(w, "%s\n%d (%s): INPUT is not supported\n", cols[1], code, codeName) + return false, nil + } + if 30 <= code && code <= 39 { + w.Header().Add("Content-Type", "text/html") + w.WriteHeader(http.StatusOK) + u := geminifyURL(host, cols[1], paths...) + w.Write([]byte( + fmt.Sprintf( + ` +%d (%s) redirection +Redirection to %s`, + code, codeName, u, u, + ))) + fifos.LogRedir <- fmt.Sprintf( + "%s %s\t%d\t%s", req.Method, req.URL, code, cols[1], + ) + return false, nil + } + if 40 <= code && code <= 49 { + w.Header().Add("Content-Type", "text/plain") + w.WriteHeader(http.StatusBadGateway) + fmt.Fprintf(w, "%s\n%d (%s)\n", cols[1], code, codeName) + return false, nil + } + if 50 <= code && code <= 59 { + w.Header().Add("Content-Type", "text/plain") + w.WriteHeader(http.StatusBadGateway) + fmt.Fprintf(w, "%s\n%d (%s)\n", cols[1], code, codeName) + return false, nil + } + if 60 <= code && code <= 69 { + w.Header().Add("Content-Type", "text/plain") + w.WriteHeader(http.StatusUnauthorized) + fmt.Fprintf(w, "%s\n%d (%s)\n", cols[1], code, codeName) + return false, nil + } + if !(20 <= code && code <= 29) { + err = fmt.Errorf("unknown response code: %d", code) + log.Printf("%s: %s\n", req.URL, err) + return false, err + } + contentType := strings.Split(strings.TrimRight(cols[1], "\r\n"), ";")[0] + if contentType == ContentTypeGemini && + !strings.Contains(req.Header.Get("Accept"), ContentTypeGemini) { + w.Header().Add("Content-Type", "text/html") + w.WriteHeader(http.StatusOK) + raw, err := io.ReadAll(br) + if err != nil { + log.Printf("%s: can not read response body: %+v\n", req.URL, err) + return false, err + } + var buf bytes.Buffer + fmt.Fprintf(&buf, ` +%d (%s) +`, code, codeName) + pre := false + for _, line := range strings.Split(string(raw), "\n") { + if strings.HasPrefix(line, "```") { + if pre { + buf.WriteString("\n") + } else { + buf.WriteString("
" + line[3:] + "\n")
+				}
+				pre = !pre
+				continue
+			}
+			if pre {
+				fmt.Fprintf(&buf, "%s\n", line)
+				continue
+			}
+			if strings.HasPrefix(line, "=> ") {
+				cols = strings.Fields(line)
+				u := geminifyURL(host, cols[1], paths...)
+				switch len(cols) {
+				case 2:
+					fmt.Fprintf(
+						&buf, "%s
\n", + u, html.EscapeString(cols[1]), + ) + default: + fmt.Fprintf( + &buf, "%s (%s)
\n", + u, html.EscapeString(strings.Join(cols[2:], " ")), cols[1], + ) + } + continue + } + if strings.HasPrefix(line, "# ") { + fmt.Fprintf(&buf, "

%s

\n", html.EscapeString(line[2:])) + continue + } + if strings.HasPrefix(line, "## ") { + fmt.Fprintf(&buf, "

%s

\n", html.EscapeString(line[3:])) + continue + } + if strings.HasPrefix(line, "### ") { + fmt.Fprintf(&buf, "

%s

\n", html.EscapeString(line[4:])) + continue + } + if strings.HasPrefix(line, "* ") { + fmt.Fprintf(&buf, "• %s\n", html.EscapeString(line[2:])) + continue + } + if strings.HasPrefix(line, "> ") { + fmt.Fprintf( + &buf, "
%s
\n", + html.EscapeString(line[2:]), + ) + continue + } + fmt.Fprintf(&buf, "%s
\n", html.EscapeString(line)) + } + buf.WriteString("\n") + _, err = w.Write(buf.Bytes()) + return false, err + } + w.Header().Add("Content-Type", contentType) + w.WriteHeader(http.StatusOK) + _, err = io.Copy(w, br) + if err != nil { + log.Printf("%s: can not read response body: %+v\n", req.URL, err) + } + return false, err +} diff --git a/rounds/habrImage.go b/rounds/habrImage.go index 941832e..c9f6fe4 100644 --- a/rounds/habrImage.go +++ b/rounds/habrImage.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/rounds/noHead.go b/rounds/noHead.go index c61af78..e1e32fe 100644 --- a/rounds/noHead.go +++ b/rounds/noHead.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/rounds/reddit.go b/rounds/reddit.go index fb126ec..8e7ecf3 100644 --- a/rounds/reddit.go +++ b/rounds/reddit.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/rounds/redirectHTML.go b/rounds/redirectHTML.go index d13047a..6a7e1fb 100644 --- a/rounds/redirectHTML.go +++ b/rounds/redirectHTML.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify @@ -77,7 +78,8 @@ func RoundRedirectHTML( location := resp.Header.Get("Location") w.Write([]byte( fmt.Sprintf( - `%d %s: %s redirection + ` +%d %s: %s redirection Redirection to %s`, resp.StatusCode, http.StatusText(resp.StatusCode), redirType, location, location, diff --git a/rounds/spy.go b/rounds/spy.go index affc1cb..fcb96f8 100644 --- a/rounds/spy.go +++ b/rounds/spy.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/rounds/transcodeAVIF.go b/rounds/transcodeAVIF.go index c315b19..a5cf4e2 100644 --- a/rounds/transcodeAVIF.go +++ b/rounds/transcodeAVIF.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/rounds/transcodeJXL.go b/rounds/transcodeJXL.go index 94fa07a..1d3a58e 100644 --- a/rounds/transcodeJXL.go +++ b/rounds/transcodeJXL.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/rounds/transcodeWebP.go b/rounds/transcodeWebP.go index 2939b08..82d3661 100644 --- a/rounds/transcodeWebP.go +++ b/rounds/transcodeWebP.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/rounds/warc.go b/rounds/warc.go index ddb566c..0ca4777 100644 --- a/rounds/warc.go +++ b/rounds/warc.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/tls.go b/tls.go index dc59862..ffd5084 100644 --- a/tls.go +++ b/tls.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify @@ -18,19 +19,14 @@ along with this program. If not, see . package tofuproxy import ( - "context" "crypto" "crypto/tls" "crypto/x509" "fmt" "log" - "net" "net/http" "strings" "time" - - "go.cypherpunks.ru/ucspi" - "go.stargrave.org/tofuproxy/fifos" ) var ( @@ -101,51 +97,3 @@ func (h *HTTPSHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { req.URL.Host = h.host roundTrip(w, req) } - -func dialTLS(ctx context.Context, network, addr string) (net.Conn, error) { - host := strings.Split(addr, ":")[0] - ccg := ClientCertificateGetter{host: host} - cfg := tls.Config{ - VerifyPeerCertificate: func( - rawCerts [][]byte, - verifiedChains [][]*x509.Certificate, - ) error { - return verifyCert(host, nil, rawCerts, verifiedChains) - }, - ClientSessionCache: sessionCache, - NextProtos: []string{"h2", "http/1.1"}, - GetClientCertificate: ccg.get, - } - conn, dialErr := tls.Dial(network, addr, &cfg) - if dialErr != nil { - if _, ok := dialErr.(ErrRejected); ok { - return nil, dialErr - } - cfg.InsecureSkipVerify = true - cfg.VerifyPeerCertificate = func( - rawCerts [][]byte, - verifiedChains [][]*x509.Certificate, - ) error { - return verifyCert(host, dialErr, rawCerts, verifiedChains) - } - var err error - conn, err = tls.Dial(network, addr, &cfg) - if err != nil { - fifos.LogErr <- fmt.Sprintf("%s\t%s", addr, dialErr.Error()) - return nil, err - } - } - connState := conn.ConnectionState() - if !connState.DidResume { - fifos.LogTLS <- fmt.Sprintf( - "%s\t%s %s %s\t%s\t%s", - addr, - ucspi.TLSVersion(connState.Version), - tls.CipherSuiteName(connState.CipherSuite), - connState.PeerCertificates[0].SignatureAlgorithm, - spkiHash(connState.PeerCertificates[0]), - connState.NegotiatedProtocol, - ) - } - return conn, nil -} diff --git a/dane.go b/tls/dane.go similarity index 94% rename from dane.go rename to tls/dane.go index 9ab9504..22fad5d 100644 --- a/dane.go +++ b/tls/dane.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/tls/dial.go b/tls/dial.go new file mode 100644 index 0000000..a22dc5f --- /dev/null +++ b/tls/dial.go @@ -0,0 +1,81 @@ +/* +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser +Copyright (C) 2021 Sergey Matveev + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package tofuproxy + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "net" + "strings" + + "go.cypherpunks.ru/ucspi" + "go.stargrave.org/tofuproxy/fifos" +) + +var sessionCache = tls.NewLRUClientSessionCache(1024) + +func DialTLS(ctx context.Context, network, addr string) (net.Conn, error) { + host := strings.Split(addr, ":")[0] + ccg := ClientCertificateGetter{host: host} + cfg := tls.Config{ + VerifyPeerCertificate: func( + rawCerts [][]byte, + verifiedChains [][]*x509.Certificate, + ) error { + return verifyCert(host, nil, rawCerts, verifiedChains) + }, + ClientSessionCache: sessionCache, + NextProtos: []string{"h2", "http/1.1"}, + GetClientCertificate: ccg.get, + } + conn, dialErr := tls.Dial(network, addr, &cfg) + if dialErr != nil { + if _, ok := dialErr.(ErrRejected); ok { + return nil, dialErr + } + cfg.InsecureSkipVerify = true + cfg.VerifyPeerCertificate = func( + rawCerts [][]byte, + verifiedChains [][]*x509.Certificate, + ) error { + return verifyCert(host, dialErr, rawCerts, verifiedChains) + } + var err error + conn, err = tls.Dial(network, addr, &cfg) + if err != nil { + fifos.LogErr <- fmt.Sprintf("%s\t%s", addr, dialErr.Error()) + return nil, err + } + } + connState := conn.ConnectionState() + if !connState.DidResume { + fifos.LogTLS <- fmt.Sprintf( + "%s\t%s %s %s\t%s\t%s", + addr, + ucspi.TLSVersion(connState.Version), + tls.CipherSuiteName(connState.CipherSuite), + connState.PeerCertificates[0].SignatureAlgorithm, + spkiHash(connState.PeerCertificates[0]), + connState.NegotiatedProtocol, + ) + } + return conn, nil +} diff --git a/tlsauth.go b/tls/tlsauth.go similarity index 86% rename from tlsauth.go rename to tls/tlsauth.go index fd9839a..1d47670 100644 --- a/tlsauth.go +++ b/tls/tlsauth.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify @@ -75,6 +76,7 @@ grid .login bind . {switch -exact %%K { q {exit 0} ; # reject once + n {puts "0:NONE" ; exit} l login }} @@ -100,6 +102,9 @@ foreach sigScheme {%s} { } certs := make([]*x509.Certificate, 0, len(ents)) tlsCerts := make([]*tls.Certificate, 0, len(ents)) + b.WriteString(".lb insert end \"0: NONE\"\n") + certs = append(certs, nil) + tlsCerts = append(tlsCerts, nil) for i, ent := range ents { p := filepath.Join(CCerts, ent.Name()) _, cert, err := ucspi.CertificateFromFile(p) @@ -115,7 +120,7 @@ foreach sigScheme {%s} { Certificate: [][]byte{cert.Raw}, PrivateKey: prv, }) - b.WriteString(fmt.Sprintf(".lb insert end \"%d: %s\"\n", i, cert.Subject)) + b.WriteString(fmt.Sprintf(".lb insert end \"%d: %s\"\n", i+1, cert.Subject)) } // ioutil.WriteFile("/tmp/tls-auth-dialog.tcl", b.Bytes(), 0666) cmd := exec.Command(CmdWish) @@ -133,6 +138,13 @@ foreach sigScheme {%s} { if err != nil { return &tls.Certificate{}, nil } + if i == 0 { + dummy := tls.Certificate{} + caches.TLSAuthCacheM.Lock() + caches.TLSAuthCache[g.host] = &dummy + caches.TLSAuthCacheM.Unlock() + return &dummy, nil + } fifos.LogTLSAuth <- fmt.Sprintf("%s\t%s", g.host, certs[i].Subject) caches.TLSAuthCacheM.Lock() caches.TLSAuthCache[g.host] = tlsCerts[i] diff --git a/verify.go b/tls/verify.go similarity index 98% rename from verify.go rename to tls/verify.go index dc852a8..467dbb3 100644 --- a/verify.go +++ b/tls/verify.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/trip.go b/trip.go index b217750..6026229 100644 --- a/trip.go +++ b/trip.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify @@ -30,6 +31,7 @@ import ( "go.stargrave.org/tofuproxy/caches" "go.stargrave.org/tofuproxy/fifos" "go.stargrave.org/tofuproxy/rounds" + ttls "go.stargrave.org/tofuproxy/tls" ) var ( @@ -41,7 +43,7 @@ var ( MaxIdleConns: http.DefaultTransport.(*http.Transport).MaxIdleConns, IdleConnTimeout: http.DefaultTransport.(*http.Transport).IdleConnTimeout * 2, TLSHandshakeTimeout: time.Minute, - DialTLSContext: dialTLS, + DialTLSContext: ttls.DialTLS, ForceAttemptHTTP2: true, } proxyHeaders = map[string]struct{}{ @@ -64,6 +66,7 @@ func roundTrip(w http.ResponseWriter, req *http.Request) { host := strings.TrimSuffix(req.URL.Host, ":443") for _, round := range []Round{ rounds.RoundNoHead, + rounds.RoundGemini, rounds.RoundWARC, rounds.RoundDenySpy, rounds.RoundRedditOld, diff --git a/warc/compressed.go b/warc/compressed.go index b1de1ce..06fc53e 100644 --- a/warc/compressed.go +++ b/warc/compressed.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/warc/gzip.go b/warc/gzip.go index 335a26e..f68da5c 100644 --- a/warc/gzip.go +++ b/warc/gzip.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/warc/header.go b/warc/header.go index cb1b7a9..53a6d41 100644 --- a/warc/header.go +++ b/warc/header.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/warc/open.go b/warc/open.go index 22792d5..f802bbd 100644 --- a/warc/open.go +++ b/warc/open.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/warc/reader.go b/warc/reader.go index 06a123b..e734136 100644 --- a/warc/reader.go +++ b/warc/reader.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/warc/record.go b/warc/record.go index a2fa6bc..3a98435 100644 --- a/warc/record.go +++ b/warc/record.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/warc/uncompressed.go b/warc/uncompressed.go index 5dd60c5..5164e3d 100644 --- a/warc/uncompressed.go +++ b/warc/uncompressed.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/warc/uris.go b/warc/uris.go index 4d8c1a3..12bf70d 100644 --- a/warc/uris.go +++ b/warc/uris.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify diff --git a/x509.go b/x509.go index 2d27b1a..f4e4e7f 100644 --- a/x509.go +++ b/x509.go @@ -1,5 +1,6 @@ /* -tofuproxy -- flexible HTTP/WARC proxy with TLS certificates management +tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates + manager, WARC/Gemini browser Copyright (C) 2021 Sergey Matveev This program is free software: you can redistribute it and/or modify -- 2.44.0