contrib/pyshop2packages.sh | 4 ++-- doc/index.texi | 2 +- doc/proxy.texi | 7 ------- doc/storage.texi | 5 +---- doc/upload.texi | 3 +-- go.mod | 2 +- list.go | 11 +++-------- list.tmpl | 3 +-- main.go | 15 +++------------ refresh.go | 39 --------------------------------------- upload.go | 36 ------------------------------------ usage.go | 3 --- diff --git a/contrib/pyshop2packages.sh b/contrib/pyshop2packages.sh index 7a419470bb43ce7c8bd6233b7d03c80d7fb5b4ffcf82ef492dd6ed0ebfbaa635..223878750361d8065024a9ff96b5b4bd26b9913fb876a13509586f29114f7843 100755 --- a/contrib/pyshop2packages.sh +++ b/contrib/pyshop2packages.sh @@ -56,8 +56,8 @@ ######################################################################## cd packages for pkg in * ; do # Assume running: - # GOCHEESE_NO_SYNC=1 gocheese -bind "[::1]:8080" -gpgupdate /gpgupdate/ - curl http://localhost:8080/gpgupdate/$pkg/ > /dev/null + # GOCHEESE_NO_SYNC=1 gocheese -bind "[::1]:8080" -refresh /simple/ + curl http://localhost:8080/simple/$pkg/ > /dev/null done ######################################################################## diff --git a/doc/index.texi b/doc/index.texi index 4f9a0367dc4e0d743e57de0455732bf96ae95315ec1b0b60bd21f36c7e300d26..19f4e1d8c3fcff23925a2a7aba7f82c6df97f56e2f9db15e68fdd257ced0fc9a 100644 --- a/doc/index.texi +++ b/doc/index.texi @@ -53,7 +53,7 @@ @item Supports uploading of internal packages through the standard Warehouse API, including signatures, metadata and checksums. -@item Supports @url{https://pythonwheels.com/, wheels}, GPG signatures, +@item Supports @url{https://pythonwheels.com/, wheels}, @url{https://packaging.python.org/specifications/core-metadata/, Metadata} with @url{https://www.python.org/dev/peps/pep-0566/, PEP-0566} compatible conversion to JSON, multiple (MD5, SHA256, SHA512, BLAKE2b-256) integrity diff --git a/doc/proxy.texi b/doc/proxy.texi index c07ae24c04f291fde60a8c862cb57c4b7d8fc1a40662a1649454498c47c0705c..ba88e88fc94593eaa7b06f57d3bc2ea0c7b83aeb3ed5df416dfa49abe1ec702e 100644 --- a/doc/proxy.texi +++ b/doc/proxy.texi @@ -19,13 +19,6 @@ @item @code{/norefresh/} (@option{-norefresh} option) Same as above, but does not refresh data from the upstream, completely read only mode. -@item @code{/gpgupdate/} (@option{-gpgupdate} option) -Refresh the package state from the upstream as above, but additionally -check and download missing GPG signatures. Intended to be used only -manually, for example after database migration. -It is probably useful to set @env{$GOCHEESE_NO_SYNC=1} environment -variable to turn off filesystem synchronization calls. - @item @code{/pypi/} (@option{-json} option) Read only (non refreshing) JSON API entrypoint, giving metadata for the packages and releases. diff --git a/doc/storage.texi b/doc/storage.texi index a3e3fe0b5285a0f97f4d8baa004afd52a0adb0bc54a2e6ba7c54c0d6f03b9c96..d641bd9cbdf0e573d34d960a42194a189a79a677260702bcd419e2dcaa2cb24f 100644 --- a/doc/storage.texi +++ b/doc/storage.texi @@ -11,14 +11,12 @@ | +- public-package-0.1.tar.gz.md5 | +- public-package-0.1.tar.gz.blake2_256 | +- public-package-0.1.1.tar.gz.blake2_256 | +- public-package-0.2.tar.gz - | +- public-package-0.2.tar.gz.asc | +- public-package-0.2.tar.gz.sha256 | +- public-package-0.2.tar.gz.blake2_256 +-- private-package | +- .internal | +- .metadata.rec | +- private-package-0.1.tar.gz - | +- private-package-0.1.tar.gz.asc | +- private-package-0.1.tar.gz.sha256 | +- private-package-0.1.tar.gz.blake2_256 |... @@ -44,8 +42,7 @@ For example @file{public-package} has @code{0.1} version, downloaded a long time ago with MD5 checksum. @code{0.1.1} version is downloaded more recently with BLAKE2b-256 checksum, also storing that checksum for @code{0.1}. @code{0.2} version is downloaded tarball, having forced -SHA256 and BLAKE2b-256 recalculated checksums. Also upstream has -corresponding @file{.asc} signature file. +SHA256 and BLAKE2b-256 recalculated checksums. @file{private-package} is private package, because it contains @file{.internal} file. It can be uploaded and queries to it are not diff --git a/doc/upload.texi b/doc/upload.texi index e4ea6d7417763a320c57cc76724e2b08ce8b7d0dfc3af066adced53f1f85e25c..f0ae472992b3fea11b2908f59374c687cfdb6fc63ae502d2eec4b20e267eca34 100644 --- a/doc/upload.texi +++ b/doc/upload.texi @@ -20,5 +20,4 @@ @end example All metadata information sent by @command{twine} is stored on the disk. Package creation time will be server's current time. If @command{twine} -send package checksums, then they are checked against. GPG signature -file is also saved. +send package checksums, then they are checked against. diff --git a/go.mod b/go.mod index 498c93d5800a99924df94dcef2809fecd05e70d2e10582748a57bb8714bbc859..7f48118bd1565994b806b0533a6ed92ec40223f40605c66f7b0a70831662dddf 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.cypherpunks.ru/gocheese/v3 +module go.cypherpunks.ru/gocheese/v4 go 1.17 diff --git a/list.go b/list.go index 60b1b248e279581b99c1a1cbcfb62a3ba0a63f1e485dd21307ee4006b20490f5..150fe052d96cc63d464865f2ff5b69a42568398f76409238684b655f4a3e3121 100644 --- a/list.go +++ b/list.go @@ -96,7 +96,6 @@ // Version format is too complicated: https://www.python.org/dev/peps/pep-0386/ // So here is very simple parser working good enough for most packages func filenameToVersion(fn string) string { - fn = strings.TrimSuffix(fn, GPGSigExt) var trimmed string for _, ext := range KnownExts { trimmed = strings.TrimSuffix(fn, ext) @@ -176,10 +175,6 @@ release.Size = fi.Size() } delete(files, fnClean) } - if _, exists := files[fnClean+GPGSigExt]; exists { - release.HasSig = true - delete(files, fnClean+GPGSigExt) - } } release.Digests[algo] = hex.EncodeToString(digest) } @@ -204,15 +199,15 @@ func serveListDir( w http.ResponseWriter, r *http.Request, pkgName string, - autorefresh, gpgUpdate bool, + autorefresh bool, ) { dirPath := filepath.Join(Root, pkgName) if autorefresh { - if !refreshDir(w, r, pkgName, "", gpgUpdate) { + if !refreshDir(w, r, pkgName, "") { return } } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && - !refreshDir(w, r, pkgName, "", false) { + !refreshDir(w, r, pkgName, "") { return } serial, releases, err := listDir(pkgName, false) diff --git a/list.tmpl b/list.tmpl index e9e2168c7a63cf42641d9a6297d98573ce887fb669fd0ed0854b4da1da72b3ae..472c3145eafbecca892480196f9ad0472ce8ddfde16988efd717f247c105576a 100644 --- a/list.tmpl +++ b/list.tmpl @@ -6,8 +6,7 @@ Links for {{.PkgName}} {{$Refresh := .RefreshURLPath}}{{$PkgName := .PkgName}}{{range .Releases}} {{.Filename}}
+ #{{range $a, $d := .Digests}}{{$a}}={{$d}}{{end -}}">{{.Filename}}
{{- end}} diff --git a/main.go b/main.go index 7b6f7e0457ba2fbb123be686c391fd020be4939211d61b1b472689f05dee800a..3cb232b76d8b2fb732417d60c93079da2d2f0af1519994b374db664788532972 100644 --- a/main.go +++ b/main.go @@ -44,7 +44,7 @@ "golang.org/x/net/netutil" ) const ( - Version = "3.7.1" + Version = "4.0.0" UserAgent = "GoCheese/" + Version ) @@ -59,7 +59,6 @@ TLSKey = flag.String("tls-key", "", "") NoRefreshURLPath = flag.String("norefresh", DefaultNoRefreshURLPath, "") RefreshURLPath = flag.String("refresh", DefaultRefreshURLPath, "") - GPGUpdateURLPath = flag.String("gpgupdate", DefaultGPGUpdateURLPath, "") JSONURLPath = flag.String("json", DefaultJSONURLPath, "") PyPIURL = flag.String("pypi", DefaultPyPIURL, "") @@ -82,7 +81,7 @@ func servePkg(w http.ResponseWriter, r *http.Request, pkgName, filename string) { log.Println(r.RemoteAddr, "get", filename) path := filepath.Join(Root, pkgName, filename) if _, err := os.Stat(path); os.IsNotExist(err) { - if !refreshDir(w, r, pkgName, filename, false) { + if !refreshDir(w, r, pkgName, filename) { return } } @@ -95,16 +94,11 @@ switch r.Method { case "GET": var path string var autorefresh bool - var gpgUpdate bool if strings.HasPrefix(r.URL.Path, *NoRefreshURLPath) { path = strings.TrimPrefix(r.URL.Path, *NoRefreshURLPath) } else if strings.HasPrefix(r.URL.Path, *RefreshURLPath) { path = strings.TrimPrefix(r.URL.Path, *RefreshURLPath) autorefresh = true - } else if strings.HasPrefix(r.URL.Path, *GPGUpdateURLPath) { - path = strings.TrimPrefix(r.URL.Path, *GPGUpdateURLPath) - autorefresh = true - gpgUpdate = true } else { http.Error(w, "unknown action", http.StatusBadRequest) return @@ -118,7 +112,7 @@ if len(parts) == 1 { if parts[0] == "" { listRoot(w, r) } else { - serveListDir(w, r, parts[0], autorefresh, gpgUpdate) + serveListDir(w, r, parts[0], autorefresh) } } else { servePkg(w, r, parts[0], parts[1]) @@ -252,9 +246,6 @@ http.HandleFunc("/hr/", serveHRPkg) http.HandleFunc(*JSONURLPath, serveJSON) http.HandleFunc(*NoRefreshURLPath, handler) http.HandleFunc(*RefreshURLPath, handler) - if *GPGUpdateURLPath != "" { - http.HandleFunc(*GPGUpdateURLPath, handler) - } if *DoUCSPI { server.SetKeepAlivesEnabled(false) diff --git a/refresh.go b/refresh.go index b9349d1b8fcef79f4a4543854f38636a0f107c3ca03139bb8a58a6e4a4837ec3..c3d8375f903f7976962dfdc758078c9b10e419a3d10953552dd32296e8a8ac3f 100644 --- a/refresh.go +++ b/refresh.go @@ -46,7 +46,6 @@ HashAlgoSHA256 = "sha256" HashAlgoBLAKE2b256 = "blake2_256" HashAlgoSHA512 = "sha512" HashAlgoMD5 = "md5" - GPGSigExt = ".asc" InternalFlag = ".internal" ) @@ -88,7 +87,6 @@ func refreshDir( w http.ResponseWriter, r *http.Request, pkgName, filenameGet string, - gpgUpdate bool, ) bool { if _, err := os.Stat(filepath.Join(Root, pkgName, InternalFlag)); err == nil { return true @@ -521,43 +519,6 @@ } } } - if filename == filenameGet || gpgUpdate { - resp, err := c.Do(agentedReq(uri + GPGSigExt)) - if err != nil { - goto GPGSigSkip - } - if resp.StatusCode != http.StatusOK { - resp.Body.Close() - goto GPGSigSkip - } - sig, err := io.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - goto GPGSigSkip - } - if !bytes.HasPrefix(sig, []byte("-----BEGIN PGP SIGNATURE-----")) { - log.Println(r.RemoteAddr, "pypi", filename+GPGSigExt, "non PGP") - goto GPGSigSkip - } - if err = WriteFileSync(dirPath, path+GPGSigExt, sig, mtime); err != nil { - log.Println("error", r.RemoteAddr, "pypi", filename+GPGSigExt, err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - log.Println(r.RemoteAddr, "pypi", filename+GPGSigExt, "downloaded") - } - if mtimeExists { - stat, err := os.Stat(path + GPGSigExt) - if err == nil && !stat.ModTime().Truncate(time.Second).Equal(mtime) { - log.Println(r.RemoteAddr, "pypi", filename+GPGSigExt, "touch") - if err = os.Chtimes(path+GPGSigExt, mtime, mtime); err != nil { - log.Println("error", r.RemoteAddr, "pypi", filename, err) - http.Error(w, err.Error(), http.StatusInternalServerError) - } - } - } - - GPGSigSkip: if digest == nil { continue } diff --git a/upload.go b/upload.go index 2b9cdb155b1d03a8da7b13cb8e83fb1aea5f8110fc729db8f2ceebef736506e4..16e4803964b371a9697c319ac79202073514a14977ece061b7b07cf6b7d5b72e 100644 --- a/upload.go +++ b/upload.go @@ -67,7 +67,6 @@ return } pkgName := strings.ToLower(NormalizationRe.ReplaceAllString(pkgNames[0], "-")) dirPath := filepath.Join(Root, pkgName) - gpgSigsExpected := make(map[string]struct{}) now := time.Now().UTC() var digestSHA256Expected []byte @@ -96,7 +95,6 @@ } for _, file := range r.MultipartForm.File["content"] { filename := file.Filename - gpgSigsExpected[filename+GPGSigExt] = struct{}{} log.Println(r.RemoteAddr, "put", filename, "by", username) path := filepath.Join(dirPath, filename) if _, err = os.Stat(path); err == nil { @@ -189,40 +187,6 @@ return } if err = WriteFileSync(dirPath, path+"."+HashAlgoBLAKE2b256, digestBLAKE2b256, now); err != nil { log.Println("error", r.RemoteAddr, path+"."+HashAlgoBLAKE2b256, err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } - for _, file := range r.MultipartForm.File["gpg_signature"] { - filename := file.Filename - if _, exists := gpgSigsExpected[filename]; !exists { - log.Println(r.RemoteAddr, filename, "unexpected GPG signature filename") - http.Error(w, "unexpected GPG signature filename", http.StatusBadRequest) - return - } - delete(gpgSigsExpected, filename) - log.Println(r.RemoteAddr, "put", filename, "by", username) - path := filepath.Join(dirPath, filename) - if _, err = os.Stat(path); err == nil { - log.Println(r.RemoteAddr, filename, "already exists") - http.Error(w, "already exists", http.StatusBadRequest) - return - } - src, err := file.Open() - if err != nil { - log.Println("error", r.RemoteAddr, filename, err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - sig, err := io.ReadAll(src) - src.Close() - if err != nil { - log.Println("error", r.RemoteAddr, filename, err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - if err = WriteFileSync(dirPath, path, sig, now); err != nil { - log.Println("error", r.RemoteAddr, path, err) http.Error(w, err.Error(), http.StatusInternalServerError) return } diff --git a/usage.go b/usage.go index 92b696311c89af1d95c6abd8d32bb9695c49ce60322faed6704a21105bc54c5d..6f7e742f97cea9bf38c875238ec9b7d3790681c45c7c579ff7132ba752e370ab 100644 --- a/usage.go +++ b/usage.go @@ -27,7 +27,6 @@ DefaultBind = "[::]:8080" DefaultMaxClients = 128 DefaultNoRefreshURLPath = "/norefresh/" DefaultRefreshURLPath = "/simple/" - DefaultGPGUpdateURLPath = "/gpgupdate/" DefaultJSONURLPath = "/pypi/" DefaultPyPIURL = "https://pypi.org/simple/" DefaultJSONURL = "https://pypi.org/pypi/" @@ -60,7 +59,6 @@ HTTP endpoints: -norefresh URLPATH -- Non-refreshing Simple API path (default: %s) -refresh URLPATH -- Auto-refreshing Simple API path (default: %s) - -gpgupdate URLPATH -- GPG forceful refreshing path (default: %s) -json URLPATH -- JSON API path (default: %s) Upstream PyPI: @@ -86,7 +84,6 @@ DefaultBind, DefaultMaxClients, DefaultNoRefreshURLPath, DefaultRefreshURLPath, - DefaultGPGUpdateURLPath, DefaultJSONURLPath, DefaultPyPIURL, DefaultJSONURLPath,