doc/Storage | 10 ++++++++++ fileutils.go | 43 ++++++++++++++++++++++++++++++++++++++++--- go.mod | 6 +++--- go.sum | 12 ++++++------ hr-pkg.tmpl | 3 ++- hr-root.tmpl | 2 +- hr.go | 33 ++++++++++++++++----------------- integrity.go | 25 +++++++++++++++++++++++++ json.go | 8 ++++---- list.go | 70 +++++++++++++++++++++++++++++++++++++++++++---------- main.go | 81 +++++++++++++++++++++++++++++++++++++++++++++++++---- refresh.go | 57 +++++++++++++++++++---------------------------------- root.tmpl | 2 +- upload.go | 4 ++-- usage.go | 6 ++++++ diff --git a/doc/Storage b/doc/Storage index 76d21a944375f8cc34fe702e5995409ce664339fb10c4c3029fdaf8c26e2bfca..64b11fbb2a55d90a7c5af54b748db7e4153100687c64873331f4ee5d7a6c034b 100644 --- a/doc/Storage +++ b/doc/Storage @@ -50,3 +50,13 @@ Each packages release file has mtime set to its upload time. Package's serial is a sum of mtime's of the directory and .metadata.rec (if it exists). Optional .reqpy file contains the PEP-345's Requires-Python string. + + Hashed names + ============ + +-hashed-names option enables hashed directories names usage. +"private-package" becomes: + 5b/bb/084be0364debf741a81516cbf97de062c4c95acabbcaa098da49e7d72f3e +It is BLAKE2b-256 hash of the name, with two subdirectories levels. +Additional "name" file is placed in each package directory with original +name. diff --git a/fileutils.go b/fileutils.go index 6cad06855a3618233d0a58bf4d4da45dbb9181da1ab7babe9a1f6664b487410f..c0d82acb952d1597cbea79b9a6cf707a51b3ec9a578db316dab61e28d5c31f4b 100644 --- a/fileutils.go +++ b/fileutils.go @@ -16,11 +16,14 @@ package main import ( + "encoding/hex" "log" "net/http" "os" "path/filepath" "time" + + "golang.org/x/crypto/blake2b" ) var ( @@ -84,13 +87,47 @@ } return DirSync(dirPath) } -func mkdirForPkg(w http.ResponseWriter, r *http.Request, pkgName string) bool { - path := filepath.Join(Root, pkgName) +type PkgName struct { + n string + fs string +} + +func (p *PkgName) String() string { + return p.n +} + +func hashedName(n string) string { + h := blake2b.Sum256([]byte(n)) + s := hex.EncodeToString(h[:]) + return s[:2] + "/" + s[2:4] + "/" + s[4:] +} + +func (p *PkgName) FS() string { + if p.fs == "" { + if !*HashedNames { + return p.n + } + p.fs = hashedName(p.n) + } + return p.fs +} + +func mkdirForPkg(w http.ResponseWriter, r *http.Request, pkgName PkgName) bool { + path := filepath.Join(Root, pkgName.FS()) if _, err := os.Stat(path); os.IsNotExist(err) { - if err = os.Mkdir(path, os.FileMode(0o777)); err != nil { + if err = os.MkdirAll(path, os.FileMode(0o777)); err != nil { log.Println("error", r.RemoteAddr, "mkdir", pkgName, err) http.Error(w, err.Error(), http.StatusInternalServerError) return false + } + if err = os.WriteFile( + filepath.Join(path, "name"), + append([]byte(pkgName.n), '\n'), + os.FileMode(0o666), + ); err != nil { + os.RemoveAll(path) + log.Println("error", r.RemoteAddr, "save name", pkgName, err) + http.Error(w, err.Error(), http.StatusInternalServerError) } log.Println(r.RemoteAddr, "mkdir", pkgName) } diff --git a/go.mod b/go.mod index c3ebb1530cd847835dad68ac3934da06a0a7a90b6deb1538c3b632e3df9a7654..c8e324447de846ed09a2d85b3637aa5584380da1c60eb3b1028da6b84d4224d8 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.26.0 require ( go.stargrave.org/recfile/v4 v4.0.0 - golang.org/x/crypto v0.57.0 - golang.org/x/net v0.60.0 + golang.org/x/crypto v0.58.0 + golang.org/x/net v0.61.0 ) -require golang.org/x/sys v0.48.0 // indirect +require golang.org/x/sys v0.49.0 // indirect diff --git a/go.sum b/go.sum index 3332189fdcc4b5e0c6bd3d471f6ef6584c1904ce9ad83a7187dd98994ef48a1a..acffcf3a4ef5b5bf6b9d25b88f37de1c04b940e1fb9156f42284da7274222d4e 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ go.stargrave.org/recfile/v4 v4.0.0 h1:v9kZnW/e64oPfOqArGMNZacuT57lDibd24xhEiBDQ/c= go.stargrave.org/recfile/v4 v4.0.0/go.mod h1:IGi8kMociIlDoR8iOssyUG/uix+wmsqvBuiDeQXsIuE= -golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= -golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= -golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= -golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= -golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= -golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= +golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= +golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw= +golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= diff --git a/hr-pkg.tmpl b/hr-pkg.tmpl index 389f61859f24aeb5ed679c22badc5f1d39245274512ec1d1f5b5af7e1f7105ed..de37ec4cc8b8ce3d3934207d10e21182c6f47fff9a7796ab8fe6ac7b85a79cb2 100644 --- a/hr-pkg.tmpl +++ b/hr-pkg.tmpl @@ -65,7 +65,7 @@ Digests {{range .Releases}}{{if .Size}} - {{.Filename}} + {{.Filename}} {{.Version}} {{.ReqPy}} {{.UploadTimeISO8601}} @@ -75,5 +75,6 @@
  • {{$a}}: {{$d}}
    • {{end}} {{end}}{{end}} + hashed name: {{.HashedName}} diff --git a/hr-root.tmpl b/hr-root.tmpl index bea8032611c6a7987a03aadf5c72a3d23fb45f5e7f2c4bdd55900347645686ae..60cd624175147d22306c203fb05b2c429c9eb85f56bc4bd0f0b67bd13498c5a9 100644 --- a/hr-root.tmpl +++ b/hr-root.tmpl @@ -5,7 +5,7 @@ {{.Version}}: packages diff --git a/hr.go b/hr.go index 02cf986f2400754d32b16f0cc618699093781b599c3041cacc59019d8fafbc42..70d36ea606820b38823589b40141e94e9f6e96cb44b48cc502fa6005824f1f2a 100644 --- a/hr.go +++ b/hr.go @@ -37,24 +37,21 @@ HRPkgTmpl = template.Must(template.New("hr-pkg").Parse(HRPkgTmplRaw)) ) func serveHRRoot(w http.ResponseWriter, r *http.Request) { - files, err := os.ReadDir(Root) + var pkgs []PkgName + err := listPkgs(&pkgs, Root) if err != nil { log.Println("error", r.RemoteAddr, "hr-root", err) http.Error(w, err.Error(), http.StatusInternalServerError) return } - packages := make([]string, 0, len(files)) - for _, f := range files { - packages = append(packages, f.Name()) - } - sort.Strings(packages) + sort.Sort(PkgNameByName(pkgs)) var buf bytes.Buffer err = HRRootTmpl.Execute(&buf, struct { Version string - Packages []string + Packages []PkgName }{ Version: UserAgent, - Packages: packages, + Packages: pkgs, }) if err != nil { log.Println("error", r.RemoteAddr, "hr-root", err) @@ -66,7 +63,7 @@ } func serveHRPkg(w http.ResponseWriter, r *http.Request) { cols := strings.Split(strings.TrimRight(r.URL.Path, "/"), "/") - pkgName := cols[len(cols)-1] + pkgName := PkgName{n: cols[len(cols)-1]} meta, releases, err := getMD(pkgName, "") if err != nil { if os.IsNotExist(err) { @@ -79,15 +76,17 @@ return } var buf bytes.Buffer err = HRPkgTmpl.Execute(&buf, struct { - Version string - PkgName string - Info PkgInfo - Releases []*PkgReleaseInfo + Version string + PkgName string + Info PkgInfo + Releases []*PkgReleaseInfo + HashedName string }{ - Version: UserAgent, - PkgName: pkgName, - Info: meta.Info, - Releases: releases, + Version: UserAgent, + PkgName: pkgName.String(), + Info: meta.Info, + Releases: releases, + HashedName: hashedName(pkgName.n), }) if err != nil { log.Println("error", r.RemoteAddr, "root", err) diff --git a/integrity.go b/integrity.go index 121c0e271e3dd848d59d24a5407840164a8108ad6215cae7247042bed916cad3..6b2d5634cc699d2df0b837f3dbfbe02a6f695d52693239249d95dd5f9c109eed 100644 --- a/integrity.go +++ b/integrity.go @@ -21,6 +21,7 @@ "bytes" "crypto/sha256" "errors" "fmt" + "hash" "io" "io/fs" "log" @@ -29,7 +30,31 @@ "path/filepath" "runtime" "strings" "sync" + + "golang.org/x/crypto/blake2b" ) + +const ( + HashAlgoBLAKE2b256 = "blake2b_256" + HashAlgoSHA256 = "sha256" + HashAlgoSHA512 = "sha512" + HashAlgoMD5 = "md5" +) + +var KnownHashAlgos []string = []string{ + HashAlgoBLAKE2b256, + HashAlgoSHA256, + HashAlgoSHA512, + HashAlgoMD5, +} + +func blake2b256New() hash.Hash { + h, err := blake2b.New256(nil) + if err != nil { + panic(err) + } + return h +} func checker(jobs chan string, isGood *bool, workers *sync.WaitGroup) { hasherBLAKE2b256 := blake2b256New() diff --git a/json.go b/json.go index 38c85b76c0f676bc3d9b41939d710717a5dd4e7a330a2f52036f98f2c0abd1e1..81f285d5166aa216ae4173ffdd283a938fd27a83fa775e27a8915c8a52e0b9b5 100644 --- a/json.go +++ b/json.go @@ -27,18 +27,18 @@ "go.stargrave.org/recfile/v4" ) -func getMD(pkgName, version string) (*PkgMeta, []*PkgReleaseInfo, error) { +func getMD(pkgName PkgName, version string) (*PkgMeta, []*PkgReleaseInfo, error) { serial, releases, err := listDir(pkgName, true) if err != nil { return nil, nil, err } - md, err := os.ReadFile(filepath.Join(Root, pkgName, MDFile)) + md, err := os.ReadFile(filepath.Join(Root, pkgName.FS(), MDFile)) if err != nil { if !os.IsNotExist(err) { return nil, nil, err } } - info := PkgInfo{Name: pkgName} + info := PkgInfo{Name: pkgName.String()} if len(md) == 0 { info.Version = releases[len(releases)-1].Version } else { @@ -121,7 +121,7 @@ var version string if len(parts) == 3 { version = parts[1] } - pkgName := parts[0] + pkgName := PkgName{n: parts[0]} meta, _, err := getMD(pkgName, version) if err != nil { diff --git a/list.go b/list.go index d55c27561bdad1162427af4312659fae7b79028d9e46faa8a02026dc3d4d1032..a716fd5132fce2a0cc96394acc1cb9585aa1bf90a599269ebd06f38776050106 100644 --- a/list.go +++ b/list.go @@ -22,6 +22,7 @@ "encoding/hex" "errors" "fmt" "html/template" + "io" "io/fs" "log" "net/http" @@ -48,25 +49,68 @@ ".exe", ".dmg", ".msi", ".rpm", ".deb", ".tgz", } ) +func listPkgs(pkgs *[]PkgName, root string) error { + fh, err := os.Open(root) + if err != nil { + return err + } + defer fh.Close() + for { + entries, err := fh.ReadDir(1 << 10) + if err != nil { + if err == io.EOF { + break + } + return err + } + for _, entry := range entries { + if entry.IsDir() { + listPkgs(pkgs, root+"/"+entry.Name()) + } else if entry.Name() == "name" { + data, err := os.ReadFile(root + "/" + entry.Name()) + if err != nil { + return err + } + p := PkgName{n: strings.TrimSuffix(string(data), "\n")} + if strings.HasSuffix(root, p.FS()) { + *pkgs = append(*pkgs, p) + } + } + } + } + return nil +} + +type PkgNameByName []PkgName + +func (a PkgNameByName) Len() int { + return len(a) +} + +func (a PkgNameByName) Swap(i, j int) { + a[i], a[j] = a[j], a[i] +} + +func (a PkgNameByName) Less(i, j int) bool { + return a[i].n < a[j].n +} + func listRoot(w http.ResponseWriter, r *http.Request) { - files, err := os.ReadDir(Root) + var pkgs []PkgName + err := listPkgs(&pkgs, Root) if err != nil { log.Println("error", r.RemoteAddr, "root", err) http.Error(w, err.Error(), http.StatusInternalServerError) return } - packages := make([]string, 0, len(files)) - for _, f := range files { - packages = append(packages, f.Name()) - } - sort.Strings(packages) + sort.Sort(PkgNameByName(pkgs)) var buf bytes.Buffer err = HTMLRootTmpl.Execute(&buf, struct { RefreshURLPath string - Packages []string + Packages []PkgName }{ RefreshURLPath: *RefreshURLPath, - Packages: packages, + Packages: pkgs, }) if err != nil { log.Println("error", r.RemoteAddr, "root", err) @@ -119,8 +163,8 @@ } return cols[0] } -func listDir(pkgName string, doSize bool) (int64, []*PkgReleaseInfo, error) { - dirPath := filepath.Join(Root, pkgName) +func listDir(pkgName PkgName, doSize bool) (int64, []*PkgReleaseInfo, error) { + dirPath := filepath.Join(Root, pkgName.FS()) entries, err := os.ReadDir(dirPath) if err != nil { return 0, nil, err @@ -216,10 +260,10 @@ func serveListDir( w http.ResponseWriter, r *http.Request, - pkgName string, + pkgName PkgName, autorefresh bool, ) { - dirPath := filepath.Join(Root, pkgName) + dirPath := filepath.Join(Root, pkgName.FS()) if autorefresh { if !refreshDir(w, r, pkgName, "") { return @@ -254,7 +298,7 @@ PkgName string Releases []*PkgReleaseInfo }{ RefreshURLPath: *RefreshURLPath, - PkgName: pkgName, + PkgName: pkgName.String(), Releases: releases, }) if err != nil { diff --git a/main.go b/main.go index edf9f9c47ea497cd771b6acc5661b12a86a1381e3385c959e38985c572e94114..0c34dd7f24506d2224824a91e1add5ae853135ca67a7355714012be73d6f02b9 100644 --- a/main.go +++ b/main.go @@ -26,6 +26,7 @@ "encoding/hex" "errors" "flag" "fmt" + "io" "log" "net" "net/http" @@ -34,6 +35,7 @@ "os" "os/signal" "path/filepath" "runtime" + "sort" "strings" "syscall" "time" @@ -42,7 +44,7 @@ "golang.org/x/net/netutil" ) const ( - Version = "6.0.0" + Version = "6.1.0" UserAgent = "GoCheese/" + Version ) @@ -55,6 +57,7 @@ TLSCert = flag.String("tls-cert", "", "") TLSKey = flag.String("tls-key", "", "") + HashedNames = flag.Bool("hashed-names", false, "Use hashed package names") NoRefreshURLPath = flag.String("norefresh", DefaultNoRefreshURLPath, "") RefreshURLPath = flag.String("refresh", DefaultRefreshURLPath, "") JSONURLPath = flag.String("json", DefaultJSONURLPath, "") @@ -76,9 +79,9 @@ Killed bool ) -func servePkg(w http.ResponseWriter, r *http.Request, pkgName, filename string) { +func servePkg(w http.ResponseWriter, r *http.Request, pkgName PkgName, filename string) { log.Println(r.RemoteAddr, "get", filename) - path := filepath.Join(Root, pkgName, filename) + path := filepath.Join(Root, pkgName.FS(), filename) if _, err := os.Stat(path); os.IsNotExist(err) { if !refreshDir(w, r, pkgName, filename) { return @@ -113,10 +116,10 @@ if len(parts) == 1 { if parts[0] == "" { listRoot(w, r) } else { - serveListDir(w, r, parts[0], autorefresh) + serveListDir(w, r, PkgName{n: parts[0]}, autorefresh) } } else { - servePkg(w, r, parts[0], parts[1]) + servePkg(w, r, PkgName{n: parts[0]}, parts[1]) } case "POST": serveUpload(w, r) @@ -126,6 +129,11 @@ } } func main() { + hashedName := flag.String("hashed-name", "", "Print hashed package name") + flatToHashed := flag.String("flat-to-hashed", "", + "Convert current flat structure directory to hashed names to specified destination") + listHashed := flag.Bool("list-hashed", false, + "List currently hashed directory names") flag.Usage = usage flag.Parse() if *DoWarranty { @@ -134,6 +142,69 @@ return } if *DoVersion { fmt.Println("GoCheese", Version, "built with", runtime.Version()) + return + } + + if *hashedName != "" { + *HashedNames = true + p := PkgName{n: *hashedName} + fmt.Println(p.FS()) + return + } + if *flatToHashed != "" { + *HashedNames = true + fh, err := os.Open(".") + if err != nil { + log.Fatal(err) + } + var entries []os.DirEntry + for { + e, err := fh.ReadDir(1 << 10) + if err != nil { + if err == io.EOF { + break + } + log.Fatal(err) + } + entries = append(entries, e...) + } + fh.Close() + for _, entry := range entries { + p := PkgName{n: entry.Name()} + if p.n == *flatToHashed { + continue + } + if err = os.WriteFile( + p.n+"/name", + append([]byte(p.n), '\n'), + os.FileMode(0o644), + ); err != nil { + log.Fatal(err) + } + dst := p.FS() + if err = os.MkdirAll( + *flatToHashed+"/"+dst[:2+1+2], + os.FileMode(0o777), + ); err != nil { + log.Fatal(err) + } + if err = os.Rename(p.n, *flatToHashed+"/"+dst); err != nil { + log.Fatal(err) + } + } + return + } + if *listHashed { + *HashedNames = true + var pkgs []PkgName + err := listPkgs(&pkgs, ".") + if err != nil { + log.Fatal(err) + } + sort.Sort(PkgNameByName(pkgs)) + for _, p := range pkgs { + fmt.Println(p.n, p.FS()) + } return } diff --git a/refresh.go b/refresh.go index 3ae8e5e221b2ac207b4079ea721e7c1fb3e7ffc611da2b3329dc36132a24e163..9530195e358dbd0202f44bbad76547dd97349e4c82342509c947c58c129095ca 100644 --- a/refresh.go +++ b/refresh.go @@ -42,12 +42,8 @@ "golang.org/x/crypto/blake2b" ) const ( - HashAlgoBLAKE2b256 = "blake2b_256" - HashAlgoSHA256 = "sha256" - HashAlgoSHA512 = "sha512" - HashAlgoMD5 = "md5" - InternalFlag = ".internal" - ReqPyExt = ".reqpy" + InternalFlag = ".internal" + ReqPyExt = ".reqpy" ) var ( @@ -55,22 +51,8 @@ PkgPyPI = regexp.MustCompile(`^.*]*>(.+).*$`) DataRequiresPy = regexp.MustCompile(`data-requires-python="([^"]+)"`) PyPIURLParsed *url.URL PyPIHTTPTransport http.Transport - KnownHashAlgos []string = []string{ - HashAlgoBLAKE2b256, - HashAlgoSHA256, - HashAlgoSHA512, - HashAlgoMD5, - } ) -func blake2b256New() hash.Hash { - h, err := blake2b.New256(nil) - if err != nil { - panic(err) - } - return h -} - func agentedReq(url string) *http.Request { req, err := http.NewRequest("GET", url, nil) if err != nil { @@ -88,27 +70,28 @@ func refreshDir( w http.ResponseWriter, r *http.Request, - pkgName, filenameGet string, + pkgName PkgName, + filenameGet string, ) bool { - if _, err := os.Stat(filepath.Join(Root, pkgName, InternalFlag)); err == nil { + if _, err := os.Stat(filepath.Join(Root, pkgName.FS(), InternalFlag)); err == nil { return true } c := http.Client{Transport: &PyPIHTTPTransport} - dirPath := filepath.Join(Root, pkgName) + dirPath := filepath.Join(Root, pkgName.FS()) now := time.Now() var allReleases map[string][]*PkgReleaseInfo if *JSONURL != "" { - resp, err := c.Do(agentedReq(*JSONURL + pkgName + "/json")) + resp, err := c.Do(agentedReq(*JSONURL + pkgName.String() + "/json")) if err != nil { - log.Println("error", r.RemoteAddr, "refresh-json", pkgName, err) + log.Println("error", r.RemoteAddr, "refresh-json", pkgName.String(), err) http.Error(w, err.Error(), http.StatusBadGateway) return false } if resp.StatusCode != http.StatusOK { resp.Body.Close() log.Println( - "error", r.RemoteAddr, "refresh-json", pkgName, + "error", r.RemoteAddr, "refresh-json", pkgName.String(), "HTTP status:", resp.Status, ) http.Error(w, "PyPI has non 200 status code", http.StatusBadGateway) @@ -117,7 +100,7 @@ } body, err := io.ReadAll(resp.Body) if err != nil { resp.Body.Close() - log.Println("error", r.RemoteAddr, "refresh-json", pkgName, err) + log.Println("error", r.RemoteAddr, "refresh-json", pkgName.String(), err) http.Error(w, "can not read body", http.StatusBadGateway) return false } @@ -178,7 +161,7 @@ var metaStripped PkgMetaStripped err = json.Unmarshal(body, &metaStripped) if err != nil { log.Println( - "error", r.RemoteAddr, "refresh-json", pkgName, + "error", r.RemoteAddr, "refresh-json", pkgName.String(), "can not parse JSON:", err, ) http.Error(w, "can not parse metadata JSON", http.StatusBadGateway) @@ -246,7 +229,7 @@ log.Println("error", r.RemoteAddr, "refresh-json", path, err) http.Error(w, err.Error(), http.StatusInternalServerError) return false } - log.Println(r.RemoteAddr, "pypi", pkgName+"/"+MDFile, "touch") + log.Println(r.RemoteAddr, "pypi", pkgName.String()+"/"+MDFile, "touch") } } mtimes := make(map[string]time.Time) @@ -261,7 +244,7 @@ } t, err := time.Parse(time.RFC3339Nano, rel.UploadTimeISO8601) if err != nil { log.Println( - "error", r.RemoteAddr, "refresh-json", pkgName, + "error", r.RemoteAddr, "refresh-json", pkgName.String(), "can not parse upload_time:", err, ) http.Error(w, "can not parse metadata JSON", http.StatusBadGateway) @@ -272,7 +255,7 @@ if d := rel.Digests[HashAlgoBLAKE2b256]; d != "" { digestsBLAKE2b256[rel.Filename], err = hex.DecodeString(d) if err != nil { log.Println( - "error", r.RemoteAddr, "refresh-json", pkgName, + "error", r.RemoteAddr, "refresh-json", pkgName.String(), "can not decode blake2b_256 digest:", err, ) http.Error(w, "can not parse metadata JSON", http.StatusBadGateway) @@ -283,7 +266,7 @@ if d := rel.Digests[HashAlgoSHA256]; d != "" { digestsSHA256[rel.Filename], err = hex.DecodeString(d) if err != nil { log.Println( - "error", r.RemoteAddr, "refresh-json", pkgName, + "error", r.RemoteAddr, "refresh-json", pkgName.String(), "can not decode sha256 digest:", err, ) http.Error(w, "can not parse metadata JSON", http.StatusBadGateway) @@ -294,7 +277,7 @@ if d := rel.Digests[HashAlgoSHA512]; d != "" { digestsSHA512[rel.Filename], err = hex.DecodeString(d) if err != nil { log.Println( - "error", r.RemoteAddr, "refresh-json", pkgName, + "error", r.RemoteAddr, "refresh-json", pkgName.String(), "can not decode sha512 digest:", err, ) http.Error(w, "can not parse metadata JSON", http.StatusBadGateway) @@ -304,16 +287,16 @@ } } } - resp, err := c.Do(agentedReq(*PyPIURL + pkgName + "/")) + resp, err := c.Do(agentedReq(*PyPIURL + pkgName.String() + "/")) if err != nil { - log.Println("error", r.RemoteAddr, "refresh", pkgName, err) + log.Println("error", r.RemoteAddr, "refresh", pkgName.String(), err) http.Error(w, err.Error(), http.StatusBadGateway) return false } if resp.StatusCode != http.StatusOK { resp.Body.Close() log.Println( - "error", r.RemoteAddr, "refresh", pkgName, + "error", r.RemoteAddr, "refresh", pkgName.String(), "HTTP status:", resp.Status, ) http.Error(w, "PyPI has non 200 status code", http.StatusBadGateway) @@ -322,7 +305,7 @@ } body, err := io.ReadAll(resp.Body) resp.Body.Close() if err != nil { - log.Println("error", r.RemoteAddr, "refresh", pkgName, err) + log.Println("error", r.RemoteAddr, "refresh", pkgName.String(), err) http.Error(w, err.Error(), http.StatusBadGateway) return false } diff --git a/root.tmpl b/root.tmpl index 374ae9b2f22648bde91beb4b96ba185769ac759bedf45dfab0ba36ce22142bbd..7ec334184b6e12feb2462231ad16e4047e291ad2366619c0439f7fa4031577e2 100644 --- a/root.tmpl +++ b/root.tmpl @@ -5,7 +5,7 @@ Simple index {{$Refresh := .RefreshURLPath}}{{range .Packages}} - {{.}}
    + {{.String}}
    {{- end}} diff --git a/upload.go b/upload.go index a0e15ba05da306a01ab74c200103a61b20702761fc9c2f6b69862e9086c287ad..1529b0020e8f761165d936ac88cd0466f370fd273ca7c9ba6d3c70f25b98fd15 100644 --- a/upload.go +++ b/upload.go @@ -60,8 +60,8 @@ if !exists || len(pkgNames) != 1 { http.Error(w, "single name is expected in request", http.StatusBadRequest) return } - pkgName := strings.ToLower(NormalizationRe.ReplaceAllString(pkgNames[0], "-")) - dirPath := filepath.Join(Root, pkgName) + pkgName := PkgName{n: strings.ToLower(NormalizationRe.ReplaceAllString(pkgNames[0], "-"))} + dirPath := filepath.Join(Root, pkgName.FS()) now := time.Now().UTC() var digestSHA256Expected []byte diff --git a/usage.go b/usage.go index 88170610faf511316a189d88f87568617201d10c94ab15eda00ef88802726dc1..79151bf7972b5eb5fad9765e75400836c629cd6c8ce34e80f9d5b9fa6808f6f4 100644 --- a/usage.go +++ b/usage.go @@ -71,6 +71,12 @@ -passwd-list PATH -- Path to writeable FIFO for listing logins -passwd-check -- Verify passwords format from stdin, then exit -auth-required -- Require authorisation even for read-only endpoints +Hashed directories names: + -hashed-names -- Enabled hashed directories names mode + -hashed-name PKG -- Print hashed directory name of PKG + -flat-to-hashed -- Convert current flat structure directory to hashed + names to specified destination + Other options: -log-timestamped -- Prepend timestamp to log messages -fsck -- Run integrity check of all packages