.gitignore | 1 + depfix.go | 214 +++++++++++++++++++++++++++++++++++++++++++++++++++++ doc/cmds.texi | 11 +++++++++++ doc/news.texi | 6 ++++++ doc/ood.texi | 6 ++++++ main.go | 4 ++++ ood.go | 4 +++- usage.go | 9 +++++++-- diff --git a/.gitignore b/.gitignore index a9ee8698bb3178b456759ab7f00597cd05e053dd4950d1b3cb905236d7c87353..9ecfe9f568477535e245c457214de5f967d62ef22636ecbbf52b17043b108277 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ /redo /redo-affects /redo-always /redo-cleanup +/redo-depfix /redo-dot /redo-ifchange /redo-ifcreate diff --git a/depfix.go b/depfix.go new file mode 100644 index 0000000000000000000000000000000000000000..c5bb1b3ae2d7d61782a4f58739c5e8cec5e294ce2a1df6735ed36bdcb19f80f4 --- /dev/null +++ b/depfix.go @@ -0,0 +1,214 @@ +/* +goredo -- djb's redo implementation on pure Go +Copyright (C) 2020-2022 Sergey Matveev + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package main + +import ( + "errors" + "io" + "os" + "path" + "strings" + + "go.cypherpunks.ru/recfile" +) + +func depFix(root string) error { + tracef(CDebug, "depfix: entering %s", root) + dir, err := os.Open(root) + if err != nil { + return err + } + defer dir.Close() + for { + fis, err := dir.Readdir(1 << 10) + if err != nil { + if err == io.EOF { + break + } + return err + } + for _, fi := range fis { + if fi.IsDir() { + if err = depFix(path.Join(root, fi.Name())); err != nil { + return err + } + } + } + } + dir.Close() + + redoDir := path.Join(root, RedoDir) + dir, err = os.Open(redoDir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + defer dir.Close() + redoDirChanged := false + for { + fis, err := dir.Readdir(1 << 10) + if err != nil { + if err == io.EOF { + break + } + return err + } + for _, fi := range fis { + if !strings.HasSuffix(fi.Name(), DepSuffix) { + continue + } + tracef(CDebug, "depfix: checking %s/%s", root, fi.Name()) + fdDepPath := path.Join(redoDir, fi.Name()) + fdDep, err := os.Open(fdDepPath) + if err != nil { + return err + } + defer fdDep.Close() + r := recfile.NewReader(fdDep) + var fieldses [][]recfile.Field + depChanged := false + for { + fields, err := r.Next() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return err + } + fieldses = append(fieldses, fields) + m := make(map[string]string, len(fields)) + for _, f := range fields { + m[f.Name] = f.Value + } + if m["Type"] != DepTypeIfchange { + continue + } + dep := m["Target"] + if dep == "" { + return ErrMissingTarget + } + tracef(CDebug, "depfix: checking %s/%s -> %s", root, fi.Name(), dep) + theirInode, err := inodeFromRec(m) + if err != nil { + return err + } + theirHsh := m["Hash"] + fd, err := os.Open(path.Join(root, dep)) + if err != nil { + if os.IsNotExist(err) { + tracef( + CDebug, "depfix: %s/%s -> %s: not exists", + root, fi.Name(), dep, + ) + continue + } + return err + } + inode, err := inodeFromFile(fd) + if err != nil { + return err + } + if inode.Size != theirInode.Size { + tracef( + CDebug, "depfix: %s/%s -> %s: size differs", + root, fi.Name(), dep, + ) + fd.Close() + continue + } + if inode.Equals(theirInode) { + tracef( + CDebug, "depfix: %s/%s -> %s: inode is equal", + root, fi.Name(), dep, + ) + fd.Close() + continue + } + hsh, err := fileHash(fd) + fd.Close() + if err != nil { + return err + } + if hsh != theirHsh { + tracef( + CDebug, "depfix: %s/%s -> %s: hash differs", + root, fi.Name(), dep, + ) + continue + } + fields = []recfile.Field{ + {Name: "Type", Value: DepTypeIfchange}, + {Name: "Target", Value: dep}, + {Name: "Hash", Value: hsh}, + } + fields = append(fields, inode.RecfileFields()...) + fieldses[len(fieldses)-1] = fields + tracef( + CDebug, "depfix: %s/%s -> %s: inode updated", + root, fi.Name(), dep, + ) + depChanged = true + } + fdDep.Close() + if !depChanged { + continue + } + redoDirChanged = true + fdDep, err = tempfile(redoDir, fi.Name()) + if err != nil { + return err + } + defer fdDep.Close() + tracef( + CDebug, "depfix: %s/%s: tmp %s", + root, fi.Name(), fdDep.Name(), + ) + w := recfile.NewWriter(fdDep) + if _, err := w.WriteFields(fieldses[0]...); err != nil { + return err + } + fieldses = fieldses[1:] + for _, fields := range fieldses { + if _, err := w.RecordStart(); err != nil { + return err + } + if _, err := w.WriteFields(fields...); err != nil { + return err + } + } + if !NoSync { + if err = fdDep.Sync(); err != nil { + return err + } + } + fdDep.Close() + if err = os.Rename(fdDep.Name(), fdDepPath); err != nil { + return err + } + tracef(CRedo, "%s", fdDepPath) + } + } + if redoDirChanged && !NoSync { + if err = syncDir(redoDir); err != nil { + return nil + } + } + return nil +} diff --git a/doc/cmds.texi b/doc/cmds.texi index f4afdefc13cf837fb70b73f449e6de28dc9df697b839ca9581ada5ce5a82c4b0..543e6191a9e643d20ae7224e0921c5bcf7c39de638cd9e1fb1db0654dff3b259 100644 --- a/doc/cmds.texi +++ b/doc/cmds.texi @@ -75,6 +75,7 @@ @table @command @item redo-cleanup Removes either temporary (@option{tmp}), log files (@option{log}), or everything related to @command{goredo} (@option{full}). + @item redo-whichdo Display @file{.do} search paths for specified target (similar to @command{apenwarr/redo}): @@ -94,6 +95,7 @@ ../default.b.o.do ../default.o.do ../default.do @end example + @item redo-dot Dependency @url{https://en.wikipedia.org/wiki/DOT_(graph_description_language), DOT} @@ -103,5 +105,14 @@ $ redo target [...] # to assure that **/.redo/*.rec are filled up $ redo-dot target [...] > whatever.dot $ dot -Tpng whatever.dot > whatever.png # possibly add -Gsplines=ortho @end example + +@item redo-depfix + When you copy your worktree to different place, then copied files + ctime will change. And because recorded dependency information + differs from updated ctimes, out-of-date algorithm will fallback to + rereading the whole files for hash calculation, that is much slower. + If you do not want to rebuild your targets from the ground, then + @command{redo-depfix} can traverse through all dependency files and + check if they have non-altered ctime values and update them in place. @end table diff --git a/doc/news.texi b/doc/news.texi index 08db9b450788573cd8f9301ad81fc1bb302b937c7057b624756f0170f05251a7..b1b08386d9dc04f690c8e02252fe7a26b2ddbf481952b73f584871ec17adf296 100644 --- a/doc/news.texi +++ b/doc/news.texi @@ -7,6 +7,12 @@ @itemize @item @code{flock} locks replaced with POSIX @code{fcntl} ones. They could be more portable. +@item + @command{redo-depfix} command appeared, that traverses through all + @file{.redo} directories and their dependency files, checks if + corresponding targets has the same content but different + @code{ctime}/@code{mtime} values and rewrites dependencies with that + updated inode information. @end itemize @anchor{Release 1_21_0} diff --git a/doc/ood.texi b/doc/ood.texi index a57cb19c0da5ba7c25b358245c660291b545d4ede0c402c6f2ef5ffc996b0517..affa14843805ed6fe6af56a86e191b86400d0cc6962dc94a687d9c5cd797c500 100644 --- a/doc/ood.texi +++ b/doc/ood.texi @@ -50,3 +50,9 @@ However GNU/Linux with @code{ext4} filesystem can easily have pretty big granularity of 10ms. @command{goredo} uses @env{$REDO_INODE_TRUST=ctime} by default. + +If you move your worktree to different place, then all @code{ctime}s +(probably @code{mtime}s if you are inaccurate) will be also changed. OOD +check will be much slower after that, because it has to fallback to +content/hash checking all the time. You can use @command{redo-depfix} +utility to rebuild dependency files. diff --git a/main.go b/main.go index 321683428d723d21ddb7a399933764736742381c8a1e7f4d4b85b3188bc14dd9..866eefbd0485af821ec75e9b9af15c5e4bf0736a5a7a2f3b5f54a0175f37468f 100644 --- a/main.go +++ b/main.go @@ -55,6 +55,7 @@ CmdNameRedoSources = "redo-sources" CmdNameRedoStamp = "redo-stamp" CmdNameRedoTargets = "redo-targets" CmdNameRedoWhichdo = "redo-whichdo" + CmdNameRedoDepFix = "redo-depfix" ) var ( @@ -112,6 +113,7 @@ CmdNameRedo, CmdNameRedoAffects, CmdNameRedoAlways, CmdNameRedoCleanup, + CmdNameRedoDepFix, CmdNameRedoDot, CmdNameRedoIfchange, CmdNameRedoIfcreate, @@ -494,6 +496,8 @@ sort.Strings(srcs) for _, src := range srcs { fmt.Println(src) } + case CmdNameRedoDepFix: + err = depFix(Cwd) default: log.Fatalln("unknown command", cmdName) } diff --git a/ood.go b/ood.go index 1ea43d22df97f002cd9a212535199cc2326a80a3b1c077dca50b5f33684746e7..5f9054caf1a0434ddd63d0f43ff86a87010b4be4a7eeb6310a1d6e033bba9f5d 100644 --- a/ood.go +++ b/ood.go @@ -46,6 +46,8 @@ var ( OODTgts map[string]struct{} FdOODTgts *os.File FdOODTgtsLock *os.File + + ErrMissingTarget = errors.New("invalid format of .rec: missing Target") ) type TgtError struct { @@ -142,7 +144,7 @@ for _, m := range depInfo.ifchanges { dep := m["Target"] if dep == "" { - return ood, TgtError{tgtOrig, errors.New("invalid format of .rec: missing Target")} + return ood, TgtError{tgtOrig, ErrMissingTarget} } theirInode, err := inodeFromRec(m) if err != nil { diff --git a/usage.go b/usage.go index c54376bfce6add3403dcc8006c649b382e5e00001945386a2a1a5137ce0feb7d..43c7a7b02d045a02a60ad80a306d37e373276691bcc174a7d81671293a1a0aed 100644 --- a/usage.go +++ b/usage.go @@ -105,13 +105,18 @@ case CmdNameRedoAffects: d = `Usage: redo-affects target [...] List all targets that will be affected by changing the specified ones.` + case CmdNameRedoDepFix: + d = `Usage: redo-depfix + +Traverse over all .redo directories beneath and check if inode's information +(ctime/mtime) differs. Update dependency if file's content is still the same.` default: d = `Usage: goredo -symlinks goredo expects to be called through the symbolic link to it. Available commands: redo, redo-affects, redo-always, redo-cleanup, -redo-dot, redo-ifchange, redo-ifcreate, redo-log, redo-ood, -redo-sources, redo-stamp, redo-targets, redo-whichdo.` +redo-depfix, redo-dot, redo-ifchange, redo-ifcreate, redo-log, +redo-ood, redo-sources, redo-stamp, redo-targets, redo-whichdo.` } fmt.Fprintf(os.Stderr, "%s\n\nCommon options:\n", d) flag.PrintDefaults()