dep.go | 40 +++++++++++++++------------------------- dep.rec | 8 +++++--- doc/features.texi | 3 ++- doc/news.texi | 7 +++++-- doc/state.texi | 4 +++- inode.go | 94 +++++++++++++++++++++++++++++++++++++++++++++++++++++ ood.go | 24 ++++++++++++++++-------- run.go | 38 +++++++++++++++++++++----------------- usage.go | 2 +- diff --git a/dep.go b/dep.go index bee6e5a39309ae7b8ee8974f5085ca519bc574f24cf84412f2401e95e050ec38..1ae67781bb00a1d7ff32dfa75159f94a30ab645ca0a5b56ba0ba25ef04a35dc6 100644 --- a/dep.go +++ b/dep.go @@ -23,14 +23,12 @@ import ( "bufio" "encoding/hex" "errors" - "fmt" "io" "os" "path" "path/filepath" "go.cypherpunks.ru/recfile" - "golang.org/x/sys/unix" "lukechampine.com/blake3" ) @@ -75,15 +73,6 @@ recfile.Field{Name: "Hash", Value: hsh}, ) } -func fileCtime(fd *os.File) (string, error) { - var stat unix.Stat_t - if err := unix.Fstat(int(fd.Fd()), &stat); err != nil { - return "", err - } - sec, nsec := stat.Ctim.Unix() - return fmt.Sprintf("%d.%d", sec, nsec), nil -} - func fileHash(fd *os.File) (string, error) { h := blake3.New(32, nil) if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil { @@ -106,7 +95,7 @@ } if fi.IsDir() { return nil } - ts, err := fileCtime(fd) + inode, err := inodeFromFile(fd) if err != nil { return err } @@ -114,13 +103,13 @@ hsh, err := fileHash(fd) if err != nil { return err } - return recfileWrite( - fdDep, - recfile.Field{Name: "Type", Value: DepTypeIfchange}, - recfile.Field{Name: "Target", Value: tgt}, - recfile.Field{Name: "Ctime", Value: ts}, - recfile.Field{Name: "Hash", Value: hsh}, - ) + fields := []recfile.Field{ + {Name: "Type", Value: DepTypeIfchange}, + {Name: "Target", Value: tgt}, + {Name: "Hash", Value: hsh}, + } + fields = append(fields, inode.RecfileFields()...) + return recfileWrite(fdDep, fields...) } func writeDeps(fdDep *os.File, tgts []string) (err error) { @@ -142,12 +131,13 @@ if _, errStat := os.Stat(tgt); errStat == nil { err = writeDep(fdDep, tgtDir, tgtRel) } else { trace(CDebug, "ifchange: %s <- %s (unexisting)", fdDep.Name(), tgtRel) - err = recfileWrite( - fdDep, - recfile.Field{Name: "Type", Value: DepTypeIfchange}, - recfile.Field{Name: "Target", Value: tgtRel}, - recfile.Field{Name: "Ctime", Value: "0.0"}, - ) + fields := []recfile.Field{ + {Name: "Type", Value: DepTypeIfchange}, + {Name: "Target", Value: tgtRel}, + } + inodeDummy := Inode{} + fields = append(fields, inodeDummy.RecfileFields()...) + err = recfileWrite(fdDep, fields...) } } return diff --git a/dep.rec b/dep.rec index b61d60ea6da5ad2e1f090af2d0c33bcb22964d71554536346e27b732b3e4618b..b60149d4ba722c91445919cd6c9a4b8cc6428ab896fb2340630aa860ea513e26 100644 --- a/dep.rec +++ b/dep.rec @@ -6,8 +6,10 @@ %rec: Dependency %doc: Dependency information %mandatory: Type -%allowed: Target Ctime Hash -%unique: Type Target Ctime Hash +%allowed: Target Size CtimeSec CtimeNsec Hash +%unique: Type Target Size CtimeSec CtimeNsec Hash %type: Type enum ifcreate ifchange always stamp -%type: Ctime regexp /[0-9]+\.[0-9]+/ +%type: Size int +%type: CtimeSec int +%type: CtimeNsec int %type: Hash regexp /[0-9a-f]{64}/ diff --git a/doc/features.texi b/doc/features.texi index 47296e440a4c0c8b74719dbcaac6aeb03a3136277a1f82038aa948b0f1e610e7..07a16737169242406547ae532b611ebf6d0b95a0fdbbcad1261cace3686799ed 100644 --- a/doc/features.texi +++ b/doc/features.texi @@ -12,7 +12,8 @@ the redo, preventing its overwriting, but continuing the build @end itemize @item targets, dependency information and their directories are explicitly synced (can be disabled, should work faster) -@item file's change is detected by comparing its @code{ctime} and BLAKE3 hash +@item file's change is detected by comparing its size, @code{ctime} + and BLAKE3 hash @item files creation is @code{umask}-friendly (unlike @code{mkstemp()} used in @command{redo-c}) @item parallel build with jobs limit, optionally in infinite mode diff --git a/doc/news.texi b/doc/news.texi index d03b05282e371a8f87bf6e1fe0a52283cee63730089685667ad0217535af7ee0..43c1527524e79c0eb28a4c3700dac750277bb429aca3401c63487424f87628f2 100644 --- a/doc/news.texi +++ b/doc/news.texi @@ -1,9 +1,12 @@ @node News @unnumbered News -@anchor{Release 0.13.0} -@section Release 0.13.0 +@anchor{Release 1.0.0} +@section Release 1.0.0 @itemize +@item + @code{Size} is stored in the state, for faster OOD detection. + Previous @command{goredo} state files won't work. @item @command{redo-whichdo} resembles @code{apenwarr/redo}'s one behaviour more. @end itemize diff --git a/doc/state.texi b/doc/state.texi index f978d8b14529d9596365a6c086c3e8566a48b121e6a21cabd73d62e8e6ca7470..d47d9de219edd3ff8a7dd86abfc68c8a05fcdde5e8fa6b37146decc4d967b2c5 100644 --- a/doc/state.texi +++ b/doc/state.texi @@ -15,7 +15,9 @@ Target: foo.o.do Type: ifchange Target: default.o.do -Ctime: 1605721341.253305000 +Size: 123 +CtimeSec: 1605721341 +CtimeNsec: 253305000 Hash: f4929732f96f11e6d4ebe94536b5edef426d00ed0146853e37a87f4295e18eda Type: always diff --git a/inode.go b/inode.go new file mode 100644 index 0000000000000000000000000000000000000000..c6e5fa0b7d4ba767ee6b0e2c360bb7c98f42334ca9fd35adfc6d1ef77e337879 --- /dev/null +++ b/inode.go @@ -0,0 +1,94 @@ +/* +goredo -- redo implementation on pure Go +Copyright (C) 2020-2021 Sergey Matveev + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Inode metainformation + +package main + +import ( + "errors" + "os" + "strconv" + + "go.cypherpunks.ru/recfile" + "golang.org/x/sys/unix" +) + +type Inode struct { + Size int64 + CtimeSec int64 + CtimeNsec int64 +} + +func (our *Inode) Equals(their *Inode) bool { + return (our.Size == their.Size) && + (our.CtimeSec == their.CtimeSec) && + (our.CtimeNsec == their.CtimeNsec) +} + +func (inode *Inode) RecfileFields() []recfile.Field { + return []recfile.Field{ + {Name: "Size", Value: strconv.FormatInt(inode.Size, 10)}, + {Name: "CtimeSec", Value: strconv.FormatInt(inode.CtimeSec, 10)}, + {Name: "CtimeNsec", Value: strconv.FormatInt(inode.CtimeNsec, 10)}, + } +} + +func inodeFromFile(fd *os.File) (*Inode, error) { + var fi os.FileInfo + fi, err := fd.Stat() + if err != nil { + return nil, err + } + var stat unix.Stat_t + err = unix.Fstat(int(fd.Fd()), &stat) + if err != nil { + return nil, err + } + sec, nsec := stat.Ctim.Unix() + return &Inode{Size: fi.Size(), CtimeSec: sec, CtimeNsec: nsec}, nil +} + +func inodeFromRec(m map[string]string) (*Inode, error) { + size := m["Size"] + ctimeSec := m["CtimeSec"] + ctimeNsec := m["CtimeNsec"] + if size == "" { + return nil, errors.New("Size is missing") + } + if ctimeSec == "" { + return nil, errors.New("CtimeSec is missing") + } + if ctimeNsec == "" { + return nil, errors.New("CtimeNsec is missing") + } + inode := Inode{} + var err error + inode.Size, err = strconv.ParseInt(size, 10, 64) + if err != nil { + return nil, err + } + inode.CtimeSec, err = strconv.ParseInt(ctimeSec, 10, 64) + if err != nil { + return nil, err + } + inode.CtimeNsec, err = strconv.ParseInt(ctimeNsec, 10, 64) + if err != nil { + return nil, err + } + return &inode, nil +} diff --git a/ood.go b/ood.go index 80d2ea9a2f6fd56d29cb7ced84670bbba413777a8d2aacd10f4670eb9dc35a06..a396c6145e2aaf364a16fb73ffe650ca7499b6da5fc779d807f769968876981b 100644 --- a/ood.go +++ b/ood.go @@ -109,11 +109,14 @@ } for _, m := range depInfo.ifchanges { dep := m["Target"] - theirTs := m["Ctime"] + if dep == "" { + return ood, TgtErr{tgtOrig, errors.New("invalid format of .dep: missing Target")} + } + theirInode, err := inodeFromRec(m) + if err != nil { + return ood, TgtErr{tgtOrig, fmt.Errorf("invalid format of .dep: %v", err)} + } theirHsh := m["Hash"] - if dep == "" || theirTs == "" { - return ood, TgtErr{tgtOrig, errors.New("invalid format of .dep")} - } trace(CDebug, "ood: %s%s -> %s: checking", indent, tgtOrig, dep) fd, err := os.Open(path.Join(cwd, dep)) @@ -127,14 +130,19 @@ return ood, TgtErr{tgtOrig, err} } defer fd.Close() - ts, err := fileCtime(fd) + inode, err := inodeFromFile(fd) if err != nil { return ood, TgtErr{tgtOrig, err} } - if theirTs == ts { - trace(CDebug, "ood: %s%s -> %s: same ctime", indent, tgtOrig, dep) + if inode.Size != theirInode.Size { + trace(CDebug, "ood: %s%s -> %s: size differs", indent, tgtOrig, dep) + ood = true + goto Done + } + if inode.Equals(theirInode) { + trace(CDebug, "ood: %s%s -> %s: same inode", indent, tgtOrig, dep) } else { - trace(CDebug, "ood: %s%s -> %s: ctime differs", indent, tgtOrig, dep) + trace(CDebug, "ood: %s%s -> %s: inode differs", indent, tgtOrig, dep) hsh, err := fileHash(fd) if err != nil { return ood, TgtErr{tgtOrig, err} diff --git a/run.go b/run.go index 9b0303243c9b9430cb69fd00dfced82259325e21135d85229e1bd05fb186575e..988e71a8a719b15e749d2ad9aaf0000fba3b829a13032062ae51b46f2fbafceb 100644 --- a/run.go +++ b/run.go @@ -101,24 +101,24 @@ } return os.MkdirAll(pth, os.FileMode(0777)) } -func isModified(cwd, redoDir, tgt string) (bool, string, error) { +func isModified(cwd, redoDir, tgt string) (bool, *Inode, error) { fdDep, err := os.Open(path.Join(redoDir, tgt+DepSuffix)) if err != nil { if os.IsNotExist(err) { - return false, "", nil + return false, nil, nil } - return false, "", err + return false, nil, err } defer fdDep.Close() r := recfile.NewReader(fdDep) - var ourTs string + var ourInode *Inode for { m, err := r.NextMap() if err != nil { if err == io.EOF { break } - return false, "", err + return false, nil, err } if m["Target"] != tgt { continue @@ -126,21 +126,25 @@ } fd, err := os.Open(path.Join(cwd, tgt)) if err != nil { if os.IsNotExist(err) { - return false, "", nil + return false, nil, nil } - return false, "", err + return false, nil, err } - defer fd.Close() - ourTs, err = fileCtime(fd) + ourInode, err = inodeFromFile(fd) + fd.Close() if err != nil { - return false, "", err + return false, nil, err + } + theirInode, err := inodeFromRec(m) + if err != nil { + return false, nil, err } - if ourTs != m["Ctime"] { - return true, ourTs, nil + if !ourInode.Equals(theirInode) { + return true, ourInode, nil } break } - return false, ourTs, nil + return false, ourInode, nil } func syncDir(dir string) error { @@ -222,7 +226,7 @@ return nil } // Check if target is not modified externally - modified, tsPrev, err := isModified(cwd, redoDir, tgt) + modified, inodePrev, err := isModified(cwd, redoDir, tgt) if err != nil { lockRelease() return TgtErr{tgtOrig, err} @@ -462,11 +466,11 @@ return } // Was $1 touched? - if tsPrev != "" { + if inodePrev != nil { if fd, err := os.Open(path.Join(cwdOrig, tgt)); err == nil { - ts, err := fileCtime(fd) + inode, err := inodeFromFile(fd) fd.Close() - if err == nil && ts != tsPrev { + if err == nil && !inode.Equals(inodePrev) { runErr.Err = errors.New("$1 was explicitly touched") errs <- runErr fd.Close() diff --git a/usage.go b/usage.go index 2d5f5a781d880159acc0c5fe738cb2320470e5e579efef2a661241d7f08ff736..afc420b68a3acb715458a193000ae4cddf77c461aacda240fd097e211294c7ce 100644 --- a/usage.go +++ b/usage.go @@ -26,7 +26,7 @@ "strings" ) const ( - Version = "0.13.0" + Version = "1.0.0" Warranty = `This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 3 of the License.