doc/news.texi | 8 ++++++++ ifchange.go | 5 +++-- main.go | 56 +++++++++++++++++++++++++++++++++++++++++++++++++---- ood.go | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++- run.go | 7 +++++++ usage.go | 2 +- diff --git a/doc/news.texi b/doc/news.texi index ad51e4de56436a19cb80abbdc2c665ee5fb913811dd003d85833c909c80dce0e..beaeb9e4414063d295a98db03a5877ff459ad142c2f12562e0c7ddadb31780cb 100644 --- a/doc/news.texi +++ b/doc/news.texi @@ -1,6 +1,14 @@ @node News @unnumbered News +@anchor{Release 1.3.0} +@section Release 1.3.0 +@itemize +@item + Repetitive OOD determination optimization: pass all already known to + be OOD targets to redoing targets. +@end itemize + @anchor{Release 1.2.0} @section Release 1.2.0 @itemize diff --git a/ifchange.go b/ifchange.go index 6f8593832a9fdc72593f104ac17fb02c85e6049fc10e03bd5afd88c136356f71..733718932353675ad0d2baefd3deaf132b162da23a5cfd0b13adf83f3e4359c9 100644 --- a/ifchange.go +++ b/ifchange.go @@ -139,7 +139,7 @@ errs = make(chan error, len(queue)) jobs := 0 queueSrc = []string{} for _, tgt := range queue { - ood, err := isOOD(Cwd, tgt, 0, seen) + ood, err := isOODWithTrace(Cwd, tgt, 0, seen) if err != nil { trace(CErr, "dependant error: %s, skipping dependants", err) return nil @@ -176,6 +176,7 @@ jsInit() defer jsAcquire("ifchange exiting") defer Jobs.Wait() seen := buildDependants(tgts) + oodTgtsClear() trace(CDebug, "building %d targets: %v", len(tgts), tgts) jobs := 0 errs := make(chan error, len(tgts)) @@ -188,7 +189,7 @@ continue } ood = true if !forced { - ood, err = isOOD(Cwd, tgt, 0, seen) + ood, err = isOODWithTrace(Cwd, tgt, 0, seen) if err != nil { return false, err } diff --git a/main.go b/main.go index 2ba9e6da15e77a7ca7a8644a0c31124eb178f1dfd2c38ade0351f2292af60347..b7d5ad6a75b8b149dcf929477c5241d393de12dbe5f9eaeff7fcf86f6de0f7bb 100644 --- a/main.go +++ b/main.go @@ -18,6 +18,8 @@ package main import ( + "bufio" + "bytes" "crypto/rand" "flag" "fmt" @@ -30,6 +32,7 @@ "path/filepath" "strconv" "go.cypherpunks.ru/recfile" + "golang.org/x/sys/unix" ) var ( @@ -159,6 +162,46 @@ traced = os.Getenv(EnvTrace) == "1" } // Those are internal envs + FdOODTgts, err = ioutil.TempFile("", "ood-tgts") + if err != nil { + panic(err) + } + if err = os.Remove(FdOODTgts.Name()); err != nil { + panic(err) + } + FdOODTgtsLock, err = ioutil.TempFile("", "ood-tgts.lock") + if err != nil { + panic(err) + } + if err = os.Remove(FdOODTgtsLock.Name()); err != nil { + panic(err) + } + + if v := os.Getenv(EnvOODTgtsFd); v != "" { + fd := mustParseFd(v, EnvOODTgtsFd) + fdLock := mustParseFd(v, EnvOODTgtsLockFd) + if err = unix.Flock(int(fdLock.Fd()), unix.LOCK_EX); err != nil { + panic(err) + } + if _, err = fd.Seek(0, os.SEEK_SET); err != nil { + panic(err) + } + tgtsRaw, err := ioutil.ReadAll(bufio.NewReader(fd)) + if err != nil { + panic(err) + } + unix.Flock(int(fdLock.Fd()), unix.LOCK_UN) + OODTgts = make(map[string]struct{}) + for _, tgtRaw := range bytes.Split(tgtsRaw, []byte{0}) { + t := string(tgtRaw) + if t == "" { + continue + } + OODTgts[t] = struct{}{} + trace(CDebug, "ood: known to be: %s", t) + } + } + StderrPrefix = os.Getenv(EnvStderrPrefix) if v := os.Getenv(EnvLevel); v != "" { Level, err = strconv.Atoi(v) @@ -169,12 +212,14 @@ if Level < 0 { panic("negative " + EnvLevel) } } + var fdDep *os.File if v := os.Getenv(EnvDepFd); v != "" { fdDep = mustParseFd(v, EnvDepFd) } + + tgts := flag.Args() BuildUUID = os.Getenv(EnvBuildUUID) - tgts := flag.Args() if BuildUUID == "" { raw := new([16]byte) if _, err = io.ReadFull(rand.Reader, raw[:]); err != nil { @@ -189,6 +234,7 @@ if len(tgts) == 0 { tgts = []string{"all"} } } + statusInit() for i, tgt := range tgts { @@ -264,17 +310,19 @@ if len(tgts) != 1 { log.Fatalln("single target expected") } var fdTmp *os.File - fdTmp, err = ioutil.TempFile("", "") + fdTmp, err = ioutil.TempFile("", "whichdo") if err != nil { break } - os.Remove(fdTmp.Name()) + if err = os.Remove(fdTmp.Name()); err != nil { + break + } cwd, tgt := cwdAndTgt(tgts[0]) doFile, upLevels, err := findDo(fdTmp, cwd, tgt) if err != nil { break } - _, err = fdTmp.Seek(0, 0) + _, err = fdTmp.Seek(0, os.SEEK_SET) if err != nil { break } diff --git a/ood.go b/ood.go index ca7e38fcca1b08dc8a9d9fa566141c4db59784c09ecccf5ec7c4993088b274e9..c20753526687484c72c7f561569ca985681fbbc671df94b6a8c6c32aca3d2238 100644 --- a/ood.go +++ b/ood.go @@ -26,6 +26,8 @@ "os" "path" "path/filepath" "strings" + + "golang.org/x/sys/unix" ) const ( @@ -33,6 +35,15 @@ DepTypeIfcreate = "ifcreate" DepTypeIfchange = "ifchange" DepTypeAlways = "always" DepTypeStamp = "stamp" + + EnvOODTgtsFd = "REDO_OOD_TGTS_FD" + EnvOODTgtsLockFd = "REDO_OOD_TGTS_LOCK_FD" +) + +var ( + OODTgts map[string]struct{} + FdOODTgts *os.File + FdOODTgtsLock *os.File ) type TgtErr struct { @@ -170,7 +181,7 @@ trace(CDebug, "ood: %s%s -> %s: was always built", indent, tgtOrig, dep) continue } - depOod, err := isOOD(cwd, dep, level+1, seen) + depOod, err := isOODWithTrace(cwd, dep, level+1, seen) if err != nil { return ood, TgtErr{tgtOrig, err} } @@ -186,3 +197,49 @@ Done: trace(CDebug, "ood: %s%s: %v", indent, tgtOrig, ood) return ood, nil } + +func isOODWithTrace( + cwd, tgtOrig string, + level int, + seen map[string]struct{}, +) (bool, error) { + p, err := filepath.Abs(path.Join(cwd, tgtOrig)) + if err != nil { + panic(err) + } + _, ood := OODTgts[p] + if ood { + trace( + CDebug, + "ood: %s%s true, external decision", + strings.Repeat(". ", level), tgtOrig, + ) + goto RecordOODTgt + } + ood, err = isOOD(cwd, tgtOrig, level, seen) + if !ood { + return ood, err + } +RecordOODTgt: + if err = unix.Flock(int(FdOODTgtsLock.Fd()), unix.LOCK_EX); err != nil { + panic(err) + } + if _, err = FdOODTgts.Seek(0, os.SEEK_END); err != nil { + panic(err) + } + if _, err := FdOODTgts.WriteString(p + "\x00"); err != nil { + panic(err) + } + unix.Flock(int(FdOODTgtsLock.Fd()), unix.LOCK_UN) + return true, nil +} + +func oodTgtsClear() { + if err := unix.Flock(int(FdOODTgtsLock.Fd()), unix.LOCK_EX); err != nil { + panic(err) + } + if err := FdOODTgts.Truncate(0); err != nil { + panic(err) + } + unix.Flock(int(FdOODTgtsLock.Fd()), unix.LOCK_UN) +} diff --git a/run.go b/run.go index bb231dda450131a578ea4dfa3970dd8e16253ed9e97038ef1b15e65bb09b3a2c..fcaa2fb355dd2e8ad3b1876eb56e35ffba2ed28869a8d615d47b4477da402927 100644 --- a/run.go +++ b/run.go @@ -345,6 +345,13 @@ "%s=%s", EnvStderrPrefix, childStderrPrefix, )) fdNum := 0 + cmd.ExtraFiles = append(cmd.ExtraFiles, FdOODTgts) + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", EnvOODTgtsFd, 3+fdNum)) + fdNum++ + cmd.ExtraFiles = append(cmd.ExtraFiles, FdOODTgtsLock) + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", EnvOODTgtsLockFd, 3+fdNum)) + fdNum++ + if FdStatus == nil { cmd.Env = append(cmd.Env, fmt.Sprintf("%s=NO", EnvStatusFd)) } else { diff --git a/usage.go b/usage.go index 9ad49630f9fcc208b71e627f90725ce1f53a56f93133dd42433b75e41d0306a4..1945e50184702b4b87a4d02a205161c772e996f7ec0850e609d2a250ff859b63 100644 --- a/usage.go +++ b/usage.go @@ -26,7 +26,7 @@ "strings" ) const ( - Version = "1.2.0" + Version = "1.3.0" Warranty = `This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 3 of the License.