From 8037011337a63a057963810aca0e71592711dcee Mon Sep 17 00:00:00 2001 From: Andrei Boar Date: Fri, 18 Nov 2022 14:10:15 +0200 Subject: [PATCH 01/10] cache nix-shell --- bob/build.go | 1 + bob/global/global.go | 3 +- bob/nix_builder.go | 10 ++++- bob/nix_builder_defaults.go | 4 +- bob/playbook/build_internal.go | 2 +- bob/playbook/options.go | 6 +++ bob/playbook/playbook.go | 5 ++- bobtask/run.go | 4 +- pkg/nix/nix.go | 22 +++++++-- pkg/nix/shell_cache.go | 82 ++++++++++++++++++++++++++++++++++ 10 files changed, 129 insertions(+), 10 deletions(-) create mode 100644 pkg/nix/shell_cache.go diff --git a/bob/build.go b/bob/build.go index 7d6eb2c4..1471c360 100644 --- a/bob/build.go +++ b/bob/build.go @@ -38,6 +38,7 @@ func (b *B) Build(ctx context.Context, taskName string) (err error) { playbook.WithPushEnabled(b.enablePush), playbook.WithPullEnabled(b.enablePull), playbook.WitNixCache(b.Nix().cache), + playbook.WitNixShellCache(b.Nix().shellCache), ) errz.Fatal(err) diff --git a/bob/global/global.go b/bob/global/global.go index ea24993a..6dfdbd7d 100644 --- a/bob/global/global.go +++ b/bob/global/global.go @@ -19,5 +19,6 @@ var ( BobCacheArtifactsDir = filepath.Join(BobCacheDir, "artifacts") BobAuthStoreDir = filepath.Join(BobCacheDir, "auth") - BobCacheNixFileName = filepath.Join(BobCacheDir, BobNixCacheFile) + BobCacheNixFileName = filepath.Join(BobCacheDir, BobNixCacheFile) + BobCacheNixShellCacheDir = filepath.Join(BobCacheDir, "env") ) diff --git a/bob/nix_builder.go b/bob/nix_builder.go index d9e8cf0a..c1d6cd87 100644 --- a/bob/nix_builder.go +++ b/bob/nix_builder.go @@ -16,6 +16,8 @@ import ( type NixBuilder struct { // cache allows caching the dependency to store path cache *nix.Cache + // shellCache allows caching of the nix-shell --command='env' output + shellCache *nix.ShellCache } type NixOption func(n *NixBuilder) @@ -26,6 +28,12 @@ func WithCache(cache *nix.Cache) NixOption { } } +func WithShellCache(cache *nix.ShellCache) NixOption { + return func(n *NixBuilder) { + n.shellCache = cache + } +} + // NewNixBuilder instantiates a new Nix builder instance func NewNixBuilder(opts ...NixOption) *NixBuilder { n := &NixBuilder{} @@ -109,5 +117,5 @@ func (n *NixBuilder) BuildDependencies(deps []nix.Dependency) error { // BuildEnvironment builds the environment with all nix deps func (n *NixBuilder) BuildEnvironment(deps []nix.Dependency, nixpkgs string) (_ []string, err error) { - return nix.BuildEnvironment(deps, nixpkgs, n.cache) + return nix.BuildEnvironment(deps, nixpkgs, n.cache, n.shellCache) } diff --git a/bob/nix_builder_defaults.go b/bob/nix_builder_defaults.go index 87b4dd0c..aa65a41f 100644 --- a/bob/nix_builder_defaults.go +++ b/bob/nix_builder_defaults.go @@ -24,5 +24,7 @@ func DefaultNix() (_ *NixBuilder, err error) { nixCache, err := nix.NewCacheStore(nix.WithPath(cacheDir)) errz.Fatal(err) - return NewNixBuilder(WithCache(nixCache)), nil + shellCache := nix.NewShellCache(filepath.Join(home, global.BobCacheNixShellCacheDir)) + + return NewNixBuilder(WithCache(nixCache), WithShellCache(shellCache)), nil } diff --git a/bob/playbook/build_internal.go b/bob/playbook/build_internal.go index ad4a9749..a0f1c4fe 100644 --- a/bob/playbook/build_internal.go +++ b/bob/playbook/build_internal.go @@ -108,7 +108,7 @@ func (p *Playbook) build(ctx context.Context, task *bobtask.Task) (err error) { err = task.Clean() errz.Fatal(err) - err = task.Run(ctx, p.namePad, p.nixCache) + err = task.Run(ctx, p.namePad, p.nixCache, p.nixShellCache) if err != nil { taskSuccessFul = false taskErr = err diff --git a/bob/playbook/options.go b/bob/playbook/options.go index 425a4869..7461830c 100644 --- a/bob/playbook/options.go +++ b/bob/playbook/options.go @@ -54,3 +54,9 @@ func WitNixCache(c *nix.Cache) Option { p.nixCache = c } } + +func WitNixShellCache(c *nix.ShellCache) Option { + return func(p *Playbook) { + p.nixShellCache = c + } +} diff --git a/bob/playbook/playbook.go b/bob/playbook/playbook.go index 986a6133..e5ed12de 100644 --- a/bob/playbook/playbook.go +++ b/bob/playbook/playbook.go @@ -76,8 +76,11 @@ type Playbook struct { // enablePull allows pulling artifacts from remote store enablePull bool - // nixCache stores the Nix store paths + // nixCache caches the Nix store paths nixCache *nix.Cache + + // nixShellCache caches the nix-shell --command='env' command output + nixShellCache *nix.ShellCache } func New(root string, opts ...Option) *Playbook { diff --git a/bobtask/run.go b/bobtask/run.go index db0d5c11..abfff6cd 100644 --- a/bobtask/run.go +++ b/bobtask/run.go @@ -19,11 +19,11 @@ import ( "github.com/benchkram/errz" ) -func (t *Task) Run(ctx context.Context, namePad int, nixCache *nix.Cache) (err error) { +func (t *Task) Run(ctx context.Context, namePad int, nixCache *nix.Cache, shellCache *nix.ShellCache) (err error) { defer errz.Recover(&err) if len(t.Env()) == 0 { - nixShellEnv, err := nix.BuildEnvironment(t.dependencies, t.nixpkgs, nixCache) + nixShellEnv, err := nix.BuildEnvironment(t.dependencies, t.nixpkgs, nixCache, shellCache) errz.Fatal(err) t.SetEnv(envutil.Merge(nixShellEnv, t.env)) } diff --git a/pkg/nix/nix.go b/pkg/nix/nix.go index b34810e2..bab18ab4 100644 --- a/pkg/nix/nix.go +++ b/pkg/nix/nix.go @@ -206,7 +206,7 @@ func source(nixpkgs string) string { // nix-shell --pure --keep NIX_SSL_CERT_FILE --keep SSL_CERT_FILE -p --command 'env' -E nixExpressionFromDeps // // nix shell can be started with empty list of packages so this method works with empty deps as well -func BuildEnvironment(deps []Dependency, nixpkgs string, cache *Cache) (_ []string, err error) { +func BuildEnvironment(deps []Dependency, nixpkgs string, cache *Cache, shellCache *ShellCache) (_ []string, err error) { defer errz.Recover(&err) // building dependencies with nix-build to display store paths to output @@ -229,8 +229,24 @@ func BuildEnvironment(deps []Dependency, nixpkgs string, cache *Cache) (_ []stri var out bytes.Buffer cmd.Stdout = &out - err = cmd.Run() - errz.Fatal(err) + + if shellCache != nil { + key, err := shellCache.GenerateKey(deps, cmd.String()) + errz.Fatal(err) + + if dat, ok := shellCache.Get(key); ok { + out.Write(dat) + } else { + err = cmd.Run() + errz.Fatal(err) + + err = shellCache.Save(key, out.Bytes()) + errz.Fatal(err) + } + } else { + err = cmd.Run() + errz.Fatal(err) + } env := strings.Split(out.String(), "\n") diff --git a/pkg/nix/shell_cache.go b/pkg/nix/shell_cache.go new file mode 100644 index 00000000..7373abc2 --- /dev/null +++ b/pkg/nix/shell_cache.go @@ -0,0 +1,82 @@ +package nix + +import ( + "bytes" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/benchkram/bob/bobtask/hash" + "github.com/benchkram/bob/pkg/file" + "github.com/benchkram/bob/pkg/filehash" + "github.com/benchkram/errz" +) + +// ShellCache caches the output of nix-shell command +type ShellCache struct { + // dir is the root directory where the cache files are stored + dir string +} + +// NewShellCache creates a new instance of ShellCache +func NewShellCache(dir string) *ShellCache { + return &ShellCache{dir} +} + +// Save caches the output inside a file named by the key cache +func (c *ShellCache) Save(key string, output []byte) (err error) { + defer errz.Recover(&err) + + err = os.MkdirAll(c.dir, 0775) + errz.Fatal(err) + + err = os.WriteFile(filepath.Join(c.dir, key), output, 0644) + errz.Fatal(err) + + return nil +} + +// Get the data by cache key +// If Reading the file returns an error, empty data is returned +func (c *ShellCache) Get(key string) ([]byte, bool) { + if !file.Exists(filepath.Join(c.dir, key)) { + return []byte{}, false + } + data, err := os.ReadFile(filepath.Join(c.dir, key)) + if err != nil { + return []byte{}, false + } + return data, true +} + +// GenerateKey generates key for the cache based on a list of Dependency and nix-shell command +// +// if Dependency it's a .nix file it will hash the nixpkgs + file contents +// if Dependency it's a package name will hash the packageName:nixpkgs content +func (c *ShellCache) GenerateKey(deps []Dependency, nixShellCmd string) (_ string, err error) { + defer errz.Recover(&err) + h := filehash.New() + + for _, dependency := range deps { + if strings.HasSuffix(dependency.Name, ".nix") { + err = h.AddBytes(bytes.NewBufferString(dependency.Nixpkgs)) + errz.Fatal(err) + + err = h.AddFile(dependency.Name) + errz.Fatal(err) + } else { + toHash := fmt.Sprintf("%s:%s", dependency.Name, dependency.Nixpkgs) + err = h.AddBytes(bytes.NewBufferString(toHash)) + errz.Fatal(err) + } + } + + err = h.AddBytes(bytes.NewBufferString(nixShellCmd)) + errz.Fatal(err) + + hashIn := hash.In(hex.EncodeToString(h.Sum())) + + return hashIn.String(), nil +} From a538d3bd5dd31fd9d23156a4ec2ea4994d97da62 Mon Sep 17 00:00:00 2001 From: Andrei Boar Date: Wed, 30 Nov 2022 18:18:14 +0200 Subject: [PATCH 02/10] new task semantics; write tests according to specs; wip --- bob/aggregate.go | 4 +- bob/bobfile/verify.go | 16 +- bob/playbook/build_internal.go | 8 +- bob/playbook/play.go | 8 +- bob/playbook/playbook.go | 6 +- bob/playbook/rebuild.go | 2 +- bob/playbook/state.go | 11 +- bob/playbook/summary.go | 2 +- bobtask/map.go | 17 ++ bobtask/task.go | 33 ++- bobtask/verify.go | 14 +- .../artifacts/artifacts_extraction_test.go | 4 +- test/e2e/artifacts/artifacts_test.go | 12 +- test/e2e/artifacts/nobuildinfo_test.go | 4 +- .../multilevelbuild/multilevelbuild_test.go | 2 +- .../target-cleanup/with_dir_target/bob.yaml | 1 + .../target-name/with_second_level/bob.yaml | 1 + .../with_second_level/second/bob.yaml | 1 + test/e2e/target-symlink/with_symlink/bob.yaml | 1 + .../task-decoration/with_decoration/bob.yaml | 1 + .../with_decoration/second/bob.yaml | 2 + .../with_invalid_decoration/bob.yaml | 1 + .../with_invalid_decoration/second/bob.yaml | 10 +- .../with_missed_decoration/bob.yaml | 1 + .../with_thirdlevel_decoration/bob.yaml | 1 + .../second/bob.yaml | 3 + .../second/third/bob.yaml | 2 + test/e2e/tasksemantics/compound_task/bob.yaml | 9 + .../tasksemantics/no_input_no_target/bob.yaml | 4 + .../bob.yaml | 5 + .../no_input_unknown_target/bob.yaml | 4 + .../no_input_with_target/bob.yaml | 5 + .../rebuild_always_with_target/bob.yaml | 6 + .../rebuild_on_input_change/bob.yaml | 6 + .../e2e/tasksemantics/semantics_suite_test.go | 110 +++++++++ test/e2e/tasksemantics/semantics_test.go | 226 ++++++++++++++++++ test/e2e/tasksemantics/setup_test.go | 55 +++++ .../with_input_no_target/bob.yaml | 5 + test/e2e/variables/variables_suite_test.go | 2 +- test/e2e/version/version_suite_test.go | 2 +- 40 files changed, 550 insertions(+), 57 deletions(-) create mode 100644 test/e2e/tasksemantics/compound_task/bob.yaml create mode 100644 test/e2e/tasksemantics/no_input_no_target/bob.yaml create mode 100644 test/e2e/tasksemantics/no_input_no_target_rebuild_always/bob.yaml create mode 100644 test/e2e/tasksemantics/no_input_unknown_target/bob.yaml create mode 100644 test/e2e/tasksemantics/no_input_with_target/bob.yaml create mode 100644 test/e2e/tasksemantics/rebuild_always_with_target/bob.yaml create mode 100644 test/e2e/tasksemantics/rebuild_on_input_change/bob.yaml create mode 100644 test/e2e/tasksemantics/semantics_suite_test.go create mode 100644 test/e2e/tasksemantics/semantics_test.go create mode 100644 test/e2e/tasksemantics/setup_test.go create mode 100644 test/e2e/tasksemantics/with_input_no_target/bob.yaml diff --git a/bob/aggregate.go b/bob/aggregate.go index f2276348..19927629 100644 --- a/bob/aggregate.go +++ b/bob/aggregate.go @@ -217,7 +217,7 @@ func (b *B) Aggregate() (aggregate *bobfile.Bobfile, err error) { aggregate.Project = aggregate.Dir() } - err = aggregate.Verify() + err = aggregate.Verify(b.enableCaching) errz.Fatal(err) err = aggregate.BTasks.IgnoreChildTargets() @@ -246,7 +246,7 @@ func collectDecorations(ag *bobfile.Bobfile) (_ map[string][]string, err error) if !task.IsDecoration() { continue } - if !task.IsValidDecoration() { + if !task.IsCompoundTask() { errz.Fatal(usererror.Wrap(fmt.Errorf("task `%s` modifies an imported task. It can only contain a `dependsOn` property", k))) } decorations[k] = task.DependsOn diff --git a/bob/bobfile/verify.go b/bob/bobfile/verify.go index e3c41563..e8f1535a 100644 --- a/bob/bobfile/verify.go +++ b/bob/bobfile/verify.go @@ -5,13 +5,8 @@ import ( ) // Verify a bobfile before task runner. -func (b *Bobfile) Verify() error { - return b.verifyBefore() -} - -// VerifyBefore a bobfile before task runner. -func (b *Bobfile) VerifyBefore() error { - return b.verifyBefore() +func (b *Bobfile) Verify(cacheEnabled bool) error { + return b.verifyBefore(cacheEnabled) } // VerifyAfter a bobfile after task runner. @@ -20,14 +15,17 @@ func (b *Bobfile) VerifyAfter() error { } // verifyBefore verifies a Bobfile before Run() is called. -func (b *Bobfile) verifyBefore() (err error) { +func (b *Bobfile) verifyBefore(cacheEnabled bool) (err error) { defer errz.Recover(&err) err = b.BTasks.VerifyDuplicateTargets() errz.Fatal(err) + err = b.BTasks.VerifyMandatoryInputs() + errz.Fatal(err) + for _, task := range b.BTasks { - err = task.VerifyBefore() + err = task.VerifyBefore(cacheEnabled) errz.Fatal(err) } diff --git a/bob/playbook/build_internal.go b/bob/playbook/build_internal.go index a0f1c4fe..cc7c52e9 100644 --- a/bob/playbook/build_internal.go +++ b/bob/playbook/build_internal.go @@ -99,10 +99,14 @@ func (p *Playbook) build(ctx context.Context, task *bobtask.Task) (err error) { } if !rebuildRequired { - status := StateNoRebuildRequired + status := StateCached + t, _ := task.Target() + if t == nil { + status = StateNoRebuildRequired + } boblog.Log.V(2).Info(fmt.Sprintf("%-*s\t%s", p.namePad, coloredName, status.Short())) taskSuccessFul = true - return p.TaskNoRebuildRequired(task.Name()) + return p.TaskNoRebuildRequired(task.Name(), status) } err = task.Clean() diff --git a/bob/playbook/play.go b/bob/playbook/play.go index 07d19732..5573b931 100644 --- a/bob/playbook/play.go +++ b/bob/playbook/play.go @@ -37,7 +37,7 @@ func (p *Playbook) play() error { return err } - //boblog.Log.V(3).Info(fmt.Sprintf("%-*s\t walking", p.namePad, taskname)) + // boblog.Log.V(3).Info(fmt.Sprintf("%-*s\t walking", p.namePad, taskname)) switch task.State() { case StatePending: @@ -45,12 +45,12 @@ func (p *Playbook) play() error { for _, dependentTaskName := range task.Task.DependsOn { t, ok := p.Tasks[dependentTaskName] if !ok { - //fmt.Printf("Task %s does not exist", dependentTaskName) + // fmt.Printf("Task %s does not exist", dependentTaskName) return usererror.Wrap(boberror.ErrTaskDoesNotExistF(dependentTaskName)) } state := t.State() - if state != StateCompleted && state != StateNoRebuildRequired { + if state != StateCompleted && state != StateCached && state != StateNoRebuildRequired { // A dependent task is not completed. // So this task is not yet ready to run. return nil @@ -60,7 +60,7 @@ func (p *Playbook) play() error { return taskFailed case StateCanceled: return nil - case StateNoRebuildRequired: + case StateCached, StateNoRebuildRequired: return nil case StateCompleted: return nil diff --git a/bob/playbook/playbook.go b/bob/playbook/playbook.go index e5ed12de..841497cf 100644 --- a/bob/playbook/playbook.go +++ b/bob/playbook/playbook.go @@ -213,10 +213,10 @@ func (p *Playbook) TaskCompleted(taskname string) (err error) { } // TaskNoRebuildRequired sets a task's state to indicate that no rebuild is required -func (p *Playbook) TaskNoRebuildRequired(taskname string) (err error) { +func (p *Playbook) TaskNoRebuildRequired(taskname string, status State) (err error) { defer errz.Recover(&err) - err = p.setTaskState(taskname, StateNoRebuildRequired, nil) + err = p.setTaskState(taskname, status, nil) errz.Fatal(err) err = p.play() @@ -300,7 +300,7 @@ func (p *Playbook) setTaskState(taskname string, state State, taskError error) e task.SetState(state, taskError) switch state { - case StateCompleted, StateCanceled, StateNoRebuildRequired, StateFailed: + case StateCompleted, StateCanceled, StateCached, StateNoRebuildRequired, StateFailed: task.SetEnd(time.Now()) } diff --git a/bob/playbook/rebuild.go b/bob/playbook/rebuild.go index 3bb84c50..08ec97ee 100644 --- a/bob/playbook/rebuild.go +++ b/bob/playbook/rebuild.go @@ -80,7 +80,7 @@ func (p *Playbook) didChildTaskChange(taskname string, namePad int, coloredName } // Check if child task changed - if t.State() != StateNoRebuildRequired { + if t.State() != StateCached && t.State() != StateNoRebuildRequired { return Done } diff --git a/bob/playbook/state.go b/bob/playbook/state.go index c61ccbdf..c19ea375 100644 --- a/bob/playbook/state.go +++ b/bob/playbook/state.go @@ -12,8 +12,10 @@ func (s *State) Summary() string { return "⌛ " case StateCompleted: return aurora.Green("✔").Bold().String() + " " - case StateNoRebuildRequired: + case StateCached: return aurora.Green("cached").String() + " " + case StateNoRebuildRequired: + return aurora.Green("no-rebuild").String() + " " case StateFailed: return aurora.Red("failed").String() + " " case StateCanceled: @@ -29,8 +31,10 @@ func (s *State) Short() string { return "pending" case StateCompleted: return "done" - case StateNoRebuildRequired: + case StateCached: return "cached" + case StateNoRebuildRequired: + return "not-rebuild" case StateFailed: return "failed" case StateCanceled: @@ -43,7 +47,8 @@ func (s *State) Short() string { const ( StatePending State = "PENDING" StateCompleted State = "COMPLETED" - StateNoRebuildRequired State = "CACHED" + StateCached State = "CACHED" + StateNoRebuildRequired State = "NO-REBUILD" StateFailed State = "FAILED" StateRunning State = "RUNNING" StateCanceled State = "CANCELED" diff --git a/bob/playbook/summary.go b/bob/playbook/summary.go index 27fa1086..fc03aa0e 100644 --- a/bob/playbook/summary.go +++ b/bob/playbook/summary.go @@ -27,7 +27,7 @@ func (p *Playbook) summary(processedTasks []*bobtask.Task) { execTime := "" status := stat.State() - if status != StateNoRebuildRequired { + if status != StateCached && status != StateNoRebuildRequired { execTime = fmt.Sprintf("\t(%s)", format.DisplayDuration(stat.ExecutionTime())) } diff --git a/bobtask/map.go b/bobtask/map.go index 12a3524a..1873137b 100644 --- a/bobtask/map.go +++ b/bobtask/map.go @@ -232,6 +232,23 @@ func (tm Map) VerifyDuplicateTargets() error { return nil } +// VerifyMandatoryInputs check that build tasks have `input` field set +// input is mandatory except when task has `rebuild:always` set or is a compound task +func (tm Map) VerifyMandatoryInputs() error { + for taskName, v := range tm { + if v.InputDirty == "" { + if v.Rebuild() == RebuildAlways { + continue + } + if v.IsCompoundTask() { + continue + } + return usererror.Wrap(fmt.Errorf("no input provided for task `%s`", taskName)) + } + } + return nil +} + func CreateErrAmbigousTargets(tasks []string, target string) error { sort.Strings(tasks) return fmt.Errorf("%w,\nmultiple tasks [%s] pointing to the same target `%s`", ErrAmbigousTargets, strings.Join(tasks, " "), target) diff --git a/bobtask/task.go b/bobtask/task.go index b172ad55..319a6b8b 100644 --- a/bobtask/task.go +++ b/bobtask/task.go @@ -20,7 +20,7 @@ const ( RebuildOnChange RebuildType = "on-change" ) -// Hint: When adding a new *Dirty field assure to update IsValidDecoration(). +// Hint: When adding a new *Dirty field assure to update IsCompoundTask(). type Task struct { // Inputs are directorys or files // the task monitors for a rebuild. @@ -143,13 +143,32 @@ func (t *Task) IsDecoration() bool { return strings.ContainsRune(t.name, TaskPathSeparator) } -// IsValidDecoration checks if the task is a valid decoration. -// tasks containing a `dependsOn` node only are considered as -// valid decoration. +// IsCompoundTask checks if the task is a compound task // -// Make sure to update IsValidDecoration() very time a new -// *Dirty field is added to the task. -func (t *Task) IsValidDecoration() bool { +// tasks containing only `dependsOn` node are considered as compound task +// +// # Compound tasks are used when decorating a task or when grouping several tasks together +// +// Grouping tasks example: +// +// build: +// dependsOn: +// - backend +// - frontend +// +// Decoration example: +// +// import: +// - backend +// build: +// backend/hello: # the task decoration +// dependsOn: +// - generateDocs +// generateDocs: +// cmd: touch docs.md +// +// Make sure to update IsCompoundTask() very time a new *Dirty field is added to the task. +func (t *Task) IsCompoundTask() bool { if t.InputDirty != "" { return false } diff --git a/bobtask/verify.go b/bobtask/verify.go index c4ff7a92..12aa8667 100644 --- a/bobtask/verify.go +++ b/bobtask/verify.go @@ -8,14 +8,9 @@ import ( "github.com/benchkram/bob/pkg/usererror" ) -// Verify a bobfile before task runner. -func (t *Task) Verify() error { - return t.verifyBefore() -} - // VerifyBefore a bobfile before task runner. -func (t *Task) VerifyBefore() error { - return t.verifyBefore() +func (t *Task) VerifyBefore(cacheEnabled bool) error { + return t.verifyBefore(cacheEnabled) } // VerifyAfter a bobfile after task runner. @@ -23,8 +18,11 @@ func (t *Task) VerifyAfter() error { return t.verifyAfter() } -func (t *Task) verifyBefore() (err error) { +func (t *Task) verifyBefore(cacheEnabled bool) (err error) { if t.target != nil { + if cacheEnabled && t.Rebuild() == RebuildAlways { + return usererror.Wrap(fmt.Errorf("`rebuild:always` not allowed in combination with `target` for task: `%s`", t.name)) + } for _, path := range t.target.FilesystemEntriesRawPlain() { if !isValidFilesystemTarget(path) { return usererror.Wrap(fmt.Errorf("invalid target `%s` for task `%s`", path, t.name)) diff --git a/test/e2e/artifacts/artifacts_extraction_test.go b/test/e2e/artifacts/artifacts_extraction_test.go index 6ef3bd1d..57bb25fa 100644 --- a/test/e2e/artifacts/artifacts_extraction_test.go +++ b/test/e2e/artifacts/artifacts_extraction_test.go @@ -57,7 +57,7 @@ var _ = Describe("Test artifact creation and extraction", func() { It("extract artifact from store on rebuild", func() { state, err := buildTask(b, bob.BuildTargetwithdirsTargetName) Expect(err).NotTo(HaveOccurred()) - Expect(state.State()).To(Equal(playbook.StateNoRebuildRequired)) + Expect(state.State()).To(Equal(playbook.StateCached)) }) It("cleanup", func() { @@ -140,7 +140,7 @@ var _ = Describe("Test artifact creation and extraction from docker targets", fu It("should extract artifact from store on rebuild", func() { state, err := buildTask(b, bob.BuildTargetDockerImageName) Expect(err).NotTo(HaveOccurred()) - Expect(state.State()).To(Equal(playbook.StateNoRebuildRequired)) + Expect(state.State()).To(Equal(playbook.StateCached)) }) It("should check that the docker image was created correctly", func() { diff --git a/test/e2e/artifacts/artifacts_test.go b/test/e2e/artifacts/artifacts_test.go index 627e7cad..862709f5 100644 --- a/test/e2e/artifacts/artifacts_test.go +++ b/test/e2e/artifacts/artifacts_test.go @@ -42,7 +42,7 @@ var _ = Describe("Test artifact and target invalidation", func() { err := artifactRemove(artifactID) Expect(err).NotTo(HaveOccurred()) - //time.Sleep(1 * time.Minute) + // time.Sleep(1 * time.Minute) state, err := buildTask(b, "build") Expect(err).NotTo(HaveOccurred()) @@ -57,7 +57,7 @@ var _ = Describe("Test artifact and target invalidation", func() { It("should not rebuild but unpack from local artifact", func() { state, err := buildTask(b, "build") Expect(err).NotTo(HaveOccurred()) - Expect(state.State()).To(Equal(playbook.StateNoRebuildRequired)) + Expect(state.State()).To(Equal(playbook.StateCached)) }) // 7) @@ -83,7 +83,7 @@ var _ = Describe("Test artifact and target invalidation", func() { state, err := buildTask(b, "build") Expect(err).NotTo(HaveOccurred()) - Expect(state.State()).To(Equal(playbook.StateNoRebuildRequired)) + Expect(state.State()).To(Equal(playbook.StateCached)) file.Exists(filepath.Join(dir, "run")) }) @@ -102,7 +102,7 @@ var _ = Describe("Test artifact and target invalidation", func() { }) }) -// docker targets +// docker targets var _ = Describe("Test artifact and docker-target invalidation", func() { Context("in a fresh playground", func() { @@ -144,7 +144,7 @@ var _ = Describe("Test artifact and docker-target invalidation", func() { It("should not rebuild but extract from local artifact", func() { state, err := buildTask(b, bob.BuildTargetDockerImageName) Expect(err).NotTo(HaveOccurred()) - Expect(state.State()).To(Equal(playbook.StateNoRebuildRequired)) + Expect(state.State()).To(Equal(playbook.StateCached)) }) // 7) @@ -183,7 +183,7 @@ var _ = Describe("Test artifact and docker-target invalidation", func() { state, err := buildTask(b, bob.BuildTargetDockerImageName) Expect(err).NotTo(HaveOccurred()) - Expect(state.State()).To(Equal(playbook.StateNoRebuildRequired)) + Expect(state.State()).To(Equal(playbook.StateCached)) exists, err := mobyClient.ImageExists(bob.BuildTargetBobTestImage) Expect(err).NotTo(HaveOccurred()) diff --git a/test/e2e/artifacts/nobuildinfo_test.go b/test/e2e/artifacts/nobuildinfo_test.go index 3d4cef83..8d1c5ef5 100644 --- a/test/e2e/artifacts/nobuildinfo_test.go +++ b/test/e2e/artifacts/nobuildinfo_test.go @@ -72,7 +72,7 @@ var _ = Describe("Test artifact and target lifecycle without existing buildinfo" state, err = buildTask(b, "build") Expect(err).NotTo(HaveOccurred()) - Expect(state.State()).To(Equal(playbook.StateNoRebuildRequired)) + Expect(state.State()).To(Equal(playbook.StateCached)) }) It("clean artifacts & buildinfo", func() { @@ -165,7 +165,7 @@ var _ = Describe("Test artifact and target lifecycle for docker images without e state, err = buildTask(b, bob.BuildTargetDockerImageName) Expect(err).NotTo(HaveOccurred()) - Expect(state.State()).To(Equal(playbook.StateNoRebuildRequired)) + Expect(state.State()).To(Equal(playbook.StateCached)) }) It("clean artifacts & buildinfo", func() { diff --git a/test/e2e/multilevelbuild/multilevelbuild_test.go b/test/e2e/multilevelbuild/multilevelbuild_test.go index 62576964..86d063aa 100644 --- a/test/e2e/multilevelbuild/multilevelbuild_test.go +++ b/test/e2e/multilevelbuild/multilevelbuild_test.go @@ -230,7 +230,7 @@ func requiresRebuildMustMatchFixtures(b *bob.B, fixtures []requiresRebuildFixtur for _, f := range fixtures { ts, err := pb.TaskStatus(f.taskname) Expect(err).NotTo(HaveOccurred()) - requiresRebuild := ts.State() != playbook.StateNoRebuildRequired + requiresRebuild := ts.State() != playbook.StateCached && ts.State() != playbook.StateNoRebuildRequired Expect(f.requiresRebuild).To(Equal(requiresRebuild), fmt.Sprintf("task's %q rebuild requirement differ, got: %t, want: %t", f.taskname, requiresRebuild, f.requiresRebuild)) } diff --git a/test/e2e/target-cleanup/with_dir_target/bob.yaml b/test/e2e/target-cleanup/with_dir_target/bob.yaml index 5471c5a1..442cf63f 100644 --- a/test/e2e/target-cleanup/with_dir_target/bob.yaml +++ b/test/e2e/target-cleanup/with_dir_target/bob.yaml @@ -1,5 +1,6 @@ build: build: + input: "*" cmd: |- mkdir sub-dir touch ./sub-dir/non-empty-file diff --git a/test/e2e/target-name/with_second_level/bob.yaml b/test/e2e/target-name/with_second_level/bob.yaml index 99439afc..8d26c956 100644 --- a/test/e2e/target-name/with_second_level/bob.yaml +++ b/test/e2e/target-name/with_second_level/bob.yaml @@ -3,5 +3,6 @@ import: build: build: + input: "*" cmd: touch hello target: hello \ No newline at end of file diff --git a/test/e2e/target-name/with_second_level/second/bob.yaml b/test/e2e/target-name/with_second_level/second/bob.yaml index 61715466..9c07ed21 100644 --- a/test/e2e/target-name/with_second_level/second/bob.yaml +++ b/test/e2e/target-name/with_second_level/second/bob.yaml @@ -1,4 +1,5 @@ build: build: + input: "*" cmd: touch hello target: hello \ No newline at end of file diff --git a/test/e2e/target-symlink/with_symlink/bob.yaml b/test/e2e/target-symlink/with_symlink/bob.yaml index 8d3fae08..eb174ed2 100644 --- a/test/e2e/target-symlink/with_symlink/bob.yaml +++ b/test/e2e/target-symlink/with_symlink/bob.yaml @@ -1,5 +1,6 @@ build: build: + input: "*" cmd: |- touch hello ln -s hello shortcut diff --git a/test/e2e/task-decoration/with_decoration/bob.yaml b/test/e2e/task-decoration/with_decoration/bob.yaml index 94fd0b24..849e9118 100644 --- a/test/e2e/task-decoration/with_decoration/bob.yaml +++ b/test/e2e/task-decoration/with_decoration/bob.yaml @@ -8,6 +8,7 @@ build: second/build: dependsOn: [ before ] before: + input: "*" cmd: |- touch textfile_before echo "Before!" > textfile_before diff --git a/test/e2e/task-decoration/with_decoration/second/bob.yaml b/test/e2e/task-decoration/with_decoration/second/bob.yaml index 81394d3c..27c04b13 100644 --- a/test/e2e/task-decoration/with_decoration/second/bob.yaml +++ b/test/e2e/task-decoration/with_decoration/second/bob.yaml @@ -2,12 +2,14 @@ nixpkgs: https://github.com/NixOS/nixpkgs/archive/eeefd01d4f630fcbab6588fe3e7fff build: build: + input: "*" cmd: |- touch textfile_build echo "Build!" > textfile_build target: textfile_build dependsOn: [ hello ] hello: + input: "*" cmd: |- touch textfile_hello echo "Hello!" > textfile_hello diff --git a/test/e2e/task-decoration/with_invalid_decoration/bob.yaml b/test/e2e/task-decoration/with_invalid_decoration/bob.yaml index a293774a..8f85a0d7 100644 --- a/test/e2e/task-decoration/with_invalid_decoration/bob.yaml +++ b/test/e2e/task-decoration/with_invalid_decoration/bob.yaml @@ -7,6 +7,7 @@ build: cmd: echo "Hello" dependsOn: [ before ] before: + input: "*" cmd: echo "before" diff --git a/test/e2e/task-decoration/with_invalid_decoration/second/bob.yaml b/test/e2e/task-decoration/with_invalid_decoration/second/bob.yaml index 3933c9a4..27c04b13 100644 --- a/test/e2e/task-decoration/with_invalid_decoration/second/bob.yaml +++ b/test/e2e/task-decoration/with_invalid_decoration/second/bob.yaml @@ -2,15 +2,17 @@ nixpkgs: https://github.com/NixOS/nixpkgs/archive/eeefd01d4f630fcbab6588fe3e7fff build: build: + input: "*" cmd: |- - touch textfile_build - echo "Build!" > textfile_build + touch textfile_build + echo "Build!" > textfile_build target: textfile_build dependsOn: [ hello ] hello: + input: "*" cmd: |- - touch textfile_hello - echo "Hello!" > textfile_hello + touch textfile_hello + echo "Hello!" > textfile_hello target: textfile_hello diff --git a/test/e2e/task-decoration/with_missed_decoration/bob.yaml b/test/e2e/task-decoration/with_missed_decoration/bob.yaml index 90eaa8a1..9294ea5e 100644 --- a/test/e2e/task-decoration/with_missed_decoration/bob.yaml +++ b/test/e2e/task-decoration/with_missed_decoration/bob.yaml @@ -6,6 +6,7 @@ build: second/build: dependsOn: [ before ] before: + input: "*" cmd: echo "before" diff --git a/test/e2e/task-decoration/with_thirdlevel_decoration/bob.yaml b/test/e2e/task-decoration/with_thirdlevel_decoration/bob.yaml index 00442f81..1b09a317 100644 --- a/test/e2e/task-decoration/with_thirdlevel_decoration/bob.yaml +++ b/test/e2e/task-decoration/with_thirdlevel_decoration/bob.yaml @@ -8,6 +8,7 @@ build: second/third/build: dependsOn: [ second/hello ] before: + input: "*" cmd: |- touch textfile_before echo "Before!" > textfile_before diff --git a/test/e2e/task-decoration/with_thirdlevel_decoration/second/bob.yaml b/test/e2e/task-decoration/with_thirdlevel_decoration/second/bob.yaml index e1a66b38..3471b7b0 100644 --- a/test/e2e/task-decoration/with_thirdlevel_decoration/second/bob.yaml +++ b/test/e2e/task-decoration/with_thirdlevel_decoration/second/bob.yaml @@ -5,17 +5,20 @@ import: build: build: + input: "*" cmd: |- touch textfile_build2 echo "Build!" > textfile_build2 target: textfile_build2 dependsOn: [ create ] hello: + input: "*" cmd: |- touch textfile_hello2 echo "Hello!" > textfile_hello2 target: textfile_hello2 create: + input: "*" cmd: echo "create!" diff --git a/test/e2e/task-decoration/with_thirdlevel_decoration/second/third/bob.yaml b/test/e2e/task-decoration/with_thirdlevel_decoration/second/third/bob.yaml index 81394d3c..27c04b13 100644 --- a/test/e2e/task-decoration/with_thirdlevel_decoration/second/third/bob.yaml +++ b/test/e2e/task-decoration/with_thirdlevel_decoration/second/third/bob.yaml @@ -2,12 +2,14 @@ nixpkgs: https://github.com/NixOS/nixpkgs/archive/eeefd01d4f630fcbab6588fe3e7fff build: build: + input: "*" cmd: |- touch textfile_build echo "Build!" > textfile_build target: textfile_build dependsOn: [ hello ] hello: + input: "*" cmd: |- touch textfile_hello echo "Hello!" > textfile_hello diff --git a/test/e2e/tasksemantics/compound_task/bob.yaml b/test/e2e/tasksemantics/compound_task/bob.yaml new file mode 100644 index 00000000..6af8040c --- /dev/null +++ b/test/e2e/tasksemantics/compound_task/bob.yaml @@ -0,0 +1,9 @@ +nixpkgs: https://github.com/NixOS/nixpkgs/archive/eeefd01d4f630fcbab6588fe3e7fffe0690fbb20.tar.gz +build: + # build is a compound task because it has only dependsOn node + build: + dependsOn: [ generated ] + generated: + input: "*" + cmd: touch hello + target: hello \ No newline at end of file diff --git a/test/e2e/tasksemantics/no_input_no_target/bob.yaml b/test/e2e/tasksemantics/no_input_no_target/bob.yaml new file mode 100644 index 00000000..0016651d --- /dev/null +++ b/test/e2e/tasksemantics/no_input_no_target/bob.yaml @@ -0,0 +1,4 @@ +nixpkgs: https://github.com/NixOS/nixpkgs/archive/eeefd01d4f630fcbab6588fe3e7fffe0690fbb20.tar.gz +build: + build: + cmd: echo Hello World diff --git a/test/e2e/tasksemantics/no_input_no_target_rebuild_always/bob.yaml b/test/e2e/tasksemantics/no_input_no_target_rebuild_always/bob.yaml new file mode 100644 index 00000000..5edb67bf --- /dev/null +++ b/test/e2e/tasksemantics/no_input_no_target_rebuild_always/bob.yaml @@ -0,0 +1,5 @@ +nixpkgs: https://github.com/NixOS/nixpkgs/archive/eeefd01d4f630fcbab6588fe3e7fffe0690fbb20.tar.gz +build: + build: + cmd: echo Hello World + rebuild: always \ No newline at end of file diff --git a/test/e2e/tasksemantics/no_input_unknown_target/bob.yaml b/test/e2e/tasksemantics/no_input_unknown_target/bob.yaml new file mode 100644 index 00000000..f97c47df --- /dev/null +++ b/test/e2e/tasksemantics/no_input_unknown_target/bob.yaml @@ -0,0 +1,4 @@ +nixpkgs: https://github.com/NixOS/nixpkgs/archive/eeefd01d4f630fcbab6588fe3e7fffe0690fbb20.tar.gz +build: + build: + cmd: touch hello \ No newline at end of file diff --git a/test/e2e/tasksemantics/no_input_with_target/bob.yaml b/test/e2e/tasksemantics/no_input_with_target/bob.yaml new file mode 100644 index 00000000..02c077bb --- /dev/null +++ b/test/e2e/tasksemantics/no_input_with_target/bob.yaml @@ -0,0 +1,5 @@ +nixpkgs: https://github.com/NixOS/nixpkgs/archive/eeefd01d4f630fcbab6588fe3e7fffe0690fbb20.tar.gz +build: + build: + cmd: touch hello + target: hello \ No newline at end of file diff --git a/test/e2e/tasksemantics/rebuild_always_with_target/bob.yaml b/test/e2e/tasksemantics/rebuild_always_with_target/bob.yaml new file mode 100644 index 00000000..5d2e6093 --- /dev/null +++ b/test/e2e/tasksemantics/rebuild_always_with_target/bob.yaml @@ -0,0 +1,6 @@ +nixpkgs: https://github.com/NixOS/nixpkgs/archive/eeefd01d4f630fcbab6588fe3e7fffe0690fbb20.tar.gz +build: + build: + cmd: touch content.json + rebuild: always + target: content.json \ No newline at end of file diff --git a/test/e2e/tasksemantics/rebuild_on_input_change/bob.yaml b/test/e2e/tasksemantics/rebuild_on_input_change/bob.yaml new file mode 100644 index 00000000..87deb644 --- /dev/null +++ b/test/e2e/tasksemantics/rebuild_on_input_change/bob.yaml @@ -0,0 +1,6 @@ +nixpkgs: https://github.com/NixOS/nixpkgs/archive/eeefd01d4f630fcbab6588fe3e7fffe0690fbb20.tar.gz +build: + build: + input: "*" + cmd: touch hello + target: hello \ No newline at end of file diff --git a/test/e2e/tasksemantics/semantics_suite_test.go b/test/e2e/tasksemantics/semantics_suite_test.go new file mode 100644 index 00000000..6cdc047f --- /dev/null +++ b/test/e2e/tasksemantics/semantics_suite_test.go @@ -0,0 +1,110 @@ +package tasksemanticstest + +import ( + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/benchkram/bob/bob/bobfile" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var ( + dir string + + stdout *os.File + stderr *os.File + pr *os.File + pw *os.File +) + +var _ = BeforeSuite(func() { + // Initialize mock bob files from local directory + bobFiles := []string{ + "rebuild_on_input_change", + "no_input_with_target", + "no_input_unknown_target", + "no_input_no_target", + "with_input_no_target", + "no_input_no_target_rebuild_always", + "compound_task", + "rebuild_always_with_target", + } + nameToBobfile := make(map[string]*bobfile.Bobfile) + for _, name := range bobFiles { + abs, err := filepath.Abs("./" + name) + Expect(err).NotTo(HaveOccurred()) + bf, err := bobfile.BobfileRead(abs) + Expect(err).NotTo(HaveOccurred()) + nameToBobfile[strings.ReplaceAll(name, "/", "_")] = bf + } + + testDir, err := ioutil.TempDir("", "bob-test-task-semantics-*") + Expect(err).NotTo(HaveOccurred()) + dir = testDir + + err = os.Chdir(dir) + Expect(err).NotTo(HaveOccurred()) + + // Save bob files in dir to have them available in tests + for name, bf := range nameToBobfile { + err = bf.BobfileSave(dir, name+".yaml") + Expect(err).NotTo(HaveOccurred()) + } +}) + +var _ = AfterSuite(func() { + err := os.RemoveAll(dir) + Expect(err).NotTo(HaveOccurred()) + + for _, file := range tmpFiles { + err = os.Remove(file) + Expect(err).NotTo(HaveOccurred()) + } +}) + +func TestBuild(t *testing.T) { + _, err := exec.LookPath("nix") + if err != nil { + // Allow to skip tests only locally. + // CI is always set to true on GitHub actions. + // https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables + if os.Getenv("CI") != "true" { + t.Skip("Test skipped because nix is not installed on your system") + } + } + RegisterFailHandler(Fail) + RunSpecs(t, "task semantics suite") +} + +func capture() { + stdout = os.Stdout + stderr = os.Stderr + + var err error + pr, pw, err = os.Pipe() + Expect(err).NotTo(HaveOccurred()) + + os.Stdout = pw + os.Stderr = pw +} + +func output() string { + pw.Close() + + b, err := io.ReadAll(pr) + Expect(err).NotTo(HaveOccurred()) + + pr.Close() + + os.Stdout = stdout + os.Stderr = stderr + + return string(b) +} diff --git a/test/e2e/tasksemantics/semantics_test.go b/test/e2e/tasksemantics/semantics_test.go new file mode 100644 index 00000000..4079e3f9 --- /dev/null +++ b/test/e2e/tasksemantics/semantics_test.go @@ -0,0 +1,226 @@ +package tasksemanticstest + +import ( + "context" + "os" + + "github.com/benchkram/bob/bob/playbook" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Testing task semantics behaviour", func() { + When("task has input * and a target", func() { + b, err := Bob() + Expect(err).NotTo(HaveOccurred()) + + It("it will first build successfully", func() { + useBobfile("rebuild_on_input_change") + defer releaseBobfile("rebuild_on_input_change") + + capture() + ctx := context.Background() + err = b.Build(ctx, "build") + Expect(err).NotTo(HaveOccurred()) + + buildCompleted := playbook.StateCompleted + Expect(output()).To(ContainSubstring(buildCompleted.Summary())) + }) + + It("it will have summary cached on the second build", func() { + capture() + useBobfile("rebuild_on_input_change") + defer releaseBobfile("rebuild_on_input_change") + + ctx := context.Background() + err = b.Build(ctx, "build") + Expect(err).NotTo(HaveOccurred()) + + buildCached := playbook.StateCached + Expect(output()).To(ContainSubstring(buildCached.Summary())) + }) + + It("it will be rebuilt when input * changes", func() { + useBobfile("rebuild_on_input_change") + defer releaseBobfile("rebuild_on_input_change") + + // invalidate target by creating a new file + err := os.WriteFile("./someFile", []byte("hello\ngo\n"), 0644) + Expect(err).NotTo(HaveOccurred()) + + ctx := context.Background() + capture() + err = b.Build(ctx, "build") + Expect(err).NotTo(HaveOccurred()) + + buildCompleted := playbook.StateCompleted + Expect(output()).To(ContainSubstring(buildCompleted.Summary())) + }) + }) + + When("task has target but no input set", func() { + It("it will trigger a no input provided error on aggregation", func() { + b, err := Bob() + Expect(err).NotTo(HaveOccurred()) + + useBobfile("no_input_with_target") + defer releaseBobfile("no_input_with_target") + + _, err = b.Aggregate() + Expect(err.Error()).To(Equal("no input provided for task `build`")) + }) + }) + + // An unknown target is a file created inside `cmd` which is not specified in `target` + When("task has no input and an unknown target to be created", func() { + It("it will trigger a no input provided error on aggregation", func() { + b, err := Bob() + Expect(err).NotTo(HaveOccurred()) + + useBobfile("no_input_unknown_target") + defer releaseBobfile("no_input_unknown_target") + + _, err = b.Aggregate() + Expect(err.Error()).To(Equal("no input provided for task `build`")) + }) + }) + + When("task has no input and no target defined", func() { + It("it will trigger a no input provided error on aggregation", func() { + useBobfile("no_input_no_target") + defer releaseBobfile("no_input_no_target") + + b, err := Bob() + Expect(err).NotTo(HaveOccurred()) + + _, err = b.Aggregate() + Expect(err.Error()).To(Equal("no input provided for task `build`")) + }) + + It("it will not trigger a no input provided error if rebuild:always", func() { + useBobfile("no_input_no_target_rebuild_always") + defer releaseBobfile("no_input_no_target_rebuild_always") + + b, err := Bob() + Expect(err).NotTo(HaveOccurred()) + + _, err = b.Aggregate() + Expect(err).NotTo(HaveOccurred()) + }) + }) + + When("task has input * and no target to be created", func() { + b, err := Bob() + Expect(err).NotTo(HaveOccurred()) + + It("it will first build successfully", func() { + useBobfile("with_input_no_target") + defer releaseBobfile("with_input_no_target") + + capture() + ctx := context.Background() + err = b.Build(ctx, "build") + Expect(err).NotTo(HaveOccurred()) + + buildCompleted := playbook.StateCompleted + Expect(output()).To(ContainSubstring(buildCompleted.Summary())) + }) + + It("it will have summary no-rebuild on second build", func() { + useBobfile("with_input_no_target") + defer releaseBobfile("with_input_no_target") + + ctx := context.Background() + capture() + err = b.Build(ctx, "build") + Expect(err).NotTo(HaveOccurred()) + + buildCached := playbook.StateNoRebuildRequired + Expect(output()).To(ContainSubstring(buildCached.Summary())) + }) + + It("it will be rebuilt when input * changes", func() { + useBobfile("with_input_no_target") + defer releaseBobfile("with_input_no_target") + + // invalidate target by creating a new file + err := os.WriteFile("./someRandomFile", []byte("hello\ngo\n"), 0644) + Expect(err).NotTo(HaveOccurred()) + + ctx := context.Background() + capture() + err = b.Build(ctx, "build") + Expect(err).NotTo(HaveOccurred()) + + buildCompleted := playbook.StateCompleted + Expect(output()).To(ContainSubstring(buildCompleted.Summary())) + }) + }) + + When("task is a compound task", func() { + b, err := Bob() + Expect(err).NotTo(HaveOccurred()) + + It("it will first build successfully", func() { + useBobfile("compound_task") + defer releaseBobfile("compound_task") + + capture() + ctx := context.Background() + err = b.Build(ctx, "build") + Expect(err).NotTo(HaveOccurred()) + + buildCompleted := playbook.StateCompleted + Expect(output()).To(ContainSubstring(buildCompleted.Summary())) + }) + + It("it will have summary no-rebuild on second build", func() { + useBobfile("compound_task") + defer releaseBobfile("compound_task") + + ctx := context.Background() + capture() + err = b.Build(ctx, "build") + Expect(err).NotTo(HaveOccurred()) + + // generated task is cached + buildCached := playbook.StateCached + out := output() + Expect(out).To(ContainSubstring(buildCached.Summary())) + + // build is marked as no-rebuild + buildNotRequired := playbook.StateNoRebuildRequired + Expect(out).To(ContainSubstring(buildNotRequired.Summary())) + }) + + It("it will be rebuilt when input * changes", func() { + useBobfile("compound_task") + defer releaseBobfile("compound_task") + + // invalidate target by creating a new file + err := os.WriteFile("./anotherRandomFile", []byte("hello\ngo\n"), 0644) + Expect(err).NotTo(HaveOccurred()) + + ctx := context.Background() + capture() + err = b.Build(ctx, "build") + Expect(err).NotTo(HaveOccurred()) + + buildCompleted := playbook.StateCompleted + Expect(output()).To(ContainSubstring(buildCompleted.Summary())) + }) + }) + + When("task has rebuild:always and target set", func() { + It("it will trigger an error on aggregation", func() { + b, err := Bob() + Expect(err).NotTo(HaveOccurred()) + + useBobfile("rebuild_always_with_target") + defer releaseBobfile("rebuild_always_with_target") + + _, err = b.Aggregate() + Expect(err.Error()).To(Equal("`rebuild:always` not allowed in combination with `target` for task: `build`")) + }) + }) +}) diff --git a/test/e2e/tasksemantics/setup_test.go b/test/e2e/tasksemantics/setup_test.go new file mode 100644 index 00000000..21b24f02 --- /dev/null +++ b/test/e2e/tasksemantics/setup_test.go @@ -0,0 +1,55 @@ +package tasksemanticstest + +import ( + "os" + + "github.com/benchkram/bob/bob" + "github.com/benchkram/bob/pkg/nix" + . "github.com/onsi/gomega" +) + +func Bob() (*bob.B, error) { + nixBuilder, err := NixBuilder() + if err != nil { + return nil, err + } + return bob.Bob( + bob.WithDir(dir), + bob.WithCachingEnabled(true), + bob.WithNixBuilder(nixBuilder), + ) +} + +// tmpFiles tracks temporarily created files in these tests +// to be cleaned up at the end. +var tmpFiles []string + +func NixBuilder() (*bob.NixBuilder, error) { + file, err := os.CreateTemp("", ".nix_cache*") + if err != nil { + return nil, err + } + name := file.Name() + file.Close() + + tmpFiles = append(tmpFiles, name) + + cache, err := nix.NewCacheStore(nix.WithPath(name)) + if err != nil { + return nil, err + } + + return bob.NewNixBuilder(bob.WithCache(cache)), nil +} + +// useBobfile sets the right bobfile to be used for test +func useBobfile(name string) { + err := os.Rename(name+".yaml", "bob.yaml") + Expect(err).NotTo(HaveOccurred()) +} + +// releaseBobfile will revert changes done in useBobfile +func releaseBobfile(name string) { + err := os.Rename("bob.yaml", name+".yaml") + Expect(err).NotTo(HaveOccurred()) +} diff --git a/test/e2e/tasksemantics/with_input_no_target/bob.yaml b/test/e2e/tasksemantics/with_input_no_target/bob.yaml new file mode 100644 index 00000000..411fd219 --- /dev/null +++ b/test/e2e/tasksemantics/with_input_no_target/bob.yaml @@ -0,0 +1,5 @@ +nixpkgs: https://github.com/NixOS/nixpkgs/archive/eeefd01d4f630fcbab6588fe3e7fffe0690fbb20.tar.gz +build: + build: + input: "*" + cmd: echo Hello World \ No newline at end of file diff --git a/test/e2e/variables/variables_suite_test.go b/test/e2e/variables/variables_suite_test.go index 9d72883e..dcca19e4 100644 --- a/test/e2e/variables/variables_suite_test.go +++ b/test/e2e/variables/variables_suite_test.go @@ -33,7 +33,7 @@ var _ = BeforeSuite(func() { nixBuilder, err := NixBuilder() Expect(err).NotTo(HaveOccurred()) - b, err = bob.BobWithBaseStoreDir(storageDir, bob.WithDir(dir), bob.WithNixBuilder(nixBuilder)) + b, err = bob.BobWithBaseStoreDir(storageDir, bob.WithDir(dir), bob.WithNixBuilder(nixBuilder), bob.WithCachingEnabled(false)) Expect(err).NotTo(HaveOccurred()) }) diff --git a/test/e2e/version/version_suite_test.go b/test/e2e/version/version_suite_test.go index 6dd06c85..81bd928f 100644 --- a/test/e2e/version/version_suite_test.go +++ b/test/e2e/version/version_suite_test.go @@ -35,7 +35,7 @@ var _ = BeforeSuite(func() { err = os.Chdir(dir) Expect(err).NotTo(HaveOccurred()) - b, err = bob.Bob(bob.WithDir(dir)) + b, err = bob.Bob(bob.WithDir(dir), bob.WithCachingEnabled(false)) Expect(err).NotTo(HaveOccurred()) }) From c6db21e8776b9fe148c0cf7ca9e23e1fcb9f27d7 Mon Sep 17 00:00:00 2001 From: equanox Date: Thu, 1 Dec 2022 16:56:07 +0100 Subject: [PATCH 03/10] prepare playground for new behavior --- bob/playground.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/bob/playground.go b/bob/playground.go index 4fe3610a..8fd4226d 100644 --- a/bob/playground.go +++ b/bob/playground.go @@ -297,7 +297,8 @@ func createPlaygroundBobfile(dir string, overwrite bool, projectName string) (er "sleep 2", "touch slowdone", }, "\n"), - TargetDirty: "slowdone", + RebuildDirty: string(bobtask.RebuildAlways), // TODO: Requires Bob to allow rebuild always together with a target. + TargetDirty: "slowdone", } // A run command to run a environment from a compose file @@ -345,7 +346,8 @@ func createPlaygroundBobfile(dir string, overwrite bool, projectName string) (er } bobfile.BTasks["print"] = bobtask.Task{ - CmdDirty: "echo ${HELLOWORLD}", + CmdDirty: "echo ${HELLOWORLD}", + RebuildDirty: string(bobtask.RebuildAlways), } bobfile.BTasks["multilinetouch"] = bobtask.Task{ @@ -354,6 +356,7 @@ func createPlaygroundBobfile(dir string, overwrite bool, projectName string) (er "touch \\\n\tmultilinefile1 \\\n\tmultilinefile2 \\\n\t\tmultilinefile3 \\\n multilinefile4", "touch \\\n multilinefile5", }, "\n"), + RebuildDirty: string(bobtask.RebuildAlways), } bobfile.BTasks["ignoredInputs"] = bobtask.Task{ @@ -369,7 +372,8 @@ func createPlaygroundBobfile(dir string, overwrite bool, projectName string) (er "touch .bbuild/dirone/dirtwo/fileone", "touch .bbuild/dirone/dirtwo/filetwo", }, "\n"), - TargetDirty: ".bbuild/dirone/", + RebuildDirty: string(bobtask.RebuildAlways), // TODO: Requires Bob to allow rebuild always together with a target. + TargetDirty: ".bbuild/dirone/", } bobfile.BTasks[SecondLevelDir+"/build2"] = bobtask.Task{ @@ -379,6 +383,7 @@ func createPlaygroundBobfile(dir string, overwrite bool, projectName string) (er m := make(map[string]interface{}) m["image"] = BuildTargetBobTestImage bobfile.BTasks[BuildTargetDockerImageName] = bobtask.Task{ + InputDirty: "Dockerfile", CmdDirty: strings.Join([]string{ fmt.Sprintf("docker build -t %s .", BuildTargetBobTestImage), }, "\n"), @@ -388,6 +393,7 @@ func createPlaygroundBobfile(dir string, overwrite bool, projectName string) (er m = make(map[string]interface{}) m["image"] = BuildTargetBobTestImagePlus bobfile.BTasks[BuildTargetDockerImagePlusName] = bobtask.Task{ + InputDirty: "Dockerfile.plus", CmdDirty: strings.Join([]string{ fmt.Sprintf("docker build -f Dockerfile.plus -t %s .", BuildTargetBobTestImagePlus), }, "\n"), @@ -444,7 +450,8 @@ func createPlaygroundBobfileThirdLevel(dir string, overwrite bool, projectName s } bobfile.BTasks["print"] = bobtask.Task{ - CmdDirty: "echo hello-third-level", + CmdDirty: "echo hello-third-level", + RebuildDirty: string(bobtask.RebuildAlways), } bobfile.Dependencies = []string{"docker", "go_1_18", "git"} From e2edddafbe7603cb74c1f584bad0f532ec27dabb Mon Sep 17 00:00:00 2001 From: Andrei Boar Date: Fri, 2 Dec 2022 11:29:21 +0200 Subject: [PATCH 04/10] option to allow redundant targets --- bob/aggregate.go | 2 +- bob/bob.go | 5 +++++ bob/bobfile/verify.go | 8 ++++---- bob/options.go | 6 ++++++ bobtask/verify.go | 8 ++++---- test/e2e/artifacts/artifacts_suite_test.go | 1 + test/e2e/build/build_suite_test.go | 2 +- test/e2e/ignore/ignore_suite_test.go | 2 +- test/e2e/multilevelbuild/multilevelbuild_suite_test.go | 2 +- test/e2e/target/target_suite_test.go | 1 + 10 files changed, 25 insertions(+), 12 deletions(-) diff --git a/bob/aggregate.go b/bob/aggregate.go index 19927629..8cdf3e31 100644 --- a/bob/aggregate.go +++ b/bob/aggregate.go @@ -217,7 +217,7 @@ func (b *B) Aggregate() (aggregate *bobfile.Bobfile, err error) { aggregate.Project = aggregate.Dir() } - err = aggregate.Verify(b.enableCaching) + err = aggregate.Verify(b.enableCaching, b.allowRedundantTargets) errz.Fatal(err) err = aggregate.BTasks.IgnoreChildTargets() diff --git a/bob/bob.go b/bob/bob.go index b2e7e6ed..5258887f 100644 --- a/bob/bob.go +++ b/bob/bob.go @@ -47,6 +47,11 @@ type B struct { // from the cache Default: true enableCaching bool + // allowRedundantTargets option allows redundant targets on build tasks + // a task has redundant target when it has a `target` used in combination with `rebuild:always` + // by default a user error will return when a task has both `target` and `rebuild:always` defined + allowRedundantTargets bool + // allowInsecure uses http protocol for accessing the remote artifact store, if any allowInsecure bool diff --git a/bob/bobfile/verify.go b/bob/bobfile/verify.go index e8f1535a..7a4c5a56 100644 --- a/bob/bobfile/verify.go +++ b/bob/bobfile/verify.go @@ -5,8 +5,8 @@ import ( ) // Verify a bobfile before task runner. -func (b *Bobfile) Verify(cacheEnabled bool) error { - return b.verifyBefore(cacheEnabled) +func (b *Bobfile) Verify(cacheEnabled, allowRedundantTargets bool) error { + return b.verifyBefore(cacheEnabled, allowRedundantTargets) } // VerifyAfter a bobfile after task runner. @@ -15,7 +15,7 @@ func (b *Bobfile) VerifyAfter() error { } // verifyBefore verifies a Bobfile before Run() is called. -func (b *Bobfile) verifyBefore(cacheEnabled bool) (err error) { +func (b *Bobfile) verifyBefore(cacheEnabled, allowRedundantTargets bool) (err error) { defer errz.Recover(&err) err = b.BTasks.VerifyDuplicateTargets() @@ -25,7 +25,7 @@ func (b *Bobfile) verifyBefore(cacheEnabled bool) (err error) { errz.Fatal(err) for _, task := range b.BTasks { - err = task.VerifyBefore(cacheEnabled) + err = task.VerifyBefore(cacheEnabled, allowRedundantTargets) errz.Fatal(err) } diff --git a/bob/options.go b/bob/options.go index c893ab3f..e707eac7 100644 --- a/bob/options.go +++ b/bob/options.go @@ -52,6 +52,12 @@ func WithCachingEnabled(enabled bool) Option { } } +func WithAllowRedundantTargets() Option { + return func(b *B) { + b.allowRedundantTargets = true + } +} + func WithPushEnabled(enabled bool) Option { return func(b *B) { b.enablePush = enabled diff --git a/bobtask/verify.go b/bobtask/verify.go index 12aa8667..fd058230 100644 --- a/bobtask/verify.go +++ b/bobtask/verify.go @@ -9,8 +9,8 @@ import ( ) // VerifyBefore a bobfile before task runner. -func (t *Task) VerifyBefore(cacheEnabled bool) error { - return t.verifyBefore(cacheEnabled) +func (t *Task) VerifyBefore(cacheEnabled, allowRedundantTargets bool) error { + return t.verifyBefore(cacheEnabled, allowRedundantTargets) } // VerifyAfter a bobfile after task runner. @@ -18,9 +18,9 @@ func (t *Task) VerifyAfter() error { return t.verifyAfter() } -func (t *Task) verifyBefore(cacheEnabled bool) (err error) { +func (t *Task) verifyBefore(cacheEnabled, allowRedundantTargets bool) (err error) { if t.target != nil { - if cacheEnabled && t.Rebuild() == RebuildAlways { + if !allowRedundantTargets && cacheEnabled && t.Rebuild() == RebuildAlways { return usererror.Wrap(fmt.Errorf("`rebuild:always` not allowed in combination with `target` for task: `%s`", t.name)) } for _, path := range t.target.FilesystemEntriesRawPlain() { diff --git a/test/e2e/artifacts/artifacts_suite_test.go b/test/e2e/artifacts/artifacts_suite_test.go index 751a3767..263b6b01 100644 --- a/test/e2e/artifacts/artifacts_suite_test.go +++ b/test/e2e/artifacts/artifacts_suite_test.go @@ -86,6 +86,7 @@ var _ = BeforeSuite(func() { bob.WithFilestore(artifactStore), bob.WithBuildinfoStore(buildinfoStore), bob.WithNixBuilder(nixBuilder), + bob.WithAllowRedundantTargets(), ) Expect(err).NotTo(HaveOccurred()) diff --git a/test/e2e/build/build_suite_test.go b/test/e2e/build/build_suite_test.go index b1a5e34e..3522bbf5 100644 --- a/test/e2e/build/build_suite_test.go +++ b/test/e2e/build/build_suite_test.go @@ -32,7 +32,7 @@ var _ = BeforeSuite(func() { nixBuilder, err := NixBuilder() Expect(err).NotTo(HaveOccurred()) - b, err = bob.BobWithBaseStoreDir(storageDir, bob.WithDir(dir), bob.WithNixBuilder(nixBuilder)) + b, err = bob.BobWithBaseStoreDir(storageDir, bob.WithDir(dir), bob.WithNixBuilder(nixBuilder), bob.WithAllowRedundantTargets()) Expect(err).NotTo(HaveOccurred()) }) diff --git a/test/e2e/ignore/ignore_suite_test.go b/test/e2e/ignore/ignore_suite_test.go index 8ff4442f..52f03340 100644 --- a/test/e2e/ignore/ignore_suite_test.go +++ b/test/e2e/ignore/ignore_suite_test.go @@ -33,7 +33,7 @@ var _ = BeforeSuite(func() { nixBuilder, err := NixBuilder() Expect(err).NotTo(HaveOccurred()) - b, err = bob.BobWithBaseStoreDir(storageDir, bob.WithDir(dir), bob.WithNixBuilder(nixBuilder)) + b, err = bob.BobWithBaseStoreDir(storageDir, bob.WithDir(dir), bob.WithNixBuilder(nixBuilder), bob.WithAllowRedundantTargets()) Expect(err).NotTo(HaveOccurred()) }) diff --git a/test/e2e/multilevelbuild/multilevelbuild_suite_test.go b/test/e2e/multilevelbuild/multilevelbuild_suite_test.go index 71805c09..110733a3 100644 --- a/test/e2e/multilevelbuild/multilevelbuild_suite_test.go +++ b/test/e2e/multilevelbuild/multilevelbuild_suite_test.go @@ -37,7 +37,7 @@ var _ = BeforeSuite(func() { nixBuilder, err := NixBuilder() Expect(err).NotTo(HaveOccurred()) - b, err = bob.BobWithBaseStoreDir(storageDir, bob.WithDir(dir), bob.WithNixBuilder(nixBuilder)) + b, err = bob.BobWithBaseStoreDir(storageDir, bob.WithDir(dir), bob.WithNixBuilder(nixBuilder), bob.WithAllowRedundantTargets()) Expect(err).NotTo(HaveOccurred()) }) diff --git a/test/e2e/target/target_suite_test.go b/test/e2e/target/target_suite_test.go index 6483855c..f839d17b 100644 --- a/test/e2e/target/target_suite_test.go +++ b/test/e2e/target/target_suite_test.go @@ -47,6 +47,7 @@ var _ = BeforeSuite(func() { bob.WithBuildinfoStore(buildInfoStore), bob.WithDir(dir), bob.WithNixBuilder(nixBuilder), + bob.WithAllowRedundantTargets(), ) Expect(err).NotTo(HaveOccurred()) From c4ff7bf6108a9366f8770b226c2bde33f3654c3f Mon Sep 17 00:00:00 2001 From: Andrei Boar Date: Fri, 2 Dec 2022 13:30:32 +0200 Subject: [PATCH 05/10] fix remaining tests --- bob/aggregate_test.go | 12 ++++++------ test/e2e/artifacts/artifacts_extraction_test.go | 2 +- test/e2e/multilevelbuild/multilevelbuild_test.go | 16 ---------------- 3 files changed, 7 insertions(+), 23 deletions(-) diff --git a/bob/aggregate_test.go b/bob/aggregate_test.go index 1eacb11a..5b1e4d7e 100644 --- a/bob/aggregate_test.go +++ b/bob/aggregate_test.go @@ -103,7 +103,7 @@ func benchmarkAggregate(b *testing.B, ignoredMultiplier int) { } // createFileSturcture creates a deep file structure. -// `multiplier`` is the number of directorys created containing the structure. +// `multiplier“ is the number of directorys created containing the structure. func createFileSturcture(dir string, multiplier int) error { for i := 0; i < multiplier; i++ { // create parent @@ -142,7 +142,7 @@ func createFileSturcture(dir string, multiplier int) error { } // createIgnoreFileSturcture creates a deep file structure witch must be ignored by Aggregate(). -// `multiplier`` is the number of directorys created containing the structure. +// `multiplier“ is the number of directorys created containing the structure. func createIgnoreFileSturcture(dir string, multiplier int) error { for i := 0; i < multiplier; i++ { // create parent @@ -190,7 +190,7 @@ func TestEmptyProjectName(t *testing.T) { err = os.Chdir(dir) assert.Nil(t, err) - testBob, err := Bob(WithDir(dir)) + testBob, err := Bob(WithDir(dir), WithAllowRedundantTargets()) assert.Nil(t, err) err = CreatePlayground(PlaygroundOptions{Dir: dir}) @@ -213,7 +213,7 @@ func TestProjectName(t *testing.T) { err = os.Chdir(dir) assert.Nil(t, err) - testBob, err := Bob(WithDir(dir)) + testBob, err := Bob(WithDir(dir), WithAllowRedundantTargets()) assert.Nil(t, err) projectName := "example.com/test-user/test-project" @@ -240,7 +240,7 @@ func TestInvalidProjectName(t *testing.T) { err = os.Chdir(dir) assert.Nil(t, err) - testBob, err := Bob(WithDir(dir)) + testBob, err := Bob(WithDir(dir), WithAllowRedundantTargets()) assert.Nil(t, err) projectName := "@" @@ -333,7 +333,7 @@ func TestMultiLevelBobfileSameProjectName(t *testing.T) { err = os.Chdir(dir) assert.Nil(t, err) - testBob, err := Bob(WithDir(dir)) + testBob, err := Bob(WithDir(dir), WithAllowRedundantTargets()) assert.Nil(t, err) projectName := "first-level" diff --git a/test/e2e/artifacts/artifacts_extraction_test.go b/test/e2e/artifacts/artifacts_extraction_test.go index 57bb25fa..540b0829 100644 --- a/test/e2e/artifacts/artifacts_extraction_test.go +++ b/test/e2e/artifacts/artifacts_extraction_test.go @@ -57,7 +57,7 @@ var _ = Describe("Test artifact creation and extraction", func() { It("extract artifact from store on rebuild", func() { state, err := buildTask(b, bob.BuildTargetwithdirsTargetName) Expect(err).NotTo(HaveOccurred()) - Expect(state.State()).To(Equal(playbook.StateCached)) + Expect(state.State()).To(Equal(playbook.StateCompleted)) }) It("cleanup", func() { diff --git a/test/e2e/multilevelbuild/multilevelbuild_test.go b/test/e2e/multilevelbuild/multilevelbuild_test.go index 86d063aa..90b9ce65 100644 --- a/test/e2e/multilevelbuild/multilevelbuild_test.go +++ b/test/e2e/multilevelbuild/multilevelbuild_test.go @@ -107,14 +107,6 @@ var _ = Describe("Test bob multilevel build", func() { It("checks that we do not require a rebuild of any of the levels", func() { fixtures := []requiresRebuildFixture{ - { - taskname: bob.BuildAllTargetName, - requiresRebuild: false, - }, - { - taskname: "second-level/build2", - requiresRebuild: false, - }, { taskname: "second-level/third-level/build3", requiresRebuild: false, @@ -162,14 +154,6 @@ var _ = Describe("Test bob multilevel build", func() { It("checks that we do not require a rebuild of any of the levels", func() { fixtures := []requiresRebuildFixture{ - { - taskname: bob.BuildAllTargetName, - requiresRebuild: false, - }, - { - taskname: "second-level/build2", - requiresRebuild: false, - }, { taskname: "second-level/third-level/build3", requiresRebuild: false, From 942ee30a3a20c3f1fe32f8f86150a4e2418dd6a3 Mon Sep 17 00:00:00 2001 From: Andrei Boar Date: Fri, 2 Dec 2022 13:31:54 +0200 Subject: [PATCH 06/10] remove todo after option to allow redundant targets --- bob/playground.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bob/playground.go b/bob/playground.go index 8fd4226d..6ac56cee 100644 --- a/bob/playground.go +++ b/bob/playground.go @@ -297,7 +297,7 @@ func createPlaygroundBobfile(dir string, overwrite bool, projectName string) (er "sleep 2", "touch slowdone", }, "\n"), - RebuildDirty: string(bobtask.RebuildAlways), // TODO: Requires Bob to allow rebuild always together with a target. + RebuildDirty: string(bobtask.RebuildAlways), TargetDirty: "slowdone", } @@ -372,7 +372,7 @@ func createPlaygroundBobfile(dir string, overwrite bool, projectName string) (er "touch .bbuild/dirone/dirtwo/fileone", "touch .bbuild/dirone/dirtwo/filetwo", }, "\n"), - RebuildDirty: string(bobtask.RebuildAlways), // TODO: Requires Bob to allow rebuild always together with a target. + RebuildDirty: string(bobtask.RebuildAlways), TargetDirty: ".bbuild/dirone/", } From f2c5cae01d903a5ab1dd2bc30302dd4314044b08 Mon Sep 17 00:00:00 2001 From: Andrei Boar Date: Fri, 2 Dec 2022 14:05:06 +0200 Subject: [PATCH 07/10] fix build: add input * to bobfile --- bob.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bob.yaml b/bob.yaml index 5f8cf4a2..ac562dbc 100644 --- a/bob.yaml +++ b/bob.yaml @@ -5,18 +5,22 @@ variables: build: build: + input: "*" cmd: go build -tags dev -ldflags="-X 'main.Version=${VERSION}'" -o ./run target: run dependson: - proto gomodtidy: + input: "*" cmd: go mod tidy lint: + input: "*" cmd: CGO_ENABLED=0 golangci-lint run --timeout=10m0s test: + input: "*" cmd: go test ./... proto: From bc7d43656d7e8c0feeb7b9c9d90e9021e6241933 Mon Sep 17 00:00:00 2001 From: Andrei Boar Date: Fri, 2 Dec 2022 15:19:06 +0200 Subject: [PATCH 08/10] gray checkmark for no-rebuild required summary --- bob/playbook/state.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bob/playbook/state.go b/bob/playbook/state.go index c19ea375..37c46570 100644 --- a/bob/playbook/state.go +++ b/bob/playbook/state.go @@ -15,7 +15,7 @@ func (s *State) Summary() string { case StateCached: return aurora.Green("cached").String() + " " case StateNoRebuildRequired: - return aurora.Green("no-rebuild").String() + " " + return aurora.Gray(10, "✔").Bold().String() + " " case StateFailed: return aurora.Red("failed").String() + " " case StateCanceled: From c2fbb6523139d69160c174f55ae98e7d0dd76a18 Mon Sep 17 00:00:00 2001 From: Andrei Boar Date: Fri, 2 Dec 2022 15:51:14 +0200 Subject: [PATCH 09/10] show dash for compound tasks in summary --- bob/playbook/summary.go | 7 +++++-- bobtask/task.go | 5 +---- test/e2e/tasksemantics/semantics_test.go | 5 ++--- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/bob/playbook/summary.go b/bob/playbook/summary.go index fc03aa0e..94d62bbe 100644 --- a/bob/playbook/summary.go +++ b/bob/playbook/summary.go @@ -11,7 +11,6 @@ import ( // summary prints the tasks processing details as a summary of the playbook. func (p *Playbook) summary(processedTasks []*bobtask.Task) { - boblog.Log.V(1).Info("") boblog.Log.V(1).Info(aurora.Bold("● ● ● ●").BrightGreen().String()) @@ -32,7 +31,11 @@ func (p *Playbook) summary(processedTasks []*bobtask.Task) { } taskName := t.Name() - boblog.Log.V(1).Info(fmt.Sprintf(" %-*s\t%s%s", p.namePad, taskName, status.Summary(), execTime)) + if t.IsCompoundTask() { + boblog.Log.V(1).Info(fmt.Sprintf(" %-*s\t%s%s", p.namePad, taskName, "-", execTime)) + } else { + boblog.Log.V(1).Info(fmt.Sprintf(" %-*s\t%s%s", p.namePad, taskName, status.Summary(), execTime)) + } } boblog.Log.V(1).Info("") } diff --git a/bobtask/task.go b/bobtask/task.go index 319a6b8b..fe7ecde5 100644 --- a/bobtask/task.go +++ b/bobtask/task.go @@ -29,7 +29,7 @@ type Task struct { InputDirty string `yaml:"input,omitempty"` // InputAdditionalIgnores is a list of ignores // usually the child targets. - InputAdditionalIgnores []string `yaml:"input_additional_ignores,omitempty"` + InputAdditionalIgnores []string // inputs is filtered by ignored & sanitized inputs []string @@ -172,9 +172,6 @@ func (t *Task) IsCompoundTask() bool { if t.InputDirty != "" { return false } - if len(t.InputAdditionalIgnores) > 0 { - return false - } if t.CmdDirty != "" { return false } diff --git a/test/e2e/tasksemantics/semantics_test.go b/test/e2e/tasksemantics/semantics_test.go index 4079e3f9..daf22909 100644 --- a/test/e2e/tasksemantics/semantics_test.go +++ b/test/e2e/tasksemantics/semantics_test.go @@ -188,9 +188,8 @@ var _ = Describe("Testing task semantics behaviour", func() { out := output() Expect(out).To(ContainSubstring(buildCached.Summary())) - // build is marked as no-rebuild - buildNotRequired := playbook.StateNoRebuildRequired - Expect(out).To(ContainSubstring(buildNotRequired.Summary())) + // build is marked as no-rebuild and shown with - + Expect(out).To(ContainSubstring("-")) }) It("it will be rebuilt when input * changes", func() { From 9bb9c7e192983d70d74d9bab845075135850fb02 Mon Sep 17 00:00:00 2001 From: Andrei Boar Date: Wed, 7 Dec 2022 10:31:08 +0200 Subject: [PATCH 10/10] rename to enableRedundatChanges --- bob/aggregate.go | 2 +- bob/aggregate_test.go | 8 ++++---- bob/bob.go | 4 ++-- bob/options.go | 4 ++-- test/e2e/artifacts/artifacts_suite_test.go | 2 +- test/e2e/build/build_suite_test.go | 2 +- test/e2e/ignore/ignore_suite_test.go | 2 +- test/e2e/multilevelbuild/multilevelbuild_suite_test.go | 2 +- test/e2e/target/target_suite_test.go | 2 +- 9 files changed, 14 insertions(+), 14 deletions(-) diff --git a/bob/aggregate.go b/bob/aggregate.go index 8cdf3e31..945fc2ec 100644 --- a/bob/aggregate.go +++ b/bob/aggregate.go @@ -217,7 +217,7 @@ func (b *B) Aggregate() (aggregate *bobfile.Bobfile, err error) { aggregate.Project = aggregate.Dir() } - err = aggregate.Verify(b.enableCaching, b.allowRedundantTargets) + err = aggregate.Verify(b.enableCaching, b.enableRedundantTargets) errz.Fatal(err) err = aggregate.BTasks.IgnoreChildTargets() diff --git a/bob/aggregate_test.go b/bob/aggregate_test.go index 5b1e4d7e..b0a2211b 100644 --- a/bob/aggregate_test.go +++ b/bob/aggregate_test.go @@ -190,7 +190,7 @@ func TestEmptyProjectName(t *testing.T) { err = os.Chdir(dir) assert.Nil(t, err) - testBob, err := Bob(WithDir(dir), WithAllowRedundantTargets()) + testBob, err := Bob(WithDir(dir), WithEnableRedundantTargets()) assert.Nil(t, err) err = CreatePlayground(PlaygroundOptions{Dir: dir}) @@ -213,7 +213,7 @@ func TestProjectName(t *testing.T) { err = os.Chdir(dir) assert.Nil(t, err) - testBob, err := Bob(WithDir(dir), WithAllowRedundantTargets()) + testBob, err := Bob(WithDir(dir), WithEnableRedundantTargets()) assert.Nil(t, err) projectName := "example.com/test-user/test-project" @@ -240,7 +240,7 @@ func TestInvalidProjectName(t *testing.T) { err = os.Chdir(dir) assert.Nil(t, err) - testBob, err := Bob(WithDir(dir), WithAllowRedundantTargets()) + testBob, err := Bob(WithDir(dir), WithEnableRedundantTargets()) assert.Nil(t, err) projectName := "@" @@ -333,7 +333,7 @@ func TestMultiLevelBobfileSameProjectName(t *testing.T) { err = os.Chdir(dir) assert.Nil(t, err) - testBob, err := Bob(WithDir(dir), WithAllowRedundantTargets()) + testBob, err := Bob(WithDir(dir), WithEnableRedundantTargets()) assert.Nil(t, err) projectName := "first-level" diff --git a/bob/bob.go b/bob/bob.go index 5258887f..7745acb4 100644 --- a/bob/bob.go +++ b/bob/bob.go @@ -47,10 +47,10 @@ type B struct { // from the cache Default: true enableCaching bool - // allowRedundantTargets option allows redundant targets on build tasks + // enableRedundantTargets option allows redundant targets on build tasks // a task has redundant target when it has a `target` used in combination with `rebuild:always` // by default a user error will return when a task has both `target` and `rebuild:always` defined - allowRedundantTargets bool + enableRedundantTargets bool // allowInsecure uses http protocol for accessing the remote artifact store, if any allowInsecure bool diff --git a/bob/options.go b/bob/options.go index e707eac7..6247455a 100644 --- a/bob/options.go +++ b/bob/options.go @@ -52,9 +52,9 @@ func WithCachingEnabled(enabled bool) Option { } } -func WithAllowRedundantTargets() Option { +func WithEnableRedundantTargets() Option { return func(b *B) { - b.allowRedundantTargets = true + b.enableRedundantTargets = true } } diff --git a/test/e2e/artifacts/artifacts_suite_test.go b/test/e2e/artifacts/artifacts_suite_test.go index 263b6b01..4201518f 100644 --- a/test/e2e/artifacts/artifacts_suite_test.go +++ b/test/e2e/artifacts/artifacts_suite_test.go @@ -86,7 +86,7 @@ var _ = BeforeSuite(func() { bob.WithFilestore(artifactStore), bob.WithBuildinfoStore(buildinfoStore), bob.WithNixBuilder(nixBuilder), - bob.WithAllowRedundantTargets(), + bob.WithEnableRedundantTargets(), ) Expect(err).NotTo(HaveOccurred()) diff --git a/test/e2e/build/build_suite_test.go b/test/e2e/build/build_suite_test.go index 3522bbf5..2a314b06 100644 --- a/test/e2e/build/build_suite_test.go +++ b/test/e2e/build/build_suite_test.go @@ -32,7 +32,7 @@ var _ = BeforeSuite(func() { nixBuilder, err := NixBuilder() Expect(err).NotTo(HaveOccurred()) - b, err = bob.BobWithBaseStoreDir(storageDir, bob.WithDir(dir), bob.WithNixBuilder(nixBuilder), bob.WithAllowRedundantTargets()) + b, err = bob.BobWithBaseStoreDir(storageDir, bob.WithDir(dir), bob.WithNixBuilder(nixBuilder), bob.WithEnableRedundantTargets()) Expect(err).NotTo(HaveOccurred()) }) diff --git a/test/e2e/ignore/ignore_suite_test.go b/test/e2e/ignore/ignore_suite_test.go index 52f03340..17611226 100644 --- a/test/e2e/ignore/ignore_suite_test.go +++ b/test/e2e/ignore/ignore_suite_test.go @@ -33,7 +33,7 @@ var _ = BeforeSuite(func() { nixBuilder, err := NixBuilder() Expect(err).NotTo(HaveOccurred()) - b, err = bob.BobWithBaseStoreDir(storageDir, bob.WithDir(dir), bob.WithNixBuilder(nixBuilder), bob.WithAllowRedundantTargets()) + b, err = bob.BobWithBaseStoreDir(storageDir, bob.WithDir(dir), bob.WithNixBuilder(nixBuilder), bob.WithEnableRedundantTargets()) Expect(err).NotTo(HaveOccurred()) }) diff --git a/test/e2e/multilevelbuild/multilevelbuild_suite_test.go b/test/e2e/multilevelbuild/multilevelbuild_suite_test.go index 110733a3..ed77b31d 100644 --- a/test/e2e/multilevelbuild/multilevelbuild_suite_test.go +++ b/test/e2e/multilevelbuild/multilevelbuild_suite_test.go @@ -37,7 +37,7 @@ var _ = BeforeSuite(func() { nixBuilder, err := NixBuilder() Expect(err).NotTo(HaveOccurred()) - b, err = bob.BobWithBaseStoreDir(storageDir, bob.WithDir(dir), bob.WithNixBuilder(nixBuilder), bob.WithAllowRedundantTargets()) + b, err = bob.BobWithBaseStoreDir(storageDir, bob.WithDir(dir), bob.WithNixBuilder(nixBuilder), bob.WithEnableRedundantTargets()) Expect(err).NotTo(HaveOccurred()) }) diff --git a/test/e2e/target/target_suite_test.go b/test/e2e/target/target_suite_test.go index f839d17b..e57aa8ea 100644 --- a/test/e2e/target/target_suite_test.go +++ b/test/e2e/target/target_suite_test.go @@ -47,7 +47,7 @@ var _ = BeforeSuite(func() { bob.WithBuildinfoStore(buildInfoStore), bob.WithDir(dir), bob.WithNixBuilder(nixBuilder), - bob.WithAllowRedundantTargets(), + bob.WithEnableRedundantTargets(), ) Expect(err).NotTo(HaveOccurred())