From 7ffd617684076b0accc97e7c7b2d02c348cab74c Mon Sep 17 00:00:00 2001 From: Daniel Ketterer <26778610+dketterer@users.noreply.github.com> Date: Fri, 8 Jul 2022 13:52:17 +0200 Subject: [PATCH 1/5] Add skeleton code for bob sync cli --- bob/bobfile/bobfile.go | 4 ++ bob/sync.go | 47 ++++++++++++++++++ bobsync/map.go | 3 ++ bobsync/sync.go | 27 +++++++++++ bobsync/verify.go | 9 ++++ cli/cmd_root.go | 7 +++ cli/cmd_sync.go | 106 +++++++++++++++++++++++++++++++++++++++++ 7 files changed, 203 insertions(+) create mode 100644 bob/sync.go create mode 100644 bobsync/map.go create mode 100644 bobsync/sync.go create mode 100644 bobsync/verify.go create mode 100644 cli/cmd_sync.go diff --git a/bob/bobfile/bobfile.go b/bob/bobfile/bobfile.go index e279e595..8e097979 100644 --- a/bob/bobfile/bobfile.go +++ b/bob/bobfile/bobfile.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strings" + "github.com/benchkram/bob/bobsync" "github.com/benchkram/bob/pkg/nix" storeclient "github.com/benchkram/bob/pkg/store-client" @@ -79,6 +80,9 @@ type Bobfile struct { // Nixpkgs specifies an optional nixpkgs source. Nixpkgs string `yaml:"nixpkgs"` + // SyncCollections are folder synchronisations through bob-server + SyncCollections bobsync.SyncMap `yaml:"sync"` + // Parent directory of the Bobfile. // Populated through BobfileRead(). dir string diff --git a/bob/sync.go b/bob/sync.go new file mode 100644 index 00000000..e700ad39 --- /dev/null +++ b/bob/sync.go @@ -0,0 +1,47 @@ +package bob + +import ( + "context" + "os" + + "github.com/benchkram/bob/bob/bobfile" + "github.com/benchkram/errz" +) + +func (b *B) SyncPush(ctx context.Context) (err error) { + defer errz.Recover(&err) + + wd, _ := os.Getwd() + aggregate, err := bobfile.BobfileRead(wd) + + for _, sync := range aggregate.SyncCollections { + err = sync.Push(ctx) + errz.Fatal(err) + } + + return nil +} + +func (b *B) SyncPull(ctx context.Context) (err error) { + defer errz.Recover(&err) + + //aggregate, err := b.Aggregate() + + return nil +} + +func (b *B) SyncListLocal(ctx context.Context) (err error) { + defer errz.Recover(&err) + + //aggregate, err := b.Aggregate() + + return nil +} + +func (b *B) SyncListRemote(ctx context.Context) (err error) { + defer errz.Recover(&err) + + //aggregate, err := b.Aggregate() + + return nil +} diff --git a/bobsync/map.go b/bobsync/map.go new file mode 100644 index 00000000..886ba9ac --- /dev/null +++ b/bobsync/map.go @@ -0,0 +1,3 @@ +package bobsync + +type SyncMap map[string]Sync diff --git a/bobsync/sync.go b/bobsync/sync.go new file mode 100644 index 00000000..581f21c8 --- /dev/null +++ b/bobsync/sync.go @@ -0,0 +1,27 @@ +package bobsync + +import "context" + +type Sync struct { + name string + + Path string `yaml:"path"` + + Version string `yaml:"version"` +} + +func (s *Sync) Push(ctx context.Context) (err error) { + return nil +} + +func (s *Sync) Pull(ctx context.Context) (err error) { + return nil +} + +func (s *Sync) ListLocal(ctx context.Context) (err error) { + return nil +} + +func (s *Sync) ListRemote(ctx context.Context) (err error) { + return nil +} diff --git a/bobsync/verify.go b/bobsync/verify.go new file mode 100644 index 00000000..ca67ebe0 --- /dev/null +++ b/bobsync/verify.go @@ -0,0 +1,9 @@ +package bobsync + +// after reading bobfile, need to verify that: +// * collection root and all underneath is in .gitignore [could be allowed with a warning] +// * collection root is somewhere in bob workspace [could be optional] +// * name and version are not allowed to include the separator char + +// after parsing the tree of a collection, need to verify that: +// * no symlinks included diff --git a/cli/cmd_root.go b/cli/cmd_root.go index d85dacee..d5c33e9f 100644 --- a/cli/cmd_root.go +++ b/cli/cmd_root.go @@ -58,6 +58,13 @@ func init() { CmdGit.AddCommand(CmdGitStatus) rootCmd.AddCommand(CmdGit) + // syncCmd + cmdSync.AddCommand(cmdSyncPush) + cmdSync.AddCommand(cmdSyncPull) + cmdSync.AddCommand(cmdSyncList) + cmdSync.AddCommand(cmdSyncListRemote) + rootCmd.AddCommand(cmdSync) + // authCmd AuthCmd.AddCommand(AuthContextCreateCmd) AuthContextCreateCmd.Flags().StringP("token", "t", "", "The token used for authentication") diff --git a/cli/cmd_sync.go b/cli/cmd_sync.go new file mode 100644 index 00000000..e9262af9 --- /dev/null +++ b/cli/cmd_sync.go @@ -0,0 +1,106 @@ +package cli + +import ( + "context" + "errors" + "os" + "os/signal" + "syscall" + + "github.com/benchkram/bob/bob" + "github.com/benchkram/bob/pkg/boblog" + "github.com/benchkram/bob/pkg/usererror" + "github.com/benchkram/errz" + "github.com/spf13/cobra" +) + +var cmdSync = &cobra.Command{ + Use: "sync", + Short: "Sync (binary) test data via a bob-server.", + Args: cobra.MinimumNArgs(0), + Long: ``, + FParseErrWhitelist: cobra.FParseErrWhitelist{ + UnknownFlags: true, + }, + Run: func(cmd *cobra.Command, args []string) { + // do nothing just show if the server can be contacted and maybe display status information + }, +} + +var cmdSyncPush = &cobra.Command{ + Use: "push", + Short: "Make server collections exactly like local", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + allowInsecure, err := cmd.Flags().GetBool("insecure") + errz.Fatal(err) + + runPush(allowInsecure) + }, +} + +var cmdSyncPull = &cobra.Command{ + Use: "pull", + Short: "Make local collections exactly like server", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + + }, +} + +var cmdSyncList = &cobra.Command{ + Use: "ls", + Short: "List files synced", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + + }, +} + +var cmdSyncListRemote = &cobra.Command{ + Use: "ls-remote", + Short: "List collections on remote", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + + }, +} + +func runPush(allowInsecure bool) { + var exitCode int + defer func() { + exit(exitCode) + }() + defer errz.Recover() + + b, err := bob.Bob( + bob.WithInsecure(allowInsecure), + ) + boblog.Log.Error(err, "Unable to initialize bob") + + if err != nil { + exitCode = 1 + errz.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go func() { + stop := make(chan os.Signal, 1) + signal.Notify(stop, os.Interrupt, syscall.SIGTERM) + + <-stop + cancel() + }() + + err = b.SyncPush(ctx) + if err != nil { + exitCode = 1 + if errors.As(err, &usererror.Err) { + boblog.Log.UserError(err) + } else { + errz.Fatal(err) + } + } +} From 5e67c718bc9a0699881a6c1036116418374d6285 Mon Sep 17 00:00:00 2001 From: Daniel Ketterer <26778610+dketterer@users.noreply.github.com> Date: Wed, 20 Jul 2022 11:28:55 +0200 Subject: [PATCH 2/5] Add basic sync with ls, ls-remote, push and pull --- bob/aggregate.go | 3 +- bob/bobfile/bobfile.go | 44 + bob/sync.go | 65 +- bobsync/delta.go | 111 ++ bobsync/hashcache.go | 156 ++ bobsync/sync.go | 212 ++- cli/cmd_root.go | 3 + cli/cmd_sync.go | 123 +- pkg/file/file.go | 9 + pkg/filehash/hash.go | 10 + pkg/store-client/client.go | 381 ++++- pkg/store-client/generated/client.gen.go | 1455 +++++++++++++++++- pkg/store-client/generated/types.gen.go | 77 + pkg/store-client/store_client.go | 10 + pkg/versionedsync/collection/collection.go | 73 + pkg/versionedsync/file/file.go | 34 + pkg/versionedsync/localsyncstore/options.go | 1 + pkg/versionedsync/localsyncstore/store.go | 41 + pkg/versionedsync/remotesyncstore/errors.go | 7 + pkg/versionedsync/remotesyncstore/options.go | 11 + pkg/versionedsync/remotesyncstore/store.go | 106 ++ 21 files changed, 2837 insertions(+), 95 deletions(-) create mode 100644 bobsync/delta.go create mode 100644 bobsync/hashcache.go create mode 100644 pkg/versionedsync/collection/collection.go create mode 100644 pkg/versionedsync/file/file.go create mode 100644 pkg/versionedsync/localsyncstore/options.go create mode 100644 pkg/versionedsync/localsyncstore/store.go create mode 100644 pkg/versionedsync/remotesyncstore/errors.go create mode 100644 pkg/versionedsync/remotesyncstore/options.go create mode 100644 pkg/versionedsync/remotesyncstore/store.go diff --git a/bob/aggregate.go b/bob/aggregate.go index b0e91efc..d14bea0e 100644 --- a/bob/aggregate.go +++ b/bob/aggregate.go @@ -232,7 +232,7 @@ func (b *B) Aggregate() (aggregate *bobfile.Bobfile, err error) { aggregate.Dependencies = make([]string, 0) aggregate.Dependencies = append(aggregate.Dependencies, allDeps...) - // Initialize remote store in case of a valid remote url / project name + // Initialize remote store for artifacts and sync in case of a valid remote url / project name if aggregate.Project != "" { projectName, err := project.Parse(aggregate.Project) if err != nil { @@ -259,6 +259,7 @@ func (b *B) Aggregate() (aggregate *bobfile.Bobfile, err error) { } else { boblog.Log.V(1).Info(fmt.Sprintf("Using remote store: %s", url.String())) aggregate.SetRemotestore(bobfile.NewRemotestore(url, b.allowInsecure, authCtx.Token)) + aggregate.SetVersionedSyncStore(bobfile.NewVersionedSyncStore(url, b.allowInsecure, authCtx.Token)) } } } else { diff --git a/bob/bobfile/bobfile.go b/bob/bobfile/bobfile.go index 8e097979..58b2de0d 100644 --- a/bob/bobfile/bobfile.go +++ b/bob/bobfile/bobfile.go @@ -3,6 +3,7 @@ package bobfile import ( "bytes" "fmt" + "github.com/benchkram/bob/pkg/versionedsync/remotesyncstore" "io/ioutil" "net/url" "path/filepath" @@ -91,6 +92,8 @@ type Bobfile struct { RemoteStoreHost string remotestore store.Store + + versionedSyncStore *remotesyncstore.S } func NewBobfile() *Bobfile { @@ -118,6 +121,14 @@ func (b *Bobfile) Remotestore() store.Store { return b.remotestore } +func (b *Bobfile) SetVersionedSyncStore(syncStore *remotesyncstore.S) { + b.versionedSyncStore = syncStore +} + +func (b *Bobfile) VersionedSyncStore() *remotesyncstore.S { + return b.versionedSyncStore +} + // bobfileRead reads a bobfile and initializes private fields. func bobfileRead(dir string) (_ *Bobfile, err error) { defer errz.Recover(&err) @@ -205,6 +216,13 @@ func bobfileRead(dir string) (_ *Bobfile, err error) { // bobfile.Project = bobfile.dir //} + // write names to Sync objects + for name, _ := range bobfile.SyncCollections { + s := bobfile.SyncCollections[name] + s.SetName(name) + bobfile.SyncCollections[name] = s + } + return bobfile, nil } @@ -249,6 +267,30 @@ func NewRemotestore(endpoint *url.URL, allowInsecure bool, token string) (s stor return s } +func NewVersionedSyncStore(endpoint *url.URL, allowInsecure bool, token string) (s *remotesyncstore.S) { + const sep = "/" + + parts := strings.Split(strings.TrimLeft(endpoint.Path, sep), sep) + + username := parts[0] + proj := strings.Join(parts[1:], sep) + + protocol := "https://" + if allowInsecure { + protocol = "http://" + } + + s = remotesyncstore.New( + username, + proj, + + remotesyncstore.WithClient( + storeclient.New(protocol+endpoint.Host, token), + ), + ) + return s +} + // BobfileRead read from a bobfile. // Calls sanitize on the result. func BobfileRead(dir string) (_ *Bobfile, err error) { @@ -339,6 +381,8 @@ func (b *Bobfile) Validate() (err error) { } } + // TODO: validate sync entries + return nil } diff --git a/bob/sync.go b/bob/sync.go index e700ad39..2a5bc855 100644 --- a/bob/sync.go +++ b/bob/sync.go @@ -2,6 +2,9 @@ package bob import ( "context" + "fmt" + "github.com/benchkram/bob/pkg/versionedsync/localsyncstore" + "github.com/logrusorgru/aurora" "os" "github.com/benchkram/bob/bob/bobfile" @@ -13,10 +16,19 @@ func (b *B) SyncPush(ctx context.Context) (err error) { wd, _ := os.Getwd() aggregate, err := bobfile.BobfileRead(wd) + aggregate, err = b.Aggregate() + errz.Fatal(err) - for _, sync := range aggregate.SyncCollections { - err = sync.Push(ctx) - errz.Fatal(err) + remoteStore := aggregate.VersionedSyncStore() + localStore := localsyncstore.New() + + if remoteStore == nil { + fmt.Println(aurora.Red("No remote project configured can not push")) + } else { + for _, sync := range aggregate.SyncCollections { + err = sync.Push(ctx, *remoteStore, *localStore, aggregate.Dir()) + errz.Fatal(err) + } } return nil @@ -25,15 +37,39 @@ func (b *B) SyncPush(ctx context.Context) (err error) { func (b *B) SyncPull(ctx context.Context) (err error) { defer errz.Recover(&err) - //aggregate, err := b.Aggregate() + wd, _ := os.Getwd() + aggregate, err := bobfile.BobfileRead(wd) + aggregate, err = b.Aggregate() + errz.Fatal(err) + + remoteStore := aggregate.VersionedSyncStore() + localStore := localsyncstore.New() + + if remoteStore == nil { + fmt.Println(aurora.Red("no remote project configured can not pull")) + } else { + for _, sync := range aggregate.SyncCollections { + err = sync.Pull(ctx, *remoteStore, *localStore, aggregate.Dir()) + errz.Fatal(err) + } + } return nil } -func (b *B) SyncListLocal(ctx context.Context) (err error) { +func (b *B) SyncListLocal(_ context.Context) (err error) { defer errz.Recover(&err) - //aggregate, err := b.Aggregate() + wd, _ := os.Getwd() + aggregate, err := bobfile.BobfileRead(wd) + errz.Fatal(err) + + fmt.Printf("bob sync ls: displaying all files ready to by synced\n\n") + + for _, sync := range aggregate.SyncCollections { + err = sync.ListLocal(aggregate.Dir()) + errz.Fatal(err) + } return nil } @@ -41,7 +77,22 @@ func (b *B) SyncListLocal(ctx context.Context) (err error) { func (b *B) SyncListRemote(ctx context.Context) (err error) { defer errz.Recover(&err) - //aggregate, err := b.Aggregate() + wd, _ := os.Getwd() + aggregate, err := bobfile.BobfileRead(wd) + errz.Fatal(err) + aggregate, err = b.Aggregate() + errz.Fatal(err) + + remoteStore := aggregate.VersionedSyncStore() + + if remoteStore == nil { + fmt.Println(aurora.Red("No remote project configured can not list remote")) + } else { + for _, sync := range aggregate.SyncCollections { + err = sync.ListRemote(ctx, *remoteStore) + errz.Fatal(err) + } + } return nil } diff --git a/bobsync/delta.go b/bobsync/delta.go new file mode 100644 index 00000000..a48ff0fb --- /dev/null +++ b/bobsync/delta.go @@ -0,0 +1,111 @@ +package bobsync + +import ( + "fmt" + "github.com/benchkram/bob/pkg/versionedsync/collection" + "github.com/benchkram/bob/pkg/versionedsync/file" +) + +type Delta struct { + // Unchanged are files which have the same hash on local and remote + // Files in this slice should always have an ID set + Unchanged []*file.F + // ToBeUpdated are files which exist on local and remote but have different hashes + // Files in this slice should always have an ID set + ToBeUpdated []*file.F + // LocalFilesMissingOnRemote can be read different for push and pull + // push: what has to be created on the remote and is only on local + // pull: what has to be removed on local since it is not on remote + // Files in this slice never have an ID set + LocalFilesMissingOnRemote []*file.F + // RemoteFilesMissingOnLocal can be read different for push and pull + // push: what has to be removed on the remote and is only on remote + // pull: what has to be created on local since it is only on remote + // Files in this slice should always have an ID set + RemoteFilesMissingOnLocal []*file.F +} + +func (d *Delta) String() string { + result := "" + + for _, f := range d.Unchanged { + result += fmt.Sprintf("(unchanged) %s\n", f.LocalPath) + } + for _, f := range d.ToBeUpdated { + result += fmt.Sprintf("(changed) %s\n", f.LocalPath) + } + for _, f := range d.LocalFilesMissingOnRemote { + result += fmt.Sprintf("(local only) %s\n", f.LocalPath) + } + for _, f := range d.RemoteFilesMissingOnLocal { + result += fmt.Sprintf("(remote only) %s\n", f.LocalPath) + } + return result +} + +func (d *Delta) PushOverview() string { + result := "" + + for _, f := range d.Unchanged { + result += fmt.Sprintf("(unchanged) %s\n", f.LocalPath) + } + for _, f := range d.ToBeUpdated { + result += fmt.Sprintf("(override server) %s\n", f.LocalPath) + } + for _, f := range d.LocalFilesMissingOnRemote { + result += fmt.Sprintf("(upload) %s\n", f.LocalPath) + } + for _, f := range d.RemoteFilesMissingOnLocal { + result += fmt.Sprintf("(delete remote) %s\n", f.LocalPath) + } + return result +} + +func (d *Delta) PullOverview() string { + result := "" + + for _, f := range d.Unchanged { + result += fmt.Sprintf("(unchanged) %s\n", f.LocalPath) + } + for _, f := range d.ToBeUpdated { + result += fmt.Sprintf("(override local) %s\n", f.LocalPath) + } + for _, f := range d.LocalFilesMissingOnRemote { + result += fmt.Sprintf("(delete local) %s\n", f.LocalPath) + } + for _, f := range d.RemoteFilesMissingOnLocal { + result += fmt.Sprintf("(download) %s\n", f.LocalPath) + } + return result + +} + +// NewDelta creates a delta that describes differences between local and remote +func NewDelta(local HashCache, remote collection.C) *Delta { + delta := &Delta{} + + for _, remoteF := range remote.Files { + fingerprint, ok := local[remoteF.LocalPath] + if ok && fingerprint.Hash == remoteF.Hash { + delta.Unchanged = append(delta.Unchanged, remoteF) + } else if ok { + // local and remote differ + delta.ToBeUpdated = append(delta.ToBeUpdated, remoteF) + } else { + // remote file non-existent on local + delta.RemoteFilesMissingOnLocal = append(delta.RemoteFilesMissingOnLocal, remoteF) + } + } + for localPath, fingerprint := range local { + _, ok := remote.FileByPath(localPath) + if !ok { + // localPath non-existent on remote + delta.LocalFilesMissingOnRemote = append(delta.LocalFilesMissingOnRemote, + &file.F{ + LocalPath: localPath, + Hash: fingerprint.Hash, + }) + } + } + return delta +} diff --git a/bobsync/hashcache.go b/bobsync/hashcache.go new file mode 100644 index 00000000..4fa78c2d --- /dev/null +++ b/bobsync/hashcache.go @@ -0,0 +1,156 @@ +package bobsync + +import ( + "encoding/json" + "fmt" + "github.com/benchkram/bob/pkg/boblog" + "github.com/benchkram/bob/pkg/file" + "github.com/benchkram/bob/pkg/filehash" + "github.com/benchkram/bob/pkg/filepathutil" + "github.com/benchkram/errz" + "github.com/logrusorgru/aurora" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "sync" + "time" +) + +type Fingerprint struct { + Hash string + CreatedAt time.Time +} + +// HashCache is a map from a file path string to a fingerprint +// paths are relative paths inside a collection to get the abs path do join (bobDir, collectionPath, thisPath) +type HashCache map[string]Fingerprint + +func FromFileOrNew(path string) (hc *HashCache, err error) { + defer errz.Recover(&err) + fileInfo, err := os.Stat(path) + if err != nil { + hc := &HashCache{} + return hc, nil + } + if fileInfo.IsDir() { + return nil, fmt.Errorf("failed to load hashcache from: %s: it is a directory", path) + } + f, err := os.Open(path) + defer f.Close() + errz.Fatal(err) + byteValue, err := ioutil.ReadAll(f) + errz.Fatal(err) + err = json.Unmarshal(byteValue, &hc) + errz.Fatal(err) + return hc, nil +} + +func (h *HashCache) SaveToFile(path string) (err error) { + defer errz.Recover(&err) + data, err := json.Marshal(*h) + errz.Fatal(err) + if file.Exists(path) { + err := os.Remove(path) + errz.Fatal(err) + } + err = ioutil.WriteFile(path, data, 0644) + errz.Fatal(err) + return nil +} + +func (h *HashCache) Update(basePath string) (err error) { + defer errz.Recover(&err) + filePaths, err := filepathutil.ListRecursive(basePath) + + saveMap := h.toInternalMap(filePaths, basePath) + + // analogue to https://gobyexample.com/worker-pools + numJobs := len(filePaths) + jobs := make(chan string, numJobs) + results := make(chan error, numJobs) + + numWorkers := runtime.NumCPU() + for w := 1; w <= numWorkers; w++ { + go updater(saveMap, jobs, results) + } + for _, f := range filePaths { + jobs <- f + } + close(jobs) + + for a := 1; a <= numJobs; a++ { + e := <-results + if e != nil { + boblog.Log.Error(e, e.Error()) + err = e + } + } + if err != nil { + fmt.Println(aurora.Red("failed to hash some files and will ignore them")) + } + err = h.overrideFromInternalMap(saveMap, basePath) + errz.Fatal(err) + return nil + +} + +func updater(saveMap *sync.Map, files <-chan string, result chan<- error) { + for f := range files { + var reHash bool + oldFingerprint, ok := saveMap.Load(f) + if ok { + var oldFingerprint = oldFingerprint.(Fingerprint) + lastMod, err := file.LastModTime(f) + if err != nil { + result <- err + continue + } + if oldFingerprint.CreatedAt.Before(lastMod) { + reHash = true + } + } else { + reHash = true + } + if reHash { + h, err := filehash.HashAsString(f) + if err != nil { + result <- err + continue + } + fp := Fingerprint{ + Hash: h, + CreatedAt: time.Now(), + } + saveMap.Store(f, fp) + } + result <- nil + } +} + +// toInternalMap copies all paths in the filePaths argument from the HashCache to a sync.Map which can be safely used in +//concurrent runs +func (h *HashCache) toInternalMap(filePaths []string, basePath string) *sync.Map { + saveMap := &sync.Map{} + for _, f := range filePaths { + fp, ok := (*h)[f] + if ok { + saveMap.Store(filepath.Join(basePath, f), fp) + } + } + return saveMap +} + +func (h *HashCache) overrideFromInternalMap(saveMap *sync.Map, basePath string) (err error) { + defer errz.Recover(&err) + for k := range *h { + delete(*h, k) + } + saveMap.Range(func(key, value interface{}) bool { + relPath, err := filepath.Rel(basePath, key.(string)) + errz.Fatal(err) + (*h)[relPath] = value.(Fingerprint) + return true + }) + return nil +} diff --git a/bobsync/sync.go b/bobsync/sync.go index 581f21c8..9830932f 100644 --- a/bobsync/sync.go +++ b/bobsync/sync.go @@ -1,27 +1,229 @@ package bobsync -import "context" +import ( + "context" + "fmt" + "github.com/benchkram/bob/pkg/versionedsync/localsyncstore" + "github.com/benchkram/bob/pkg/versionedsync/remotesyncstore" + "github.com/benchkram/errz" + "github.com/logrusorgru/aurora" + "path/filepath" +) +const ( + hashCachePath = ".bob.hashcache" +) + +// Sync is a collection of versioned synced files type Sync struct { name string Path string `yaml:"path"` Version string `yaml:"version"` + + remoteCollectionId string + + cache *HashCache +} + +func (s *Sync) SetName(name string) { + s.name = name } -func (s *Sync) Push(ctx context.Context) (err error) { +func (s *Sync) Push(ctx context.Context, remoteStore remotesyncstore.S, localStore localsyncstore.S, bobDir string) (err error) { + defer errz.Recover(&err) + + var collectionMustBeCreated bool + // get collectionId ready + var collectionId string + collectionId, err = remoteStore.CollectionIdByName(ctx, s.name, s.Version) + + // check if collections exists + switch err { + case nil: + case remotesyncstore.ErrCollectionNotFound: + collectionMustBeCreated = true + collectionId, err = remoteStore.CollectionCreate(ctx, s.name, s.Version, s.Path) + errz.Fatal(err) + default: + errz.Fatal(err) + } + + absHashCachePath := filepath.Join(bobDir, hashCachePath) + if s.cache == nil { + s.cache, err = FromFileOrNew(absHashCachePath) + errz.Fatal(err) + } + + // TODO: run list local and remote in parallel + err = s.cache.Update(filepath.Join(bobDir, s.Path)) + errz.Fatal(err) + err = s.cache.SaveToFile(absHashCachePath) + errz.Fatal(err) + + remoteCollection, err := remoteStore.Collection(ctx, collectionId) + errz.Fatal(err) + + // create the delta + delta := NewDelta(*s.cache, *remoteCollection) + + fmt.Printf("Local-Remote delta for %s\n", aurora.Bold(s.name)) + fmt.Println(delta.PushOverview()) + + if !collectionMustBeCreated { + // TODO: prompt user and seek confirmation + } + + for _, f := range delta.LocalFilesMissingOnRemote { + srcReader, err := localStore.ReadFile(bobDir, s.Path, f.LocalPath) + errz.Fatal(err) + err = remoteStore.FileUpload(ctx, collectionId, f.LocalPath, srcReader) + errz.Fatal(err) + } + for _, f := range delta.RemoteFilesMissingOnLocal { + if f.ID == nil { + return fmt.Errorf("ID not available can not delete from remote") + } + err = remoteStore.FileDelete(ctx, collectionId, *f.ID) + errz.Fatal(err) + } + for _, f := range delta.ToBeUpdated { + if f.ID == nil { + return fmt.Errorf("ID not available can not update on remote") + } + srcReader, err := localStore.ReadFile(bobDir, s.Path, f.LocalPath) + errz.Fatal(err) + err = remoteStore.FileUpdate(ctx, collectionId, *f.ID, srcReader) + errz.Fatal(err) + } + return nil } -func (s *Sync) Pull(ctx context.Context) (err error) { +func (s *Sync) Pull(ctx context.Context, remoteStore remotesyncstore.S, localStore localsyncstore.S, bobDir string) (err error) { + defer errz.Recover(&err) + + var collectionMustBeCreated bool + // get collectionId ready + var collectionId string + collectionId, err = remoteStore.CollectionIdByName(ctx, s.name, s.Version) + + // check if collections exists + switch err { + case nil: + case remotesyncstore.ErrCollectionNotFound: + collectionMustBeCreated = true + collectionId, err = remoteStore.CollectionCreate(ctx, s.name, s.Version, s.Path) + errz.Fatal(err) + default: + errz.Fatal(err) + } + + absHashCachePath := filepath.Join(bobDir, hashCachePath) + if s.cache == nil { + s.cache, err = FromFileOrNew(absHashCachePath) + errz.Fatal(err) + } + + // TODO: run list local and remote in parallel + err = s.cache.Update(filepath.Join(bobDir, s.Path)) + errz.Fatal(err) + err = s.cache.SaveToFile(absHashCachePath) + errz.Fatal(err) + + remoteCollection, err := remoteStore.Collection(ctx, collectionId) + errz.Fatal(err) + + // create the delta + delta := NewDelta(*s.cache, *remoteCollection) + + fmt.Printf("Local-Remote delta for %s\n", aurora.Bold(s.name)) + fmt.Println(delta.PullOverview()) + + if !collectionMustBeCreated { + // TODO: prompt user and seek confirmation + } + + for _, f := range delta.LocalFilesMissingOnRemote { + err := localStore.DeleteFile(bobDir, s.Path, f.LocalPath) + errz.Fatal(err) + } + for _, f := range delta.RemoteFilesMissingOnLocal { + if f.ID == nil { + return fmt.Errorf("ID not available can not downlaod from remote") + } + srcReader, err := remoteStore.File(ctx, collectionId, *f.ID) + errz.Fatal(err) + err = localStore.WriteFile(bobDir, s.Path, f.LocalPath, srcReader) + errz.Fatal(err) + } + for _, f := range delta.ToBeUpdated { + if f.ID == nil { + return fmt.Errorf("ID not available can not downlaod from remote") + } + srcReader, err := remoteStore.File(ctx, collectionId, *f.ID) + errz.Fatal(err) + err = localStore.WriteFile(bobDir, s.Path, f.LocalPath, srcReader) + errz.Fatal(err) + } + return nil } -func (s *Sync) ListLocal(ctx context.Context) (err error) { +func (s *Sync) ListLocal(bobDir string) (err error) { + defer errz.Recover(&err) + + // sync files in collection + + absHashCachPath := filepath.Join(bobDir, hashCachePath) + if s.cache == nil { + s.cache, err = FromFileOrNew(absHashCachPath) + errz.Fatal(err) + } + + err = s.cache.Update(s.Path) + errz.Fatal(err) + err = s.cache.SaveToFile(absHashCachPath) + errz.Fatal(err) + + fmt.Printf("%s@%s (./%s)\n", aurora.Bold(s.name), aurora.Italic(s.Version), s.Path) + for p, _ := range *s.cache { + fmt.Printf("\t%s\n", p) + } + return nil + } -func (s *Sync) ListRemote(ctx context.Context) (err error) { +func (s *Sync) ListRemote(ctx context.Context, store remotesyncstore.S) (err error) { + defer errz.Recover(&err) + + // get collectionId ready + var collectionId string + collectionId, err = store.CollectionIdByName(ctx, s.name, s.Version) + + // check if collections exists + switch err { + case nil: + case remotesyncstore.ErrCollectionNotFound: + fmt.Printf("Sync collection %s with version %s does not exist on the server.\n", + aurora.Bold(s.name), aurora.Bold(s.Version)) + default: + errz.Fatal(err) + } + collections, err := store.Collections(ctx) + errz.Fatal(err) + + for _, c := range collections { + fmt.Printf("%s@%s (./%s)", aurora.Bold(c.Name), aurora.Italic(c.Version), c.LocalPath) + if c.ID == collectionId { + fmt.Printf(" [synced to local]") + } + fmt.Println() + for _, f := range c.Files { + fmt.Printf("\t%s\n", f.LocalPath) + } + } return nil } diff --git a/cli/cmd_root.go b/cli/cmd_root.go index d5c33e9f..7750b3fd 100644 --- a/cli/cmd_root.go +++ b/cli/cmd_root.go @@ -59,6 +59,9 @@ func init() { rootCmd.AddCommand(CmdGit) // syncCmd + cmdSyncListRemote.Flags().Bool("insecure", false, "Set to true to use http instead of https when accessing bob-server") + cmdSyncPush.Flags().Bool("insecure", false, "Set to true to use http instead of https when accessing bob-server") + cmdSyncPull.Flags().Bool("insecure", false, "Set to true to use http instead of https when accessing bob-server") cmdSync.AddCommand(cmdSyncPush) cmdSync.AddCommand(cmdSyncPull) cmdSync.AddCommand(cmdSyncList) diff --git a/cli/cmd_sync.go b/cli/cmd_sync.go index e9262af9..fc296432 100644 --- a/cli/cmd_sync.go +++ b/cli/cmd_sync.go @@ -44,7 +44,10 @@ var cmdSyncPull = &cobra.Command{ Short: "Make local collections exactly like server", Long: ``, Run: func(cmd *cobra.Command, args []string) { + allowInsecure, err := cmd.Flags().GetBool("insecure") + errz.Fatal(err) + runPull(allowInsecure) }, } @@ -53,7 +56,7 @@ var cmdSyncList = &cobra.Command{ Short: "List files synced", Long: ``, Run: func(cmd *cobra.Command, args []string) { - + runList() }, } @@ -62,7 +65,9 @@ var cmdSyncListRemote = &cobra.Command{ Short: "List collections on remote", Long: ``, Run: func(cmd *cobra.Command, args []string) { - + allowInsecure, err := cmd.Flags().GetBool("insecure") + errz.Fatal(err) + runListRemote(allowInsecure) }, } @@ -104,3 +109,117 @@ func runPush(allowInsecure bool) { } } } + +func runPull(allowInsecure bool) { + var exitCode int + defer func() { + exit(exitCode) + }() + defer errz.Recover() + + b, err := bob.Bob( + bob.WithInsecure(allowInsecure), + ) + boblog.Log.Error(err, "Unable to initialize bob") + + if err != nil { + exitCode = 1 + errz.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go func() { + stop := make(chan os.Signal, 1) + signal.Notify(stop, os.Interrupt, syscall.SIGTERM) + + <-stop + cancel() + }() + + err = b.SyncPull(ctx) + if err != nil { + exitCode = 1 + if errors.As(err, &usererror.Err) { + boblog.Log.UserError(err) + } else { + errz.Fatal(err) + } + } +} + +func runList() { + var exitCode int + defer func() { + exit(exitCode) + }() + defer errz.Recover() + + b, err := bob.Bob() + boblog.Log.Error(err, "Unable to initialize bob") + + if err != nil { + exitCode = 1 + errz.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go func() { + stop := make(chan os.Signal, 1) + signal.Notify(stop, os.Interrupt, syscall.SIGTERM) + + <-stop + cancel() + }() + + err = b.SyncListLocal(ctx) + if err != nil { + exitCode = 1 + if errors.As(err, &usererror.Err) { + boblog.Log.UserError(err) + } else { + errz.Fatal(err) + errz.Log(err) + } + } +} + +func runListRemote(allowInsecure bool) { + var exitCode int + defer func() { + exit(exitCode) + }() + defer errz.Recover() + + b, err := bob.Bob(bob.WithInsecure(allowInsecure)) + boblog.Log.Error(err, "Unable to initialize bob") + + if err != nil { + exitCode = 1 + errz.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go func() { + stop := make(chan os.Signal, 1) + signal.Notify(stop, os.Interrupt, syscall.SIGTERM) + + <-stop + cancel() + }() + + err = b.SyncListRemote(ctx) + if err != nil { + exitCode = 1 + if errors.As(err, &usererror.Err) { + boblog.Log.UserError(err) + } else { + errz.Fatal(err) + } + } +} diff --git a/pkg/file/file.go b/pkg/file/file.go index 1d7bbbd5..4f9851f3 100644 --- a/pkg/file/file.go +++ b/pkg/file/file.go @@ -3,6 +3,7 @@ package file import ( "io/ioutil" "os" + "time" ) // Exists return true when a file exists, false otherwise. @@ -26,3 +27,11 @@ func Copy(dst, src string) error { return nil } + +func LastModTime(filePath string) (time.Time, error) { + file, err := os.Stat(filePath) + if err != nil { + return time.Time{}, err + } + return file.ModTime(), nil +} diff --git a/pkg/filehash/hash.go b/pkg/filehash/hash.go index 031ee02d..52bfaa35 100644 --- a/pkg/filehash/hash.go +++ b/pkg/filehash/hash.go @@ -1,6 +1,7 @@ package filehash import ( + "encoding/hex" "fmt" "io" "os" @@ -30,3 +31,12 @@ func HashBytes(r io.Reader) ([]byte, error) { return h.Sum(nil), nil } + +func HashAsString(file string) (string, error) { + b, err := Hash(file) + if err != nil { + return "", err + } + encryptedHash := hex.EncodeToString(b) + return encryptedHash, nil +} diff --git a/pkg/store-client/client.go b/pkg/store-client/client.go index 1956f92e..a1e26a0f 100644 --- a/pkg/store-client/client.go +++ b/pkg/store-client/client.go @@ -3,18 +3,29 @@ package storeclient import ( "context" "fmt" + "github.com/benchkram/bob/pkg/store-client/generated" + "github.com/benchkram/errz" + "github.com/pkg/errors" "io" "mime/multipart" "net/http" "net/textproto" - - "github.com/benchkram/errz" - "github.com/pkg/errors" + "path/filepath" + "syscall" "github.com/benchkram/bob/pkg/usererror" ) -var ErrProjectNotFound = errors.New("project not found") +var ( + ErrProjectNotFound = errors.New("project not found") + ErrCollectionNotFound = errors.New("collection not found") + ErrFileNotFound = errors.New("file not found") + ErrAuthorizationFailed = errors.New("authorization failed") + ErrResourceForbidden = errors.New("accessed to resource forbidden") + ErrEmptyResponse = errors.New("empty response") + ErrDownloadFailed = errors.New("binary download failed") + ErrConnectionRefused = errors.New("connection to server failed (connection refused)") +) func (c *c) UploadArtifact( ctx context.Context, @@ -99,7 +110,7 @@ func (c *c) ListArtifacts(ctx context.Context, project string) (ids []string, er } if res.JSON200 == nil { - errz.Fatal(errors.New("invalid response")) + errz.Fatal(ErrEmptyResponse) } return *res.JSON200, nil @@ -120,7 +131,7 @@ func (c *c) GetArtifact(ctx context.Context, projectId string, artifactId string } if res.JSON200 == nil { - errz.Fatal(errors.New("invalid response")) + errz.Fatal(ErrEmptyResponse) } res2, err := http.Get(*res.JSON200.Location) @@ -132,3 +143,361 @@ func (c *c) GetArtifact(ctx context.Context, projectId string, artifactId string return res2.Body, nil } + +func (c *c) CollectionCreate(ctx context.Context, projectName, name, localPath string) (collection *generated.SyncCollection, err error) { + defer errz.Recover(&err) + body := generated.CreateSyncCollectionJSONRequestBody{ + LocalPath: localPath, + Name: name, + } + + res, err := c.clientWithResponses.CreateSyncCollectionWithResponse( + ctx, + projectName, + body, + ) + errMsg := "creation of sync collection on remote failed" + if errors.Is(err, syscall.ECONNREFUSED) { + errz.Fatal(usererror.Wrapm(ErrConnectionRefused, errMsg)) + } + errz.Fatal(err) + + switch res.StatusCode() { + case http.StatusOK: + case http.StatusNotFound: + errz.Fatal(usererror.Wrapm(ErrProjectNotFound, errMsg)) + case http.StatusUnauthorized: + err = usererror.Wrapm(ErrAuthorizationFailed, errMsg) + errz.Fatal(err) + case http.StatusForbidden: + err = usererror.Wrapm(ErrResourceForbidden, errMsg) + errz.Fatal(err) + default: + err = errors.Errorf("request failed [status: %d, msg: %q]", res.StatusCode(), res.Body) + errz.Fatal(err) + } + + if res.JSON200 == nil { + errz.Fatal(ErrEmptyResponse) + } + + return res.JSON200, nil + +} + +func (c *c) Collection(ctx context.Context, projectName, collectionId string) (collection *generated.SyncCollection, err error) { + defer errz.Recover(&err) + + res, err := c.clientWithResponses.GetSyncCollectionWithResponse( + ctx, + projectName, + collectionId, + ) + errMsg := "reading of sync collection from remote failed" + if errors.Is(err, syscall.ECONNREFUSED) { + errz.Fatal(usererror.Wrapm(ErrConnectionRefused, errMsg)) + } + errz.Fatal(err) + + switch res.StatusCode() { + case http.StatusOK: + case http.StatusUnauthorized: + errz.Fatal(usererror.Wrapm(ErrAuthorizationFailed, errMsg)) + case http.StatusForbidden: + errz.Fatal(usererror.Wrapm(ErrResourceForbidden, errMsg)) + case http.StatusNotFound: + errz.Fatal(usererror.Wrapm(ErrCollectionNotFound, errMsg)) + default: + err = errors.Errorf("request failed [status: %d, msg: %q]", res.StatusCode(), res.Body) + errz.Fatal(err) + } + + if res.JSON200 == nil { + errz.Fatal(ErrEmptyResponse) + } + + return res.JSON200, nil +} + +func (c *c) Collections(ctx context.Context, projectName string) (collections []generated.SyncCollection, err error) { + defer errz.Recover(&err) + + res, err := c.clientWithResponses.GetSyncCollectionsWithResponse( + ctx, + projectName, + ) + errMsg := "reading of sync collections from remote failed" + if errors.Is(err, syscall.ECONNREFUSED) { + errz.Fatal(usererror.Wrapm(ErrConnectionRefused, errMsg)) + } + + errz.Fatal(err) + + switch res.StatusCode() { + case http.StatusOK: + case http.StatusUnauthorized: + err = usererror.Wrapm(ErrAuthorizationFailed, errMsg) + errz.Fatal(err) + case http.StatusForbidden: + errz.Fatal(usererror.Wrapm(ErrResourceForbidden, errMsg)) + case http.StatusNotFound: + err = usererror.Wrapm(ErrProjectNotFound, errMsg) + errz.Fatal(err) + default: + err = errors.Errorf("request failed [status: %d, msg: %q]", res.StatusCode(), res.Body) + errz.Fatal(err) + } + if res.JSON200 == nil { + errz.Fatal(ErrEmptyResponse) + } + + return *res.JSON200, nil +} + +func (c *c) FileCreate(ctx context.Context, projectName, collectionId, localPath string, src io.Reader) (f *generated.SyncFile, err error) { + r, w := io.Pipe() + mpw := multipart.NewWriter(w) + + go func() { + + err0 := mpw.WriteField("local_path", localPath) + if err0 != nil { + _ = w.CloseWithError(err0) + } + + fieldWriter, err0 := mpw.CreateFormFile("file", filepath.Base(localPath)) + if err0 != nil { + _ = w.CloseWithError(err0) + } + _, err0 = io.Copy(fieldWriter, src) + if err0 != nil { + _ = w.CloseWithError(err0) + } + + err0 = mpw.Close() + if err0 != nil { + _ = w.CloseWithError(err0) + } + _ = w.Close() + }() + + resp, err := c.clientWithResponses.CreateSyncFileWithBodyWithResponse( + ctx, + projectName, + collectionId, + mpw.FormDataContentType(), + r, + ) + errMsg := "creation of file on remote failed" + if errors.Is(err, syscall.ECONNREFUSED) { + errz.Fatal(usererror.Wrapm(ErrConnectionRefused, errMsg)) + } + errz.Fatal(err) + + switch resp.StatusCode() { + case http.StatusOK: + case http.StatusUnauthorized: + err = usererror.Wrapm(ErrAuthorizationFailed, errMsg) + errz.Fatal(err) + case http.StatusForbidden: + err = usererror.Wrapm(ErrResourceForbidden, errMsg) + errz.Fatal(err) + case http.StatusNotFound: + err = usererror.Wrapm(ErrCollectionNotFound, errMsg) + errz.Fatal(err) + default: + //TODO: add specific error handling for http.StatusConflict and http.StatusBadRequest + err = errors.Errorf("request failed [status: %d, msg: %q]", resp.StatusCode(), resp.Body) + errz.Fatal(err) + } + if resp.JSON200 == nil { + errz.Fatal(ErrEmptyResponse) + } + + return resp.JSON200, nil +} + +func (c *c) File(ctx context.Context, projectName, collectionId, fileId string) (f *generated.SyncFile, rc io.ReadCloser, err error) { + defer errz.Recover(&err) + + res, err := c.clientWithResponses.GetSyncFileWithResponse( + ctx, + projectName, + collectionId, + fileId, + ) + errMsg := "reading of sync file from remote failed" + if errors.Is(err, syscall.ECONNREFUSED) { + errz.Fatal(usererror.Wrapm(ErrConnectionRefused, errMsg)) + } + errz.Fatal(err) + + switch res.StatusCode() { + case http.StatusOK: + case http.StatusUnauthorized: + err = usererror.Wrapm(ErrAuthorizationFailed, errMsg) + errz.Fatal(err) + case http.StatusForbidden: + err = usererror.Wrapm(ErrResourceForbidden, errMsg) + errz.Fatal(err) + case http.StatusNotFound: + errz.Fatal(usererror.Wrapm(ErrFileNotFound, errMsg)) + default: + err = errors.Errorf("request failed [status: %d, msg: %q]", res.StatusCode(), res.Body) + errz.Fatal(err) + } + if res.JSON200 == nil { + errz.Fatal(ErrEmptyResponse) + } + + res2, err := http.Get(*res.JSON200.Location) + errz.Fatal(err) + + if res2.StatusCode != http.StatusOK { + errz.Fatal(usererror.Wrapm(ErrDownloadFailed, fmt.Sprintf("reading from storage failed (Status %d)", res2.StatusCode))) + } + if res2.Body == nil { + errz.Fatal(ErrEmptyResponse) + } + + return res.JSON200, res2.Body, nil +} + +func (c *c) Files(ctx context.Context, projectName, collectionId string, withLocation bool) (files []generated.SyncFile, err error) { + defer errz.Recover(&err) + + params := generated.GetSyncFilesParams{ + WithLocation: &withLocation, + } + res, err := c.clientWithResponses.GetSyncFilesWithResponse( + ctx, + projectName, + collectionId, + ¶ms, + ) + errMsg := "reading of sync files from remote failed" + if errors.Is(err, syscall.ECONNREFUSED) { + errz.Fatal(usererror.Wrapm(ErrConnectionRefused, errMsg)) + } + errz.Fatal(err) + + switch res.StatusCode() { + case http.StatusOK: + case http.StatusUnauthorized: + err = usererror.Wrapm(ErrAuthorizationFailed, errMsg) + errz.Fatal(err) + case http.StatusForbidden: + err = usererror.Wrapm(ErrResourceForbidden, errMsg) + errz.Fatal(err) + case http.StatusNotFound: + errz.Fatal(usererror.Wrapm(ErrCollectionNotFound, errMsg)) + default: + err = errors.Errorf("request failed [status: %d, msg: %q]", res.StatusCode(), res.Body) + errz.Fatal(err) + } + if res.JSON200 == nil { + errz.Fatal(ErrEmptyResponse) + } + + return *res.JSON200, nil +} + +func (c *c) FileUpdate(ctx context.Context, projectName, collectionId, fileId, localPath string, src *io.Reader) (file *generated.SyncFile, err error) { + r, w := io.Pipe() + mpw := multipart.NewWriter(w) + + go func() { + if localPath != "" { + err0 := mpw.WriteField("local_path", localPath) + if err0 != nil { + _ = w.CloseWithError(err0) + } + } + + if src != nil { + fieldWriter, err0 := mpw.CreateFormFile("file", filepath.Base(localPath)) + if err0 != nil { + _ = w.CloseWithError(err0) + } + _, err0 = io.Copy(fieldWriter, *src) + if err0 != nil { + _ = w.CloseWithError(err0) + } + } + err0 := mpw.Close() + if err0 != nil { + _ = w.CloseWithError(err0) + } + _ = w.Close() + }() + + resp, err := c.clientWithResponses.PutSyncFileWithBodyWithResponse( + ctx, + projectName, + collectionId, + fileId, + mpw.FormDataContentType(), + r, + ) + errMsg := "update of sync file on remote failed" + if errors.Is(err, syscall.ECONNREFUSED) { + errz.Fatal(usererror.Wrapm(ErrConnectionRefused, errMsg)) + } + errz.Fatal(err) + + switch resp.StatusCode() { + case http.StatusOK: + case http.StatusUnauthorized: + err = usererror.Wrapm(ErrAuthorizationFailed, errMsg) + errz.Fatal(err) + case http.StatusForbidden: + err = usererror.Wrapm(ErrResourceForbidden, errMsg) + errz.Fatal(err) + case http.StatusNotFound: + err = usererror.Wrapm(ErrFileNotFound, errMsg) + errz.Fatal(err) + default: + //TODO: add specific error handling for http.StatusConflict and http.StatusBadRequest + err = errors.Errorf("request failed [status: %d, msg: %q]", resp.StatusCode(), resp.Body) + errz.Fatal(err) + } + + if resp.JSON200 == nil { + errz.Fatal(ErrEmptyResponse) + } + + return resp.JSON200, nil +} + +func (c *c) FileDelete(ctx context.Context, projectName, collectionId, fileId string) (err error) { + defer errz.Recover(&err) + + res, err := c.client.DeleteSyncFile( + ctx, + projectName, + collectionId, + fileId, + ) + errMsg := "delete from remote failed" + if errors.Is(err, syscall.ECONNREFUSED) { + errz.Fatal(usererror.Wrapm(ErrConnectionRefused, errMsg)) + } + errz.Fatal(err) + + switch res.StatusCode { + case http.StatusOK: + case http.StatusUnauthorized: + err = usererror.Wrapm(ErrAuthorizationFailed, errMsg) + errz.Fatal(err) + case http.StatusForbidden: + err = usererror.Wrapm(ErrResourceForbidden, errMsg) + errz.Fatal(err) + case http.StatusNotFound: + errz.Fatal(usererror.Wrapm(ErrFileNotFound, errMsg)) + default: + err = errors.Errorf("request failed [status: %d, msg: %q]", res.StatusCode, res.Body) + errz.Fatal(err) + } + + return nil +} diff --git a/pkg/store-client/generated/client.gen.go b/pkg/store-client/generated/client.gen.go index 30031256..ec58dca8 100644 --- a/pkg/store-client/generated/client.gen.go +++ b/pkg/store-client/generated/client.gen.go @@ -4,6 +4,7 @@ package generated import ( + "bytes" "context" "encoding/json" "fmt" @@ -92,6 +93,12 @@ type ClientInterface interface { // GetHealth request GetHealth(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetProject request + GetProject(ctx context.Context, projectName string, reqEditors ...RequestEditorFn) (*http.Response, error) + + // ProjectExists request + ProjectExists(ctx context.Context, projectName string, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetProjectArtifact request GetProjectArtifact(ctx context.Context, projectName string, artifactId string, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -103,6 +110,35 @@ type ClientInterface interface { // UploadArtifact request with any body UploadArtifactWithBody(ctx context.Context, projectName string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + // DeleteSyncCollection request + DeleteSyncCollection(ctx context.Context, projectName string, collectionId string, reqEditors ...RequestEditorFn) (*http.Response, error) + + // GetSyncCollection request + GetSyncCollection(ctx context.Context, projectName string, collectionId string, reqEditors ...RequestEditorFn) (*http.Response, error) + + // DeleteSyncFile request + DeleteSyncFile(ctx context.Context, projectName string, collectionId string, fileId string, reqEditors ...RequestEditorFn) (*http.Response, error) + + // GetSyncFile request + GetSyncFile(ctx context.Context, projectName string, collectionId string, fileId string, reqEditors ...RequestEditorFn) (*http.Response, error) + + // PutSyncFile request with any body + PutSyncFileWithBody(ctx context.Context, projectName string, collectionId string, fileId string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + // GetSyncFiles request + GetSyncFiles(ctx context.Context, projectName string, collectionId string, params *GetSyncFilesParams, reqEditors ...RequestEditorFn) (*http.Response, error) + + // CreateSyncFile request with any body + CreateSyncFileWithBody(ctx context.Context, projectName string, collectionId string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + // GetSyncCollections request + GetSyncCollections(ctx context.Context, projectName string, reqEditors ...RequestEditorFn) (*http.Response, error) + + // CreateSyncCollection request with any body + CreateSyncCollectionWithBody(ctx context.Context, projectName string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + CreateSyncCollection(ctx context.Context, projectName string, body CreateSyncCollectionJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) } func (c *Client) GetHealth(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) { @@ -117,6 +153,30 @@ func (c *Client) GetHealth(ctx context.Context, reqEditors ...RequestEditorFn) ( return c.Client.Do(req) } +func (c *Client) GetProject(ctx context.Context, projectName string, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewGetProjectRequest(c.Server, projectName) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) ProjectExists(ctx context.Context, projectName string, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewProjectExistsRequest(c.Server, projectName) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) GetProjectArtifact(ctx context.Context, projectName string, artifactId string, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewGetProjectArtifactRequest(c.Server, projectName, artifactId) if err != nil { @@ -165,6 +225,126 @@ func (c *Client) UploadArtifactWithBody(ctx context.Context, projectName string, return c.Client.Do(req) } +func (c *Client) DeleteSyncCollection(ctx context.Context, projectName string, collectionId string, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewDeleteSyncCollectionRequest(c.Server, projectName, collectionId) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) GetSyncCollection(ctx context.Context, projectName string, collectionId string, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewGetSyncCollectionRequest(c.Server, projectName, collectionId) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) DeleteSyncFile(ctx context.Context, projectName string, collectionId string, fileId string, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewDeleteSyncFileRequest(c.Server, projectName, collectionId, fileId) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) GetSyncFile(ctx context.Context, projectName string, collectionId string, fileId string, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewGetSyncFileRequest(c.Server, projectName, collectionId, fileId) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) PutSyncFileWithBody(ctx context.Context, projectName string, collectionId string, fileId string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewPutSyncFileRequestWithBody(c.Server, projectName, collectionId, fileId, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) GetSyncFiles(ctx context.Context, projectName string, collectionId string, params *GetSyncFilesParams, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewGetSyncFilesRequest(c.Server, projectName, collectionId, params) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) CreateSyncFileWithBody(ctx context.Context, projectName string, collectionId string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewCreateSyncFileRequestWithBody(c.Server, projectName, collectionId, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) GetSyncCollections(ctx context.Context, projectName string, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewGetSyncCollectionsRequest(c.Server, projectName) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) CreateSyncCollectionWithBody(ctx context.Context, projectName string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewCreateSyncCollectionRequestWithBody(c.Server, projectName, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) CreateSyncCollection(ctx context.Context, projectName string, body CreateSyncCollectionJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewCreateSyncCollectionRequest(c.Server, projectName, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + // NewGetHealthRequest generates requests for GetHealth func NewGetHealthRequest(server string) (*http.Request, error) { var err error @@ -192,6 +372,74 @@ func NewGetHealthRequest(server string) (*http.Request, error) { return req, nil } +// NewGetProjectRequest generates requests for GetProject +func NewGetProjectRequest(server string, projectName string) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "projectName", runtime.ParamLocationPath, projectName) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/api/project/%s", pathParam0) + if operationPath[0] == '/' { + operationPath = operationPath[1:] + } + operationURL := url.URL{ + Path: operationPath, + } + + queryURL := serverURL.ResolveReference(&operationURL) + + req, err := http.NewRequest("GET", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + +// NewProjectExistsRequest generates requests for ProjectExists +func NewProjectExistsRequest(server string, projectName string) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "projectName", runtime.ParamLocationPath, projectName) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/api/project/%s", pathParam0) + if operationPath[0] == '/' { + operationPath = operationPath[1:] + } + operationURL := url.URL{ + Path: operationPath, + } + + queryURL := serverURL.ResolveReference(&operationURL) + + req, err := http.NewRequest("HEAD", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + // NewGetProjectArtifactRequest generates requests for GetProjectArtifact func NewGetProjectArtifactRequest(server string, projectName string, artifactId string) (*http.Request, error) { var err error @@ -344,74 +592,565 @@ func NewUploadArtifactRequestWithBody(server string, projectName string, content return req, nil } -func (c *Client) applyEditors(ctx context.Context, req *http.Request, additionalEditors []RequestEditorFn) error { - for _, r := range c.RequestEditors { - if err := r(ctx, req); err != nil { - return err - } - } - for _, r := range additionalEditors { - if err := r(ctx, req); err != nil { - return err - } +// NewDeleteSyncCollectionRequest generates requests for DeleteSyncCollection +func NewDeleteSyncCollectionRequest(server string, projectName string, collectionId string) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "projectName", runtime.ParamLocationPath, projectName) + if err != nil { + return nil, err } - return nil -} -// ClientWithResponses builds on ClientInterface to offer response payloads -type ClientWithResponses struct { - ClientInterface -} + var pathParam1 string -// NewClientWithResponses creates a new ClientWithResponses, which wraps -// Client with return type handling -func NewClientWithResponses(server string, opts ...ClientOption) (*ClientWithResponses, error) { - client, err := NewClient(server, opts...) + pathParam1, err = runtime.StyleParamWithLocation("simple", false, "collectionId", runtime.ParamLocationPath, collectionId) if err != nil { return nil, err } - return &ClientWithResponses{client}, nil -} -// WithBaseURL overrides the baseURL. -func WithBaseURL(baseURL string) ClientOption { - return func(c *Client) error { - newBaseURL, err := url.Parse(baseURL) - if err != nil { - return err - } - c.Server = newBaseURL.String() - return nil + serverURL, err := url.Parse(server) + if err != nil { + return nil, err } -} -// ClientWithResponsesInterface is the interface specification for the client with responses above. -type ClientWithResponsesInterface interface { - // GetHealth request - GetHealthWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*GetHealthResponse, error) + operationPath := fmt.Sprintf("/api/project/%s/sync/collection/%s", pathParam0, pathParam1) + if operationPath[0] == '/' { + operationPath = operationPath[1:] + } + operationURL := url.URL{ + Path: operationPath, + } - // GetProjectArtifact request - GetProjectArtifactWithResponse(ctx context.Context, projectName string, artifactId string, reqEditors ...RequestEditorFn) (*GetProjectArtifactResponse, error) + queryURL := serverURL.ResolveReference(&operationURL) - // ProjectArtifactExists request - ProjectArtifactExistsWithResponse(ctx context.Context, projectName string, artifactId string, reqEditors ...RequestEditorFn) (*ProjectArtifactExistsResponse, error) + req, err := http.NewRequest("DELETE", queryURL.String(), nil) + if err != nil { + return nil, err + } - // GetProjectArtifacts request + return req, nil +} + +// NewGetSyncCollectionRequest generates requests for GetSyncCollection +func NewGetSyncCollectionRequest(server string, projectName string, collectionId string) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "projectName", runtime.ParamLocationPath, projectName) + if err != nil { + return nil, err + } + + var pathParam1 string + + pathParam1, err = runtime.StyleParamWithLocation("simple", false, "collectionId", runtime.ParamLocationPath, collectionId) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/api/project/%s/sync/collection/%s", pathParam0, pathParam1) + if operationPath[0] == '/' { + operationPath = operationPath[1:] + } + operationURL := url.URL{ + Path: operationPath, + } + + queryURL := serverURL.ResolveReference(&operationURL) + + req, err := http.NewRequest("GET", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + +// NewDeleteSyncFileRequest generates requests for DeleteSyncFile +func NewDeleteSyncFileRequest(server string, projectName string, collectionId string, fileId string) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "projectName", runtime.ParamLocationPath, projectName) + if err != nil { + return nil, err + } + + var pathParam1 string + + pathParam1, err = runtime.StyleParamWithLocation("simple", false, "collectionId", runtime.ParamLocationPath, collectionId) + if err != nil { + return nil, err + } + + var pathParam2 string + + pathParam2, err = runtime.StyleParamWithLocation("simple", false, "fileId", runtime.ParamLocationPath, fileId) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/api/project/%s/sync/collection/%s/file/%s", pathParam0, pathParam1, pathParam2) + if operationPath[0] == '/' { + operationPath = operationPath[1:] + } + operationURL := url.URL{ + Path: operationPath, + } + + queryURL := serverURL.ResolveReference(&operationURL) + + req, err := http.NewRequest("DELETE", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + +// NewGetSyncFileRequest generates requests for GetSyncFile +func NewGetSyncFileRequest(server string, projectName string, collectionId string, fileId string) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "projectName", runtime.ParamLocationPath, projectName) + if err != nil { + return nil, err + } + + var pathParam1 string + + pathParam1, err = runtime.StyleParamWithLocation("simple", false, "collectionId", runtime.ParamLocationPath, collectionId) + if err != nil { + return nil, err + } + + var pathParam2 string + + pathParam2, err = runtime.StyleParamWithLocation("simple", false, "fileId", runtime.ParamLocationPath, fileId) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/api/project/%s/sync/collection/%s/file/%s", pathParam0, pathParam1, pathParam2) + if operationPath[0] == '/' { + operationPath = operationPath[1:] + } + operationURL := url.URL{ + Path: operationPath, + } + + queryURL := serverURL.ResolveReference(&operationURL) + + req, err := http.NewRequest("GET", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + +// NewPutSyncFileRequestWithBody generates requests for PutSyncFile with any type of body +func NewPutSyncFileRequestWithBody(server string, projectName string, collectionId string, fileId string, contentType string, body io.Reader) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "projectName", runtime.ParamLocationPath, projectName) + if err != nil { + return nil, err + } + + var pathParam1 string + + pathParam1, err = runtime.StyleParamWithLocation("simple", false, "collectionId", runtime.ParamLocationPath, collectionId) + if err != nil { + return nil, err + } + + var pathParam2 string + + pathParam2, err = runtime.StyleParamWithLocation("simple", false, "fileId", runtime.ParamLocationPath, fileId) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/api/project/%s/sync/collection/%s/file/%s", pathParam0, pathParam1, pathParam2) + if operationPath[0] == '/' { + operationPath = operationPath[1:] + } + operationURL := url.URL{ + Path: operationPath, + } + + queryURL := serverURL.ResolveReference(&operationURL) + + req, err := http.NewRequest("PUT", queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + +// NewGetSyncFilesRequest generates requests for GetSyncFiles +func NewGetSyncFilesRequest(server string, projectName string, collectionId string, params *GetSyncFilesParams) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "projectName", runtime.ParamLocationPath, projectName) + if err != nil { + return nil, err + } + + var pathParam1 string + + pathParam1, err = runtime.StyleParamWithLocation("simple", false, "collectionId", runtime.ParamLocationPath, collectionId) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/api/project/%s/sync/collection/%s/files", pathParam0, pathParam1) + if operationPath[0] == '/' { + operationPath = operationPath[1:] + } + operationURL := url.URL{ + Path: operationPath, + } + + queryURL := serverURL.ResolveReference(&operationURL) + + queryValues := queryURL.Query() + + if params.WithLocation != nil { + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "withLocation", runtime.ParamLocationQuery, *params.WithLocation); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + } + + queryURL.RawQuery = queryValues.Encode() + + req, err := http.NewRequest("GET", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + +// NewCreateSyncFileRequestWithBody generates requests for CreateSyncFile with any type of body +func NewCreateSyncFileRequestWithBody(server string, projectName string, collectionId string, contentType string, body io.Reader) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "projectName", runtime.ParamLocationPath, projectName) + if err != nil { + return nil, err + } + + var pathParam1 string + + pathParam1, err = runtime.StyleParamWithLocation("simple", false, "collectionId", runtime.ParamLocationPath, collectionId) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/api/project/%s/sync/collection/%s/files", pathParam0, pathParam1) + if operationPath[0] == '/' { + operationPath = operationPath[1:] + } + operationURL := url.URL{ + Path: operationPath, + } + + queryURL := serverURL.ResolveReference(&operationURL) + + req, err := http.NewRequest("POST", queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + +// NewGetSyncCollectionsRequest generates requests for GetSyncCollections +func NewGetSyncCollectionsRequest(server string, projectName string) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "projectName", runtime.ParamLocationPath, projectName) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/api/project/%s/sync/collections", pathParam0) + if operationPath[0] == '/' { + operationPath = operationPath[1:] + } + operationURL := url.URL{ + Path: operationPath, + } + + queryURL := serverURL.ResolveReference(&operationURL) + + req, err := http.NewRequest("GET", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + +// NewCreateSyncCollectionRequest calls the generic CreateSyncCollection builder with application/json body +func NewCreateSyncCollectionRequest(server string, projectName string, body CreateSyncCollectionJSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewCreateSyncCollectionRequestWithBody(server, projectName, "application/json", bodyReader) +} + +// NewCreateSyncCollectionRequestWithBody generates requests for CreateSyncCollection with any type of body +func NewCreateSyncCollectionRequestWithBody(server string, projectName string, contentType string, body io.Reader) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "projectName", runtime.ParamLocationPath, projectName) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/api/project/%s/sync/collections", pathParam0) + if operationPath[0] == '/' { + operationPath = operationPath[1:] + } + operationURL := url.URL{ + Path: operationPath, + } + + queryURL := serverURL.ResolveReference(&operationURL) + + req, err := http.NewRequest("POST", queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + +func (c *Client) applyEditors(ctx context.Context, req *http.Request, additionalEditors []RequestEditorFn) error { + for _, r := range c.RequestEditors { + if err := r(ctx, req); err != nil { + return err + } + } + for _, r := range additionalEditors { + if err := r(ctx, req); err != nil { + return err + } + } + return nil +} + +// ClientWithResponses builds on ClientInterface to offer response payloads +type ClientWithResponses struct { + ClientInterface +} + +// NewClientWithResponses creates a new ClientWithResponses, which wraps +// Client with return type handling +func NewClientWithResponses(server string, opts ...ClientOption) (*ClientWithResponses, error) { + client, err := NewClient(server, opts...) + if err != nil { + return nil, err + } + return &ClientWithResponses{client}, nil +} + +// WithBaseURL overrides the baseURL. +func WithBaseURL(baseURL string) ClientOption { + return func(c *Client) error { + newBaseURL, err := url.Parse(baseURL) + if err != nil { + return err + } + c.Server = newBaseURL.String() + return nil + } +} + +// ClientWithResponsesInterface is the interface specification for the client with responses above. +type ClientWithResponsesInterface interface { + // GetHealth request + GetHealthWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*GetHealthResponse, error) + + // GetProject request + GetProjectWithResponse(ctx context.Context, projectName string, reqEditors ...RequestEditorFn) (*GetProjectResponse, error) + + // ProjectExists request + ProjectExistsWithResponse(ctx context.Context, projectName string, reqEditors ...RequestEditorFn) (*ProjectExistsResponse, error) + + // GetProjectArtifact request + GetProjectArtifactWithResponse(ctx context.Context, projectName string, artifactId string, reqEditors ...RequestEditorFn) (*GetProjectArtifactResponse, error) + + // ProjectArtifactExists request + ProjectArtifactExistsWithResponse(ctx context.Context, projectName string, artifactId string, reqEditors ...RequestEditorFn) (*ProjectArtifactExistsResponse, error) + + // GetProjectArtifacts request GetProjectArtifactsWithResponse(ctx context.Context, projectName string, reqEditors ...RequestEditorFn) (*GetProjectArtifactsResponse, error) - // UploadArtifact request with any body - UploadArtifactWithBodyWithResponse(ctx context.Context, projectName string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*UploadArtifactResponse, error) + // UploadArtifact request with any body + UploadArtifactWithBodyWithResponse(ctx context.Context, projectName string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*UploadArtifactResponse, error) + + // DeleteSyncCollection request + DeleteSyncCollectionWithResponse(ctx context.Context, projectName string, collectionId string, reqEditors ...RequestEditorFn) (*DeleteSyncCollectionResponse, error) + + // GetSyncCollection request + GetSyncCollectionWithResponse(ctx context.Context, projectName string, collectionId string, reqEditors ...RequestEditorFn) (*GetSyncCollectionResponse, error) + + // DeleteSyncFile request + DeleteSyncFileWithResponse(ctx context.Context, projectName string, collectionId string, fileId string, reqEditors ...RequestEditorFn) (*DeleteSyncFileResponse, error) + + // GetSyncFile request + GetSyncFileWithResponse(ctx context.Context, projectName string, collectionId string, fileId string, reqEditors ...RequestEditorFn) (*GetSyncFileResponse, error) + + // PutSyncFile request with any body + PutSyncFileWithBodyWithResponse(ctx context.Context, projectName string, collectionId string, fileId string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*PutSyncFileResponse, error) + + // GetSyncFiles request + GetSyncFilesWithResponse(ctx context.Context, projectName string, collectionId string, params *GetSyncFilesParams, reqEditors ...RequestEditorFn) (*GetSyncFilesResponse, error) + + // CreateSyncFile request with any body + CreateSyncFileWithBodyWithResponse(ctx context.Context, projectName string, collectionId string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*CreateSyncFileResponse, error) + + // GetSyncCollections request + GetSyncCollectionsWithResponse(ctx context.Context, projectName string, reqEditors ...RequestEditorFn) (*GetSyncCollectionsResponse, error) + + // CreateSyncCollection request with any body + CreateSyncCollectionWithBodyWithResponse(ctx context.Context, projectName string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*CreateSyncCollectionResponse, error) + + CreateSyncCollectionWithResponse(ctx context.Context, projectName string, body CreateSyncCollectionJSONRequestBody, reqEditors ...RequestEditorFn) (*CreateSyncCollectionResponse, error) +} + +type GetHealthResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *Success + JSONDefault *Error +} + +// Status returns HTTPResponse.Status +func (r GetHealthResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r GetHealthResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +type GetProjectResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *Project +} + +// Status returns HTTPResponse.Status +func (r GetProjectResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r GetProjectResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 } -type GetHealthResponse struct { +type ProjectExistsResponse struct { Body []byte HTTPResponse *http.Response - JSON200 *Success - JSONDefault *Error } // Status returns HTTPResponse.Status -func (r GetHealthResponse) Status() string { +func (r ProjectExistsResponse) Status() string { if r.HTTPResponse != nil { return r.HTTPResponse.Status } @@ -419,7 +1158,7 @@ func (r GetHealthResponse) Status() string { } // StatusCode returns HTTPResponse.StatusCode -func (r GetHealthResponse) StatusCode() int { +func (r ProjectExistsResponse) StatusCode() int { if r.HTTPResponse != nil { return r.HTTPResponse.StatusCode } @@ -501,60 +1240,363 @@ func (r UploadArtifactResponse) Status() string { if r.HTTPResponse != nil { return r.HTTPResponse.Status } - return http.StatusText(0) + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r UploadArtifactResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +type DeleteSyncCollectionResponse struct { + Body []byte + HTTPResponse *http.Response +} + +// Status returns HTTPResponse.Status +func (r DeleteSyncCollectionResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r DeleteSyncCollectionResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +type GetSyncCollectionResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *SyncCollection +} + +// Status returns HTTPResponse.Status +func (r GetSyncCollectionResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r GetSyncCollectionResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +type DeleteSyncFileResponse struct { + Body []byte + HTTPResponse *http.Response +} + +// Status returns HTTPResponse.Status +func (r DeleteSyncFileResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r DeleteSyncFileResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +type GetSyncFileResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *SyncFile +} + +// Status returns HTTPResponse.Status +func (r GetSyncFileResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r GetSyncFileResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +type PutSyncFileResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *SyncFile +} + +// Status returns HTTPResponse.Status +func (r PutSyncFileResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r PutSyncFileResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +type GetSyncFilesResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *[]SyncFile +} + +// Status returns HTTPResponse.Status +func (r GetSyncFilesResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r GetSyncFilesResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +type CreateSyncFileResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *SyncFile +} + +// Status returns HTTPResponse.Status +func (r CreateSyncFileResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r CreateSyncFileResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +type GetSyncCollectionsResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *[]SyncCollection +} + +// Status returns HTTPResponse.Status +func (r GetSyncCollectionsResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r GetSyncCollectionsResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +type CreateSyncCollectionResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *SyncCollection +} + +// Status returns HTTPResponse.Status +func (r CreateSyncCollectionResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r CreateSyncCollectionResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +// GetHealthWithResponse request returning *GetHealthResponse +func (c *ClientWithResponses) GetHealthWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*GetHealthResponse, error) { + rsp, err := c.GetHealth(ctx, reqEditors...) + if err != nil { + return nil, err + } + return ParseGetHealthResponse(rsp) +} + +// GetProjectWithResponse request returning *GetProjectResponse +func (c *ClientWithResponses) GetProjectWithResponse(ctx context.Context, projectName string, reqEditors ...RequestEditorFn) (*GetProjectResponse, error) { + rsp, err := c.GetProject(ctx, projectName, reqEditors...) + if err != nil { + return nil, err + } + return ParseGetProjectResponse(rsp) +} + +// ProjectExistsWithResponse request returning *ProjectExistsResponse +func (c *ClientWithResponses) ProjectExistsWithResponse(ctx context.Context, projectName string, reqEditors ...RequestEditorFn) (*ProjectExistsResponse, error) { + rsp, err := c.ProjectExists(ctx, projectName, reqEditors...) + if err != nil { + return nil, err + } + return ParseProjectExistsResponse(rsp) +} + +// GetProjectArtifactWithResponse request returning *GetProjectArtifactResponse +func (c *ClientWithResponses) GetProjectArtifactWithResponse(ctx context.Context, projectName string, artifactId string, reqEditors ...RequestEditorFn) (*GetProjectArtifactResponse, error) { + rsp, err := c.GetProjectArtifact(ctx, projectName, artifactId, reqEditors...) + if err != nil { + return nil, err + } + return ParseGetProjectArtifactResponse(rsp) +} + +// ProjectArtifactExistsWithResponse request returning *ProjectArtifactExistsResponse +func (c *ClientWithResponses) ProjectArtifactExistsWithResponse(ctx context.Context, projectName string, artifactId string, reqEditors ...RequestEditorFn) (*ProjectArtifactExistsResponse, error) { + rsp, err := c.ProjectArtifactExists(ctx, projectName, artifactId, reqEditors...) + if err != nil { + return nil, err + } + return ParseProjectArtifactExistsResponse(rsp) +} + +// GetProjectArtifactsWithResponse request returning *GetProjectArtifactsResponse +func (c *ClientWithResponses) GetProjectArtifactsWithResponse(ctx context.Context, projectName string, reqEditors ...RequestEditorFn) (*GetProjectArtifactsResponse, error) { + rsp, err := c.GetProjectArtifacts(ctx, projectName, reqEditors...) + if err != nil { + return nil, err + } + return ParseGetProjectArtifactsResponse(rsp) +} + +// UploadArtifactWithBodyWithResponse request with arbitrary body returning *UploadArtifactResponse +func (c *ClientWithResponses) UploadArtifactWithBodyWithResponse(ctx context.Context, projectName string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*UploadArtifactResponse, error) { + rsp, err := c.UploadArtifactWithBody(ctx, projectName, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseUploadArtifactResponse(rsp) +} + +// DeleteSyncCollectionWithResponse request returning *DeleteSyncCollectionResponse +func (c *ClientWithResponses) DeleteSyncCollectionWithResponse(ctx context.Context, projectName string, collectionId string, reqEditors ...RequestEditorFn) (*DeleteSyncCollectionResponse, error) { + rsp, err := c.DeleteSyncCollection(ctx, projectName, collectionId, reqEditors...) + if err != nil { + return nil, err + } + return ParseDeleteSyncCollectionResponse(rsp) +} + +// GetSyncCollectionWithResponse request returning *GetSyncCollectionResponse +func (c *ClientWithResponses) GetSyncCollectionWithResponse(ctx context.Context, projectName string, collectionId string, reqEditors ...RequestEditorFn) (*GetSyncCollectionResponse, error) { + rsp, err := c.GetSyncCollection(ctx, projectName, collectionId, reqEditors...) + if err != nil { + return nil, err + } + return ParseGetSyncCollectionResponse(rsp) +} + +// DeleteSyncFileWithResponse request returning *DeleteSyncFileResponse +func (c *ClientWithResponses) DeleteSyncFileWithResponse(ctx context.Context, projectName string, collectionId string, fileId string, reqEditors ...RequestEditorFn) (*DeleteSyncFileResponse, error) { + rsp, err := c.DeleteSyncFile(ctx, projectName, collectionId, fileId, reqEditors...) + if err != nil { + return nil, err + } + return ParseDeleteSyncFileResponse(rsp) +} + +// GetSyncFileWithResponse request returning *GetSyncFileResponse +func (c *ClientWithResponses) GetSyncFileWithResponse(ctx context.Context, projectName string, collectionId string, fileId string, reqEditors ...RequestEditorFn) (*GetSyncFileResponse, error) { + rsp, err := c.GetSyncFile(ctx, projectName, collectionId, fileId, reqEditors...) + if err != nil { + return nil, err + } + return ParseGetSyncFileResponse(rsp) } -// StatusCode returns HTTPResponse.StatusCode -func (r UploadArtifactResponse) StatusCode() int { - if r.HTTPResponse != nil { - return r.HTTPResponse.StatusCode +// PutSyncFileWithBodyWithResponse request with arbitrary body returning *PutSyncFileResponse +func (c *ClientWithResponses) PutSyncFileWithBodyWithResponse(ctx context.Context, projectName string, collectionId string, fileId string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*PutSyncFileResponse, error) { + rsp, err := c.PutSyncFileWithBody(ctx, projectName, collectionId, fileId, contentType, body, reqEditors...) + if err != nil { + return nil, err } - return 0 + return ParsePutSyncFileResponse(rsp) } -// GetHealthWithResponse request returning *GetHealthResponse -func (c *ClientWithResponses) GetHealthWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*GetHealthResponse, error) { - rsp, err := c.GetHealth(ctx, reqEditors...) +// GetSyncFilesWithResponse request returning *GetSyncFilesResponse +func (c *ClientWithResponses) GetSyncFilesWithResponse(ctx context.Context, projectName string, collectionId string, params *GetSyncFilesParams, reqEditors ...RequestEditorFn) (*GetSyncFilesResponse, error) { + rsp, err := c.GetSyncFiles(ctx, projectName, collectionId, params, reqEditors...) if err != nil { return nil, err } - return ParseGetHealthResponse(rsp) + return ParseGetSyncFilesResponse(rsp) } -// GetProjectArtifactWithResponse request returning *GetProjectArtifactResponse -func (c *ClientWithResponses) GetProjectArtifactWithResponse(ctx context.Context, projectName string, artifactId string, reqEditors ...RequestEditorFn) (*GetProjectArtifactResponse, error) { - rsp, err := c.GetProjectArtifact(ctx, projectName, artifactId, reqEditors...) +// CreateSyncFileWithBodyWithResponse request with arbitrary body returning *CreateSyncFileResponse +func (c *ClientWithResponses) CreateSyncFileWithBodyWithResponse(ctx context.Context, projectName string, collectionId string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*CreateSyncFileResponse, error) { + rsp, err := c.CreateSyncFileWithBody(ctx, projectName, collectionId, contentType, body, reqEditors...) if err != nil { return nil, err } - return ParseGetProjectArtifactResponse(rsp) + return ParseCreateSyncFileResponse(rsp) } -// ProjectArtifactExistsWithResponse request returning *ProjectArtifactExistsResponse -func (c *ClientWithResponses) ProjectArtifactExistsWithResponse(ctx context.Context, projectName string, artifactId string, reqEditors ...RequestEditorFn) (*ProjectArtifactExistsResponse, error) { - rsp, err := c.ProjectArtifactExists(ctx, projectName, artifactId, reqEditors...) +// GetSyncCollectionsWithResponse request returning *GetSyncCollectionsResponse +func (c *ClientWithResponses) GetSyncCollectionsWithResponse(ctx context.Context, projectName string, reqEditors ...RequestEditorFn) (*GetSyncCollectionsResponse, error) { + rsp, err := c.GetSyncCollections(ctx, projectName, reqEditors...) if err != nil { return nil, err } - return ParseProjectArtifactExistsResponse(rsp) + return ParseGetSyncCollectionsResponse(rsp) } -// GetProjectArtifactsWithResponse request returning *GetProjectArtifactsResponse -func (c *ClientWithResponses) GetProjectArtifactsWithResponse(ctx context.Context, projectName string, reqEditors ...RequestEditorFn) (*GetProjectArtifactsResponse, error) { - rsp, err := c.GetProjectArtifacts(ctx, projectName, reqEditors...) +// CreateSyncCollectionWithBodyWithResponse request with arbitrary body returning *CreateSyncCollectionResponse +func (c *ClientWithResponses) CreateSyncCollectionWithBodyWithResponse(ctx context.Context, projectName string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*CreateSyncCollectionResponse, error) { + rsp, err := c.CreateSyncCollectionWithBody(ctx, projectName, contentType, body, reqEditors...) if err != nil { return nil, err } - return ParseGetProjectArtifactsResponse(rsp) + return ParseCreateSyncCollectionResponse(rsp) } -// UploadArtifactWithBodyWithResponse request with arbitrary body returning *UploadArtifactResponse -func (c *ClientWithResponses) UploadArtifactWithBodyWithResponse(ctx context.Context, projectName string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*UploadArtifactResponse, error) { - rsp, err := c.UploadArtifactWithBody(ctx, projectName, contentType, body, reqEditors...) +func (c *ClientWithResponses) CreateSyncCollectionWithResponse(ctx context.Context, projectName string, body CreateSyncCollectionJSONRequestBody, reqEditors ...RequestEditorFn) (*CreateSyncCollectionResponse, error) { + rsp, err := c.CreateSyncCollection(ctx, projectName, body, reqEditors...) if err != nil { return nil, err } - return ParseUploadArtifactResponse(rsp) + return ParseCreateSyncCollectionResponse(rsp) } // ParseGetHealthResponse parses an HTTP response from a GetHealthWithResponse call @@ -590,6 +1632,51 @@ func ParseGetHealthResponse(rsp *http.Response) (*GetHealthResponse, error) { return response, nil } +// ParseGetProjectResponse parses an HTTP response from a GetProjectWithResponse call +func ParseGetProjectResponse(rsp *http.Response) (*GetProjectResponse, error) { + bodyBytes, err := ioutil.ReadAll(rsp.Body) + defer rsp.Body.Close() + if err != nil { + return nil, err + } + + response := &GetProjectResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest Project + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + } + + return response, nil +} + +// ParseProjectExistsResponse parses an HTTP response from a ProjectExistsWithResponse call +func ParseProjectExistsResponse(rsp *http.Response) (*ProjectExistsResponse, error) { + bodyBytes, err := ioutil.ReadAll(rsp.Body) + defer rsp.Body.Close() + if err != nil { + return nil, err + } + + response := &ProjectExistsResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + } + + return response, nil +} + // ParseGetProjectArtifactResponse parses an HTTP response from a GetProjectArtifactWithResponse call func ParseGetProjectArtifactResponse(rsp *http.Response) (*GetProjectArtifactResponse, error) { bodyBytes, err := ioutil.ReadAll(rsp.Body) @@ -679,3 +1766,223 @@ func ParseUploadArtifactResponse(rsp *http.Response) (*UploadArtifactResponse, e return response, nil } + +// ParseDeleteSyncCollectionResponse parses an HTTP response from a DeleteSyncCollectionWithResponse call +func ParseDeleteSyncCollectionResponse(rsp *http.Response) (*DeleteSyncCollectionResponse, error) { + bodyBytes, err := ioutil.ReadAll(rsp.Body) + defer rsp.Body.Close() + if err != nil { + return nil, err + } + + response := &DeleteSyncCollectionResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + } + + return response, nil +} + +// ParseGetSyncCollectionResponse parses an HTTP response from a GetSyncCollectionWithResponse call +func ParseGetSyncCollectionResponse(rsp *http.Response) (*GetSyncCollectionResponse, error) { + bodyBytes, err := ioutil.ReadAll(rsp.Body) + defer rsp.Body.Close() + if err != nil { + return nil, err + } + + response := &GetSyncCollectionResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest SyncCollection + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + } + + return response, nil +} + +// ParseDeleteSyncFileResponse parses an HTTP response from a DeleteSyncFileWithResponse call +func ParseDeleteSyncFileResponse(rsp *http.Response) (*DeleteSyncFileResponse, error) { + bodyBytes, err := ioutil.ReadAll(rsp.Body) + defer rsp.Body.Close() + if err != nil { + return nil, err + } + + response := &DeleteSyncFileResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + } + + return response, nil +} + +// ParseGetSyncFileResponse parses an HTTP response from a GetSyncFileWithResponse call +func ParseGetSyncFileResponse(rsp *http.Response) (*GetSyncFileResponse, error) { + bodyBytes, err := ioutil.ReadAll(rsp.Body) + defer rsp.Body.Close() + if err != nil { + return nil, err + } + + response := &GetSyncFileResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest SyncFile + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + } + + return response, nil +} + +// ParsePutSyncFileResponse parses an HTTP response from a PutSyncFileWithResponse call +func ParsePutSyncFileResponse(rsp *http.Response) (*PutSyncFileResponse, error) { + bodyBytes, err := ioutil.ReadAll(rsp.Body) + defer rsp.Body.Close() + if err != nil { + return nil, err + } + + response := &PutSyncFileResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest SyncFile + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + } + + return response, nil +} + +// ParseGetSyncFilesResponse parses an HTTP response from a GetSyncFilesWithResponse call +func ParseGetSyncFilesResponse(rsp *http.Response) (*GetSyncFilesResponse, error) { + bodyBytes, err := ioutil.ReadAll(rsp.Body) + defer rsp.Body.Close() + if err != nil { + return nil, err + } + + response := &GetSyncFilesResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest []SyncFile + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + } + + return response, nil +} + +// ParseCreateSyncFileResponse parses an HTTP response from a CreateSyncFileWithResponse call +func ParseCreateSyncFileResponse(rsp *http.Response) (*CreateSyncFileResponse, error) { + bodyBytes, err := ioutil.ReadAll(rsp.Body) + defer rsp.Body.Close() + if err != nil { + return nil, err + } + + response := &CreateSyncFileResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest SyncFile + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + } + + return response, nil +} + +// ParseGetSyncCollectionsResponse parses an HTTP response from a GetSyncCollectionsWithResponse call +func ParseGetSyncCollectionsResponse(rsp *http.Response) (*GetSyncCollectionsResponse, error) { + bodyBytes, err := ioutil.ReadAll(rsp.Body) + defer rsp.Body.Close() + if err != nil { + return nil, err + } + + response := &GetSyncCollectionsResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest []SyncCollection + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + } + + return response, nil +} + +// ParseCreateSyncCollectionResponse parses an HTTP response from a CreateSyncCollectionWithResponse call +func ParseCreateSyncCollectionResponse(rsp *http.Response) (*CreateSyncCollectionResponse, error) { + bodyBytes, err := ioutil.ReadAll(rsp.Body) + defer rsp.Body.Close() + if err != nil { + return nil, err + } + + response := &CreateSyncCollectionResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest SyncCollection + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + } + + return response, nil +} diff --git a/pkg/store-client/generated/types.gen.go b/pkg/store-client/generated/types.gen.go index c5f1efe0..83c21317 100644 --- a/pkg/store-client/generated/types.gen.go +++ b/pkg/store-client/generated/types.gen.go @@ -19,7 +19,84 @@ type Error struct { Id string `json:"id"` } +// Project defines model for Project. +type Project struct { + Description string `json:"description"` + Hashes *[]Artifact `json:"hashes,omitempty"` + Id string `json:"id"` + Name string `json:"name"` + SyncCollections *[]SyncCollectionStub `json:"syncCollections,omitempty"` +} + // Success defines model for Success. type Success struct { Message string `json:"message"` } + +// It represents (one of possibly many) sync root folders in a bob.yaml +type SyncCollection struct { + + // a list of syncFiles in this collection + Files *[]SyncFileStub `json:"files,omitempty"` + Id string `json:"id"` + + // relative path to the bob.yaml on the client to the collection folder, DO NOT TRUST this on the client, always check if it is not malicious + LocalPath string `json:"local_path"` + Name string `json:"name"` +} + +// SyncCollectionCreate defines model for SyncCollectionCreate. +type SyncCollectionCreate struct { + LocalPath string `json:"local_path"` + Name string `json:"name"` +} + +// It represents (one of possibly many) sync root folders in a bob.yaml +type SyncCollectionStub struct { + Id string `json:"id"` + + // relative path to the bob.yaml on the client to the collection folder, DO NOT TRUST this on the client, always check if it is not malicious + LocalPath string `json:"local_path"` + Name string `json:"name"` +} + +// SyncFile defines model for SyncFile. +type SyncFile struct { + EncryptedHash *string `json:"encrypted_hash,omitempty"` + Id string `json:"id"` + LocalPath string `json:"local_path"` + + // location to download the file using a GET request. + Location *string `json:"location,omitempty"` +} + +// SyncFileCreate defines model for SyncFileCreate. +type SyncFileCreate struct { + File string `json:"file"` + LocalPath string `json:"local_path"` +} + +// SyncFileStub defines model for SyncFileStub. +type SyncFileStub struct { + EncryptedHash *string `json:"encrypted_hash,omitempty"` + Id string `json:"id"` + LocalPath string `json:"local_path"` +} + +// Updated version of the file, properties which are omitted will not be changed. At least one property has be included. +type SyncFileUpdate struct { + File *string `json:"file,omitempty"` + Id string `json:"id"` + LocalPath *string `json:"local_path,omitempty"` +} + +// GetSyncFilesParams defines parameters for GetSyncFiles. +type GetSyncFilesParams struct { + WithLocation *bool `json:"withLocation,omitempty"` +} + +// CreateSyncCollectionJSONBody defines parameters for CreateSyncCollection. +type CreateSyncCollectionJSONBody SyncCollectionCreate + +// CreateSyncCollectionJSONRequestBody defines body for CreateSyncCollection for application/json ContentType. +type CreateSyncCollectionJSONRequestBody CreateSyncCollectionJSONBody diff --git a/pkg/store-client/store_client.go b/pkg/store-client/store_client.go index 2483b122..08a8e37d 100644 --- a/pkg/store-client/store_client.go +++ b/pkg/store-client/store_client.go @@ -12,6 +12,16 @@ type I interface { UploadArtifact(ctx context.Context, projectName string, artifactID string, src io.Reader) (err error) ListArtifacts(ctx context.Context, projectName string) (artifactIds []string, err error) GetArtifact(ctx context.Context, projectName string, artifactId string) (rc io.ReadCloser, err error) + + CollectionCreate(ctx context.Context, projectName, name, localPath string) (*generated.SyncCollection, error) + Collection(ctx context.Context, projectName, collectionId string) (*generated.SyncCollection, error) + Collections(ctx context.Context, projectName string) ([]generated.SyncCollection, error) + + FileCreate(ctx context.Context, projectName, collectionId, localPath string, src io.Reader) (*generated.SyncFile, error) + File(ctx context.Context, projectName, collectionId, fileId string) (*generated.SyncFile, io.ReadCloser, error) + Files(ctx context.Context, projectName, collectionId string, withLocation bool) ([]generated.SyncFile, error) + FileUpdate(ctx context.Context, projectName, collectionId, fileId, localPath string, src *io.Reader) (*generated.SyncFile, error) + FileDelete(ctx context.Context, projectName, collectionId, fileId string) error } type c struct { diff --git a/pkg/versionedsync/collection/collection.go b/pkg/versionedsync/collection/collection.go new file mode 100644 index 00000000..6fed1c22 --- /dev/null +++ b/pkg/versionedsync/collection/collection.go @@ -0,0 +1,73 @@ +package collection + +import ( + "fmt" + "github.com/benchkram/bob/pkg/store-client/generated" + "github.com/benchkram/bob/pkg/versionedsync/file" + "github.com/benchkram/errz" + "strings" +) + +const ( + divider = ";;" +) + +// C is the representation of a collection of synced files +type C struct { + ID string + + // Name of the collection + Name string + // Version + Version string + // LocalPath is the path to the collections root folder + LocalPath string + // Files are a set of individual blobs belonging to this collection + Files []*file.F +} + +func JoinNameAndVersion(name, tag string) string { + return name + divider + tag +} + +func SplitName(combinedName string) (name, version string, _ error) { + parts := strings.Split(combinedName, divider) + if len(parts) == 1 { + return combinedName, "", nil + } else if len(parts) == 2 { + return parts[0], parts[1], nil + } else { + return "", "", fmt.Errorf("invalid combined collection name") + } +} + +func FromRestType(genC *generated.SyncCollection) (_ *C, err error) { + defer errz.Recover(&err) + var files []*file.F + if genC.Files != nil { + for _, f := range *genC.Files { + files = append(files, file.FileFromRestStubType(f)) + } + } + name, version, err := SplitName(genC.Name) + errz.Fatal(err) + return &C{ + ID: genC.Id, + Name: name, + Version: version, + LocalPath: genC.LocalPath, + Files: files, + }, nil +} + +func (c *C) FileByPath(localPath string) (*file.F, bool) { + if c.Files == nil { + return nil, false + } + for _, f := range c.Files { + if f.LocalPath == localPath { + return f, true + } + } + return nil, false +} diff --git a/pkg/versionedsync/file/file.go b/pkg/versionedsync/file/file.go new file mode 100644 index 00000000..b25448f7 --- /dev/null +++ b/pkg/versionedsync/file/file.go @@ -0,0 +1,34 @@ +package file + +import ( + "github.com/benchkram/bob/pkg/store-client/generated" +) + +// F represents a synced file +type F struct { + // ID is the optional identifier on the server + ID *string + + // LocalPath is the path of the file on the client relative to the collection root + LocalPath string + + // Hash is the hash generated on the server or client over the file content + // it is used to detect changes in the content when comparing local and remote + Hash string +} + +func FileFromRestType(f generated.SyncFile) *F { + return &F{ + ID: &f.Id, + LocalPath: f.LocalPath, + Hash: *f.EncryptedHash, + } +} + +func FileFromRestStubType(f generated.SyncFileStub) *F { + return &F{ + ID: &f.Id, + LocalPath: f.LocalPath, + Hash: *f.EncryptedHash, + } +} diff --git a/pkg/versionedsync/localsyncstore/options.go b/pkg/versionedsync/localsyncstore/options.go new file mode 100644 index 00000000..a054f3bc --- /dev/null +++ b/pkg/versionedsync/localsyncstore/options.go @@ -0,0 +1 @@ +package localsyncstore diff --git a/pkg/versionedsync/localsyncstore/store.go b/pkg/versionedsync/localsyncstore/store.go new file mode 100644 index 00000000..6d60074c --- /dev/null +++ b/pkg/versionedsync/localsyncstore/store.go @@ -0,0 +1,41 @@ +package localsyncstore + +import ( + "github.com/benchkram/bob/pkg/file" + "github.com/benchkram/errz" + "io" + "os" + "path/filepath" +) + +type S struct { +} + +func (s *S) ReadFile(bobDir, collectionPath, localPath string) (r io.ReadCloser, err error) { + return os.Open(filepath.Join(bobDir, collectionPath, localPath)) +} + +func (s *S) DeleteFile(bobDir, collectionPath, localPath string) (err error) { + return os.Remove(filepath.Join(bobDir, collectionPath, localPath)) +} + +// WriteFile writes contents from read closer to a file +// if the file exists it is replaced +func (s *S) WriteFile(bobDir, collectionPath, localPath string, reader io.ReadCloser) (err error) { + defer errz.Recover(&err) + absPath := filepath.Join(bobDir, collectionPath, localPath) + if file.Exists(absPath) { + err = s.DeleteFile(bobDir, collectionPath, localPath) + errz.Fatal(err) + } + outFile, err := os.Create(absPath) + errz.Fatal(err) + _, err = io.Copy(outFile, reader) + err = reader.Close() + errz.Fatal(err) + return nil +} + +func New() *S { + return &S{} +} diff --git a/pkg/versionedsync/remotesyncstore/errors.go b/pkg/versionedsync/remotesyncstore/errors.go new file mode 100644 index 00000000..96765ccf --- /dev/null +++ b/pkg/versionedsync/remotesyncstore/errors.go @@ -0,0 +1,7 @@ +package remotesyncstore + +import "fmt" + +var ( + ErrCollectionNotFound = fmt.Errorf("sync collection not found") +) diff --git a/pkg/versionedsync/remotesyncstore/options.go b/pkg/versionedsync/remotesyncstore/options.go new file mode 100644 index 00000000..d363d4c2 --- /dev/null +++ b/pkg/versionedsync/remotesyncstore/options.go @@ -0,0 +1,11 @@ +package remotesyncstore + +import storeclient "github.com/benchkram/bob/pkg/store-client" + +type Option func(s *S) + +func WithClient(client storeclient.I) Option { + return func(s *S) { + s.client = client + } +} diff --git a/pkg/versionedsync/remotesyncstore/store.go b/pkg/versionedsync/remotesyncstore/store.go new file mode 100644 index 00000000..b947c31a --- /dev/null +++ b/pkg/versionedsync/remotesyncstore/store.go @@ -0,0 +1,106 @@ +package remotesyncstore + +import ( + "context" + "fmt" + storeclient "github.com/benchkram/bob/pkg/store-client" + "github.com/benchkram/bob/pkg/versionedsync/collection" + "github.com/benchkram/errz" + "io" +) + +// S is a versioned sync store that handles the logic of accessing the bob-server to store collections and files +// it uses the store client interface to achieve that +type S struct { + username string + client storeclient.I + project string +} + +func New(username, project string, opts ...Option) *S { + s := &S{ + username: username, + project: project, + } + + for _, opt := range opts { + if opt == nil { + continue + } + opt(s) + } + + if s.client == nil { + panic(fmt.Errorf("no client")) + } + + return s +} + +func (s *S) CollectionCreate(ctx context.Context, name, tag, path string) (cId string, err error) { + defer errz.Recover(&err) + genC, err := s.client.CollectionCreate(ctx, s.project, collection.JoinNameAndVersion(name, tag), path) + errz.Fatal(err) + c, err := collection.FromRestType(genC) + errz.Fatal(err) + return c.ID, nil +} +func (s *S) Collection(ctx context.Context, collectionId string) (c *collection.C, err error) { + defer errz.Recover(&err) + genC, err := s.client.Collection(ctx, s.project, collectionId) + errz.Fatal(err) + c, err = collection.FromRestType(genC) + errz.Fatal(err) + return c, nil +} +func (s *S) CollectionIdByName(ctx context.Context, name, tag string) (cId string, err error) { + remoteName := collection.JoinNameAndVersion(name, tag) + defer errz.Recover(&err) + collections, err := s.client.Collections(ctx, s.project) + errz.Fatal(err) + for _, c := range collections { + if c.Name == remoteName { + return c.Id, nil + } + } + return "", ErrCollectionNotFound +} + +func (s *S) Collections(ctx context.Context) (collections []*collection.C, err error) { + defer errz.Recover(&err) + genCs, err := s.client.Collections(ctx, s.project) + errz.Fatal(err) + for _, gc := range genCs { + c, err := collection.FromRestType(&gc) + errz.Fatal(err) + collections = append(collections, c) + } + + return collections, nil +} +func (s *S) FileUpload(ctx context.Context, collectionId, localPath string, srcReader io.Reader) (err error) { + defer errz.Recover(&err) + _, err = s.client.FileCreate(ctx, s.project, collectionId, localPath, srcReader) + errz.Fatal(err) + return nil + +} +func (s *S) File(ctx context.Context, collectionId, fileId string) (r io.ReadCloser, err error) { + defer errz.Recover(&err) + + _, r, err = s.client.File(ctx, s.project, collectionId, fileId) + errz.Fatal(err) + return r, nil +} +func (s *S) FileUpdate(ctx context.Context, collectionId, fileId string, srcReader io.Reader) (err error) { + defer errz.Recover(&err) + _, err = s.client.FileUpdate(ctx, s.project, collectionId, fileId, "", &srcReader) + errz.Fatal(err) + return nil +} +func (s *S) FileDelete(ctx context.Context, collectionId, fileId string) (err error) { + defer errz.Recover(&err) + err = s.client.FileDelete(ctx, s.project, collectionId, fileId) + errz.Fatal(err) + return nil +} From a22ee961b861bc0aaf24c9c83c1707b9e4d6bffc Mon Sep 17 00:00:00 2001 From: Daniel Ketterer <26778610+dketterer@users.noreply.github.com> Date: Wed, 20 Jul 2022 13:14:40 +0200 Subject: [PATCH 3/5] Fix linting issues, catch errors --- bob/bobfile/bobfile.go | 2 +- bob/sync.go | 33 ++++++++++++++++++-------- bobsync/hashcache.go | 2 +- bobsync/sync.go | 54 +++++++++++++++++++----------------------- 4 files changed, 50 insertions(+), 41 deletions(-) diff --git a/bob/bobfile/bobfile.go b/bob/bobfile/bobfile.go index 58b2de0d..4384cc43 100644 --- a/bob/bobfile/bobfile.go +++ b/bob/bobfile/bobfile.go @@ -217,7 +217,7 @@ func bobfileRead(dir string) (_ *Bobfile, err error) { //} // write names to Sync objects - for name, _ := range bobfile.SyncCollections { + for name := range bobfile.SyncCollections { s := bobfile.SyncCollections[name] s.SetName(name) bobfile.SyncCollections[name] = s diff --git a/bob/sync.go b/bob/sync.go index 2a5bc855..301dc923 100644 --- a/bob/sync.go +++ b/bob/sync.go @@ -3,6 +3,7 @@ package bob import ( "context" "fmt" + "github.com/benchkram/bob/pkg/boblog" "github.com/benchkram/bob/pkg/versionedsync/localsyncstore" "github.com/logrusorgru/aurora" "os" @@ -15,8 +16,9 @@ func (b *B) SyncPush(ctx context.Context) (err error) { defer errz.Recover(&err) wd, _ := os.Getwd() - aggregate, err := bobfile.BobfileRead(wd) - aggregate, err = b.Aggregate() + _, err = bobfile.BobfileRead(wd) + errz.Fatal(err) + aggregate, err := b.Aggregate() errz.Fatal(err) remoteStore := aggregate.VersionedSyncStore() @@ -27,7 +29,10 @@ func (b *B) SyncPush(ctx context.Context) (err error) { } else { for _, sync := range aggregate.SyncCollections { err = sync.Push(ctx, *remoteStore, *localStore, aggregate.Dir()) - errz.Fatal(err) + if err != nil { + boblog.Log.V(1).Error(err, fmt.Sprintf("failed to sync from local to remote [collection: %s@%s]", sync.GetName(), sync.Version)) + + } } } @@ -38,8 +43,9 @@ func (b *B) SyncPull(ctx context.Context) (err error) { defer errz.Recover(&err) wd, _ := os.Getwd() - aggregate, err := bobfile.BobfileRead(wd) - aggregate, err = b.Aggregate() + _, err = bobfile.BobfileRead(wd) + errz.Fatal(err) + aggregate, err := b.Aggregate() errz.Fatal(err) remoteStore := aggregate.VersionedSyncStore() @@ -50,7 +56,9 @@ func (b *B) SyncPull(ctx context.Context) (err error) { } else { for _, sync := range aggregate.SyncCollections { err = sync.Pull(ctx, *remoteStore, *localStore, aggregate.Dir()) - errz.Fatal(err) + if err != nil { + boblog.Log.V(1).Error(err, fmt.Sprintf("failed to sync from remote to local [collection: %s@%s]", sync.GetName(), sync.Version)) + } } } @@ -68,7 +76,9 @@ func (b *B) SyncListLocal(_ context.Context) (err error) { for _, sync := range aggregate.SyncCollections { err = sync.ListLocal(aggregate.Dir()) - errz.Fatal(err) + if err != nil { + boblog.Log.V(1).Error(err, fmt.Sprintf("failed list local [collection: %s@%s]", sync.GetName(), sync.Version)) + } } return nil @@ -78,9 +88,9 @@ func (b *B) SyncListRemote(ctx context.Context) (err error) { defer errz.Recover(&err) wd, _ := os.Getwd() - aggregate, err := bobfile.BobfileRead(wd) + _, err = bobfile.BobfileRead(wd) errz.Fatal(err) - aggregate, err = b.Aggregate() + aggregate, err := b.Aggregate() errz.Fatal(err) remoteStore := aggregate.VersionedSyncStore() @@ -88,9 +98,12 @@ func (b *B) SyncListRemote(ctx context.Context) (err error) { if remoteStore == nil { fmt.Println(aurora.Red("No remote project configured can not list remote")) } else { + // FIXME: list remote should only be run once for _, sync := range aggregate.SyncCollections { err = sync.ListRemote(ctx, *remoteStore) - errz.Fatal(err) + if err != nil { + boblog.Log.V(1).Error(err, fmt.Sprintf("failed list remote [collection: %s@%s]", sync.GetName(), sync.Version)) + } } } diff --git a/bobsync/hashcache.go b/bobsync/hashcache.go index 4fa78c2d..edfd4629 100644 --- a/bobsync/hashcache.go +++ b/bobsync/hashcache.go @@ -37,8 +37,8 @@ func FromFileOrNew(path string) (hc *HashCache, err error) { return nil, fmt.Errorf("failed to load hashcache from: %s: it is a directory", path) } f, err := os.Open(path) - defer f.Close() errz.Fatal(err) + defer f.Close() byteValue, err := ioutil.ReadAll(f) errz.Fatal(err) err = json.Unmarshal(byteValue, &hc) diff --git a/bobsync/sync.go b/bobsync/sync.go index 9830932f..65af0c11 100644 --- a/bobsync/sync.go +++ b/bobsync/sync.go @@ -27,6 +27,10 @@ type Sync struct { cache *HashCache } +func (s *Sync) GetName() string { + return s.name +} + func (s *Sync) SetName(name string) { s.name = name } @@ -34,17 +38,17 @@ func (s *Sync) SetName(name string) { func (s *Sync) Push(ctx context.Context, remoteStore remotesyncstore.S, localStore localsyncstore.S, bobDir string) (err error) { defer errz.Recover(&err) - var collectionMustBeCreated bool + //var collectionMustBeCreated bool // get collectionId ready - var collectionId string - collectionId, err = remoteStore.CollectionIdByName(ctx, s.name, s.Version) + + s.remoteCollectionId, err = remoteStore.CollectionIdByName(ctx, s.name, s.Version) // check if collections exists switch err { case nil: case remotesyncstore.ErrCollectionNotFound: - collectionMustBeCreated = true - collectionId, err = remoteStore.CollectionCreate(ctx, s.name, s.Version, s.Path) + //collectionMustBeCreated = true + s.remoteCollectionId, err = remoteStore.CollectionCreate(ctx, s.name, s.Version, s.Path) errz.Fatal(err) default: errz.Fatal(err) @@ -62,7 +66,7 @@ func (s *Sync) Push(ctx context.Context, remoteStore remotesyncstore.S, localSto err = s.cache.SaveToFile(absHashCachePath) errz.Fatal(err) - remoteCollection, err := remoteStore.Collection(ctx, collectionId) + remoteCollection, err := remoteStore.Collection(ctx, s.remoteCollectionId) errz.Fatal(err) // create the delta @@ -71,21 +75,19 @@ func (s *Sync) Push(ctx context.Context, remoteStore remotesyncstore.S, localSto fmt.Printf("Local-Remote delta for %s\n", aurora.Bold(s.name)) fmt.Println(delta.PushOverview()) - if !collectionMustBeCreated { - // TODO: prompt user and seek confirmation - } + // TODO: prompt user and seek confirmation for _, f := range delta.LocalFilesMissingOnRemote { srcReader, err := localStore.ReadFile(bobDir, s.Path, f.LocalPath) errz.Fatal(err) - err = remoteStore.FileUpload(ctx, collectionId, f.LocalPath, srcReader) + err = remoteStore.FileUpload(ctx, s.remoteCollectionId, f.LocalPath, srcReader) errz.Fatal(err) } for _, f := range delta.RemoteFilesMissingOnLocal { if f.ID == nil { return fmt.Errorf("ID not available can not delete from remote") } - err = remoteStore.FileDelete(ctx, collectionId, *f.ID) + err = remoteStore.FileDelete(ctx, s.remoteCollectionId, *f.ID) errz.Fatal(err) } for _, f := range delta.ToBeUpdated { @@ -94,7 +96,7 @@ func (s *Sync) Push(ctx context.Context, remoteStore remotesyncstore.S, localSto } srcReader, err := localStore.ReadFile(bobDir, s.Path, f.LocalPath) errz.Fatal(err) - err = remoteStore.FileUpdate(ctx, collectionId, *f.ID, srcReader) + err = remoteStore.FileUpdate(ctx, s.remoteCollectionId, *f.ID, srcReader) errz.Fatal(err) } @@ -104,17 +106,16 @@ func (s *Sync) Push(ctx context.Context, remoteStore remotesyncstore.S, localSto func (s *Sync) Pull(ctx context.Context, remoteStore remotesyncstore.S, localStore localsyncstore.S, bobDir string) (err error) { defer errz.Recover(&err) - var collectionMustBeCreated bool + //var collectionMustBeCreated bool // get collectionId ready - var collectionId string - collectionId, err = remoteStore.CollectionIdByName(ctx, s.name, s.Version) + s.remoteCollectionId, err = remoteStore.CollectionIdByName(ctx, s.name, s.Version) // check if collections exists switch err { case nil: case remotesyncstore.ErrCollectionNotFound: - collectionMustBeCreated = true - collectionId, err = remoteStore.CollectionCreate(ctx, s.name, s.Version, s.Path) + //collectionMustBeCreated = true + s.remoteCollectionId, err = remoteStore.CollectionCreate(ctx, s.name, s.Version, s.Path) errz.Fatal(err) default: errz.Fatal(err) @@ -132,7 +133,7 @@ func (s *Sync) Pull(ctx context.Context, remoteStore remotesyncstore.S, localSto err = s.cache.SaveToFile(absHashCachePath) errz.Fatal(err) - remoteCollection, err := remoteStore.Collection(ctx, collectionId) + remoteCollection, err := remoteStore.Collection(ctx, s.remoteCollectionId) errz.Fatal(err) // create the delta @@ -141,9 +142,7 @@ func (s *Sync) Pull(ctx context.Context, remoteStore remotesyncstore.S, localSto fmt.Printf("Local-Remote delta for %s\n", aurora.Bold(s.name)) fmt.Println(delta.PullOverview()) - if !collectionMustBeCreated { - // TODO: prompt user and seek confirmation - } + // TODO: prompt user and seek confirmation for _, f := range delta.LocalFilesMissingOnRemote { err := localStore.DeleteFile(bobDir, s.Path, f.LocalPath) @@ -153,7 +152,7 @@ func (s *Sync) Pull(ctx context.Context, remoteStore remotesyncstore.S, localSto if f.ID == nil { return fmt.Errorf("ID not available can not downlaod from remote") } - srcReader, err := remoteStore.File(ctx, collectionId, *f.ID) + srcReader, err := remoteStore.File(ctx, s.remoteCollectionId, *f.ID) errz.Fatal(err) err = localStore.WriteFile(bobDir, s.Path, f.LocalPath, srcReader) errz.Fatal(err) @@ -162,7 +161,7 @@ func (s *Sync) Pull(ctx context.Context, remoteStore remotesyncstore.S, localSto if f.ID == nil { return fmt.Errorf("ID not available can not downlaod from remote") } - srcReader, err := remoteStore.File(ctx, collectionId, *f.ID) + srcReader, err := remoteStore.File(ctx, s.remoteCollectionId, *f.ID) errz.Fatal(err) err = localStore.WriteFile(bobDir, s.Path, f.LocalPath, srcReader) errz.Fatal(err) @@ -174,8 +173,6 @@ func (s *Sync) Pull(ctx context.Context, remoteStore remotesyncstore.S, localSto func (s *Sync) ListLocal(bobDir string) (err error) { defer errz.Recover(&err) - // sync files in collection - absHashCachPath := filepath.Join(bobDir, hashCachePath) if s.cache == nil { s.cache, err = FromFileOrNew(absHashCachPath) @@ -188,7 +185,7 @@ func (s *Sync) ListLocal(bobDir string) (err error) { errz.Fatal(err) fmt.Printf("%s@%s (./%s)\n", aurora.Bold(s.name), aurora.Italic(s.Version), s.Path) - for p, _ := range *s.cache { + for p := range *s.cache { fmt.Printf("\t%s\n", p) } @@ -200,8 +197,7 @@ func (s *Sync) ListRemote(ctx context.Context, store remotesyncstore.S) (err err defer errz.Recover(&err) // get collectionId ready - var collectionId string - collectionId, err = store.CollectionIdByName(ctx, s.name, s.Version) + s.remoteCollectionId, err = store.CollectionIdByName(ctx, s.name, s.Version) // check if collections exists switch err { @@ -217,7 +213,7 @@ func (s *Sync) ListRemote(ctx context.Context, store remotesyncstore.S) (err err for _, c := range collections { fmt.Printf("%s@%s (./%s)", aurora.Bold(c.Name), aurora.Italic(c.Version), c.LocalPath) - if c.ID == collectionId { + if c.ID == s.remoteCollectionId { fmt.Printf(" [synced to local]") } fmt.Println() From 490a1004ac2f7c6cf1efb25d6062498d69236fc8 Mon Sep 17 00:00:00 2001 From: Daniel Ketterer <26778610+dketterer@users.noreply.github.com> Date: Tue, 26 Jul 2022 12:14:42 +0200 Subject: [PATCH 4/5] Change CLI, add dirs, add more validation --- bob/aggregate.go | 5 + bob/bobfile/bobfile.go | 23 ++- bob/sync.go | 89 +++++++-- bobsync/delta.go | 42 +++- bobsync/errors.go | 12 ++ bobsync/hashcache.go | 46 ++++- bobsync/map.go | 2 +- bobsync/sync.go | 217 ++++++++++++++++----- bobsync/verify.go | 15 +- bobtask/input.go | 4 +- cli/cmd_root.go | 12 +- cli/cmd_sync.go | 55 +++--- pkg/filehash/hash.go | 20 +- pkg/filepathutil/list.go | 10 +- pkg/store-client/client.go | 52 +++-- pkg/store-client/generated/types.gen.go | 14 +- pkg/store-client/store_client.go | 6 +- pkg/userprompt/userprompt.go | 28 +++ pkg/versionedsync/collection/collection.go | 26 ++- pkg/versionedsync/file/file.go | 20 +- pkg/versionedsync/localsyncstore/store.go | 31 ++- pkg/versionedsync/remotesyncstore/store.go | 32 ++- 22 files changed, 575 insertions(+), 186 deletions(-) create mode 100644 bobsync/errors.go create mode 100644 pkg/userprompt/userprompt.go diff --git a/bob/aggregate.go b/bob/aggregate.go index d14bea0e..f9a8b602 100644 --- a/bob/aggregate.go +++ b/bob/aggregate.go @@ -266,6 +266,11 @@ func (b *B) Aggregate() (aggregate *bobfile.Bobfile, err error) { aggregate.Project = aggregate.Dir() } + for _, sync := range aggregate.SyncCollections { + err = sync.Validate(aggregate.Dir()) + errz.Fatal(err) + } + return aggregate, aggregate.Verify() } diff --git a/bob/bobfile/bobfile.go b/bob/bobfile/bobfile.go index 4384cc43..b9093ea3 100644 --- a/bob/bobfile/bobfile.go +++ b/bob/bobfile/bobfile.go @@ -51,6 +51,8 @@ var ( ErrInvalidRunType = fmt.Errorf("Invalid run type") + ErrDuplicateSyncPath = fmt.Errorf("found duplicate sync path in bob.yaml") + ProjectNameFormatHint = "project name should be in the form 'project' or 'registry.com/user/project'" ) @@ -82,7 +84,7 @@ type Bobfile struct { Nixpkgs string `yaml:"nixpkgs"` // SyncCollections are folder synchronisations through bob-server - SyncCollections bobsync.SyncMap `yaml:"sync"` + SyncCollections bobsync.SyncList `yaml:"syncCollections"` // Parent directory of the Bobfile. // Populated through BobfileRead(). @@ -215,14 +217,6 @@ func bobfileRead(dir string) (_ *Bobfile, err error) { //} else { // bobfile.Project = bobfile.dir //} - - // write names to Sync objects - for name := range bobfile.SyncCollections { - s := bobfile.SyncCollections[name] - s.SetName(name) - bobfile.SyncCollections[name] = s - } - return bobfile, nil } @@ -381,7 +375,16 @@ func (b *Bobfile) Validate() (err error) { } } - // TODO: validate sync entries + // no duplicate paths allowed + existingPaths := make(map[string]int) + for _, sync := range b.SyncCollections { + err = sync.Validate(b.Dir()) + errz.Fatal(err) + existingPaths[sync.Path]++ + if existingPaths[sync.Path] > 1 { + return usererror.Wrapm(ErrDuplicateSyncPath, fmt.Sprintf("invalid collection path %s", sync.Path)) + } + } return nil } diff --git a/bob/sync.go b/bob/sync.go index 301dc923..1d5de5f4 100644 --- a/bob/sync.go +++ b/bob/sync.go @@ -2,17 +2,22 @@ package bob import ( "context" + "errors" "fmt" + "github.com/benchkram/bob/bobsync" "github.com/benchkram/bob/pkg/boblog" + "github.com/benchkram/bob/pkg/usererror" "github.com/benchkram/bob/pkg/versionedsync/localsyncstore" + "github.com/benchkram/bob/pkg/versionedsync/remotesyncstore" "github.com/logrusorgru/aurora" "os" + "sort" "github.com/benchkram/bob/bob/bobfile" "github.com/benchkram/errz" ) -func (b *B) SyncPush(ctx context.Context) (err error) { +func (b *B) SyncCreatePush(ctx context.Context, collectionName, version, path string, dry bool) (err error) { defer errz.Recover(&err) wd, _ := os.Getwd() @@ -27,19 +32,39 @@ func (b *B) SyncPush(ctx context.Context) (err error) { if remoteStore == nil { fmt.Println(aurora.Red("No remote project configured can not push")) } else { - for _, sync := range aggregate.SyncCollections { - err = sync.Push(ctx, *remoteStore, *localStore, aggregate.Dir()) - if err != nil { - boblog.Log.V(1).Error(err, fmt.Sprintf("failed to sync from local to remote [collection: %s@%s]", sync.GetName(), sync.Version)) + // create only sync object and check if exists and is inside bobdir + // TODO: check if it conflicts with any git tracked file + sync, err := bobsync.NewSync(collectionName, version, path, aggregate.Dir()) + errz.Fatal(err) + // check if path is the same as in another sync in bob.yaml + err = bobsync.CheckForConflicts(aggregate.SyncCollections, *sync) + if errors.Is(err, bobsync.ErrSyncPathTaken) { + errz.Fatal(usererror.Wrap(err)) + } else { + errz.Fatal(err) + } + // create collection on remote, if name-version exists on remote => fail + err = sync.CreateOnRemote(ctx, *remoteStore, dry) + errz.Fatal(err) + // TODO: automatically add sync to bob file + // add it to the bobfile and check for conflicts + // if path exists => replace sync entry in bobfile + // err = aggregate.AddSync(sync) + // errz.Fatal(err) + fmt.Printf("Add that to your bob file so that others can use it:\n"+ + "syncCollections:\n - name: %s\n version: %s\n path: %s\n", sync.Name, sync.Version, sync.Path) + + err = sync.Push(ctx, *remoteStore, *localStore, aggregate.Dir(), dry) + if err != nil { + boblog.Log.V(1).Error(err, fmt.Sprintf("failed to sync from local to remote [collection: %s@%s]", sync.Name, sync.Version)) - } } } return nil } -func (b *B) SyncPull(ctx context.Context) (err error) { +func (b *B) SyncPull(ctx context.Context, force bool) (err error) { defer errz.Recover(&err) wd, _ := os.Getwd() @@ -55,9 +80,9 @@ func (b *B) SyncPull(ctx context.Context) (err error) { fmt.Println(aurora.Red("no remote project configured can not pull")) } else { for _, sync := range aggregate.SyncCollections { - err = sync.Pull(ctx, *remoteStore, *localStore, aggregate.Dir()) + err = sync.Pull(ctx, *remoteStore, *localStore, aggregate.Dir(), force) if err != nil { - boblog.Log.V(1).Error(err, fmt.Sprintf("failed to sync from remote to local [collection: %s@%s]", sync.GetName(), sync.Version)) + boblog.Log.V(1).Error(err, fmt.Sprintf("failed to sync from remote to local [collection: %s@%s]", sync.Name, sync.Version)) } } } @@ -72,12 +97,18 @@ func (b *B) SyncListLocal(_ context.Context) (err error) { aggregate, err := bobfile.BobfileRead(wd) errz.Fatal(err) - fmt.Printf("bob sync ls: displaying all files ready to by synced\n\n") + // call validate separate since aggregate is not called + for _, sync := range aggregate.SyncCollections { + err = sync.Validate(aggregate.Dir()) + if err != nil { + errz.Fatal(usererror.Wrapm(err, "can not validate defined sync")) + } + } for _, sync := range aggregate.SyncCollections { err = sync.ListLocal(aggregate.Dir()) if err != nil { - boblog.Log.V(1).Error(err, fmt.Sprintf("failed list local [collection: %s@%s]", sync.GetName(), sync.Version)) + boblog.Log.V(1).Error(err, fmt.Sprintf("failed list local [collection: %s@%s]", sync.Name, sync.Version)) } } @@ -98,13 +129,41 @@ func (b *B) SyncListRemote(ctx context.Context) (err error) { if remoteStore == nil { fmt.Println(aurora.Red("No remote project configured can not list remote")) } else { - // FIXME: list remote should only be run once - for _, sync := range aggregate.SyncCollections { - err = sync.ListRemote(ctx, *remoteStore) + for i := range aggregate.SyncCollections { + err = aggregate.SyncCollections[i].FillRemoteId(ctx, *remoteStore) if err != nil { - boblog.Log.V(1).Error(err, fmt.Sprintf("failed list remote [collection: %s@%s]", sync.GetName(), sync.Version)) + if errors.Is(err, remotesyncstore.ErrCollectionNotFound) { + boblog.Log.V(1).Error(err, fmt.Sprintf("sync collection %s@%s does not exist on the server", + aggregate.SyncCollections[i].Name, aggregate.SyncCollections[i].Version)) + } else { + errz.Fatal(err) + } + } + } + + collections, err := (*remoteStore).Collections(ctx) + errz.Fatal(err) + for _, c := range collections { + sort.Sort(c) + fmt.Printf("%s@%s", aurora.Bold(c.Name), aurora.Italic(c.Version)) + referenced, path := func() (bool, string) { + for _, sync := range aggregate.SyncCollections { + if c.ID == sync.GetRemoteId() { + return true, sync.Path + } + } + return false, "" + }() + if referenced { + fmt.Printf(" [referenced in bob.yaml] (./%s)", path) + } + fmt.Println() + for _, f := range c.Files { + fmt.Printf("\t%s\n", f.LocalPath) } + } + } return nil diff --git a/bobsync/delta.go b/bobsync/delta.go index a48ff0fb..e1e412a3 100644 --- a/bobsync/delta.go +++ b/bobsync/delta.go @@ -4,25 +4,46 @@ import ( "fmt" "github.com/benchkram/bob/pkg/versionedsync/collection" "github.com/benchkram/bob/pkg/versionedsync/file" + "sort" ) +type FileList []*file.F + +// Len is the number of elements in the collection. +func (f FileList) Len() int { + return len(f) +} + +// Less reports whether the element with +// index i should sort before the element with index j. +func (f FileList) Less(i, j int) bool { + return (f)[i].LocalPath < (f)[j].LocalPath +} + +// Swap swaps the elements with indexes i and j. +func (f FileList) Swap(i, j int) { + tmpF := (f)[i] + (f)[i] = (f)[j] + (f)[j] = tmpF +} + type Delta struct { // Unchanged are files which have the same hash on local and remote // Files in this slice should always have an ID set - Unchanged []*file.F + Unchanged FileList // ToBeUpdated are files which exist on local and remote but have different hashes // Files in this slice should always have an ID set - ToBeUpdated []*file.F + ToBeUpdated FileList // LocalFilesMissingOnRemote can be read different for push and pull // push: what has to be created on the remote and is only on local // pull: what has to be removed on local since it is not on remote // Files in this slice never have an ID set - LocalFilesMissingOnRemote []*file.F + LocalFilesMissingOnRemote FileList // RemoteFilesMissingOnLocal can be read different for push and pull // push: what has to be removed on the remote and is only on remote // pull: what has to be created on local since it is only on remote // Files in this slice should always have an ID set - RemoteFilesMissingOnLocal []*file.F + RemoteFilesMissingOnLocal FileList } func (d *Delta) String() string { @@ -102,10 +123,19 @@ func NewDelta(local HashCache, remote collection.C) *Delta { // localPath non-existent on remote delta.LocalFilesMissingOnRemote = append(delta.LocalFilesMissingOnRemote, &file.F{ - LocalPath: localPath, - Hash: fingerprint.Hash, + LocalPath: localPath, + Hash: fingerprint.Hash, + IsDirectory: fingerprint.IsDir, }) } } + delta.Sort() return delta } + +func (d *Delta) Sort() { + sort.Sort(d.Unchanged) + sort.Sort(d.ToBeUpdated) + sort.Sort(d.RemoteFilesMissingOnLocal) + sort.Sort(d.LocalFilesMissingOnRemote) +} diff --git a/bobsync/errors.go b/bobsync/errors.go new file mode 100644 index 00000000..dd5af7c7 --- /dev/null +++ b/bobsync/errors.go @@ -0,0 +1,12 @@ +package bobsync + +import ( + "errors" +) + +var ( + ErrCollectionVersionExists = errors.New("collection version exists on remote") + ErrSyncPathTaken = errors.New("sync collection path already in use") + ErrInvalidCollectionName = errors.New("invalid collection name") + ErrInvalidCollectionVersion = errors.New("invalid collection version") +) diff --git a/bobsync/hashcache.go b/bobsync/hashcache.go index edfd4629..27acff73 100644 --- a/bobsync/hashcache.go +++ b/bobsync/hashcache.go @@ -6,18 +6,19 @@ import ( "github.com/benchkram/bob/pkg/boblog" "github.com/benchkram/bob/pkg/file" "github.com/benchkram/bob/pkg/filehash" - "github.com/benchkram/bob/pkg/filepathutil" "github.com/benchkram/errz" "github.com/logrusorgru/aurora" "io/ioutil" "os" "path/filepath" "runtime" + "sort" "sync" "time" ) type Fingerprint struct { + IsDir bool Hash string CreatedAt time.Time } @@ -61,7 +62,19 @@ func (h *HashCache) SaveToFile(path string) (err error) { func (h *HashCache) Update(basePath string) (err error) { defer errz.Recover(&err) - filePaths, err := filepathutil.ListRecursive(basePath) + var filePaths []string + err = filepath.Walk(basePath, + func(path string, _ os.FileInfo, err error) error { + if err != nil { + return err + } + if path == basePath { + return nil + } + filePaths = append(filePaths, path) + return nil + }) + errz.Fatal(err) saveMap := h.toInternalMap(filePaths, basePath) @@ -113,14 +126,32 @@ func updater(saveMap *sync.Map, files <-chan string, result chan<- error) { reHash = true } if reHash { - h, err := filehash.HashAsString(f) + fi, err := os.Stat(f) if err != nil { result <- err continue } + var h string + isDir := fi.IsDir() + if isDir { + hashBytes, err := filehash.HashString(filepath.Base(f)) + if err != nil { + result <- err + continue + } + h = filehash.HashToString(hashBytes) + } else { + hashBytes, err := filehash.Hash(f) + h = filehash.HashToString(hashBytes) + if err != nil { + result <- err + continue + } + } fp := Fingerprint{ Hash: h, CreatedAt: time.Now(), + IsDir: isDir, } saveMap.Store(f, fp) } @@ -154,3 +185,12 @@ func (h *HashCache) overrideFromInternalMap(saveMap *sync.Map, basePath string) }) return nil } + +func (h *HashCache) SortedKeys() []string { + keys := make([]string, 0) + for k := range *h { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/bobsync/map.go b/bobsync/map.go index 886ba9ac..7e1b2403 100644 --- a/bobsync/map.go +++ b/bobsync/map.go @@ -1,3 +1,3 @@ package bobsync -type SyncMap map[string]Sync +type SyncList []Sync diff --git a/bobsync/sync.go b/bobsync/sync.go index 65af0c11..083a4472 100644 --- a/bobsync/sync.go +++ b/bobsync/sync.go @@ -3,11 +3,17 @@ package bobsync import ( "context" "fmt" + "github.com/benchkram/bob/pkg/file" + "github.com/benchkram/bob/pkg/usererror" + "github.com/benchkram/bob/pkg/userprompt" + "github.com/benchkram/bob/pkg/versionedsync/collection" "github.com/benchkram/bob/pkg/versionedsync/localsyncstore" "github.com/benchkram/bob/pkg/versionedsync/remotesyncstore" "github.com/benchkram/errz" "github.com/logrusorgru/aurora" "path/filepath" + "sort" + "strings" ) const ( @@ -16,7 +22,7 @@ const ( // Sync is a collection of versioned synced files type Sync struct { - name string + Name string `yaml:"name"` Path string `yaml:"path"` @@ -27,28 +33,84 @@ type Sync struct { cache *HashCache } -func (s *Sync) GetName() string { - return s.name +func NewSync(collectionName, version, path, bobDir string) (s *Sync, err error) { + s = &Sync{ + Name: collectionName, + Version: version, + Path: path, + } + return s, s.Validate(bobDir) } -func (s *Sync) SetName(name string) { - s.name = name +func (s *Sync) GetRemoteId() string { + return s.remoteCollectionId } -func (s *Sync) Push(ctx context.Context, remoteStore remotesyncstore.S, localStore localsyncstore.S, bobDir string) (err error) { +// Validate checks if the Sync definition is ok +// path must be inside the bobDir. +func (s *Sync) Validate(bobDir string) (err error) { + defer errz.Recover(&err) + if s.Path == "" { + return usererror.Wrapm(fmt.Errorf("path is empty"), "invalid collection path") + } + // fileInfo, err := os.Stat(filepath.Join(bobDir, s.Path)) + // if err != nil { + // return usererror.Wrapm(err, "invalid collection path") + //} + //if !fileInfo.IsDir() { + // return usererror.Wrapm(fmt.Errorf("%s is not a directory", s.Path), "invalid collection path") + //} + rel, err := filepath.Rel(bobDir, filepath.Join(bobDir, s.Path)) + errz.Fatal(err) + if strings.HasPrefix(rel, "..") { + return usererror.Wrapm(fmt.Errorf("%s is not in the bob directory", s.Path), "invalid collection path") + } + + if strings.Contains(s.Name, collection.Divider) { + return usererror.Wrapm(ErrInvalidCollectionName, fmt.Sprintf("can not contain \"%s\"", collection.Divider)) + } + if strings.Contains(s.Version, collection.Divider) { + return usererror.Wrapm(ErrInvalidCollectionVersion, fmt.Sprintf("can not contain \"%s\"", collection.Divider)) + } + + return nil +} + +func (s *Sync) CreateOnRemote(ctx context.Context, remoteStore remotesyncstore.S, dry bool) (err error) { + defer errz.Recover(&err) + + _, err = remoteStore.CollectionIdByName(ctx, s.Name, s.Version) + switch err { + case nil: + return usererror.Wrapm(ErrCollectionVersionExists, fmt.Sprintf("%s@%s can not be created on remote", s.Name, s.Version)) + case remotesyncstore.ErrCollectionNotFound: + default: + errz.Fatal(err) + } + + if dry { + return nil + } + s.remoteCollectionId, err = remoteStore.CollectionCreate(ctx, s.Name, s.Version) + errz.Fatal(err) + + return nil +} + +func (s *Sync) Push(ctx context.Context, remoteStore remotesyncstore.S, localStore localsyncstore.S, bobDir string, dry bool) (err error) { defer errz.Recover(&err) //var collectionMustBeCreated bool // get collectionId ready - s.remoteCollectionId, err = remoteStore.CollectionIdByName(ctx, s.name, s.Version) + s.remoteCollectionId, err = remoteStore.CollectionIdByName(ctx, s.Name, s.Version) // check if collections exists switch err { case nil: case remotesyncstore.ErrCollectionNotFound: //collectionMustBeCreated = true - s.remoteCollectionId, err = remoteStore.CollectionCreate(ctx, s.name, s.Version, s.Path) + s.remoteCollectionId, err = remoteStore.CollectionCreate(ctx, s.Name, s.Version) errz.Fatal(err) default: errz.Fatal(err) @@ -61,10 +123,10 @@ func (s *Sync) Push(ctx context.Context, remoteStore remotesyncstore.S, localSto } // TODO: run list local and remote in parallel + // TODO: no symlinks allowed + // TODO: collection root and all underneath is in .gitignore [could be allowed with a warning] err = s.cache.Update(filepath.Join(bobDir, s.Path)) errz.Fatal(err) - err = s.cache.SaveToFile(absHashCachePath) - errz.Fatal(err) remoteCollection, err := remoteStore.Collection(ctx, s.remoteCollectionId) errz.Fatal(err) @@ -72,16 +134,27 @@ func (s *Sync) Push(ctx context.Context, remoteStore remotesyncstore.S, localSto // create the delta delta := NewDelta(*s.cache, *remoteCollection) - fmt.Printf("Local-Remote delta for %s\n", aurora.Bold(s.name)) + if !dry { + fmt.Printf("Creating collection %s@%s (%s) on remote\n", aurora.Bold(s.Name), aurora.Italic(s.Version), s.Path) + } else { + fmt.Printf("Simulated collection %s@%s (%s) on remote\n", aurora.Bold(s.Name), aurora.Italic(s.Version), s.Path) + } fmt.Println(delta.PushOverview()) - // TODO: prompt user and seek confirmation + if dry { + return nil + } for _, f := range delta.LocalFilesMissingOnRemote { - srcReader, err := localStore.ReadFile(bobDir, s.Path, f.LocalPath) - errz.Fatal(err) - err = remoteStore.FileUpload(ctx, s.remoteCollectionId, f.LocalPath, srcReader) - errz.Fatal(err) + if !f.IsDirectory { + srcReader, err := localStore.ReadFile(filepath.Join(bobDir, s.Path, f.LocalPath)) + errz.Fatal(err) + err = remoteStore.FileUpload(ctx, s.remoteCollectionId, f.LocalPath, srcReader) + errz.Fatal(err) + } else { + err = remoteStore.MakeDir(ctx, s.remoteCollectionId, f.LocalPath) + errz.Fatal(err) + } } for _, f := range delta.RemoteFilesMissingOnLocal { if f.ID == nil { @@ -94,29 +167,36 @@ func (s *Sync) Push(ctx context.Context, remoteStore remotesyncstore.S, localSto if f.ID == nil { return fmt.Errorf("ID not available can not update on remote") } - srcReader, err := localStore.ReadFile(bobDir, s.Path, f.LocalPath) - errz.Fatal(err) - err = remoteStore.FileUpdate(ctx, s.remoteCollectionId, *f.ID, srcReader) - errz.Fatal(err) + if !f.IsDirectory { + srcReader, err := localStore.ReadFile(filepath.Join(bobDir, s.Path, f.LocalPath)) + errz.Fatal(err) + err = remoteStore.FileUpdate(ctx, s.remoteCollectionId, *f.ID, false, srcReader) + errz.Fatal(err) + } else { + err = remoteStore.FileUpdate(ctx, s.remoteCollectionId, *f.ID, true, nil) + errz.Fatal(err) + } } + err = s.cache.SaveToFile(absHashCachePath) + errz.Fatal(err) + return nil } -func (s *Sync) Pull(ctx context.Context, remoteStore remotesyncstore.S, localStore localsyncstore.S, bobDir string) (err error) { +func (s *Sync) Pull(ctx context.Context, remoteStore remotesyncstore.S, localStore localsyncstore.S, bobDir string, force bool) (err error) { defer errz.Recover(&err) //var collectionMustBeCreated bool // get collectionId ready - s.remoteCollectionId, err = remoteStore.CollectionIdByName(ctx, s.name, s.Version) + s.remoteCollectionId, err = remoteStore.CollectionIdByName(ctx, s.Name, s.Version) // check if collections exists switch err { case nil: case remotesyncstore.ErrCollectionNotFound: //collectionMustBeCreated = true - s.remoteCollectionId, err = remoteStore.CollectionCreate(ctx, s.name, s.Version, s.Path) - errz.Fatal(err) + errz.Fatal(usererror.Wrapm(err, fmt.Sprintf("can not sync: %s@%s does not exist on the server", s.Name, s.Version))) default: errz.Fatal(err) } @@ -127,10 +207,16 @@ func (s *Sync) Pull(ctx context.Context, remoteStore remotesyncstore.S, localSto errz.Fatal(err) } + // create collection dir if it does not exist + absCollectionPath := filepath.Join(bobDir, s.Path) + if !file.Exists(absCollectionPath) { + fmt.Printf("creating sync collection directory: %s\n", absCollectionPath) + err = localStore.MakeDir(absCollectionPath) + errz.Fatal(err) + } + // TODO: run list local and remote in parallel - err = s.cache.Update(filepath.Join(bobDir, s.Path)) - errz.Fatal(err) - err = s.cache.SaveToFile(absHashCachePath) + err = s.cache.Update(absCollectionPath) errz.Fatal(err) remoteCollection, err := remoteStore.Collection(ctx, s.remoteCollectionId) @@ -139,54 +225,80 @@ func (s *Sync) Pull(ctx context.Context, remoteStore remotesyncstore.S, localSto // create the delta delta := NewDelta(*s.cache, *remoteCollection) - fmt.Printf("Local-Remote delta for %s\n", aurora.Bold(s.name)) + fmt.Printf("Sync %s@%s from remote to %s\n", aurora.Bold(s.Name), aurora.Italic(s.Version), absCollectionPath) fmt.Println(delta.PullOverview()) - // TODO: prompt user and seek confirmation + if !force { + confirm, err := userprompt.Confirm() + errz.Fatal(err) + if !confirm { + return nil + } + } for _, f := range delta.LocalFilesMissingOnRemote { - err := localStore.DeleteFile(bobDir, s.Path, f.LocalPath) + err := localStore.DeleteFile(filepath.Join(bobDir, s.Path, f.LocalPath)) errz.Fatal(err) } for _, f := range delta.RemoteFilesMissingOnLocal { if f.ID == nil { return fmt.Errorf("ID not available can not downlaod from remote") } - srcReader, err := remoteStore.File(ctx, s.remoteCollectionId, *f.ID) - errz.Fatal(err) - err = localStore.WriteFile(bobDir, s.Path, f.LocalPath, srcReader) - errz.Fatal(err) + if !f.IsDirectory { + srcReader, err := remoteStore.File(ctx, s.remoteCollectionId, *f.ID) + errz.Fatal(err) + err = localStore.WriteFile(filepath.Join(bobDir, s.Path, f.LocalPath), srcReader) + errz.Fatal(err) + } else { + err = localStore.MakeDir(filepath.Join(bobDir, s.Path, f.LocalPath)) + errz.Fatal(err) + } } for _, f := range delta.ToBeUpdated { if f.ID == nil { return fmt.Errorf("ID not available can not downlaod from remote") } - srcReader, err := remoteStore.File(ctx, s.remoteCollectionId, *f.ID) - errz.Fatal(err) - err = localStore.WriteFile(bobDir, s.Path, f.LocalPath, srcReader) - errz.Fatal(err) + if !f.IsDirectory { + srcReader, err := remoteStore.File(ctx, s.remoteCollectionId, *f.ID) + errz.Fatal(err) + err = localStore.WriteFile(filepath.Join(bobDir, s.Path, f.LocalPath), srcReader) + errz.Fatal(err) + } else { + err = localStore.MakeDir(filepath.Join(bobDir, s.Path, f.LocalPath)) + errz.Fatal(err) + } } + err = s.cache.Update(filepath.Join(bobDir, s.Path)) + errz.Fatal(err) + err = s.cache.SaveToFile(absHashCachePath) + errz.Fatal(err) + return nil } func (s *Sync) ListLocal(bobDir string) (err error) { defer errz.Recover(&err) - absHashCachPath := filepath.Join(bobDir, hashCachePath) + absHashCachePath := filepath.Join(bobDir, hashCachePath) if s.cache == nil { - s.cache, err = FromFileOrNew(absHashCachPath) + s.cache, err = FromFileOrNew(absHashCachePath) errz.Fatal(err) } err = s.cache.Update(s.Path) errz.Fatal(err) - err = s.cache.SaveToFile(absHashCachPath) + err = s.cache.SaveToFile(absHashCachePath) errz.Fatal(err) - fmt.Printf("%s@%s (./%s)\n", aurora.Bold(s.name), aurora.Italic(s.Version), s.Path) - for p := range *s.cache { - fmt.Printf("\t%s\n", p) + fmt.Printf("%s@%s (./%s)\n", aurora.Bold(s.Name), aurora.Italic(s.Version), s.Path) + for _, k := range (*s.cache).SortedKeys() { + var suffix string + if (*s.cache)[k].IsDir { + suffix = "/" + } + + fmt.Printf("\t%s%s\n", k, suffix) } return nil @@ -197,14 +309,14 @@ func (s *Sync) ListRemote(ctx context.Context, store remotesyncstore.S) (err err defer errz.Recover(&err) // get collectionId ready - s.remoteCollectionId, err = store.CollectionIdByName(ctx, s.name, s.Version) + s.remoteCollectionId, err = store.CollectionIdByName(ctx, s.Name, s.Version) // check if collections exists switch err { case nil: case remotesyncstore.ErrCollectionNotFound: - fmt.Printf("Sync collection %s with version %s does not exist on the server.\n", - aurora.Bold(s.name), aurora.Bold(s.Version)) + fmt.Printf("Sync collection %s@%s does not exist on the server.\n", + aurora.Bold(s.Name), aurora.Italic(s.Version)) default: errz.Fatal(err) } @@ -212,6 +324,7 @@ func (s *Sync) ListRemote(ctx context.Context, store remotesyncstore.S) (err err errz.Fatal(err) for _, c := range collections { + sort.Sort(c) fmt.Printf("%s@%s (./%s)", aurora.Bold(c.Name), aurora.Italic(c.Version), c.LocalPath) if c.ID == s.remoteCollectionId { fmt.Printf(" [synced to local]") @@ -223,3 +336,13 @@ func (s *Sync) ListRemote(ctx context.Context, store remotesyncstore.S) (err err } return nil } + +func (s *Sync) FillRemoteId(ctx context.Context, store remotesyncstore.S) (err error) { + defer errz.Recover(&err) + + // get collectionId ready + s.remoteCollectionId, err = store.CollectionIdByName(ctx, s.Name, s.Version) + + errz.Fatal(err) + return nil +} diff --git a/bobsync/verify.go b/bobsync/verify.go index ca67ebe0..782df2ca 100644 --- a/bobsync/verify.go +++ b/bobsync/verify.go @@ -1,9 +1,10 @@ package bobsync -// after reading bobfile, need to verify that: -// * collection root and all underneath is in .gitignore [could be allowed with a warning] -// * collection root is somewhere in bob workspace [could be optional] -// * name and version are not allowed to include the separator char - -// after parsing the tree of a collection, need to verify that: -// * no symlinks included +func CheckForConflicts(current []Sync, new Sync) error { + for _, s := range current { + if s.Path == new.Path { + return ErrSyncPathTaken + } + } + return nil +} diff --git a/bobtask/input.go b/bobtask/input.go index c6235c8c..d30229ce 100644 --- a/bobtask/input.go +++ b/bobtask/input.go @@ -45,7 +45,7 @@ func (t *Task) filteredInputs() ([]string, error) { // Ignore starts with ! if strings.HasPrefix(input, "!") { input = strings.TrimPrefix(input, "!") - list, err := filepathutil.ListRecursive(input) + list, err := filepathutil.ListRecursive(input, false) if err != nil { return nil, fmt.Errorf("failed to list input: %w", err) } @@ -54,7 +54,7 @@ func (t *Task) filteredInputs() ([]string, error) { continue } - list, err := filepathutil.ListRecursive(input) + list, err := filepathutil.ListRecursive(input, false) if err != nil { return nil, fmt.Errorf("failed to list input: %w", err) } diff --git a/cli/cmd_root.go b/cli/cmd_root.go index 7750b3fd..36b925d5 100644 --- a/cli/cmd_root.go +++ b/cli/cmd_root.go @@ -59,11 +59,13 @@ func init() { rootCmd.AddCommand(CmdGit) // syncCmd - cmdSyncListRemote.Flags().Bool("insecure", false, "Set to true to use http instead of https when accessing bob-server") - cmdSyncPush.Flags().Bool("insecure", false, "Set to true to use http instead of https when accessing bob-server") - cmdSyncPull.Flags().Bool("insecure", false, "Set to true to use http instead of https when accessing bob-server") - cmdSync.AddCommand(cmdSyncPush) - cmdSync.AddCommand(cmdSyncPull) + cmdSync.PersistentFlags().Bool("insecure", false, "Set to true to use http instead of https when accessing bob-server") + cmdSync.Flags().BoolP("force", "f", false, "Set to true to not prompt if files are deleted or overwritten") + //cmdSyncListRemote.Flags().Bool("insecure", false, "Set to true to use http instead of https when accessing bob-server") + //cmdSyncCreate.Flags().Bool("insecure", false, "Set to true to use http instead of https when accessing bob-server") + cmdSyncCreate.Flags().Bool("dry", false, "Set to true to show which files would be added to the new collection") + cmdSyncCreate.Flags().String("set-version", "v1", "Set the version tag") + cmdSync.AddCommand(cmdSyncCreate) cmdSync.AddCommand(cmdSyncList) cmdSync.AddCommand(cmdSyncListRemote) rootCmd.AddCommand(cmdSync) diff --git a/cli/cmd_sync.go b/cli/cmd_sync.go index fc296432..82552491 100644 --- a/cli/cmd_sync.go +++ b/cli/cmd_sync.go @@ -15,45 +15,48 @@ import ( ) var cmdSync = &cobra.Command{ - Use: "sync", - Short: "Sync (binary) test data via a bob-server.", - Args: cobra.MinimumNArgs(0), + Use: "sync [--force] [--insecure]", + Short: "Pull the sync collections defined in bobfile", Long: ``, FParseErrWhitelist: cobra.FParseErrWhitelist{ UnknownFlags: true, }, Run: func(cmd *cobra.Command, args []string) { - // do nothing just show if the server can be contacted and maybe display status information - }, -} - -var cmdSyncPush = &cobra.Command{ - Use: "push", - Short: "Make server collections exactly like local", - Long: ``, - Run: func(cmd *cobra.Command, args []string) { - allowInsecure, err := cmd.Flags().GetBool("insecure") + allowInsecure, err := cmd.PersistentFlags().GetBool("insecure") + errz.Fatal(err) + force, err := cmd.Flags().GetBool("force") errz.Fatal(err) - runPush(allowInsecure) + runPull(allowInsecure, force) }, } -var cmdSyncPull = &cobra.Command{ - Use: "pull", - Short: "Make local collections exactly like server", - Long: ``, +var cmdSyncCreate = &cobra.Command{ + Use: "create collection_name path/to/dir", + Short: "Create a new collection or collection version", + Long: ``, + Args: cobra.ExactArgs(2), + ArgAliases: []string{"collectionName", "path"}, Run: func(cmd *cobra.Command, args []string) { allowInsecure, err := cmd.Flags().GetBool("insecure") errz.Fatal(err) + dry, err := cmd.Flags().GetBool("dry") + errz.Fatal(err) + version, err := cmd.Flags().GetString("set-version") + errz.Fatal(err) - runPull(allowInsecure) + // collection_name can be anything but not empty + collectionName := args[0] + // path is validated in createPush + path := args[1] + + runCreatePush(collectionName, version, path, dry, allowInsecure) }, } var cmdSyncList = &cobra.Command{ - Use: "ls", - Short: "List files synced", + Use: "ls-local", + Short: "List files tracked by sync", Long: ``, Run: func(cmd *cobra.Command, args []string) { runList() @@ -71,7 +74,7 @@ var cmdSyncListRemote = &cobra.Command{ }, } -func runPush(allowInsecure bool) { +func runCreatePush(collectionName, version, path string, dry, allowInsecure bool) { var exitCode int defer func() { exit(exitCode) @@ -99,7 +102,7 @@ func runPush(allowInsecure bool) { cancel() }() - err = b.SyncPush(ctx) + err = b.SyncCreatePush(ctx, collectionName, version, path, dry) if err != nil { exitCode = 1 if errors.As(err, &usererror.Err) { @@ -110,7 +113,7 @@ func runPush(allowInsecure bool) { } } -func runPull(allowInsecure bool) { +func runPull(allowInsecure bool, force bool) { var exitCode int defer func() { exit(exitCode) @@ -138,7 +141,7 @@ func runPull(allowInsecure bool) { cancel() }() - err = b.SyncPull(ctx) + err = b.SyncPull(ctx, force) if err != nil { exitCode = 1 if errors.As(err, &usererror.Err) { @@ -181,8 +184,8 @@ func runList() { if errors.As(err, &usererror.Err) { boblog.Log.UserError(err) } else { - errz.Fatal(err) errz.Log(err) + errz.Fatal(err) } } } diff --git a/pkg/filehash/hash.go b/pkg/filehash/hash.go index 52bfaa35..bdbaa3fb 100644 --- a/pkg/filehash/hash.go +++ b/pkg/filehash/hash.go @@ -1,12 +1,12 @@ package filehash import ( + "bytes" "encoding/hex" "fmt" + "github.com/cespare/xxhash/v2" "io" "os" - - "github.com/cespare/xxhash/v2" ) var ( @@ -32,11 +32,17 @@ func HashBytes(r io.Reader) ([]byte, error) { return h.Sum(nil), nil } -func HashAsString(file string) (string, error) { - b, err := Hash(file) +func HashToString(bytes []byte) string { + return hex.EncodeToString(bytes) +} + +func HashString(inp string) ([]byte, error) { + hash := New() + var buf bytes.Buffer + buf.WriteString(inp) + err := hash.AddBytes(bytes.NewReader(buf.Bytes())) if err != nil { - return "", err + return nil, err } - encryptedHash := hex.EncodeToString(b) - return encryptedHash, nil + return hash.Sum(), nil } diff --git a/pkg/filepathutil/list.go b/pkg/filepathutil/list.go index 414c9c3f..ce5902b8 100644 --- a/pkg/filepathutil/list.go +++ b/pkg/filepathutil/list.go @@ -23,7 +23,7 @@ func ClearListRecursiveCache() { // listRecursiveCache = make(map[string][]string, 1024) } -func ListRecursive(inp string) (all []string, err error) { +func ListRecursive(inp string, includeDirs bool) (all []string, err error) { // if result, ok := listRecursiveCache[inp]; ok { // return result, nil // } @@ -67,7 +67,9 @@ func ListRecursive(inp string) (all []string, err error) { if err != nil { return nil, fmt.Errorf("failed to list dir: %w", err) } - + if includeDirs { + all = append(all, m) + } all = append(all, files...) } } @@ -77,7 +79,9 @@ func ListRecursive(inp string) (all []string, err error) { if err != nil { return nil, fmt.Errorf("failed to list dir: %w", err) } - + if includeDirs { + all = append(all, inp) + } all = append(all, files...) } diff --git a/pkg/store-client/client.go b/pkg/store-client/client.go index a1e26a0f..4e027c71 100644 --- a/pkg/store-client/client.go +++ b/pkg/store-client/client.go @@ -11,6 +11,7 @@ import ( "net/http" "net/textproto" "path/filepath" + "strconv" "syscall" "github.com/benchkram/bob/pkg/usererror" @@ -254,26 +255,30 @@ func (c *c) Collections(ctx context.Context, projectName string) (collections [] return *res.JSON200, nil } -func (c *c) FileCreate(ctx context.Context, projectName, collectionId, localPath string, src io.Reader) (f *generated.SyncFile, err error) { +func (c *c) FileCreate(ctx context.Context, projectName, collectionId, localPath string, isDir bool, src *io.Reader) (f *generated.SyncFile, err error) { r, w := io.Pipe() mpw := multipart.NewWriter(w) go func() { - err0 := mpw.WriteField("local_path", localPath) if err0 != nil { _ = w.CloseWithError(err0) } - - fieldWriter, err0 := mpw.CreateFormFile("file", filepath.Base(localPath)) - if err0 != nil { - _ = w.CloseWithError(err0) - } - _, err0 = io.Copy(fieldWriter, src) + err0 = mpw.WriteField("is_directory", strconv.FormatBool(isDir)) if err0 != nil { _ = w.CloseWithError(err0) } + if src != nil { + fieldWriter, err0 := mpw.CreateFormFile("file", filepath.Base(localPath)) + if err0 != nil { + _ = w.CloseWithError(err0) + } + _, err0 = io.Copy(fieldWriter, *src) + if err0 != nil { + _ = w.CloseWithError(err0) + } + } err0 = mpw.Close() if err0 != nil { _ = w.CloseWithError(err0) @@ -317,7 +322,7 @@ func (c *c) FileCreate(ctx context.Context, projectName, collectionId, localPath return resp.JSON200, nil } -func (c *c) File(ctx context.Context, projectName, collectionId, fileId string) (f *generated.SyncFile, rc io.ReadCloser, err error) { +func (c *c) File(ctx context.Context, projectName, collectionId, fileId string) (f *generated.SyncFile, rc *io.ReadCloser, err error) { defer errz.Recover(&err) res, err := c.clientWithResponses.GetSyncFileWithResponse( @@ -349,18 +354,22 @@ func (c *c) File(ctx context.Context, projectName, collectionId, fileId string) if res.JSON200 == nil { errz.Fatal(ErrEmptyResponse) } + if !res.JSON200.IsDirectory { - res2, err := http.Get(*res.JSON200.Location) - errz.Fatal(err) + res2, err := http.Get(*res.JSON200.Location) + errz.Fatal(err) - if res2.StatusCode != http.StatusOK { - errz.Fatal(usererror.Wrapm(ErrDownloadFailed, fmt.Sprintf("reading from storage failed (Status %d)", res2.StatusCode))) - } - if res2.Body == nil { - errz.Fatal(ErrEmptyResponse) + if res2.StatusCode != http.StatusOK { + errz.Fatal(usererror.Wrapm(ErrDownloadFailed, fmt.Sprintf("reading from storage failed (Status %d)", res2.StatusCode))) + } + if res2.Body == nil { + errz.Fatal(ErrEmptyResponse) + } + return res.JSON200, &res2.Body, nil + } else { + return res.JSON200, nil, nil } - return res.JSON200, res2.Body, nil } func (c *c) Files(ctx context.Context, projectName, collectionId string, withLocation bool) (files []generated.SyncFile, err error) { @@ -402,7 +411,7 @@ func (c *c) Files(ctx context.Context, projectName, collectionId string, withLoc return *res.JSON200, nil } -func (c *c) FileUpdate(ctx context.Context, projectName, collectionId, fileId, localPath string, src *io.Reader) (file *generated.SyncFile, err error) { +func (c *c) FileUpdate(ctx context.Context, projectName, collectionId, fileId, localPath string, isDir bool, src *io.Reader) (file *generated.SyncFile, err error) { r, w := io.Pipe() mpw := multipart.NewWriter(w) @@ -414,6 +423,11 @@ func (c *c) FileUpdate(ctx context.Context, projectName, collectionId, fileId, l } } + err0 := mpw.WriteField("is_directory", strconv.FormatBool(isDir)) + if err0 != nil { + _ = w.CloseWithError(err0) + } + if src != nil { fieldWriter, err0 := mpw.CreateFormFile("file", filepath.Base(localPath)) if err0 != nil { @@ -424,7 +438,7 @@ func (c *c) FileUpdate(ctx context.Context, projectName, collectionId, fileId, l _ = w.CloseWithError(err0) } } - err0 := mpw.Close() + err0 = mpw.Close() if err0 != nil { _ = w.CloseWithError(err0) } diff --git a/pkg/store-client/generated/types.gen.go b/pkg/store-client/generated/types.gen.go index 83c21317..12b6cc34 100644 --- a/pkg/store-client/generated/types.gen.go +++ b/pkg/store-client/generated/types.gen.go @@ -64,6 +64,7 @@ type SyncCollectionStub struct { type SyncFile struct { EncryptedHash *string `json:"encrypted_hash,omitempty"` Id string `json:"id"` + IsDirectory bool `json:"is_directory"` LocalPath string `json:"local_path"` // location to download the file using a GET request. @@ -72,22 +73,25 @@ type SyncFile struct { // SyncFileCreate defines model for SyncFileCreate. type SyncFileCreate struct { - File string `json:"file"` - LocalPath string `json:"local_path"` + File string `json:"file"` + IsDirectory bool `json:"is_directory"` + LocalPath string `json:"local_path"` } // SyncFileStub defines model for SyncFileStub. type SyncFileStub struct { EncryptedHash *string `json:"encrypted_hash,omitempty"` Id string `json:"id"` + IsDirectory bool `json:"is_directory"` LocalPath string `json:"local_path"` } // Updated version of the file, properties which are omitted will not be changed. At least one property has be included. type SyncFileUpdate struct { - File *string `json:"file,omitempty"` - Id string `json:"id"` - LocalPath *string `json:"local_path,omitempty"` + File *string `json:"file,omitempty"` + Id string `json:"id"` + IsDirectory bool `json:"is_directory"` + LocalPath *string `json:"local_path,omitempty"` } // GetSyncFilesParams defines parameters for GetSyncFiles. diff --git a/pkg/store-client/store_client.go b/pkg/store-client/store_client.go index 08a8e37d..ae92ebdd 100644 --- a/pkg/store-client/store_client.go +++ b/pkg/store-client/store_client.go @@ -17,10 +17,10 @@ type I interface { Collection(ctx context.Context, projectName, collectionId string) (*generated.SyncCollection, error) Collections(ctx context.Context, projectName string) ([]generated.SyncCollection, error) - FileCreate(ctx context.Context, projectName, collectionId, localPath string, src io.Reader) (*generated.SyncFile, error) - File(ctx context.Context, projectName, collectionId, fileId string) (*generated.SyncFile, io.ReadCloser, error) + FileCreate(ctx context.Context, projectName, collectionId, localPath string, isDir bool, src *io.Reader) (*generated.SyncFile, error) + File(ctx context.Context, projectName, collectionId, fileId string) (*generated.SyncFile, *io.ReadCloser, error) Files(ctx context.Context, projectName, collectionId string, withLocation bool) ([]generated.SyncFile, error) - FileUpdate(ctx context.Context, projectName, collectionId, fileId, localPath string, src *io.Reader) (*generated.SyncFile, error) + FileUpdate(ctx context.Context, projectName, collectionId, fileId, localPath string, isDir bool, src *io.Reader) (*generated.SyncFile, error) FileDelete(ctx context.Context, projectName, collectionId, fileId string) error } diff --git a/pkg/userprompt/userprompt.go b/pkg/userprompt/userprompt.go new file mode 100644 index 00000000..45107c74 --- /dev/null +++ b/pkg/userprompt/userprompt.go @@ -0,0 +1,28 @@ +package userprompt + +import ( + "bufio" + "fmt" + "github.com/benchkram/errz" + "os" + "strings" +) + +func Confirm() (_ bool, err error) { + defer errz.Recover(&err) + writeMsg := func() { fmt.Println("Confirm [Y/n] ?") } + writeMsg() + reader := bufio.NewReader(os.Stdin) + for { + text, err := reader.ReadString('\n') + text = strings.TrimRight(text, "\n") + errz.Fatal(err) + if strings.ToLower(text) == "y" || text == "" { + return true, nil + } else if strings.ToLower(text) == "n" { + return false, nil + } else { + writeMsg() + } + } +} diff --git a/pkg/versionedsync/collection/collection.go b/pkg/versionedsync/collection/collection.go index 6fed1c22..dfbadf62 100644 --- a/pkg/versionedsync/collection/collection.go +++ b/pkg/versionedsync/collection/collection.go @@ -9,7 +9,7 @@ import ( ) const ( - divider = ";;" + Divider = ";;;;;" ) // C is the representation of a collection of synced files @@ -27,11 +27,11 @@ type C struct { } func JoinNameAndVersion(name, tag string) string { - return name + divider + tag + return name + Divider + tag } func SplitName(combinedName string) (name, version string, _ error) { - parts := strings.Split(combinedName, divider) + parts := strings.Split(combinedName, Divider) if len(parts) == 1 { return combinedName, "", nil } else if len(parts) == 2 { @@ -46,7 +46,7 @@ func FromRestType(genC *generated.SyncCollection) (_ *C, err error) { var files []*file.F if genC.Files != nil { for _, f := range *genC.Files { - files = append(files, file.FileFromRestStubType(f)) + files = append(files, file.FromRestStubType(f)) } } name, version, err := SplitName(genC.Name) @@ -71,3 +71,21 @@ func (c *C) FileByPath(localPath string) (*file.F, bool) { } return nil, false } + +// Len is the number of elements in the collection. +func (c C) Len() int { + return len(c.Files) +} + +// Less reports whether the element with +// index i should sort before the element with index j. +func (c C) Less(i, j int) bool { + return c.Files[i].LocalPath < c.Files[j].LocalPath +} + +// Swap swaps the elements with indexes i and j. +func (c C) Swap(i, j int) { + tmpF := c.Files[i] + c.Files[i] = c.Files[j] + c.Files[j] = tmpF +} diff --git a/pkg/versionedsync/file/file.go b/pkg/versionedsync/file/file.go index b25448f7..e9830c71 100644 --- a/pkg/versionedsync/file/file.go +++ b/pkg/versionedsync/file/file.go @@ -15,20 +15,24 @@ type F struct { // Hash is the hash generated on the server or client over the file content // it is used to detect changes in the content when comparing local and remote Hash string + + IsDirectory bool } -func FileFromRestType(f generated.SyncFile) *F { +func FromRestType(f generated.SyncFile) *F { return &F{ - ID: &f.Id, - LocalPath: f.LocalPath, - Hash: *f.EncryptedHash, + ID: &f.Id, + LocalPath: f.LocalPath, + Hash: *f.EncryptedHash, + IsDirectory: f.IsDirectory, } } -func FileFromRestStubType(f generated.SyncFileStub) *F { +func FromRestStubType(f generated.SyncFileStub) *F { return &F{ - ID: &f.Id, - LocalPath: f.LocalPath, - Hash: *f.EncryptedHash, + ID: &f.Id, + LocalPath: f.LocalPath, + Hash: *f.EncryptedHash, + IsDirectory: f.IsDirectory, } } diff --git a/pkg/versionedsync/localsyncstore/store.go b/pkg/versionedsync/localsyncstore/store.go index 6d60074c..05ea210f 100644 --- a/pkg/versionedsync/localsyncstore/store.go +++ b/pkg/versionedsync/localsyncstore/store.go @@ -11,21 +11,22 @@ import ( type S struct { } -func (s *S) ReadFile(bobDir, collectionPath, localPath string) (r io.ReadCloser, err error) { - return os.Open(filepath.Join(bobDir, collectionPath, localPath)) +func (s *S) ReadFile(path string) (r io.ReadCloser, err error) { + return os.Open(path) } -func (s *S) DeleteFile(bobDir, collectionPath, localPath string) (err error) { - return os.Remove(filepath.Join(bobDir, collectionPath, localPath)) +func (s *S) DeleteFile(path string) (err error) { + return os.Remove(path) } // WriteFile writes contents from read closer to a file // if the file exists it is replaced -func (s *S) WriteFile(bobDir, collectionPath, localPath string, reader io.ReadCloser) (err error) { +func (s *S) WriteFile(path string, reader io.ReadCloser) (err error) { defer errz.Recover(&err) - absPath := filepath.Join(bobDir, collectionPath, localPath) + absPath, err := filepath.Abs(path) + errz.Fatal(err) if file.Exists(absPath) { - err = s.DeleteFile(bobDir, collectionPath, localPath) + err = s.DeleteFile(absPath) errz.Fatal(err) } outFile, err := os.Create(absPath) @@ -36,6 +37,22 @@ func (s *S) WriteFile(bobDir, collectionPath, localPath string, reader io.ReadCl return nil } +func (s *S) MakeDir(path string) (err error) { + defer errz.Recover(&err) + // if path is a file: remove it + fi, err := os.Stat(path) + if os.IsExist(err) { + if !fi.IsDir() { + err = s.DeleteFile(path) + } // else do nothing, dir exists + } else if os.IsNotExist(err) { + return os.MkdirAll(path, 0774) + } else { + errz.Fatal(err) + } + return nil +} + func New() *S { return &S{} } diff --git a/pkg/versionedsync/remotesyncstore/store.go b/pkg/versionedsync/remotesyncstore/store.go index b947c31a..21d96b63 100644 --- a/pkg/versionedsync/remotesyncstore/store.go +++ b/pkg/versionedsync/remotesyncstore/store.go @@ -2,6 +2,7 @@ package remotesyncstore import ( "context" + "errors" "fmt" storeclient "github.com/benchkram/bob/pkg/store-client" "github.com/benchkram/bob/pkg/versionedsync/collection" @@ -9,6 +10,10 @@ import ( "io" ) +var ( + ErrFileIsDirectory = errors.New("collection file is a directory and can not be downloaded") +) + // S is a versioned sync store that handles the logic of accessing the bob-server to store collections and files // it uses the store client interface to achieve that type S struct { @@ -37,9 +42,9 @@ func New(username, project string, opts ...Option) *S { return s } -func (s *S) CollectionCreate(ctx context.Context, name, tag, path string) (cId string, err error) { +func (s *S) CollectionCreate(ctx context.Context, name, tag string) (cId string, err error) { defer errz.Recover(&err) - genC, err := s.client.CollectionCreate(ctx, s.project, collection.JoinNameAndVersion(name, tag), path) + genC, err := s.client.CollectionCreate(ctx, s.project, collection.JoinNameAndVersion(name, tag), "") errz.Fatal(err) c, err := collection.FromRestType(genC) errz.Fatal(err) @@ -80,21 +85,32 @@ func (s *S) Collections(ctx context.Context) (collections []*collection.C, err e } func (s *S) FileUpload(ctx context.Context, collectionId, localPath string, srcReader io.Reader) (err error) { defer errz.Recover(&err) - _, err = s.client.FileCreate(ctx, s.project, collectionId, localPath, srcReader) + _, err = s.client.FileCreate(ctx, s.project, collectionId, localPath, false, &srcReader) + errz.Fatal(err) + return nil +} + +func (s *S) MakeDir(ctx context.Context, collectionId, localPath string) (err error) { + defer errz.Recover(&err) + _, err = s.client.FileCreate(ctx, s.project, collectionId, localPath, true, nil) errz.Fatal(err) return nil } -func (s *S) File(ctx context.Context, collectionId, fileId string) (r io.ReadCloser, err error) { + +func (s *S) File(ctx context.Context, collectionId, fileId string) (_ io.ReadCloser, err error) { defer errz.Recover(&err) - _, r, err = s.client.File(ctx, s.project, collectionId, fileId) + _, rc, err := s.client.File(ctx, s.project, collectionId, fileId) errz.Fatal(err) - return r, nil + if rc == nil { + return nil, ErrFileIsDirectory + } + return *rc, nil } -func (s *S) FileUpdate(ctx context.Context, collectionId, fileId string, srcReader io.Reader) (err error) { +func (s *S) FileUpdate(ctx context.Context, collectionId, fileId string, isDir bool, srcReader io.Reader) (err error) { defer errz.Recover(&err) - _, err = s.client.FileUpdate(ctx, s.project, collectionId, fileId, "", &srcReader) + _, err = s.client.FileUpdate(ctx, s.project, collectionId, fileId, "", isDir, &srcReader) errz.Fatal(err) return nil } From 1a6578eff38a584895a02f714cc1bcb83f1b1cc6 Mon Sep 17 00:00:00 2001 From: Daniel Ketterer <26778610+dketterer@users.noreply.github.com> Date: Thu, 22 Jun 2023 12:12:05 +0200 Subject: [PATCH 5/5] Fix merge problems --- bob/bobfile/bobfile.go | 4 ++-- pkg/store-client/client.go | 13 ++++++------- pkg/store-client/generated/client.gen.go | 1 + 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/bob/bobfile/bobfile.go b/bob/bobfile/bobfile.go index fe9e0d80..8816c78a 100644 --- a/bob/bobfile/bobfile.go +++ b/bob/bobfile/bobfile.go @@ -3,13 +3,13 @@ package bobfile import ( "bytes" "fmt" - "github.com/benchkram/bob/pkg/versionedsync/remotesyncstore" - "io/ioutil" "net/url" "os" "path/filepath" "strings" + "github.com/benchkram/bob/pkg/versionedsync/remotesyncstore" + "github.com/benchkram/bob/bobsync" "github.com/benchkram/bob/pkg/nix" storeclient "github.com/benchkram/bob/pkg/store-client" diff --git a/pkg/store-client/client.go b/pkg/store-client/client.go index 266e5234..f37c6cac 100644 --- a/pkg/store-client/client.go +++ b/pkg/store-client/client.go @@ -3,23 +3,22 @@ package storeclient import ( "context" "fmt" - "github.com/benchkram/bob/pkg/store-client/generated" - "github.com/benchkram/errz" - "github.com/pkg/errors" "io" "mime/multipart" "net/http" "net/textproto" + "os" "path/filepath" "strconv" "syscall" - "os" "time" - "github.com/benchkram/bob/bob/playbook" - progress2 "github.com/benchkram/bob/pkg/progress" + "github.com/benchkram/bob/pkg/store-client/generated" "github.com/benchkram/errz" "github.com/pkg/errors" + + "github.com/benchkram/bob/bob/playbook" + progress2 "github.com/benchkram/bob/pkg/progress" "github.com/schollz/progressbar/v3" "github.com/benchkram/bob/pkg/usererror" @@ -34,7 +33,7 @@ var ( ErrEmptyResponse = errors.New("empty response") ErrDownloadFailed = errors.New("binary download failed") ErrConnectionRefused = errors.New("connection to server failed (connection refused)") - ErrNotAuthorized = errors.New("not authorized") + ErrNotAuthorized = errors.New("not authorized") ) func (c *c) UploadArtifact( diff --git a/pkg/store-client/generated/client.gen.go b/pkg/store-client/generated/client.gen.go index 0138144e..6deee174 100644 --- a/pkg/store-client/generated/client.gen.go +++ b/pkg/store-client/generated/client.gen.go @@ -9,6 +9,7 @@ import ( "encoding/json" "fmt" "io" + "io/ioutil" "net/http" "net/url" "strings"