diff --git a/common/config/config.go b/common/config/config.go index 7da6197..ef65416 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -7,15 +7,25 @@ // its own, because the vault APIs differ between products. package config +import "time" + +// Env is the Skyflow environment a vault runs in. The zero value means "not set": it +// resolves to PROD when the vault URL is built, and UpdateVaultConfig leaves the current +// Env unchanged when the update doesn't set one. type Env int const ( - PROD Env = iota + PROD Env = iota + 1 STAGE SANDBOX DEV ) +// IsSet reports whether the Env was set explicitly. +func (e Env) IsSet() bool { + return e != 0 +} + type Credentials struct { Path string Roles []string @@ -33,6 +43,32 @@ type VaultConfig struct { BaseVaultURL string Env Env Credentials Credentials + + // Optional HTTP settings for this vault. Zero values (and a nil MaxRetries) + // mean "not set": the client-level HttpConfig applies, then the SDK default. + Timeout time.Duration + MaxRetries *int + InitialRetryDelay time.Duration + MaxRetryDelay time.Duration +} + +// HttpConfig holds client-level call timeout and retry settings. A value set on +// VaultConfig wins over the one here. +type HttpConfig struct { + Timeout time.Duration + MaxRetries *int + InitialRetryDelay time.Duration + MaxRetryDelay time.Duration +} + +// HttpConfig returns the vault's own HTTP settings. +func (c VaultConfig) HttpConfig() HttpConfig { + return HttpConfig{ + Timeout: c.Timeout, + MaxRetries: c.MaxRetries, + InitialRetryDelay: c.InitialRetryDelay, + MaxRetryDelay: c.MaxRetryDelay, + } } type ConnectionConfig struct { diff --git a/common/config/messages.go b/common/config/messages.go index e7cbedf..f1c285c 100644 --- a/common/config/messages.go +++ b/common/config/messages.go @@ -12,6 +12,7 @@ const ( errInvalidApiKey = " Initialization failed. Invalid credentials. Specify valid api key." errEmptyRoles = " Initialization failed. Invalid roles. Specify at least one role." errEmptyRoleInRoles = " Initialization failed. Invalid role. Specify a valid role." + errNegativeHttpConfig = " Initialization failed. Invalid HTTP config. Timeout, retry count and retry delays must not be negative." ) const ( @@ -24,4 +25,5 @@ const ( logInvalidApiKey = "Invalid credentials. Api key is invalid." logEmptyRoles = "Invalid credentials. Roles can not be empty." logEmptyOrNullRoleInRoles = "Invalid credentials. Role can not be null or empty in roles at index %v." + logNegativeHttpConfig = "Invalid HTTP config. Timeout, retry count and retry delays must not be negative." ) diff --git a/common/config/validate.go b/common/config/validate.go index e562fe7..7f34626 100644 --- a/common/config/validate.go +++ b/common/config/validate.go @@ -55,6 +55,15 @@ func (v Validator) ValidateVaultConfig(vaultConfig VaultConfig) *skyflowError.Sk } else if !isValidHTTPURL(baseVaultUrl) { return v.invalid(logVaultUrlIsInvalid, errInvalidVaultUrl) } + return v.ValidateHttpConfig(vaultConfig.HttpConfig()) +} + +// ValidateHttpConfig checks that timeout, retry count and retry delays are not negative. +func (v Validator) ValidateHttpConfig(httpConfig HttpConfig) *skyflowError.SkyflowError { + if httpConfig.Timeout < 0 || httpConfig.InitialRetryDelay < 0 || httpConfig.MaxRetryDelay < 0 || + (httpConfig.MaxRetries != nil && *httpConfig.MaxRetries < 0) { + return v.invalid(logNegativeHttpConfig, errNegativeHttpConfig) + } return nil } diff --git a/common/vault/operations.go b/common/vault/operations.go new file mode 100644 index 0000000..8e8204a --- /dev/null +++ b/common/vault/operations.go @@ -0,0 +1,63 @@ +// Package vault defines the vault operations shared by the Skyflow Go SDKs. +// +// The SDKs (v2 for SkyVault, flowvault for FlowVault) expose the same methods but with +// their own request, options and response types, because the vault APIs differ. Each +// interface here is generic over those types, so both SDKs' vault services satisfy the +// same contract: +// +// var _ vault.Inserter[common.InsertRequest, common.InsertOptions, common.InsertResponse] = (*vaultService)(nil) +// +// Code that works with either SDK can depend on these interfaces instead of a concrete +// vault service. +package vault + +import ( + "context" + + skyflowError "github.com/skyflowapi/skyflow-go/common/errors" +) + +// Inserter inserts records into a vault. +type Inserter[Req, Opts, Resp any] interface { + Insert(ctx context.Context, request Req, options Opts) (*Resp, *skyflowError.SkyflowError) +} + +// Updater updates existing records in a vault. +type Updater[Req, Opts, Resp any] interface { + Update(ctx context.Context, request Req, options Opts) (*Resp, *skyflowError.SkyflowError) +} + +// Getter retrieves records from a vault. +type Getter[Req, Opts, Resp any] interface { + Get(ctx context.Context, request Req, options Opts) (*Resp, *skyflowError.SkyflowError) +} + +// Deleter deletes records from a vault. +type Deleter[Req, Opts, Resp any] interface { + Delete(ctx context.Context, request Req, options Opts) (*Resp, *skyflowError.SkyflowError) +} + +// Querier runs a query against a vault. +type Querier[Req, Opts, Resp any] interface { + Query(ctx context.Context, request Req, options Opts) (*Resp, *skyflowError.SkyflowError) +} + +// Detokenizer returns the values behind tokens. +type Detokenizer[Req, Opts, Resp any] interface { + Detokenize(ctx context.Context, request Req, options Opts) (*Resp, *skyflowError.SkyflowError) +} + +// TokenGetter returns existing tokens for values. +type TokenGetter[Req, Opts, Resp any] interface { + GetTokens(ctx context.Context, request Req, options Opts) (*Resp, *skyflowError.SkyflowError) +} + +// FilesUploader uploads files into file columns. +type FilesUploader[Req, Opts, Resp any] interface { + UploadFiles(ctx context.Context, request Req, options Opts) (*Resp, *skyflowError.SkyflowError) +} + +// FilesDeleter deletes files from file columns. +type FilesDeleter[Req, Opts, Resp any] interface { + DeleteFiles(ctx context.Context, request Req, options Opts) (*Resp, *skyflowError.SkyflowError) +} diff --git a/flowvault/client/client.go b/flowvault/client/client.go index de774d1..60bc336 100644 --- a/flowvault/client/client.go +++ b/flowvault/client/client.go @@ -2,20 +2,24 @@ package client import ( "fmt" + "sync" fvhelpers "github.com/skyflowapi/skyflow-go/flowvault/internal/helpers" + "github.com/skyflowapi/skyflow-go/flowvault/internal/vault/controller" fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" fverrors "github.com/skyflowapi/skyflow-go/flowvault/utils/error" fvlogger "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" logs "github.com/skyflowapi/skyflow-go/flowvault/utils/messages" ) -// Skyflow is the flowvault client entry point. +// Skyflow is the flowvault client entry point. It is safe for concurrent use: vaults and +// client settings can be changed while calls are running. type Skyflow struct { + mu sync.RWMutex vaultServices map[string]*VaultService credentials *fvcommon.Credentials logLevel fvlogger.LogLevel - customHeaders map[fvcommon.CustomHeaderKey]string + httpConfig fvcommon.HttpConfig } type Option func(*Skyflow) *fverrors.SkyflowError @@ -60,22 +64,55 @@ func WithCredentials(credentials fvcommon.Credentials) Option { // WithLogLevel sets the logging level. func WithLogLevel(logLevel fvlogger.LogLevel) Option { return func(s *Skyflow) *fverrors.SkyflowError { + s.mu.Lock() + defer s.mu.Unlock() s.logLevel = logLevel fvlogger.SetLogLevel(logLevel) return nil } } -// WithCustomHeaders sets custom headers sent with every request from this client. -func WithCustomHeaders(headers map[fvcommon.CustomHeaderKey]string) Option { +// WithHttpConfig sets client-level timeout and retry defaults. A value set on a +// VaultConfig wins over the one set here. +func WithHttpConfig(httpConfig fvcommon.HttpConfig) Option { return func(s *Skyflow) *fverrors.SkyflowError { - s.customHeaders = headers + if err := fvhelpers.ConfigValidator().ValidateHttpConfig(httpConfig); err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + s.httpConfig = copyHttpConfig(httpConfig) + s.syncVaults() return nil } } +// clientSettings returns the settings vaults fall back to. The caller holds s.mu. +func (s *Skyflow) clientSettings() controller.ClientSettings { + return controller.ClientSettings{Credentials: s.credentials, HttpConfig: s.httpConfig} +} + +// syncVaults pushes client-level settings to every registered vault. The caller holds s.mu. +func (s *Skyflow) syncVaults() { + settings := s.clientSettings() + for _, svc := range s.vaultServices { + svc.controller.SetClientSettings(settings) + } +} + +// copyHttpConfig copies MaxRetries so later changes to the caller's value don't leak in. +func copyHttpConfig(config fvcommon.HttpConfig) fvcommon.HttpConfig { + if config.MaxRetries != nil { + retries := *config.MaxRetries + config.MaxRetries = &retries + } + return config +} + // Vault returns the vault service registered under vaultId. func (s *Skyflow) Vault(vaultId string) (*VaultService, *fverrors.SkyflowError) { + s.mu.RLock() + defer s.mu.RUnlock() svc, ok := s.vaultServices[vaultId] if !ok { return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, fverrors.VAULT_ID_NOT_IN_CONFIG_LIST) @@ -94,11 +131,17 @@ func (s *Skyflow) AddVaultConfig(cfg fvcommon.VaultConfig) *fverrors.SkyflowErro if err := fvhelpers.ConfigValidator().ValidateVaultConfig(cfg); err != nil { return err } + s.mu.Lock() + defer s.mu.Unlock() if _, exists := s.vaultServices[cfg.VaultId]; exists { fvlogger.Error(fmt.Sprintf(logs.VAULT_CONFIG_EXISTS, cfg.VaultId)) return fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, fmt.Sprintf(fverrors.VAULT_ID_EXISTS_IN_CONFIG_LIST, cfg.VaultId)) } - s.vaultServices[cfg.VaultId] = newVaultService(s, &cfg) + if cfg.MaxRetries != nil { + retries := *cfg.MaxRetries + cfg.MaxRetries = &retries + } + s.vaultServices[cfg.VaultId] = newVaultService(controller.NewVaultController(cfg, s.clientSettings())) fvlogger.Info(fmt.Sprintf(logs.VAULT_CONTROLLER_INITIALIZED, cfg.VaultId)) return nil } @@ -110,30 +153,54 @@ func (s *Skyflow) UpdateVaultConfig(updatedConfig fvcommon.VaultConfig) *fverror if err := fvhelpers.ConfigValidator().ValidateUpdateVaultConfig(updatedConfig); err != nil { return err } + if err := fvhelpers.ConfigValidator().ValidateHttpConfig(updatedConfig.HttpConfig()); err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() svc, exists := s.vaultServices[updatedConfig.VaultId] if !exists { fvlogger.Error(fmt.Sprintf(logs.VAULT_CONFIG_DOES_NOT_EXIST, updatedConfig.VaultId)) return fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, fverrors.VAULT_ID_NOT_IN_CONFIG_LIST) } - // The service holds a pointer to its config, so in-place updates reach it - // and any controller it has built; cached tokens are dropped on change. - if !fvcommon.IsCredentialsEmpty(updatedConfig.Credentials) { - svc.config.Credentials = updatedConfig.Credentials - svc.resetAuth() + // Only the fields set on updatedConfig change. The merged config replaces the old one in + // a single step, so a call that is running keeps the config it started with. + config := svc.controller.Config() + credentialsChanged := !fvcommon.IsCredentialsEmpty(updatedConfig.Credentials) + if credentialsChanged { + config.Credentials = updatedConfig.Credentials } if updatedConfig.ClusterId != "" { - svc.config.ClusterId = updatedConfig.ClusterId + config.ClusterId = updatedConfig.ClusterId } if updatedConfig.BaseVaultUrl != "" { - svc.config.BaseVaultUrl = updatedConfig.BaseVaultUrl + config.BaseVaultUrl = updatedConfig.BaseVaultUrl + } + if updatedConfig.Env.IsSet() { + config.Env = updatedConfig.Env + } + if updatedConfig.Timeout != 0 { + config.Timeout = updatedConfig.Timeout + } + if updatedConfig.MaxRetries != nil { + retries := *updatedConfig.MaxRetries + config.MaxRetries = &retries + } + if updatedConfig.InitialRetryDelay != 0 { + config.InitialRetryDelay = updatedConfig.InitialRetryDelay } - svc.config.Env = updatedConfig.Env + if updatedConfig.MaxRetryDelay != 0 { + config.MaxRetryDelay = updatedConfig.MaxRetryDelay + } + svc.controller.SetConfig(config, credentialsChanged) return nil } // RemoveVaultConfig removes the vault config registered under vaultId. // Returns an error if the vault ID is not registered. func (s *Skyflow) RemoveVaultConfig(vaultId string) *fverrors.SkyflowError { + s.mu.Lock() + defer s.mu.Unlock() if _, exists := s.vaultServices[vaultId]; !exists { fvlogger.Error(fmt.Sprintf(logs.VAULT_CONFIG_DOES_NOT_EXIST, vaultId)) return fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, fverrors.VAULT_ID_NOT_IN_CONFIG_LIST) @@ -142,13 +209,17 @@ func (s *Skyflow) RemoveVaultConfig(vaultId string) *fverrors.SkyflowError { return nil } -// GetVaultConfig returns the VaultConfig registered under vaultId, or an error if it is not found. +// GetVaultConfig returns a copy of the VaultConfig registered under vaultId, or an error if +// it is not found. Use UpdateVaultConfig to change it. func (s *Skyflow) GetVaultConfig(vaultId string) (*fvcommon.VaultConfig, *fverrors.SkyflowError) { + s.mu.RLock() + defer s.mu.RUnlock() svc, exists := s.vaultServices[vaultId] if !exists { return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, fverrors.VAULT_ID_NOT_IN_CONFIG_LIST) } - return svc.config, nil + config := svc.controller.Config() + return &config, nil } // --------------------------------------------------------------------------- @@ -161,10 +232,11 @@ func (s *Skyflow) AddSkyflowCredentials(credentials fvcommon.Credentials) *fverr if err := fvhelpers.ConfigValidator().ValidateCredentials(credentials); err != nil { return err } + s.mu.Lock() + defer s.mu.Unlock() + // A new pointer, so every vault drops its cached token. s.credentials = &credentials - for _, svc := range s.vaultServices { - svc.resetAuth() - } + s.syncVaults() return nil } @@ -175,7 +247,13 @@ func (s *Skyflow) UpdateSkyflowCredentials(credentials fvcommon.Credentials) *fv // GetSkyflowCredentials returns the client-level credentials, or nil if none are set. func (s *Skyflow) GetSkyflowCredentials() *fvcommon.Credentials { - return s.credentials + s.mu.RLock() + defer s.mu.RUnlock() + if s.credentials == nil { + return nil + } + credentials := *s.credentials + return &credentials } // --------------------------------------------------------------------------- @@ -184,6 +262,8 @@ func (s *Skyflow) GetSkyflowCredentials() *fvcommon.Credentials { // UpdateLogLevel changes the log level for the client. func (s *Skyflow) UpdateLogLevel(logLevel fvlogger.LogLevel) { + s.mu.Lock() + defer s.mu.Unlock() fvlogger.Info(fmt.Sprintf(logs.CURRENT_LOG_LEVEL, s.logLevel)) s.logLevel = logLevel fvlogger.SetLogLevel(logLevel) @@ -191,6 +271,8 @@ func (s *Skyflow) UpdateLogLevel(logLevel fvlogger.LogLevel) { // GetLoglevel returns the current log level of the client. func (s *Skyflow) GetLoglevel() *fvlogger.LogLevel { + s.mu.RLock() + defer s.mu.RUnlock() logLevel := s.logLevel return &logLevel } diff --git a/flowvault/client/client_test.go b/flowvault/client/client_test.go new file mode 100644 index 0000000..593e65e --- /dev/null +++ b/flowvault/client/client_test.go @@ -0,0 +1,210 @@ +package client + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/skyflowapi/skyflow-go/flowvault/internal/testutil" + fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" +) + +func TestClient(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Flowvault Client Suite") +} + +const testVaultID = "vault123" + +var testApiKey = testutil.Resource("client_api_key.txt") + +func newTestClient(opts ...Option) *Skyflow { + options := append([]Option{ + WithVaults(fvcommon.VaultConfig{VaultId: testVaultID, ClusterId: "cluster", Env: fvcommon.DEV}), + WithCredentials(fvcommon.Credentials{ApiKey: testApiKey}), + }, opts...) + client, err := NewSkyflow(options...) + Expect(err).To(BeNil()) + return client +} + +var _ = Describe("Skyflow client", func() { + It("rejects an empty vault list and duplicate vault IDs", func() { + _, err := NewSkyflow(WithVaults()) + Expect(err.GetMessage()).To(ContainSubstring("No vault configurations available")) + + config := fvcommon.VaultConfig{VaultId: testVaultID, ClusterId: "cluster"} + _, err = NewSkyflow(WithVaults(config, config)) + Expect(err.GetMessage()).To(ContainSubstring("already exists")) + }) + + It("rejects negative HTTP settings on the client and on a vault", func() { + _, err := NewSkyflow(WithHttpConfig(fvcommon.HttpConfig{Timeout: -time.Second})) + Expect(err.GetMessage()).To(ContainSubstring("must not be negative")) + + retries := -1 + _, err = NewSkyflow(WithVaults(fvcommon.VaultConfig{VaultId: testVaultID, ClusterId: "cluster", MaxRetries: &retries})) + Expect(err.GetMessage()).To(ContainSubstring("must not be negative")) + }) + + It("passes client settings to vaults registered before or after them", func() { + retries := 2 + client := newTestClient( + WithHttpConfig(fvcommon.HttpConfig{MaxRetries: &retries}), + ) + vault, err := client.Vault(testVaultID) + Expect(err).To(BeNil()) + settings := vault.controller.ClientSettings() + Expect(settings.Credentials.ApiKey).To(Equal(testApiKey)) + Expect(*settings.HttpConfig.MaxRetries).To(Equal(2)) + + Expect(client.AddVaultConfig(fvcommon.VaultConfig{VaultId: "other", ClusterId: "cluster"})).To(BeNil()) + other, _ := client.Vault("other") + Expect(*other.controller.ClientSettings().HttpConfig.MaxRetries).To(Equal(2)) + }) + + It("keeps the vault's Env when an update doesn't set one", func() { + client := newTestClient() + Expect(client.UpdateVaultConfig(fvcommon.VaultConfig{VaultId: testVaultID, ClusterId: "other"})).To(BeNil()) + config, _ := client.GetVaultConfig(testVaultID) + Expect(config.Env).To(Equal(fvcommon.DEV)) + Expect(config.ClusterId).To(Equal("other")) + + Expect(client.UpdateVaultConfig(fvcommon.VaultConfig{VaultId: testVaultID, Env: fvcommon.STAGE})).To(BeNil()) + config, _ = client.GetVaultConfig(testVaultID) + Expect(config.Env).To(Equal(fvcommon.STAGE)) + }) + + It("returns copies, so changing them doesn't change the client", func() { + client := newTestClient() + config, _ := client.GetVaultConfig(testVaultID) + config.ClusterId = "changed" + again, _ := client.GetVaultConfig(testVaultID) + Expect(again.ClusterId).To(Equal("cluster")) + + creds := client.GetSkyflowCredentials() + creds.ApiKey = "changed" + Expect(client.GetSkyflowCredentials().ApiKey).To(Equal(testApiKey)) + }) + + It("can be changed while calls are running", func() { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{}`)) + })) + defer server.Close() + client := newTestClient(WithVaults(fvcommon.VaultConfig{VaultId: "local", BaseVaultUrl: server.URL})) + vault, err := client.Vault("local") + Expect(err).To(BeNil()) + + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(2) + go func() { + defer wg.Done() + defer GinkgoRecover() + for j := 0; j < 20; j++ { + _, err := vault.Query(context.Background(), fvcommon.QueryRequest{Query: "SELECT 1"}, fvcommon.QueryOptions{}) + Expect(err).To(BeNil()) + } + }() + go func(i int) { + defer wg.Done() + defer GinkgoRecover() + id := fmt.Sprintf("extra-%d", i) + for j := 0; j < 20; j++ { + Expect(client.AddVaultConfig(fvcommon.VaultConfig{VaultId: id, ClusterId: "c"})).To(BeNil()) + Expect(client.UpdateVaultConfig(fvcommon.VaultConfig{VaultId: "local", Timeout: time.Duration(j+1) * time.Second})).To(BeNil()) + Expect(client.UpdateSkyflowCredentials(fvcommon.Credentials{ApiKey: testApiKey})).To(BeNil()) + _, _ = client.GetVaultConfig("local") + Expect(client.RemoveVaultConfig(id)).To(BeNil()) + } + }(i) + } + wg.Wait() + }) + + It("updates a vault's HTTP settings in place", func() { + client := newTestClient() + retries := 3 + Expect(client.UpdateVaultConfig(fvcommon.VaultConfig{VaultId: testVaultID, Timeout: 5 * time.Second, MaxRetries: &retries})).To(BeNil()) + config, _ := client.GetVaultConfig(testVaultID) + Expect(config.Timeout).To(Equal(5 * time.Second)) + Expect(*config.MaxRetries).To(Equal(3)) + }) +}) + +var _ = Describe("VaultService", func() { + var ( + vault *VaultService + ctx = context.Background() + ) + BeforeEach(func() { + var err error + vault, err = newTestClient().Vault(testVaultID) + Expect(err).To(BeNil()) + }) + + It("validates before anything else", func() { + _, err := vault.Insert(ctx, fvcommon.InsertRequest{}, fvcommon.InsertOptions{}) + Expect(err.GetMessage()).To(ContainSubstring("Records must not be empty")) + _, err = vault.Query(ctx, fvcommon.QueryRequest{}, fvcommon.QueryOptions{}) + Expect(err.GetMessage()).To(ContainSubstring("Query must not be empty")) + }) + + It("rejects interceptor headers that aren't supported", func() { + _, err := vault.Query(ctx, fvcommon.QueryRequest{Query: "SELECT 1"}, fvcommon.QueryOptions{ + Interceptor: func(rc *fvcommon.RequestContext) { rc.AddHeader("x-other", "v") }, + }) + Expect(err.GetMessage()).To(ContainSubstring("is not supported")) + }) + + It("sends valid requests to the vault", func() { + var paths []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + paths = append(paths, r.URL.Path) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{}`)) + })) + defer server.Close() + client := newTestClient(WithVaults(fvcommon.VaultConfig{VaultId: "local", BaseVaultUrl: server.URL})) + vault, err := client.Vault("local") + Expect(err).To(BeNil()) + + data := map[string]interface{}{"a": 1} + _, err = vault.Insert(ctx, fvcommon.InsertRequest{TableName: "t", Records: []fvcommon.InsertRequestRecord{{Data: data}}}, fvcommon.InsertOptions{}) + Expect(err).To(BeNil()) + _, err = vault.Update(ctx, fvcommon.UpdateRequest{TableName: "t", Records: []fvcommon.UpdateRequestRecord{{SkyflowId: "id", Data: data}}}, fvcommon.UpdateOptions{}) + Expect(err).To(BeNil()) + _, err = vault.Get(ctx, fvcommon.GetRequest{TableName: "t", SkyflowIds: []string{"id"}}, fvcommon.GetOptions{}) + Expect(err).To(BeNil()) + _, err = vault.Delete(ctx, fvcommon.DeleteRequest{TableName: "t", SkyflowIds: []string{"id"}}, fvcommon.DeleteOptions{}) + Expect(err).To(BeNil()) + _, err = vault.Detokenize(ctx, fvcommon.DetokenizeRequest{Tokens: []string{"tok"}}, fvcommon.DetokenizeOptions{}) + Expect(err).To(BeNil()) + _, err = vault.GetTokens(ctx, fvcommon.GetTokensRequest{Records: []fvcommon.GetTokensRequestRecord{{Value: "v", TokenGroupName: "g"}}}, fvcommon.GetTokensOptions{}) + Expect(err).To(BeNil()) + _, err = vault.Query(ctx, fvcommon.QueryRequest{Query: "SELECT 1"}, fvcommon.QueryOptions{}) + Expect(err).To(BeNil()) + _, err = vault.UploadFiles(ctx, fvcommon.UploadFilesRequest{Records: []fvcommon.UploadFilesRequestRecord{ + {TableName: "t", Columns: []fvcommon.UploadFilesRequestColumn{{Column: "c", Base64: "YQ==", FileName: "a.txt"}}}, + }}, fvcommon.UploadFilesOptions{}) + Expect(err).To(BeNil()) + _, err = vault.DeleteFiles(ctx, fvcommon.DeleteFilesRequest{Records: []fvcommon.DeleteFilesRequestRecord{ + {TableName: "t", Columns: []string{"c"}, SkyflowId: "id"}, + }}, fvcommon.DeleteFilesOptions{}) + Expect(err).To(BeNil()) + + Expect(paths).To(Equal([]string{ + "/v2/records/insert", "/v2/records/update", "/v2/records/get", "/v2/records/delete", + "/v2/tokens/detokenize", "/v2/records/getTokens", "/v2/query", "/v2/files/upload", "/v2/files/delete", + })) + }) +}) diff --git a/flowvault/client/operations.go b/flowvault/client/operations.go new file mode 100644 index 0000000..9264e6c --- /dev/null +++ b/flowvault/client/operations.go @@ -0,0 +1,20 @@ +package client + +import ( + "github.com/skyflowapi/skyflow-go/common/vault" + fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" +) + +// Compile-time checks that the flowvault vault service implements the operations shared +// with the other Skyflow Go SDKs. +var ( + _ vault.Inserter[fvcommon.InsertRequest, fvcommon.InsertOptions, fvcommon.InsertResponse] = (*VaultService)(nil) + _ vault.Updater[fvcommon.UpdateRequest, fvcommon.UpdateOptions, fvcommon.UpdateResponse] = (*VaultService)(nil) + _ vault.Getter[fvcommon.GetRequest, fvcommon.GetOptions, fvcommon.GetResponse] = (*VaultService)(nil) + _ vault.Deleter[fvcommon.DeleteRequest, fvcommon.DeleteOptions, fvcommon.DeleteResponse] = (*VaultService)(nil) + _ vault.Querier[fvcommon.QueryRequest, fvcommon.QueryOptions, fvcommon.QueryResponse] = (*VaultService)(nil) + _ vault.Detokenizer[fvcommon.DetokenizeRequest, fvcommon.DetokenizeOptions, fvcommon.DetokenizeResponse] = (*VaultService)(nil) + _ vault.TokenGetter[fvcommon.GetTokensRequest, fvcommon.GetTokensOptions, fvcommon.GetTokensResponse] = (*VaultService)(nil) + _ vault.FilesUploader[fvcommon.UploadFilesRequest, fvcommon.UploadFilesOptions, fvcommon.UploadFilesResponse] = (*VaultService)(nil) + _ vault.FilesDeleter[fvcommon.DeleteFilesRequest, fvcommon.DeleteFilesOptions, fvcommon.DeleteFilesResponse] = (*VaultService)(nil) +) diff --git a/flowvault/client/vault_service.go b/flowvault/client/vault_service.go index 13b6703..7a83c4c 100644 --- a/flowvault/client/vault_service.go +++ b/flowvault/client/vault_service.go @@ -1,105 +1,67 @@ package client import ( + "context" + + "github.com/skyflowapi/skyflow-go/flowvault/internal/vault/controller" fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" fverrors "github.com/skyflowapi/skyflow-go/flowvault/utils/error" - fvlogger "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" ) -// VaultService handles all vault operations for a single vault. +// VaultService handles all vault operations for a single vault. Every method is unary: +// one request, one response. // -// It keeps a pointer to its config and a back-reference to the client, so -// UpdateVaultConfig and credential updates on the client are visible to it -// without any copying. +// The client updates the controller's settings in place, so a VaultService obtained +// earlier sees UpdateVaultConfig and credential changes on its next call. type VaultService struct { - client *Skyflow - config *fvcommon.VaultConfig + controller *controller.VaultController } -func newVaultService(client *Skyflow, config *fvcommon.VaultConfig) *VaultService { - return &VaultService{client: client, config: config} +func newVaultService(controller *controller.VaultController) *VaultService { + return &VaultService{controller: controller} } -// credentials returns the client-level credentials (nil if none are set). -func (v *VaultService) credentials() *fvcommon.Credentials { - return v.client.credentials +// Insert inserts records into the vault, optionally upserting on unique columns. +func (v *VaultService) Insert(ctx context.Context, request fvcommon.InsertRequest, options fvcommon.InsertOptions) (*fvcommon.InsertResponse, *fverrors.SkyflowError) { + return v.controller.Insert(ctx, request, options) } -// resetAuth drops any cached token so the next call re-authenticates with -// the current credentials. The vault controller is not wired yet; once it is, -// this is where its Token/ApiKey are cleared. -func (v *VaultService) resetAuth() {} +// Update updates existing records by Skyflow ID. +func (v *VaultService) Update(ctx context.Context, request fvcommon.UpdateRequest, options fvcommon.UpdateOptions) (*fvcommon.UpdateResponse, *fverrors.SkyflowError) { + return v.controller.Update(ctx, request, options) +} -// Insert inserts records into the vault. -func (v *VaultService) Insert(request fvcommon.InsertRequest, options fvcommon.InsertOptions) (*fvcommon.InsertResponse, *fverrors.SkyflowError) { - fvlogger.Info("Insert triggered for vault: " + v.config.VaultId + ", table: " + request.Table) - fvlogger.Debug("Insert request: ", request) - if request.Table == "" { - return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, "table must not be empty") - } - if len(request.Values) == 0 { - return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, "values must not be empty") - } - // TODO: implement actual HTTP call to Skyflow vault API - return &fvcommon.InsertResponse{}, nil +// Get retrieves records by Skyflow IDs or unique values. +func (v *VaultService) Get(ctx context.Context, request fvcommon.GetRequest, options fvcommon.GetOptions) (*fvcommon.GetResponse, *fverrors.SkyflowError) { + return v.controller.Get(ctx, request, options) } -// Detokenize detokenizes the given tokens. -func (v *VaultService) Detokenize(request fvcommon.DetokenizeRequest, options fvcommon.DetokenizeOptions) (*fvcommon.DetokenizeResponse, *fverrors.SkyflowError) { - fvlogger.Info("Detokenize triggered for vault: " + v.config.VaultId) - if len(request.DetokenizeData) == 0 { - return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, "detokenize data must not be empty") - } - // TODO: implement actual HTTP call - return &fvcommon.DetokenizeResponse{}, nil +// Delete deletes records by Skyflow IDs or unique values. +func (v *VaultService) Delete(ctx context.Context, request fvcommon.DeleteRequest, options fvcommon.DeleteOptions) (*fvcommon.DeleteResponse, *fverrors.SkyflowError) { + return v.controller.Delete(ctx, request, options) } -// Get retrieves records from the vault. -func (v *VaultService) Get(request fvcommon.GetRequest, options fvcommon.GetOptions) (*fvcommon.GetResponse, *fverrors.SkyflowError) { - fvlogger.Info("Get triggered for vault: " + v.config.VaultId + ", table: " + request.Table) - if request.Table == "" { - return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, "table must not be empty") - } - // TODO: implement actual HTTP call - return &fvcommon.GetResponse{}, nil +// Detokenize returns the values behind tokens. +func (v *VaultService) Detokenize(ctx context.Context, request fvcommon.DetokenizeRequest, options fvcommon.DetokenizeOptions) (*fvcommon.DetokenizeResponse, *fverrors.SkyflowError) { + return v.controller.Detokenize(ctx, request, options) } -// Delete deletes records from the vault. -func (v *VaultService) Delete(request fvcommon.DeleteRequest, options fvcommon.DeleteOptions) (*fvcommon.DeleteResponse, *fverrors.SkyflowError) { - fvlogger.Info("Delete triggered for vault: " + v.config.VaultId + ", table: " + request.Table) - if request.Table == "" { - return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, "table must not be empty") - } - // TODO: implement actual HTTP call - return &fvcommon.DeleteResponse{}, nil +// GetTokens returns existing tokens for values in deterministic token groups. +func (v *VaultService) GetTokens(ctx context.Context, request fvcommon.GetTokensRequest, options fvcommon.GetTokensOptions) (*fvcommon.GetTokensResponse, *fverrors.SkyflowError) { + return v.controller.GetTokens(ctx, request, options) } -// Update updates records in the vault. -func (v *VaultService) Update(request fvcommon.UpdateRequest, options fvcommon.UpdateOptions) (*fvcommon.UpdateResponse, *fverrors.SkyflowError) { - fvlogger.Info("Update triggered for vault: " + v.config.VaultId + ", table: " + request.Table) - if request.Table == "" { - return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, "table must not be empty") - } - // TODO: implement actual HTTP call - return &fvcommon.UpdateResponse{}, nil +// Query runs a SELECT query against the vault. +func (v *VaultService) Query(ctx context.Context, request fvcommon.QueryRequest, options fvcommon.QueryOptions) (*fvcommon.QueryResponse, *fverrors.SkyflowError) { + return v.controller.Query(ctx, request, options) } -// Query executes a query against the vault. -func (v *VaultService) Query(request fvcommon.QueryRequest, options fvcommon.QueryOptions) (*fvcommon.QueryResponse, *fverrors.SkyflowError) { - fvlogger.Info("Query triggered for vault: " + v.config.VaultId) - if request.Query == "" { - return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, "query must not be empty") - } - // TODO: implement actual HTTP call - return &fvcommon.QueryResponse{}, nil +// UploadFiles uploads files into file columns. +func (v *VaultService) UploadFiles(ctx context.Context, request fvcommon.UploadFilesRequest, options fvcommon.UploadFilesOptions) (*fvcommon.UploadFilesResponse, *fverrors.SkyflowError) { + return v.controller.UploadFiles(ctx, request, options) } -// Tokenize tokenizes values using the vault. -func (v *VaultService) Tokenize(requests []fvcommon.TokenizeRequest, options fvcommon.TokenizeOptions) (*fvcommon.TokenizeResponse, *fverrors.SkyflowError) { - fvlogger.Info("Tokenize triggered for vault: " + v.config.VaultId) - if len(requests) == 0 { - return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, "tokenize requests must not be empty") - } - // TODO: implement actual HTTP call - return &fvcommon.TokenizeResponse{}, nil +// DeleteFiles deletes files from file columns of existing records. +func (v *VaultService) DeleteFiles(ctx context.Context, request fvcommon.DeleteFilesRequest, options fvcommon.DeleteFilesOptions) (*fvcommon.DeleteFilesResponse, *fverrors.SkyflowError) { + return v.controller.DeleteFiles(ctx, request, options) } diff --git a/flowvault/go.mod b/flowvault/go.mod index def0414..f4327d1 100644 --- a/flowvault/go.mod +++ b/flowvault/go.mod @@ -4,11 +4,30 @@ go 1.25.0 require ( github.com/golang-jwt/jwt/v4 v4.5.2 // indirect - github.com/skyflowapi/skyflow-go/common v0.0.0-20261014151608-a64b46a2705c + github.com/skyflowapi/skyflow-go/common v0.0.0-20261019104456-6115e81fcef9 ) require ( - github.com/google/uuid v1.6.0 // indirect - github.com/onsi/ginkgo/v2 v2.33.1 // indirect - github.com/onsi/gomega v1.44.0 // indirect + github.com/google/uuid v1.6.0 + github.com/onsi/ginkgo/v2 v2.33.1 + github.com/onsi/gomega v1.44.0 + github.com/stretchr/testify v1.8.4 +) + +require ( + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/mod v0.36.0 // indirect + golang.org/x/net v0.56.0 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/text v0.38.0 // indirect + golang.org/x/tools v0.45.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/flowvault/go.sum b/flowvault/go.sum index 11985f8..cf05d87 100644 --- a/flowvault/go.sum +++ b/flowvault/go.sum @@ -1,9 +1,19 @@ github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= +github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= +github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -12,23 +22,53 @@ github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oX github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= github.com/onsi/ginkgo/v2 v2.33.1 h1:62XfWJbEIjPSCSDYrsCPP/SHcmH7O0M8ne8zfqQ2iz4= github.com/onsi/ginkgo/v2 v2.33.1/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= github.com/onsi/gomega v1.44.0 h1:eAiGl3Pw5jz5GQdDff0BcxYpAX1JxW8xD7mFUuwNfZQ= github.com/onsi/gomega v1.44.0/go.mod h1:e/C2HwaZ1DhvjzXXuFhcR7hY7Sh9pl7MmoWKEjzwcdA= -github.com/skyflowapi/skyflow-go/common v0.0.0-20261014151608-a64b46a2705c h1:oWxTNYJ++iBGbcVpJ2oNhyB36WNNVMyBpusEAcSL7pw= -github.com/skyflowapi/skyflow-go/common v0.0.0-20261014151608-a64b46a2705c/go.mod h1:T9roW4Qpv1WFo6tgLjbfLrdOCOz20lVE8ENf2bGb7zI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/skyflowapi/skyflow-go/common v0.0.0-20261019104456-6115e81fcef9 h1:0SNuvxkS+mr+sy7lDZvwmgjgAIC+HrqEr81lsTTXEgQ= +github.com/skyflowapi/skyflow-go/common v0.0.0-20261019104456-6115e81fcef9/go.mod h1:T9roW4Qpv1WFo6tgLjbfLrdOCOz20lVE8ENf2bGb7zI= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= -golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= -golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= -golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= -golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= +google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/flowvault/internal/constants/constants.go b/flowvault/internal/constants/constants.go index 322e304..38a1210 100644 --- a/flowvault/internal/constants/constants.go +++ b/flowvault/internal/constants/constants.go @@ -1,16 +1,18 @@ package internal -// internal private constants — flowvault copy (cannot import v2's internal) +// internal private constants. FlowVault (v3) vaults are served from the skyvault subdomain, +// not v2's vault subdomain. const ( SDK_METRICS_HEADER_KEY = "sky-metadata" SECURE_PROTOCOL = "https://" - DEV_DOMAIN = ".vault.skyflowapis.dev" - STAGE_DOMAIN = ".vault.skyflowapis.tech" - SANDBOX_DOMAIN = ".vault.skyflowapis-preview.com" - PROD_DOMAIN = ".vault.skyflowapis.com" + DEV_DOMAIN = ".skyvault.skyflowapis.dev" + STAGE_DOMAIN = ".skyvault.skyflowapis.tech" + SANDBOX_DOMAIN = ".skyvault.skyflowapis-preview.com" + PROD_DOMAIN = ".skyvault.skyflowapis.com" GRANT_TYPE = "urn:ietf:params:oauth:grant-type:jwt-bearer" - SDK_NAME = "Skyflow Go SDK " - SDK_VERSION = "v1.0.0" + SDK_NAME = "Skyflow FlowVault Go SDK " + SDK_VERSION = "v0.1.0" // flowvault module version; bump with each flowvault/vX.Y.Z tag + METRICS_SDK_NAME = "skyflow-flowvault-go" SDK_PREFIX = SDK_NAME + SDK_VERSION SDK_LOG_PREFIX = "[ " + SDK_PREFIX + " ] " ERROR_FROM_CLIENT = "error-from-client" @@ -34,3 +36,46 @@ const ( ERROR_KEY_DETAILS = "details" ERROR_KEY_FROM_CLIENT = "from_client" ) + +// Method names used in logs and error messages, and operation names passed to interceptors. +const ( + METHOD_INSERT = "Insert" + METHOD_UPDATE = "Update" + METHOD_GET = "Get" + METHOD_DELETE = "Delete" + METHOD_DETOKENIZE = "Detokenize" + METHOD_GET_TOKENS = "GetTokens" + METHOD_QUERY = "Query" + METHOD_UPLOAD_FILES = "UploadFiles" + METHOD_DELETE_FILES = "DeleteFiles" + + OPERATION_INSERT = "INSERT" + OPERATION_UPDATE = "UPDATE" + OPERATION_GET = "GET" + OPERATION_DELETE = "DELETE" + OPERATION_DETOKENIZE = "DETOKENIZE" + OPERATION_GET_TOKENS = "GET_TOKENS" + OPERATION_QUERY = "QUERY" + OPERATION_UPLOAD_FILES = "UPLOAD_FILES" + OPERATION_DELETE_FILES = "DELETE_FILES" + + MAX_DELETE_FILES_RECORDS = 20000 + MAX_CONCURRENT_UPLOADS = 5 + RESPONSE_HEADER_REQUEST_ID = "x-request-id" + SKYFLOW_CREDENTIALS_ENV = "SKYFLOW_CREDENTIALS" +) + +// HTTP defaults, used when neither the vault nor the client sets a value. +const ( + DEFAULT_TIMEOUT_SECONDS = 60 + DEFAULT_MAX_RETRIES = 0 + DEFAULT_INITIAL_RETRY_DELAY_MILLIS = 500 + DEFAULT_MAX_RETRY_DELAY_MILLIS = 2000 + RETRY_JITTER_FACTOR = 0.2 + + // Signed-URL uploads: bounds for connecting and for the storage service's response, + // matching the Java and Python defaults. There is no overall upload timeout. + UPLOAD_CONNECT_TIMEOUT_SECONDS = 10 + UPLOAD_RESPONSE_HEADER_TIMEOUT_SECONDS = 10 + UPLOAD_KEEP_ALIVE_SECONDS = 30 +) diff --git a/flowvault/internal/helpers/files.go b/flowvault/internal/helpers/files.go new file mode 100644 index 0000000..f8777aa --- /dev/null +++ b/flowvault/internal/helpers/files.go @@ -0,0 +1,104 @@ +package helpers + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + "io" + "mime" + "net/http" + "os" + "path/filepath" + "sync" + + fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + fverrors "github.com/skyflowapi/skyflow-go/flowvault/utils/error" +) + +const ( + putFailedStatus = "PUT failed: %d" + putFailedRequest = "PUT failed: %s" + contentType = "Content-Type" +) + +// ResolvedFile is a file column's content plus the name and content type to upload it with. +type ResolvedFile struct { + Name string + ContentType string + Content []byte +} + +// ResolveFile reads the column's file from FilePath, Base64 or FileObject and works out +// its name and content type. +func ResolveFile(column fvcommon.UploadFilesRequestColumn) (ResolvedFile, *fverrors.SkyflowError) { + file := ResolvedFile{Name: column.FileName} + switch { + case column.FilePath != "": + content, err := os.ReadFile(column.FilePath) + if err != nil { + return file, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, fmt.Sprintf(fverrors.FILE_NOT_READABLE, column.Column, err.Error())) + } + file.Content = content + if file.Name == "" { + file.Name = filepath.Base(column.FilePath) + } + case column.Base64 != "": + content, err := base64.StdEncoding.DecodeString(column.Base64) + if err != nil { + return file, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, fmt.Sprintf(fverrors.INVALID_BASE64, column.Column)) + } + file.Content = content + default: + content, err := io.ReadAll(column.FileObject) + if err != nil { + return file, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, fmt.Sprintf(fverrors.FILE_NOT_READABLE, column.Column, err.Error())) + } + file.Content = content + if file.Name == "" && column.FileObject.Name() != "" { + file.Name = filepath.Base(column.FileObject.Name()) + } + } + if file.Name != "" { + file.ContentType = mime.TypeByExtension(filepath.Ext(file.Name)) + } + return file, nil +} + +// PutFile uploads one file to its signed URL. The signature is in the URL, so no +// Authorization header is sent. +func PutFile(ctx context.Context, client *http.Client, signedURL string, file ResolvedFile) (fvcommon.UploadStatus, string) { + req, err := http.NewRequestWithContext(ctx, http.MethodPut, signedURL, bytes.NewReader(file.Content)) + if err != nil { + return fvcommon.UploadStatusFailed, fmt.Sprintf(putFailedRequest, err.Error()) + } + if file.ContentType != "" { + req.Header.Set(contentType, file.ContentType) + } + resp, err := client.Do(req) + if err != nil { + return fvcommon.UploadStatusFailed, fmt.Sprintf(putFailedRequest, err.Error()) + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, resp.Body) + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return fvcommon.UploadStatusFailed, fmt.Sprintf(putFailedStatus, resp.StatusCode) + } + return fvcommon.UploadStatusUploaded, "" +} + +// RunConcurrently runs tasks with at most limit running at once, and waits for all of them. +func RunConcurrently(tasks []func(), limit int) { + var wg sync.WaitGroup + slots := make(chan struct{}, limit) + for _, task := range tasks { + wg.Add(1) + slots <- struct{}{} + go func(run func()) { + defer wg.Done() + defer func() { <-slots }() + run() + }(task) + } + wg.Wait() +} diff --git a/flowvault/internal/helpers/helpers.go b/flowvault/internal/helpers/helpers.go index 7fc53cd..427f7e3 100644 --- a/flowvault/internal/helpers/helpers.go +++ b/flowvault/internal/helpers/helpers.go @@ -30,7 +30,7 @@ func GetURLWithEnv(env fvcommon.Env, clusterId string) string { // CreateSDKMetadata builds the sky-metadata header value. func CreateSDKMetadata() string { data := map[string]string{ - "sdk_name_version": fmt.Sprintf("skyflow-go@%s", fvconsts.SDK_VERSION), + "sdk_name_version": fmt.Sprintf("%s@%s", fvconsts.METRICS_SDK_NAME, fvconsts.SDK_VERSION), "device_model": runtime.GOOS, "os_details": fmt.Sprintf("%s %s", runtime.GOOS, runtime.GOARCH), "runtime_details": runtime.Version(), diff --git a/flowvault/internal/helpers/helpers_test.go b/flowvault/internal/helpers/helpers_test.go new file mode 100644 index 0000000..fc80eed --- /dev/null +++ b/flowvault/internal/helpers/helpers_test.go @@ -0,0 +1,224 @@ +package helpers + +import ( + "context" + "encoding/base64" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" +) + +func TestHelpers(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Flowvault Helpers Suite") +} + +var _ = Describe("HTTP config", func() { + It("prefers vault, then client, then defaults", func() { + vaultRetries, clientRetries := 2, 5 + resolved := ResolveHttpConfig( + fvcommon.HttpConfig{Timeout: 5 * time.Second, MaxRetries: &vaultRetries}, + fvcommon.HttpConfig{Timeout: 9 * time.Second, MaxRetries: &clientRetries, MaxRetryDelay: time.Second}, + ) + Expect(resolved).To(Equal(ResolvedHttpConfig{ + Timeout: 5 * time.Second, + MaxRetries: 2, + InitialRetryDelay: 500 * time.Millisecond, + MaxRetryDelay: time.Second, + })) + + defaults := ResolveHttpConfig(fvcommon.HttpConfig{}, fvcommon.HttpConfig{}) + Expect(defaults.Timeout).To(Equal(60 * time.Second)) + Expect(defaults.MaxRetries).To(Equal(0)) + Expect(defaults.MaxRetryDelay).To(Equal(2000 * time.Millisecond)) + }) + + It("keeps backoff within the jittered bounds", func() { + for attempt := 1; attempt <= 5; attempt++ { + delay := backoff(attempt, 500*time.Millisecond, 2*time.Second) + Expect(delay).To(BeNumerically("<=", 2400*time.Millisecond)) + Expect(delay).To(BeNumerically(">=", 400*time.Millisecond)) + } + }) + + It("retries retryable statuses up to MaxRetries and resends the body", func() { + var calls int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + Expect(string(body)).To(Equal("payload")) + if atomic.AddInt32(&calls, 1) < 3 { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + transport := &retryTransport{ + base: http.DefaultTransport, + config: ResolvedHttpConfig{MaxRetries: 3, InitialRetryDelay: time.Millisecond, MaxRetryDelay: time.Millisecond}, + sleep: func(time.Duration) {}, + } + req, _ := http.NewRequest(http.MethodPost, server.URL, strings.NewReader("payload")) + resp, err := transport.RoundTrip(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + Expect(atomic.LoadInt32(&calls)).To(Equal(int32(3))) + }) + + It("does not retry when retries are off", func() { + var calls int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&calls, 1) + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + resp, err := NewHTTPClient(ResolveHttpConfig(fvcommon.HttpConfig{}, fvcommon.HttpConfig{})).Get(server.URL) + Expect(err).To(BeNil()) + Expect(resp.StatusCode).To(Equal(http.StatusTooManyRequests)) + Expect(atomic.LoadInt32(&calls)).To(Equal(int32(1))) + }) +}) + +var _ = Describe("Files", func() { + It("resolves files from a path, base64 and an *os.File", func() { + dir := GinkgoT().TempDir() + path := filepath.Join(dir, "resume.pdf") + Expect(os.WriteFile(path, []byte("pdf"), 0o600)).To(Succeed()) + + file, err := ResolveFile(fvcommon.UploadFilesRequestColumn{Column: "c", FilePath: path}) + Expect(err).To(BeNil()) + Expect(file).To(Equal(ResolvedFile{Name: "resume.pdf", ContentType: "application/pdf", Content: []byte("pdf")})) + + file, err = ResolveFile(fvcommon.UploadFilesRequestColumn{Column: "c", Base64: base64.StdEncoding.EncodeToString([]byte("txt")), FileName: "notes.txt"}) + Expect(err).To(BeNil()) + Expect(file.Content).To(Equal([]byte("txt"))) + Expect(file.ContentType).To(HavePrefix("text/plain")) + + handle, _ := os.Open(path) + defer handle.Close() + file, err = ResolveFile(fvcommon.UploadFilesRequestColumn{Column: "c", FileObject: handle}) + Expect(err).To(BeNil()) + Expect(file.Name).To(Equal("resume.pdf")) + }) + + It("rejects an unreadable path and invalid base64", func() { + _, err := ResolveFile(fvcommon.UploadFilesRequestColumn{Column: "c", FilePath: "/does/not/exist"}) + Expect(err.GetMessage()).To(ContainSubstring("Could not read file")) + _, err = ResolveFile(fvcommon.UploadFilesRequestColumn{Column: "c", Base64: "%%%", FileName: "a.txt"}) + Expect(err.GetMessage()).To(ContainSubstring("invalid base64")) + }) + + It("PUTs a file without an Authorization header and reports failures", func() { + var got *http.Request + var body string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + raw, _ := io.ReadAll(r.Body) + got, body = r, string(raw) + if r.URL.Path == "/forbidden" { + w.WriteHeader(http.StatusForbidden) + return + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + file := ResolvedFile{Name: "a.pdf", ContentType: "application/pdf", Content: []byte("pdf")} + + status, msg := PutFile(context.Background(), http.DefaultClient, server.URL+"/ok", file) + Expect(status).To(Equal(fvcommon.UploadStatusUploaded)) + Expect(msg).To(BeEmpty()) + Expect(got.Method).To(Equal(http.MethodPut)) + Expect(got.Header.Get("Content-Type")).To(Equal("application/pdf")) + Expect(got.Header.Get("Authorization")).To(BeEmpty()) + Expect(body).To(Equal("pdf")) + + status, msg = PutFile(context.Background(), http.DefaultClient, server.URL+"/forbidden", file) + Expect(status).To(Equal(fvcommon.UploadStatusFailed)) + Expect(msg).To(Equal("PUT failed: 403")) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + status, _ = PutFile(ctx, http.DefaultClient, server.URL+"/ok", file) + Expect(status).To(Equal(fvcommon.UploadStatusFailed)) + }) + + It("uploads with no retries and no overall timeout, but bounded connect and response waits", func() { + client := UploadHTTPClient() + Expect(client).To(BeIdenticalTo(UploadHTTPClient())) + Expect(client.Timeout).To(BeZero()) + transport, ok := client.Transport.(*http.Transport) + Expect(ok).To(BeTrue()) + Expect(transport.TLSHandshakeTimeout).To(Equal(10 * time.Second)) + Expect(transport.ResponseHeaderTimeout).To(Equal(10 * time.Second)) + }) + + It("runs at most limit tasks at once", func() { + var running, peak int32 + tasks := make([]func(), 20) + for i := range tasks { + tasks[i] = func() { + now := atomic.AddInt32(&running, 1) + for { + old := atomic.LoadInt32(&peak) + if now <= old || atomic.CompareAndSwapInt32(&peak, old, now) { + break + } + } + time.Sleep(2 * time.Millisecond) + atomic.AddInt32(&running, -1) + } + } + RunConcurrently(tasks, 5) + Expect(atomic.LoadInt32(&peak)).To(BeNumerically("<=", 5)) + Expect(atomic.LoadInt32(&running)).To(Equal(int32(0))) + }) +}) + +var _ = Describe("Parsing", func() { + It("parses tokens and hashed data in object, list and string form", func() { + tokens := ParseTokens(map[string]interface{}{ + "email": []interface{}{map[string]interface{}{"token": "tok", "tokenGroupName": "det", "path": "$.a"}}, + "ssn": "plain-token", + }) + Expect(tokens["email"]).To(Equal([]fvcommon.Token{{Token: "tok", TokenGroupName: "det", Path: "$.a"}})) + Expect(tokens["ssn"]).To(Equal([]fvcommon.Token{{Token: "plain-token"}})) + Expect(ParseTokens(nil)).To(BeNil()) + + hashes := ParseHashedData(map[string]interface{}{"email": map[string]interface{}{"data": "abc", "hashName": "h1"}}) + Expect(hashes["email"]).To(Equal([]fvcommon.HashedValue{{Data: "abc", HashName: "h1"}})) + }) + + It("accepts table or tableName in detokenize metadata", func() { + Expect(ParseDetokenizeMetadata(map[string]interface{}{"skyflowID": "id", "table": "t1"})). + To(Equal(&fvcommon.DetokenizeRecordMetadata{SkyflowId: "id", TableName: "t1"})) + Expect(ParseDetokenizeMetadata(map[string]interface{}{"skyflowId": "id", "tableName": "t2"})). + To(Equal(&fvcommon.DetokenizeRecordMetadata{SkyflowId: "id", TableName: "t2"})) + Expect(ParseDetokenizeMetadata(nil)).To(BeNil()) + }) + + It("parses delete-files columns sorted by name", func() { + Expect(ParseDeleteFilesColumns(map[string]interface{}{ + "resumePDF": map[string]interface{}{"status": "DELETED"}, + "photoID": "DELETED", + })).To(Equal([]fvcommon.DeleteFilesColumnResult{{Column: "photoID", Status: "DELETED"}, {Column: "resumePDF", Status: "DELETED"}})) + Expect(ParseDeleteFilesColumns(nil)).To(BeNil()) + }) + + It("sets the request ID only for failed records", func() { + header := http.Header{} + header.Set("x-request-id", "req-1") + Expect(RequestIdFor(header, "boom")).To(Equal("req-1")) + Expect(RequestIdFor(header, "")).To(BeEmpty()) + }) +}) diff --git a/flowvault/internal/helpers/http.go b/flowvault/internal/helpers/http.go new file mode 100644 index 0000000..a521886 --- /dev/null +++ b/flowvault/internal/helpers/http.go @@ -0,0 +1,147 @@ +package helpers + +import ( + "io" + "math/rand" + "net" + "net/http" + "time" + + fvconsts "github.com/skyflowapi/skyflow-go/flowvault/internal/constants" + fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" +) + +// ResolvedHttpConfig is the effective timeout and retry policy for a call. +type ResolvedHttpConfig struct { + Timeout time.Duration + MaxRetries int + InitialRetryDelay time.Duration + MaxRetryDelay time.Duration +} + +// ResolveHttpConfig picks each setting from the vault config, then the client config, +// then the SDK default. +func ResolveHttpConfig(vault, client fvcommon.HttpConfig) ResolvedHttpConfig { + resolved := ResolvedHttpConfig{ + Timeout: firstDuration(vault.Timeout, client.Timeout, fvconsts.DEFAULT_TIMEOUT_SECONDS*time.Second), + MaxRetries: fvconsts.DEFAULT_MAX_RETRIES, + InitialRetryDelay: firstDuration(vault.InitialRetryDelay, client.InitialRetryDelay, fvconsts.DEFAULT_INITIAL_RETRY_DELAY_MILLIS*time.Millisecond), + MaxRetryDelay: firstDuration(vault.MaxRetryDelay, client.MaxRetryDelay, fvconsts.DEFAULT_MAX_RETRY_DELAY_MILLIS*time.Millisecond), + } + if vault.MaxRetries != nil { + resolved.MaxRetries = *vault.MaxRetries + } else if client.MaxRetries != nil { + resolved.MaxRetries = *client.MaxRetries + } + return resolved +} + +func firstDuration(values ...time.Duration) time.Duration { + for _, value := range values { + if value > 0 { + return value + } + } + return 0 +} + +// NewHTTPClient builds the client used for API calls. Timeout covers the whole call, +// including retries. +func NewHTTPClient(config ResolvedHttpConfig) *http.Client { + return &http.Client{ + Timeout: config.Timeout, + Transport: &retryTransport{ + base: http.DefaultTransport, + config: config, + }, + } +} + +// uploadClient is shared by all uploads so they reuse one connection pool. +var uploadClient = newUploadHTTPClient() + +// UploadHTTPClient returns the client that PUTs files to signed URLs. Like the Java and +// Python SDKs, it doesn't retry and has no overall timeout, because a large file can take +// longer than one API call's timeout while still uploading. Connecting and waiting for the +// storage service's response are bounded, so a stalled connection still fails, and the +// caller's context can cancel an upload at any time. +func UploadHTTPClient() *http.Client { + return uploadClient +} + +func newUploadHTTPClient() *http.Client { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.DialContext = (&net.Dialer{ + Timeout: fvconsts.UPLOAD_CONNECT_TIMEOUT_SECONDS * time.Second, + KeepAlive: fvconsts.UPLOAD_KEEP_ALIVE_SECONDS * time.Second, + }).DialContext + transport.TLSHandshakeTimeout = fvconsts.UPLOAD_CONNECT_TIMEOUT_SECONDS * time.Second + transport.ResponseHeaderTimeout = fvconsts.UPLOAD_RESPONSE_HEADER_TIMEOUT_SECONDS * time.Second + return &http.Client{Transport: transport} +} + +// retryTransport retries 408, 429 and 5xx responses with exponential backoff and jitter. +type retryTransport struct { + base http.RoundTripper + config ResolvedHttpConfig + sleep func(time.Duration) // overridden in tests +} + +func (t *retryTransport) RoundTrip(req *http.Request) (*http.Response, error) { + resp, err := t.base.RoundTrip(req) + for retry := 0; err == nil && shouldRetry(resp.StatusCode) && retry < t.config.MaxRetries; retry++ { + if req.Body != nil && req.GetBody == nil { + break + } + retryReq := req.Clone(req.Context()) + if req.GetBody != nil { + body, bodyErr := req.GetBody() + if bodyErr != nil { + break + } + retryReq.Body = body + } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + if waitErr := t.wait(req, backoff(retry+1, t.config.InitialRetryDelay, t.config.MaxRetryDelay)); waitErr != nil { + return nil, waitErr + } + resp, err = t.base.RoundTrip(retryReq) + } + return resp, err +} + +func (t *retryTransport) wait(req *http.Request, delay time.Duration) error { + if t.sleep != nil { + t.sleep(delay) + return nil + } + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-req.Context().Done(): + return req.Context().Err() + case <-timer.C: + return nil + } +} + +func shouldRetry(status int) bool { + return status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= http.StatusInternalServerError +} + +// backoff doubles the delay per attempt up to max, then applies +/-20% jitter. +func backoff(attempt int, initial, max time.Duration) time.Duration { + delay := initial + for step := 1; step < attempt && delay < max; step++ { + delay *= 2 + } + if delay > max { + delay = max + } + jitter := time.Duration(float64(delay) * fvconsts.RETRY_JITTER_FACTOR) + if jitter <= 0 { + return delay + } + return delay - jitter + time.Duration(rand.Int63n(int64(2*jitter)+1)) //nolint:gosec // jitter, not security +} diff --git a/flowvault/internal/helpers/parse.go b/flowvault/internal/helpers/parse.go new file mode 100644 index 0000000..e051a8c --- /dev/null +++ b/flowvault/internal/helpers/parse.go @@ -0,0 +1,123 @@ +package helpers + +import ( + "fmt" + "net/http" + "sort" + + fvconsts "github.com/skyflowapi/skyflow-go/flowvault/internal/constants" + fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" +) + +const statusKey = "status" + +// ParseTokens accepts a column value that is a token string, a token object or a list of either. +func ParseTokens(raw map[string]interface{}) map[string][]fvcommon.Token { + if raw == nil { + return nil + } + parsed := make(map[string][]fvcommon.Token, len(raw)) + for column, value := range raw { + var tokens []fvcommon.Token + for _, item := range asList(value) { + switch entry := item.(type) { + case map[string]interface{}: + tokens = append(tokens, fvcommon.Token{ + Token: stringField(entry, "token"), + TokenGroupName: stringField(entry, "tokenGroupName"), + Path: stringField(entry, "path"), + }) + case nil: + default: + tokens = append(tokens, fvcommon.Token{Token: fmt.Sprint(entry)}) + } + } + if tokens != nil { + parsed[column] = tokens + } + } + return parsed +} + +// ParseHashedData accepts a column value that is a hash string, a {data, hashName} object or a list of either. +func ParseHashedData(raw map[string]interface{}) map[string][]fvcommon.HashedValue { + if raw == nil { + return nil + } + parsed := make(map[string][]fvcommon.HashedValue, len(raw)) + for column, value := range raw { + var hashes []fvcommon.HashedValue + for _, item := range asList(value) { + switch entry := item.(type) { + case map[string]interface{}: + hashes = append(hashes, fvcommon.HashedValue{Data: stringField(entry, "data"), HashName: stringField(entry, "hashName")}) + case nil: + default: + hashes = append(hashes, fvcommon.HashedValue{Data: fmt.Sprint(entry)}) + } + } + if hashes != nil { + parsed[column] = hashes + } + } + return parsed +} + +// ParseDetokenizeMetadata accepts both "table" and "tableName" keys. +func ParseDetokenizeMetadata(raw map[string]interface{}) *fvcommon.DetokenizeRecordMetadata { + if raw == nil { + return nil + } + return &fvcommon.DetokenizeRecordMetadata{ + SkyflowId: stringField(raw, "skyflowID", "skyflowId"), + TableName: stringField(raw, "tableName", "table"), + } +} + +func asList(value interface{}) []interface{} { + if list, ok := value.([]interface{}); ok { + return list + } + return []interface{}{value} +} + +// stringField returns the first of keys present in entry as a string. +func stringField(entry map[string]interface{}, keys ...string) string { + for _, key := range keys { + if value, ok := entry[key]; ok && value != nil { + if text, isString := value.(string); isString { + return text + } + return fmt.Sprint(value) + } + } + return "" +} + +// ParseDeleteFilesColumns converts {column: {"status": "DELETED"}} into results sorted by column. +func ParseDeleteFilesColumns(data map[string]interface{}) []fvcommon.DeleteFilesColumnResult { + if data == nil { + return nil + } + columns := make([]fvcommon.DeleteFilesColumnResult, 0, len(data)) + for name, value := range data { + status := "" + switch entry := value.(type) { + case map[string]interface{}: + status = stringField(entry, statusKey) + case string: + status = entry + } + columns = append(columns, fvcommon.DeleteFilesColumnResult{Column: name, Status: status}) + } + sort.Slice(columns, func(i, j int) bool { return columns[i].Column < columns[j].Column }) + return columns +} + +// RequestIdFor returns the call's request ID for a record that failed, and "" otherwise. +func RequestIdFor(header http.Header, recordErr string) string { + if recordErr == "" || header == nil { + return "" + } + return header.Get(fvconsts.RESPONSE_HEADER_REQUEST_ID) +} diff --git a/flowvault/internal/testutil/resources.go b/flowvault/internal/testutil/resources.go new file mode 100644 index 0000000..08eb9f1 --- /dev/null +++ b/flowvault/internal/testutil/resources.go @@ -0,0 +1,22 @@ +// Package testutil loads test fixtures from flowvault/test/resources. Fake credentials live +// there instead of in test source, so secret scanners don't flag the tests. +package testutil + +import ( + "os" + "path/filepath" + "runtime" + "strings" +) + +// Resource returns the trimmed contents of a file in flowvault/test/resources. It panics if +// the file can't be read, which fails the test that asked for it. +func Resource(name string) string { + _, thisFile, _, _ := runtime.Caller(0) + path := filepath.Join(filepath.Dir(thisFile), "..", "..", "test", "resources", name) + content, err := os.ReadFile(path) + if err != nil { + panic(err) + } + return strings.TrimSpace(string(content)) +} diff --git a/flowvault/internal/validation/validations.go b/flowvault/internal/validation/validations.go new file mode 100644 index 0000000..d684fca --- /dev/null +++ b/flowvault/internal/validation/validations.go @@ -0,0 +1,307 @@ +package validation + +import ( + "fmt" + "strings" + + fvconsts "github.com/skyflowapi/skyflow-go/flowvault/internal/constants" + fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + fverrors "github.com/skyflowapi/skyflow-go/flowvault/utils/error" +) + +func invalid(format string, args ...interface{}) *fverrors.SkyflowError { + message := format + if len(args) > 0 { + message = fmt.Sprintf(format, args...) + } + return fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, message) +} + +func blank(value string) bool { + return strings.TrimSpace(value) == "" +} + +func ValidateInsert(req fvcommon.InsertRequest) *fverrors.SkyflowError { + name := fvconsts.METHOD_INSERT + if len(req.Records) == 0 { + return invalid(fverrors.EMPTY_RECORDS, name) + } + tableAtRequest := !blank(req.TableName) + for i, record := range req.Records { + if err := validateData(name, i, record.Data); err != nil { + return err + } + tableAtRecord := !blank(record.TableName) + if tableAtRequest && tableAtRecord { + return invalid(fverrors.TABLE_IN_BOTH_PLACES, name) + } + if !tableAtRequest && !tableAtRecord { + return invalid(fverrors.TABLE_MISSING, name) + } + if tableAtRequest && record.Upsert != nil { + return invalid(fverrors.RECORD_UPSERT_NOT_ALLOWED) + } + if err := validateUpsert(record.Upsert); err != nil { + return err + } + } + if !tableAtRequest && req.Upsert != nil { + return invalid(fverrors.REQUEST_UPSERT_NOT_ALLOWED) + } + return validateUpsert(req.Upsert) +} + +func validateUpsert(upsert *fvcommon.UpsertOptions) *fverrors.SkyflowError { + if upsert == nil { + return nil + } + if len(upsert.UniqueColumns) == 0 { + return invalid(fverrors.EMPTY_UNIQUE_COLUMNS) + } + for _, column := range upsert.UniqueColumns { + if blank(column) { + return invalid(fverrors.EMPTY_UNIQUE_COLUMNS) + } + } + return validateUpdateType(fvconsts.METHOD_INSERT, upsert.UpdateType) +} + +func validateUpdateType(name string, updateType fvcommon.UpdateType) *fverrors.SkyflowError { + switch updateType { + case "", fvcommon.UpdateTypeUpdate, fvcommon.UpdateTypeReplace: + return nil + } + return invalid(fverrors.INVALID_UPDATE_TYPE, name, string(updateType)) +} + +func validateData(name string, index int, data map[string]interface{}) *fverrors.SkyflowError { + if len(data) == 0 { + return invalid(fverrors.EMPTY_DATA, name, index) + } + for key := range data { + if blank(key) { + return invalid(fverrors.EMPTY_KEY_IN_DATA, name, index) + } + } + return nil +} + +func ValidateUpdate(req fvcommon.UpdateRequest) *fverrors.SkyflowError { + name := fvconsts.METHOD_UPDATE + if len(req.Records) == 0 { + return invalid(fverrors.EMPTY_RECORDS, name) + } + if err := validateUpdateType(name, req.UpdateType); err != nil { + return err + } + tableAtRequest := !blank(req.TableName) + for i, record := range req.Records { + if blank(record.SkyflowId) { + return invalid(fverrors.MISSING_SKYFLOW_ID, i) + } + if err := validateData(name, i, record.Data); err != nil { + return err + } + tableAtRecord := !blank(record.TableName) + if tableAtRequest && tableAtRecord { + return invalid(fverrors.TABLE_IN_BOTH_PLACES, name) + } + if !tableAtRequest && !tableAtRecord { + return invalid(fverrors.TABLE_MISSING, name) + } + } + return nil +} + +func ValidateGet(req fvcommon.GetRequest) *fverrors.SkyflowError { + name := fvconsts.METHOD_GET + if req.Limit < 0 || req.Offset < 0 { + return invalid(fverrors.NEGATIVE_LIMIT_OFFSET) + } + if req.Records != nil { + if req.TableName != "" || req.SkyflowIds != nil || req.UniqueValues != nil || req.Columns != nil || + req.ColumnRedactions != nil || req.Limit != 0 || req.Offset != 0 { + return invalid(fverrors.GET_MODE_CONFLICT) + } + if len(req.Records) == 0 { + return invalid(fverrors.EMPTY_RECORDS, name) + } + for _, record := range req.Records { + if err := validateTableTarget(name, record.TableName, record.SkyflowIds, record.UniqueValues); err != nil { + return err + } + if err := validateColumnRedactions(name, record.ColumnRedactions); err != nil { + return err + } + } + return nil + } + if err := validateTableTarget(name, req.TableName, req.SkyflowIds, req.UniqueValues); err != nil { + return err + } + return validateColumnRedactions(name, req.ColumnRedactions) +} + +func ValidateDelete(req fvcommon.DeleteRequest) *fverrors.SkyflowError { + return validateTableTarget(fvconsts.METHOD_DELETE, req.TableName, req.SkyflowIds, req.UniqueValues) +} + +// validateTableTarget checks a table name plus exactly one of IDs / unique values. +func validateTableTarget(name, table string, ids []string, uniqueValues []map[string]interface{}) *fverrors.SkyflowError { + if blank(table) { + return invalid(fverrors.EMPTY_TABLE_NAME, name) + } + if (len(ids) > 0) == (len(uniqueValues) > 0) { + return invalid(fverrors.IDS_OR_UNIQUE_VALUES, name) + } + for _, id := range ids { + if blank(id) { + return invalid(fverrors.EMPTY_ID, name) + } + } + return validateUniqueValues(name, uniqueValues) +} + +func validateUniqueValues(name string, uniqueValues []map[string]interface{}) *fverrors.SkyflowError { + for _, value := range uniqueValues { + if len(value) == 0 { + return invalid(fverrors.EMPTY_UNIQUE_VALUE, name) + } + } + return nil +} + +func validateColumnRedactions(name string, redactions []fvcommon.ColumnRedaction) *fverrors.SkyflowError { + for _, redaction := range redactions { + if blank(redaction.ColumnName) || blank(redaction.Redaction) { + return invalid(fverrors.INVALID_COLUMN_REDACTION, name) + } + } + return nil +} + +func ValidateDetokenize(req fvcommon.DetokenizeRequest) *fverrors.SkyflowError { + if len(req.Tokens) == 0 { + return invalid(fverrors.EMPTY_TOKENS) + } + for _, token := range req.Tokens { + if blank(token) { + return invalid(fverrors.EMPTY_TOKEN) + } + } + for _, redaction := range req.TokenGroupRedactions { + if blank(redaction.TokenGroupName) || blank(redaction.Redaction) { + return invalid(fverrors.INVALID_TOKEN_GROUP_REDACTION) + } + } + return nil +} + +func ValidateGetTokens(req fvcommon.GetTokensRequest) *fverrors.SkyflowError { + if len(req.Records) == 0 { + return invalid(fverrors.EMPTY_RECORDS, fvconsts.METHOD_GET_TOKENS) + } + for i, record := range req.Records { + if record.Value == nil { + return invalid(fverrors.MISSING_VALUE, i) + } + if blank(record.TokenGroupName) { + return invalid(fverrors.MISSING_TOKEN_GROUP_NAME, i) + } + } + return nil +} + +func ValidateQuery(req fvcommon.QueryRequest) *fverrors.SkyflowError { + if blank(req.Query) { + return invalid(fverrors.EMPTY_QUERY) + } + return nil +} + +func ValidateUploadFiles(req fvcommon.UploadFilesRequest) *fverrors.SkyflowError { + name := fvconsts.METHOD_UPLOAD_FILES + if len(req.Records) == 0 { + return invalid(fverrors.EMPTY_RECORDS, name) + } + for i, record := range req.Records { + if blank(record.TableName) { + return invalid(fverrors.EMPTY_TABLE_NAME, name) + } + if len(record.Columns) == 0 { + return invalid(fverrors.EMPTY_COLUMNS, name, i) + } + for _, column := range record.Columns { + if blank(column.Column) { + return invalid(fverrors.EMPTY_COLUMN_NAME, name, i) + } + sources := 0 + if column.FilePath != "" { + sources++ + } + if column.Base64 != "" { + sources++ + } + if column.FileObject != nil { + sources++ + } + if sources != 1 { + return invalid(fverrors.FILE_SOURCE_COUNT, column.Column) + } + if column.Base64 != "" && blank(column.FileName) { + return invalid(fverrors.BASE64_NEEDS_FILE_NAME, column.Column) + } + } + } + return nil +} + +func ValidateDeleteFiles(req fvcommon.DeleteFilesRequest) *fverrors.SkyflowError { + name := fvconsts.METHOD_DELETE_FILES + if len(req.Records) == 0 { + return invalid(fverrors.EMPTY_RECORDS, name) + } + if len(req.Records) > fvconsts.MAX_DELETE_FILES_RECORDS { + return invalid(fverrors.DELETE_FILES_LIMIT, fvconsts.MAX_DELETE_FILES_RECORDS) + } + for i, record := range req.Records { + if blank(record.TableName) { + return invalid(fverrors.EMPTY_TABLE_NAME, name) + } + if len(record.Columns) == 0 { + return invalid(fverrors.EMPTY_COLUMNS, name, i) + } + seen := make(map[string]bool, len(record.Columns)) + for _, column := range record.Columns { + if blank(column) { + return invalid(fverrors.EMPTY_COLUMN_NAME, name, i) + } + if seen[column] { + return invalid(fverrors.DUPLICATE_COLUMN, name, column, i) + } + seen[column] = true + } + if blank(record.SkyflowId) == (len(record.UniqueValues) == 0) { + return invalid(fverrors.DELETE_FILES_TARGET, i) + } + if err := validateUniqueValues(name, record.UniqueValues); err != nil { + return err + } + } + return nil +} + +// ValidateCustomHeaders checks that only supported header keys are set, each with a value. +func ValidateCustomHeaders(headers map[fvcommon.CustomHeaderKey]string, tag string) *fverrors.SkyflowError { + for key, value := range headers { + switch key { + case fvcommon.SkyflowAccountId, fvcommon.SkyflowAccountName, fvcommon.RequestIdHeader: + default: + return invalid(fverrors.INVALID_HEADER_KEY, tag, string(key)) + } + if blank(value) { + return invalid(fverrors.EMPTY_HEADER_VALUE, tag, string(key)) + } + } + return nil +} diff --git a/flowvault/internal/validation/validations_test.go b/flowvault/internal/validation/validations_test.go new file mode 100644 index 0000000..ed35b8c --- /dev/null +++ b/flowvault/internal/validation/validations_test.go @@ -0,0 +1,124 @@ +package validation_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + . "github.com/skyflowapi/skyflow-go/flowvault/internal/validation" + fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + fverrors "github.com/skyflowapi/skyflow-go/flowvault/utils/error" +) + +func TestValidation(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Flowvault Validation Suite") +} + +func expectError(err *fverrors.SkyflowError, message string) { + if message == "" { + Expect(err).To(BeNil()) + return + } + Expect(err).ToNot(BeNil()) + Expect(err.GetMessage()).To(ContainSubstring(message)) +} + +var data = map[string]interface{}{"name": "john"} + +var _ = Describe("ValidateInsert", func() { + DescribeTable("table and upsert placement", + func(req fvcommon.InsertRequest, message string) { expectError(ValidateInsert(req), message) }, + Entry("request-level table", fvcommon.InsertRequest{TableName: "t", Records: []fvcommon.InsertRequestRecord{{Data: data}}}, ""), + Entry("record-level table and upsert", fvcommon.InsertRequest{Records: []fvcommon.InsertRequestRecord{{Data: data, TableName: "t", Upsert: &fvcommon.UpsertOptions{UniqueColumns: []string{"name"}}}}}, ""), + Entry("no records", fvcommon.InsertRequest{TableName: "t"}, "Records must not be empty"), + Entry("empty data", fvcommon.InsertRequest{TableName: "t", Records: []fvcommon.InsertRequestRecord{{}}}, "Data must not be empty"), + Entry("empty column name", fvcommon.InsertRequest{TableName: "t", Records: []fvcommon.InsertRequestRecord{{Data: map[string]interface{}{" ": 1}}}}, "empty column name"), + Entry("table in both places", fvcommon.InsertRequest{TableName: "t", Records: []fvcommon.InsertRequestRecord{{Data: data, TableName: "t"}}}, "not both"), + Entry("table missing on a record", fvcommon.InsertRequest{Records: []fvcommon.InsertRequestRecord{{Data: data, TableName: "t"}, {Data: data}}}, "Table name is missing"), + Entry("record upsert with request table", fvcommon.InsertRequest{TableName: "t", Records: []fvcommon.InsertRequestRecord{{Data: data, Upsert: &fvcommon.UpsertOptions{UniqueColumns: []string{"a"}}}}}, "Set upsert on the request"), + Entry("request upsert with record tables", fvcommon.InsertRequest{Upsert: &fvcommon.UpsertOptions{UniqueColumns: []string{"a"}}, Records: []fvcommon.InsertRequestRecord{{Data: data, TableName: "t"}}}, "Set upsert on each record"), + Entry("upsert without unique columns", fvcommon.InsertRequest{TableName: "t", Upsert: &fvcommon.UpsertOptions{}, Records: []fvcommon.InsertRequestRecord{{Data: data}}}, "unique columns"), + Entry("invalid upsert update type", fvcommon.InsertRequest{TableName: "t", Upsert: &fvcommon.UpsertOptions{UniqueColumns: []string{"a"}, UpdateType: "MERGE"}, Records: []fvcommon.InsertRequestRecord{{Data: data}}}, "Invalid update type"), + ) +}) + +var _ = Describe("ValidateUpdate", func() { + DescribeTable("requests", + func(req fvcommon.UpdateRequest, message string) { expectError(ValidateUpdate(req), message) }, + Entry("valid", fvcommon.UpdateRequest{TableName: "t", UpdateType: fvcommon.UpdateTypeReplace, Records: []fvcommon.UpdateRequestRecord{{SkyflowId: "id", Data: data}}}, ""), + Entry("missing skyflow ID", fvcommon.UpdateRequest{TableName: "t", Records: []fvcommon.UpdateRequestRecord{{Data: data}}}, "Skyflow ID is missing"), + Entry("table in both places", fvcommon.UpdateRequest{TableName: "t", Records: []fvcommon.UpdateRequestRecord{{SkyflowId: "id", Data: data, TableName: "t"}}}, "not both"), + Entry("table missing", fvcommon.UpdateRequest{Records: []fvcommon.UpdateRequestRecord{{SkyflowId: "id", Data: data}}}, "Table name is missing"), + Entry("invalid update type", fvcommon.UpdateRequest{TableName: "t", UpdateType: "MERGE", Records: []fvcommon.UpdateRequestRecord{{SkyflowId: "id", Data: data}}}, "Invalid update type"), + ) +}) + +var _ = Describe("ValidateGet", func() { + DescribeTable("requests", + func(req fvcommon.GetRequest, message string) { expectError(ValidateGet(req), message) }, + Entry("ids", fvcommon.GetRequest{TableName: "t", SkyflowIds: []string{"id"}}, ""), + Entry("multi-table", fvcommon.GetRequest{Records: []fvcommon.GetRequestRecord{{TableName: "t", UniqueValues: []map[string]interface{}{data}}}}, ""), + Entry("both modes", fvcommon.GetRequest{TableName: "t", Records: []fvcommon.GetRequestRecord{{TableName: "t", SkyflowIds: []string{"id"}}}}, "not both"), + Entry("empty multi-table records", fvcommon.GetRequest{Records: []fvcommon.GetRequestRecord{}}, "Records must not be empty"), + Entry("ids and unique values", fvcommon.GetRequest{TableName: "t", SkyflowIds: []string{"id"}, UniqueValues: []map[string]interface{}{data}}, "exactly one"), + Entry("neither ids nor unique values", fvcommon.GetRequest{TableName: "t"}, "exactly one"), + Entry("empty unique value", fvcommon.GetRequest{TableName: "t", UniqueValues: []map[string]interface{}{{}}}, "empty entries"), + Entry("negative limit", fvcommon.GetRequest{TableName: "t", SkyflowIds: []string{"id"}, Limit: -1}, "must not be negative"), + Entry("bad redaction", fvcommon.GetRequest{TableName: "t", SkyflowIds: []string{"id"}, ColumnRedactions: []fvcommon.ColumnRedaction{{ColumnName: "a"}}}, "Column redactions"), + ) +}) + +var _ = Describe("Other validators", func() { + It("validates Delete", func() { + expectError(ValidateDelete(fvcommon.DeleteRequest{SkyflowIds: []string{"id"}}), "Table name must not be empty") + expectError(ValidateDelete(fvcommon.DeleteRequest{TableName: "t", SkyflowIds: []string{""}}), "empty values") + expectError(ValidateDelete(fvcommon.DeleteRequest{TableName: "t", UniqueValues: []map[string]interface{}{data}}), "") + }) + + It("validates Detokenize", func() { + expectError(ValidateDetokenize(fvcommon.DetokenizeRequest{}), "Tokens must not be empty") + expectError(ValidateDetokenize(fvcommon.DetokenizeRequest{Tokens: []string{" "}}), "empty values") + expectError(ValidateDetokenize(fvcommon.DetokenizeRequest{Tokens: []string{"tok"}, TokenGroupRedactions: []fvcommon.TokenGroupRedaction{{TokenGroupName: "g"}}}), "Token group redactions") + expectError(ValidateDetokenize(fvcommon.DetokenizeRequest{Tokens: []string{"tok"}}), "") + }) + + It("validates GetTokens and Query", func() { + expectError(ValidateGetTokens(fvcommon.GetTokensRequest{}), "Records must not be empty") + expectError(ValidateGetTokens(fvcommon.GetTokensRequest{Records: []fvcommon.GetTokensRequestRecord{{TokenGroupName: "g"}}}), "Value is missing") + expectError(ValidateGetTokens(fvcommon.GetTokensRequest{Records: []fvcommon.GetTokensRequestRecord{{Value: "v"}}}), "Token group name is missing") + expectError(ValidateQuery(fvcommon.QueryRequest{Query: " "}), "Query must not be empty") + expectError(ValidateQuery(fvcommon.QueryRequest{Query: "SELECT 1"}), "") + }) + + It("validates UploadFiles", func() { + column := func(c fvcommon.UploadFilesRequestColumn) fvcommon.UploadFilesRequest { + return fvcommon.UploadFilesRequest{Records: []fvcommon.UploadFilesRequestRecord{{TableName: "t", Columns: []fvcommon.UploadFilesRequestColumn{c}}}} + } + expectError(ValidateUploadFiles(fvcommon.UploadFilesRequest{Records: []fvcommon.UploadFilesRequestRecord{{TableName: "t"}}}), "Columns must not be empty") + expectError(ValidateUploadFiles(column(fvcommon.UploadFilesRequestColumn{FilePath: "a"})), "Column name must not be empty") + expectError(ValidateUploadFiles(column(fvcommon.UploadFilesRequestColumn{Column: "c"})), "exactly one of FilePath") + expectError(ValidateUploadFiles(column(fvcommon.UploadFilesRequestColumn{Column: "c", FilePath: "a", Base64: "b"})), "exactly one of FilePath") + expectError(ValidateUploadFiles(column(fvcommon.UploadFilesRequestColumn{Column: "c", Base64: "b"})), "FileName is required") + expectError(ValidateUploadFiles(column(fvcommon.UploadFilesRequestColumn{Column: "c", Base64: "b", FileName: "a.txt"})), "") + }) + + It("validates DeleteFiles", func() { + record := func(r fvcommon.DeleteFilesRequestRecord) fvcommon.DeleteFilesRequest { + return fvcommon.DeleteFilesRequest{Records: []fvcommon.DeleteFilesRequestRecord{r}} + } + expectError(ValidateDeleteFiles(record(fvcommon.DeleteFilesRequestRecord{TableName: "t", Columns: []string{"c"}})), "exactly one of SkyflowId") + expectError(ValidateDeleteFiles(record(fvcommon.DeleteFilesRequestRecord{TableName: "t", Columns: []string{"c"}, SkyflowId: "id", UniqueValues: []map[string]interface{}{data}})), "exactly one of SkyflowId") + expectError(ValidateDeleteFiles(record(fvcommon.DeleteFilesRequestRecord{TableName: "t", Columns: []string{""}, SkyflowId: "id"})), "Column name must not be empty") + expectError(ValidateDeleteFiles(record(fvcommon.DeleteFilesRequestRecord{TableName: "t", Columns: []string{"c", "d", "c"}, SkyflowId: "id"})), `Column "c" is listed more than once for record at index 0`) + expectError(ValidateDeleteFiles(fvcommon.DeleteFilesRequest{Records: make([]fvcommon.DeleteFilesRequestRecord, 20001)}), "At most 20000") + expectError(ValidateDeleteFiles(record(fvcommon.DeleteFilesRequestRecord{TableName: "t", Columns: []string{"c"}, SkyflowId: "id"})), "") + }) + + It("validates custom headers", func() { + expectError(ValidateCustomHeaders(map[fvcommon.CustomHeaderKey]string{"x-other": "v"}, "Insert"), "is not supported") + expectError(ValidateCustomHeaders(map[fvcommon.CustomHeaderKey]string{fvcommon.RequestIdHeader: " "}, "Insert"), "must have a value") + expectError(ValidateCustomHeaders(map[fvcommon.CustomHeaderKey]string{fvcommon.SkyflowAccountId: "acc"}, "Insert"), "") + }) +}) diff --git a/flowvault/internal/vault/controller/api_test.go b/flowvault/internal/vault/controller/api_test.go new file mode 100644 index 0000000..b9eee4d --- /dev/null +++ b/flowvault/internal/vault/controller/api_test.go @@ -0,0 +1,305 @@ +package controller + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" +) + +// fakeFlowService records each request and answers with the response set for its path. +type fakeFlowService struct { + mu sync.Mutex + server *httptest.Server + requests map[string]map[string]interface{} + headers map[string]http.Header + responses map[string]fakeResponse + puts map[string]string + putCalls map[string]int +} + +type fakeResponse struct { + status int + body string +} + +func newFakeFlowService() *fakeFlowService { + f := &fakeFlowService{ + requests: map[string]map[string]interface{}{}, + headers: map[string]http.Header{}, + responses: map[string]fakeResponse{}, + puts: map[string]string{}, + putCalls: map[string]int{}, + } + f.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + raw, _ := io.ReadAll(r.Body) + f.mu.Lock() + defer f.mu.Unlock() + if r.Method == http.MethodPut { + f.puts[r.URL.Path] = string(raw) + f.putCalls[r.URL.Path]++ + f.headers[r.URL.Path] = r.Header.Clone() + if strings.HasPrefix(r.URL.Path, "/signed/unavailable") { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + return + } + var body map[string]interface{} + _ = json.Unmarshal(raw, &body) + f.requests[r.URL.Path] = body + f.headers[r.URL.Path] = r.Header.Clone() + response, ok := f.responses[r.URL.Path] + if !ok { + response = fakeResponse{status: http.StatusOK, body: `{}`} + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("x-request-id", "req-123") + w.WriteHeader(response.status) + _, _ = io.WriteString(w, response.body) + })) + return f +} + +func (f *fakeFlowService) respond(path string, status int, body string) { + f.responses[path] = fakeResponse{status: status, body: body} +} + +var _ = Describe("VaultController API calls", func() { + var ( + fake *fakeFlowService + v *VaultController + ctx = context.Background() + ) + BeforeEach(func() { + fake = newFakeFlowService() + DeferCleanup(fake.server.Close) + v = NewVaultController(fvcommon.VaultConfig{VaultId: "vault1", BaseVaultUrl: fake.server.URL}, ClientSettings{Credentials: &fvcommon.Credentials{ApiKey: testApiKey}}) + }) + + It("inserts with request-level table and upsert, and parses tokens and per-record errors", func() { + fake.respond("/v2/records/insert", http.StatusOK, `{"records":[ + {"skyflowID":"id1","tableName":"users","httpCode":200,"tokens":{"email":[{"token":"tok1","tokenGroupName":"det"}]},"hashedData":{"email":[{"data":"h","hashName":"sha"}]}}, + {"httpCode":400,"error":"bad record"}]}`) + resp, err := v.Insert(ctx, fvcommon.InsertRequest{ + TableName: "users", + Upsert: &fvcommon.UpsertOptions{UniqueColumns: []string{"email"}, UpdateType: fvcommon.UpdateTypeReplace}, + Records: []fvcommon.InsertRequestRecord{{Data: map[string]interface{}{"email": "a@b.c"}}, {Data: map[string]interface{}{"email": "x"}}}, + }, fvcommon.InsertOptions{}) + Expect(err).To(BeNil()) + + sent := fake.requests["/v2/records/insert"] + Expect(sent["vaultID"]).To(Equal("vault1")) + Expect(sent["tableName"]).To(Equal("users")) + Expect(sent["upsert"]).To(Equal(map[string]interface{}{"uniqueColumns": []interface{}{"email"}, "updateType": "REPLACE"})) + Expect(sent["records"]).To(HaveLen(2)) + Expect(fake.headers["/v2/records/insert"].Get("Authorization")).To(Equal("Bearer " + testApiKey)) + Expect(fake.headers["/v2/records/insert"].Get("sky-metadata")).ToNot(BeEmpty()) + + Expect(resp.Records).To(HaveLen(2)) + Expect(resp.Records[0].SkyflowId).To(Equal("id1")) + Expect(resp.Records[0].Tokens["email"]).To(Equal([]fvcommon.Token{{Token: "tok1", TokenGroupName: "det"}})) + Expect(resp.Records[0].HashedData["email"]).To(Equal([]fvcommon.HashedValue{{Data: "h", HashName: "sha"}})) + Expect(resp.Records[0].RequestId).To(BeEmpty()) + Expect(resp.Records[1].Error).To(Equal("bad record")) + Expect(resp.Records[1].HttpCode).To(Equal(400)) + Expect(resp.Records[1].RequestId).To(Equal("req-123")) + }) + + It("sends record-level tables and upserts", func() { + _, err := v.Insert(ctx, fvcommon.InsertRequest{Records: []fvcommon.InsertRequestRecord{ + {Data: map[string]interface{}{"a": 1}, TableName: "t1", Upsert: &fvcommon.UpsertOptions{UniqueColumns: []string{"a"}}}, + }}, fvcommon.InsertOptions{}) + Expect(err).To(BeNil()) + record := fake.requests["/v2/records/insert"]["records"].([]interface{})[0].(map[string]interface{}) + Expect(record["tableName"]).To(Equal("t1")) + Expect(record["upsert"]).To(Equal(map[string]interface{}{"uniqueColumns": []interface{}{"a"}})) + }) + + It("returns per-record results from a non-2xx response that carries records", func() { + fake.respond("/v2/records/update", http.StatusBadRequest, `{"records":[{"skyflowID":"id1","httpCode":400,"error":"column missing"}]}`) + resp, err := v.Update(ctx, fvcommon.UpdateRequest{ + TableName: "users", UpdateType: fvcommon.UpdateTypeUpdate, + Records: []fvcommon.UpdateRequestRecord{{SkyflowId: "id1", Data: map[string]interface{}{"a": 1}}}, + }, fvcommon.UpdateOptions{}) + Expect(err).To(BeNil()) + Expect(fake.requests["/v2/records/update"]["updateType"]).To(Equal("UPDATE")) + Expect(resp.Records[0].Error).To(Equal("column missing")) + Expect(resp.Records[0].RequestId).To(Equal("req-123")) + }) + + It("maps a whole-call API error to a SkyflowError", func() { + fake.respond("/v2/query", http.StatusBadRequest, `{"error":{"grpc_code":3,"http_code":400,"http_status":"Bad Request","message":"Invalid query"}}`) + _, err := v.Query(ctx, fvcommon.QueryRequest{Query: "SELECT 1"}, fvcommon.QueryOptions{}) + Expect(err).ToNot(BeNil()) + Expect(err.GetMessage()).To(ContainSubstring("Invalid query")) + Expect(err.GetHttpCode()).To(Equal("400")) + Expect(err.GetRequestId()).To(Equal("req-123")) + }) + + It("maps an error with an unexpected status code", func() { + fake.respond("/v2/records/get", http.StatusBadGateway, `{"error":{"http_code":502,"message":"upstream down"}}`) + _, err := v.Get(ctx, fvcommon.GetRequest{TableName: "t", SkyflowIds: []string{"id"}}, fvcommon.GetOptions{}) + Expect(err).ToNot(BeNil()) + Expect(err.GetMessage()).To(ContainSubstring("upstream down")) + }) + + It("gets from one table with limit, offset and redactions", func() { + fake.respond("/v2/records/get", http.StatusOK, `{"records":[{"skyflowID":"id1","data":{"name":"j"},"httpCode":200}]}`) + resp, err := v.Get(ctx, fvcommon.GetRequest{ + TableName: "t", SkyflowIds: []string{"id1"}, Columns: []string{"name"}, Limit: 10, Offset: 5, + ColumnRedactions: []fvcommon.ColumnRedaction{{ColumnName: "name", Redaction: "MASKED"}}, + }, fvcommon.GetOptions{}) + Expect(err).To(BeNil()) + sent := fake.requests["/v2/records/get"] + Expect(sent["tableName"]).To(Equal("t")) + Expect(sent["limit"]).To(BeEquivalentTo(10)) + Expect(sent["offset"]).To(BeEquivalentTo(5)) + Expect(sent["columnRedactions"]).To(Equal([]interface{}{map[string]interface{}{"columnName": "name", "redaction": "MASKED"}})) + Expect(resp.Records[0].Data).To(Equal(map[string]interface{}{"name": "j"})) + }) + + It("gets from several tables", func() { + _, err := v.Get(ctx, fvcommon.GetRequest{Records: []fvcommon.GetRequestRecord{ + {TableName: "t1", SkyflowIds: []string{"a"}}, + {TableName: "t2", UniqueValues: []map[string]interface{}{{"email": "x"}}}, + }}, fvcommon.GetOptions{}) + Expect(err).To(BeNil()) + sent := fake.requests["/v2/records/get"] + Expect(sent).ToNot(HaveKey("tableName")) + records := sent["records"].([]interface{}) + Expect(records[1].(map[string]interface{})["uniqueValues"]).To(Equal([]interface{}{map[string]interface{}{"data": map[string]interface{}{"email": "x"}}})) + }) + + It("deletes records", func() { + fake.respond("/v2/records/delete", http.StatusOK, `{"records":[{"skyflowID":"id1","httpCode":200}]}`) + resp, err := v.Delete(ctx, fvcommon.DeleteRequest{TableName: "t", SkyflowIds: []string{"id1"}}, fvcommon.DeleteOptions{}) + Expect(err).To(BeNil()) + Expect(fake.requests["/v2/records/delete"]["skyflowIDs"]).To(Equal([]interface{}{"id1"})) + Expect(resp.Records).To(Equal([]fvcommon.DeleteResponseRecord{{SkyflowId: "id1", HttpCode: 200}})) + }) + + It("detokenizes with redactions and metadata", func() { + fake.respond("/v2/tokens/detokenize", http.StatusOK, `{"response":[ + {"token":"tok1","value":"john","tokenGroupName":"g","httpCode":200,"metadata":{"skyflowID":"id1","tableName":"users"}}, + {"token":"tok2","httpCode":404,"error":"not found"}]}`) + resp, err := v.Detokenize(ctx, fvcommon.DetokenizeRequest{ + Tokens: []string{"tok1", "tok2"}, + TokenGroupRedactions: []fvcommon.TokenGroupRedaction{{TokenGroupName: "g", Redaction: "PLAIN_TEXT"}}, + }, fvcommon.DetokenizeOptions{}) + Expect(err).To(BeNil()) + Expect(fake.requests["/v2/tokens/detokenize"]["tokenGroupRedactions"]).To(Equal([]interface{}{map[string]interface{}{"tokenGroupName": "g", "redaction": "PLAIN_TEXT"}})) + Expect(resp.Records[0].Value).To(Equal("john")) + Expect(resp.Records[0].Metadata).To(Equal(&fvcommon.DetokenizeRecordMetadata{SkyflowId: "id1", TableName: "users"})) + Expect(resp.Records[1].HttpCode).To(Equal(404)) + Expect(resp.Records[1].RequestId).To(Equal("req-123")) + }) + + It("gets tokens for values", func() { + fake.respond("/v2/records/getTokens", http.StatusOK, `{"records":[{"token":"tok1","value":"a@b.c","tokenGroupName":"det","httpCode":200}]}`) + resp, err := v.GetTokens(ctx, fvcommon.GetTokensRequest{Records: []fvcommon.GetTokensRequestRecord{{Value: "a@b.c", TokenGroupName: "det"}}}, fvcommon.GetTokensOptions{}) + Expect(err).To(BeNil()) + Expect(fake.requests["/v2/records/getTokens"]["records"]).To(Equal([]interface{}{map[string]interface{}{"value": "a@b.c", "tokenGroupName": "det"}})) + Expect(resp.Records).To(Equal([]fvcommon.GetTokensResponseRecord{{Value: "a@b.c", TokenGroupName: "det", Token: "tok1", HttpCode: 200}})) + }) + + It("runs a query and returns rows, columns and the request ID", func() { + fake.respond("/v2/query", http.StatusOK, `{"records":[{"data":{"name":"j"}}],"metadata":{"columns":["name"]}}`) + resp, err := v.Query(ctx, fvcommon.QueryRequest{Query: "SELECT name FROM users"}, fvcommon.QueryOptions{}) + Expect(err).To(BeNil()) + Expect(fake.requests["/v2/query"]["query"]).To(Equal("SELECT name FROM users")) + Expect(resp.Fields).To(Equal([]map[string]interface{}{{"name": "j"}})) + Expect(resp.Metadata.Columns).To(Equal([]string{"name"})) + Expect(resp.RequestId).To(Equal("req-123")) + }) + + It("uploads files to their signed URLs and reports each column", func() { + fake.respond("/v2/files/upload", http.StatusOK, `{"records":[ + {"skyflowID":"new1","tableName":"docs","httpCode":200,"data":{"resume":"`+fake.server.URL+`/signed/resume"}}, + {"skyflowID":"id2","tableName":"docs","httpCode":404,"error":"record not found"}]}`) + resp, err := v.UploadFiles(ctx, fvcommon.UploadFilesRequest{Records: []fvcommon.UploadFilesRequestRecord{ + {TableName: "docs", Columns: []fvcommon.UploadFilesRequestColumn{ + {Column: "resume", Base64: "cGRm", FileName: "resume.pdf"}, + {Column: "photo", Base64: "aW1n", FileName: "photo.png"}, + }}, + {TableName: "docs", SkyflowId: "id2", Columns: []fvcommon.UploadFilesRequestColumn{{Column: "resume", Base64: "cGRm", FileName: "b.pdf"}}}, + }}, fvcommon.UploadFilesOptions{}) + Expect(err).To(BeNil()) + + sent := fake.requests["/v2/files/upload"]["records"].([]interface{}) + Expect(sent[0].(map[string]interface{})).ToNot(HaveKey("skyflowID")) + Expect(sent[0].(map[string]interface{})["columns"]).To(Equal([]interface{}{ + map[string]interface{}{"column": "resume", "fileName": "resume.pdf"}, + map[string]interface{}{"column": "photo", "fileName": "photo.png"}, + })) + Expect(sent[1].(map[string]interface{})["skyflowID"]).To(Equal("id2")) + + Expect(fake.puts["/signed/resume"]).To(Equal("pdf")) + Expect(fake.headers["/signed/resume"].Get("Authorization")).To(BeEmpty()) + Expect(resp.Records[0].SkyflowId).To(Equal("new1")) + Expect(resp.Records[0].Columns).To(Equal([]fvcommon.UploadFilesColumnResult{ + {Column: "resume", FileName: "resume.pdf", UploadStatus: fvcommon.UploadStatusUploaded}, + {Column: "photo", FileName: "photo.png", UploadStatus: fvcommon.UploadStatusSkipped}, + })) + Expect(resp.Records[1].Error).To(Equal("record not found")) + Expect(resp.Records[1].RequestId).To(Equal("req-123")) + Expect(resp.Records[1].Columns[0].UploadStatus).To(Equal(fvcommon.UploadStatusSkipped)) + Expect(resp.Records[1].Columns[0].Error).To(Equal("record not found")) + }) + + It("doesn't retry a failed file upload, even with retries on", func() { + retries := 3 + v.SetConfig(fvcommon.VaultConfig{VaultId: "vault1", BaseVaultUrl: fake.server.URL, MaxRetries: &retries, InitialRetryDelay: time.Millisecond}, false) + fake.respond("/v2/files/upload", http.StatusOK, `{"records":[ + {"skyflowID":"id1","tableName":"docs","httpCode":200,"data":{"resume":"`+fake.server.URL+`/signed/unavailable"}}]}`) + resp, err := v.UploadFiles(ctx, fvcommon.UploadFilesRequest{Records: []fvcommon.UploadFilesRequestRecord{ + {TableName: "docs", Columns: []fvcommon.UploadFilesRequestColumn{{Column: "resume", Base64: "cGRm", FileName: "resume.pdf"}}}, + }}, fvcommon.UploadFilesOptions{ + Interceptor: func(rc *fvcommon.RequestContext) { rc.AddHeader(fvcommon.RequestIdHeader, "mine") }, + }) + Expect(err).To(BeNil()) + Expect(fake.putCalls["/signed/unavailable"]).To(Equal(1)) + Expect(fake.headers["/signed/unavailable"].Get("x-request-id")).To(BeEmpty()) + Expect(fake.headers["/signed/unavailable"].Get("sky-metadata")).To(BeEmpty()) + Expect(resp.Records[0].Columns[0]).To(Equal(fvcommon.UploadFilesColumnResult{ + Column: "resume", FileName: "resume.pdf", UploadStatus: fvcommon.UploadStatusFailed, Error: "PUT failed: 503", + })) + }) + + It("deletes files", func() { + fake.respond("/v2/files/delete", http.StatusOK, `{"records":[{"skyflowID":"id1","tableName":"docs","httpCode":200,"data":{"resume":{"status":"DELETED"}}}]}`) + resp, err := v.DeleteFiles(ctx, fvcommon.DeleteFilesRequest{Records: []fvcommon.DeleteFilesRequestRecord{ + {TableName: "docs", Columns: []string{"resume"}, UniqueValues: []map[string]interface{}{{"email": "x"}}}, + }}, fvcommon.DeleteFilesOptions{}) + Expect(err).To(BeNil()) + record := fake.requests["/v2/files/delete"]["records"].([]interface{})[0].(map[string]interface{}) + Expect(record).ToNot(HaveKey("skyflowID")) + Expect(record["columns"]).To(Equal([]interface{}{"resume"})) + Expect(resp.Records[0].Columns).To(Equal([]fvcommon.DeleteFilesColumnResult{{Column: "resume", Status: "DELETED"}})) + }) + + It("sends interceptor headers", func() { + _, err := v.Query(ctx, fvcommon.QueryRequest{Query: "SELECT 1"}, fvcommon.QueryOptions{ + Interceptor: func(rc *fvcommon.RequestContext) { rc.AddHeader(fvcommon.RequestIdHeader, "mine") }, + }) + Expect(err).To(BeNil()) + Expect(fake.headers["/v2/query"].Get("x-request-id")).To(Equal("mine")) + }) + + It("reports a network failure as a SkyflowError", func() { + fake.server.Close() + _, err := v.Query(ctx, fvcommon.QueryRequest{Query: "SELECT 1"}, fvcommon.QueryOptions{}) + Expect(err).ToNot(BeNil()) + Expect(err.GetCode()).To(ContainSubstring("500")) + }) +}) diff --git a/flowvault/internal/vault/controller/base_controller.go b/flowvault/internal/vault/controller/base_controller.go index 9ea2088..74f4274 100644 --- a/flowvault/internal/vault/controller/base_controller.go +++ b/flowvault/internal/vault/controller/base_controller.go @@ -3,28 +3,60 @@ package controller import ( "net/http" "os" + "sync" fvconsts "github.com/skyflowapi/skyflow-go/flowvault/internal/constants" + apiclient "github.com/skyflowapi/skyflow-go/flowvault/internal/generated/client" + "github.com/skyflowapi/skyflow-go/flowvault/internal/generated/option" fvhelpers "github.com/skyflowapi/skyflow-go/flowvault/internal/helpers" + fvvalidation "github.com/skyflowapi/skyflow-go/flowvault/internal/validation" fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" fverrors "github.com/skyflowapi/skyflow-go/flowvault/utils/error" fvlogger "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" + logs "github.com/skyflowapi/skyflow-go/flowvault/utils/messages" ) // BaseController holds shared auth state and HTTP client logic for flowvault vault operations. // flowvault's VaultController embeds this — equivalent to Java's AbstractBaseVaultController. // -// Can only use v2's EXPORTED packages (serviceaccount, utils/*). -// v2's internal/* are module-private and cannot be imported from flowvault. +// It is safe for concurrent use: calls read a snapshot of the settings, and the client can +// change them (SetConfig, SetClientSettings) while calls are running. type BaseController struct { - Config *fvcommon.VaultConfig - CommonCreds *fvcommon.Credentials - Token string - ApiKey string - HTTPClient *http.Client - BaseURL string - AuthHeader string - CustomHeaders map[fvcommon.CustomHeaderKey]string + settingsMu sync.RWMutex + config fvcommon.VaultConfig + client ClientSettings + // authVersion changes whenever the credentials may have changed, so a token generated + // from older credentials is never reused. + authVersion uint64 + + tokenMu sync.Mutex + token string + tokenVersion uint64 +} + +// ClientSettings are the client-level settings every vault falls back to. +type ClientSettings struct { + Credentials *fvcommon.Credentials + HttpConfig fvcommon.HttpConfig +} + +// settings is a consistent copy of the controller's settings for one call. +type settings struct { + config fvcommon.VaultConfig + client ClientSettings + authVersion uint64 +} + +// RequestSetup is everything one API call needs: the vault ID, where to send it, how to +// authenticate, which headers to add, the HTTP client (timeout and retries applied) and +// the generated API client built from all of these. +type RequestSetup struct { + VaultID string + BaseURL string + AuthToken string + Headers http.Header + HTTPClient *http.Client + API *apiclient.Client } // GenerateToken resolves a bearer token from credentials. @@ -56,93 +88,166 @@ func GenerateToken(creds fvcommon.Credentials) (*string, *fverrors.SkyflowError) } token = resp.AccessToken default: - return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, "no valid credentials provided") + return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, fverrors.EMPTY_CREDENTIALS) } return &token, nil } -// RefreshToken checks token expiry and regenerates if needed. -// fvhelpers.Auth().IsExpired is exported — reused directly. -func (b *BaseController) RefreshToken() *fverrors.SkyflowError { - creds, err := b.resolveCredentials() - if err != nil { - return err - } - if b.Token == "" || fvhelpers.Auth().IsExpired(b.Token) { - fvlogger.Info("generating new bearer token") - token, err := GenerateToken(*creds) - if err != nil { - return err - } - b.Token = *token - } else { - fvlogger.Info("reusing existing bearer token") - } - return nil +// Config returns a copy of the vault config. +func (b *BaseController) Config() fvcommon.VaultConfig { + b.settingsMu.RLock() + defer b.settingsMu.RUnlock() + return b.config } -// SetupClient prepares the authenticated HTTP client for a vault request. -func (b *BaseController) SetupClient(customHeaders map[string]string) *fverrors.SkyflowError { - // 1. resolve auth token - if b.Config.Credentials.ApiKey != "" { - b.ApiKey = b.Config.Credentials.ApiKey - } else if b.Config.Credentials.Token != "" { - if fvhelpers.Auth().IsExpired(b.Config.Credentials.Token) { - return fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, "bearer token is expired") - } - b.Token = b.Config.Credentials.Token - } else { - if err := b.RefreshToken(); err != nil { - return err - } +// SetConfig replaces the vault config. Set resetAuth when the credentials changed. +func (b *BaseController) SetConfig(config fvcommon.VaultConfig, resetAuth bool) { + b.settingsMu.Lock() + defer b.settingsMu.Unlock() + b.config = config + if resetAuth { + b.authVersion++ } +} - b.AuthHeader = b.Token - if b.ApiKey != "" { - b.AuthHeader = b.ApiKey - } +// ClientSettings returns the client-level settings this vault uses. +func (b *BaseController) ClientSettings() ClientSettings { + b.settingsMu.RLock() + defer b.settingsMu.RUnlock() + return b.client +} - // 2. build base URL - if b.Config.BaseVaultUrl != "" { - b.BaseURL = b.Config.BaseVaultUrl - } else { - b.BaseURL = fvhelpers.GetURLWithEnv(b.Config.Env, b.Config.ClusterId) +// SetClientSettings replaces the client-level settings. A change of client credentials +// drops the cached token. +func (b *BaseController) SetClientSettings(client ClientSettings) { + b.settingsMu.Lock() + defer b.settingsMu.Unlock() + if client.Credentials != b.client.Credentials { + b.authVersion++ } + b.client = client +} + +// ResetAuth drops the cached bearer token so the next call re-authenticates. +func (b *BaseController) ResetAuth() { + b.settingsMu.Lock() + defer b.settingsMu.Unlock() + b.authVersion++ +} - // 3. build default headers applied to every request - b.HTTPClient = &http.Client{} - return nil +func (b *BaseController) snapshot() settings { + b.settingsMu.RLock() + defer b.settingsMu.RUnlock() + return settings{config: b.config, client: b.client, authVersion: b.authVersion} } -// NewRequest builds an authenticated *http.Request for a vault API call. -func (b *BaseController) NewRequest(method, path string, body interface{}) (*http.Request, error) { - req, err := http.NewRequest(method, b.BaseURL+path, nil) +// SetupRequest resolves auth, headers and HTTP settings for one call. The interceptor, +// when set, can add per-call headers. +func (b *BaseController) SetupRequest(operation string, interceptor fvcommon.Interceptor) (*RequestSetup, *fverrors.SkyflowError) { + current := b.snapshot() + headers, err := buildHeaders(operation, interceptor) if err != nil { return nil, err } - req.Header.Set("Authorization", "Bearer "+b.AuthHeader) - req.Header.Set("Content-Type", "application/json") - req.Header.Set(fvconsts.SDK_METRICS_HEADER_KEY, fvhelpers.CreateSDKMetadata()) - for k, val := range b.CustomHeaders { - req.Header.Set(string(k), val) + authToken, err := b.authToken(current) + if err != nil { + return nil, err + } + httpConfig := fvhelpers.ResolveHttpConfig(current.config.HttpConfig(), current.client.HttpConfig) + setup := &RequestSetup{ + VaultID: current.config.VaultId, + BaseURL: baseURL(current.config), + AuthToken: authToken, + Headers: headers, + HTTPClient: fvhelpers.NewHTTPClient(httpConfig), + } + // Retries happen in setup.HTTPClient, so the generated client makes a single attempt. + setup.API = apiclient.NewClient( + option.WithBaseURL(setup.BaseURL), + option.WithHTTPClient(setup.HTTPClient), + option.WithHTTPHeader(setup.Headers), + option.WithToken(setup.AuthToken), + option.WithMaxAttempts(1), + ) + return setup, nil +} + +// buildHeaders merges the interceptor's per-call headers and the SDK metadata header. +func buildHeaders(operation string, interceptor fvcommon.Interceptor) (http.Header, *fverrors.SkyflowError) { + headers := http.Header{} + if interceptor != nil { + rc := &fvcommon.RequestContext{Operation: operation} + interceptor(rc) + if added := rc.Headers(); len(added) > 0 { + if err := fvvalidation.ValidateCustomHeaders(added, operation); err != nil { + return nil, err + } + for key, value := range added { + headers.Set(string(key), value) + } + } } - return req, nil + headers.Set(fvconsts.SDK_METRICS_HEADER_KEY, fvhelpers.CreateSDKMetadata()) + return headers, nil } -func (b *BaseController) resolveCredentials() (*fvcommon.Credentials, *fverrors.SkyflowError) { - if b.Config != nil && !isEmpty(b.Config.Credentials) { - return &b.Config.Credentials, nil +// authToken returns the API key or bearer token to send, generating and caching a +// bearer token from service-account credentials when needed. +func (b *BaseController) authToken(current settings) (string, *fverrors.SkyflowError) { + creds, err := resolveCredentials(current) + if err != nil { + return "", err + } + if creds.ApiKey != "" { + return creds.ApiKey, nil + } + if creds.Token != "" { + if fvhelpers.Auth().IsExpired(creds.Token) { + fvlogger.Error(logs.BEARER_TOKEN_EXPIRED) + return "", fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, fverrors.BEARER_TOKEN_EXPIRED) + } + return creds.Token, nil + } + b.tokenMu.Lock() + defer b.tokenMu.Unlock() + if b.token != "" && b.tokenVersion == current.authVersion && !fvhelpers.Auth().IsExpired(b.token) { + fvlogger.Info(logs.REUSE_BEARER_TOKEN) + return b.token, nil + } + fvlogger.Info(logs.GENERATE_BEARER_TOKEN_TRIGGERED) + token, err := GenerateToken(*creds) + if err != nil { + return "", err + } + // A call that started before a credentials change must not replace a newer token. + if current.authVersion >= b.tokenVersion { + b.token, b.tokenVersion = *token, current.authVersion } - if b.CommonCreds != nil && !isEmpty(*b.CommonCreds) { - return b.CommonCreds, nil + return *token, nil +} + +func baseURL(config fvcommon.VaultConfig) string { + if config.BaseVaultUrl != "" { + return config.BaseVaultUrl } - if envCreds := os.Getenv("SKYFLOW_CREDENTIALS"); envCreds != "" { - c := fvcommon.Credentials{CredentialsString: envCreds} - return &c, nil + if config.BaseVaultURL != "" { + return config.BaseVaultURL } - return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, "no credentials found") + return fvhelpers.GetURLWithEnv(config.Env, config.ClusterId) } -func isEmpty(c fvcommon.Credentials) bool { - return c.Path == "" && c.CredentialsString == "" && c.Token == "" && c.ApiKey == "" +// resolveCredentials picks vault credentials, then client credentials, then SKYFLOW_CREDENTIALS. +func resolveCredentials(current settings) (*fvcommon.Credentials, *fverrors.SkyflowError) { + if !fvcommon.IsCredentialsEmpty(current.config.Credentials) { + creds := current.config.Credentials + return &creds, nil + } + if current.client.Credentials != nil && !fvcommon.IsCredentialsEmpty(*current.client.Credentials) { + return current.client.Credentials, nil + } + if envCreds := os.Getenv(fvconsts.SKYFLOW_CREDENTIALS_ENV); envCreds != "" { + return &fvcommon.Credentials{CredentialsString: envCreds}, nil + } + fvlogger.Error(logs.EMPTY_CREDENTIALS) + return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, fverrors.EMPTY_CREDENTIALS) } diff --git a/flowvault/internal/vault/controller/controller_test.go b/flowvault/internal/vault/controller/controller_test.go new file mode 100644 index 0000000..02e69fb --- /dev/null +++ b/flowvault/internal/vault/controller/controller_test.go @@ -0,0 +1,87 @@ +package controller + +import ( + "os" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/skyflowapi/skyflow-go/flowvault/internal/testutil" + fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" +) + +func TestController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Flowvault Controller Suite") +} + +var ( + testApiKey = testutil.Resource("client_api_key.txt") + vaultApiKey = testutil.Resource("vault_api_key.txt") +) + +var _ = Describe("SetupRequest", func() { + It("uses the client API key and the interceptor's headers", func() { + config := fvcommon.VaultConfig{VaultId: "v", ClusterId: "cluster", Env: fvcommon.DEV, Timeout: 5 * time.Second} + v := NewVaultController(config, ClientSettings{Credentials: &fvcommon.Credentials{ApiKey: testApiKey}}) + + var operation string + setup, err := v.SetupRequest("QUERY", func(rc *fvcommon.RequestContext) { + operation = rc.Operation + rc.AddHeader(fvcommon.RequestIdHeader, "custom-id") + rc.AddHeader(fvcommon.SkyflowAccountId, "acc") + }) + Expect(err).To(BeNil()) + Expect(operation).To(Equal("QUERY")) + Expect(setup.AuthToken).To(Equal(testApiKey)) + Expect(setup.BaseURL).To(Equal("https://cluster.skyvault.skyflowapis.dev")) + Expect(setup.Headers.Get("x-skyflow-account-id")).To(Equal("acc")) + Expect(setup.Headers.Get("x-request-id")).To(Equal("custom-id")) + Expect(setup.Headers.Get("sky-metadata")).To(ContainSubstring(`"sdk_name_version":"skyflow-flowvault-go@`)) + Expect(setup.HTTPClient.Timeout).To(Equal(5 * time.Second)) + }) + + It("prefers vault credentials and BaseVaultUrl", func() { + config := fvcommon.VaultConfig{ + VaultId: "v", BaseVaultUrl: "https://example.com", + Credentials: fvcommon.Credentials{ApiKey: vaultApiKey}, + } + setup, err := NewVaultController(config, ClientSettings{Credentials: &fvcommon.Credentials{ApiKey: testApiKey}}).SetupRequest("GET", nil) + Expect(err).To(BeNil()) + Expect(setup.AuthToken).To(Equal(vaultApiKey)) + Expect(setup.BaseURL).To(Equal("https://example.com")) + }) + + It("uses PROD when Env isn't set", func() { + setup, err := NewVaultController(fvcommon.VaultConfig{VaultId: "v", ClusterId: "cluster"}, ClientSettings{Credentials: &fvcommon.Credentials{ApiKey: testApiKey}}).SetupRequest("GET", nil) + Expect(err).To(BeNil()) + Expect(setup.BaseURL).To(Equal("https://cluster.skyvault.skyflowapis.com")) + }) + + It("drops the cached token when credentials change", func() { + v := NewVaultController(fvcommon.VaultConfig{VaultId: "v"}, ClientSettings{}) + version := v.snapshot().authVersion + v.SetClientSettings(ClientSettings{Credentials: &fvcommon.Credentials{ApiKey: testApiKey}}) + Expect(v.snapshot().authVersion).To(Equal(version + 1)) + v.SetClientSettings(ClientSettings{Credentials: v.ClientSettings().Credentials, HttpConfig: fvcommon.HttpConfig{Timeout: time.Second}}) + Expect(v.snapshot().authVersion).To(Equal(version + 1)) + v.SetConfig(v.Config(), true) + Expect(v.snapshot().authVersion).To(Equal(version + 2)) + v.ResetAuth() + Expect(v.snapshot().authVersion).To(Equal(version + 3)) + }) + + It("fails when no credentials are available", func() { + Expect(os.Unsetenv("SKYFLOW_CREDENTIALS")).To(Succeed()) + _, err := NewVaultController(fvcommon.VaultConfig{VaultId: "v", ClusterId: "c"}, ClientSettings{}).SetupRequest("GET", nil) + Expect(err.GetMessage()).To(ContainSubstring("Invalid credentials")) + }) + + It("rejects an expired bearer token", func() { + config := fvcommon.VaultConfig{VaultId: "v", ClusterId: "c", Credentials: fvcommon.Credentials{Token: "not-a-jwt"}} + _, err := NewVaultController(config, ClientSettings{}).SetupRequest("GET", nil) + Expect(err.GetMessage()).To(ContainSubstring("expired")) + }) +}) diff --git a/flowvault/internal/vault/controller/convert.go b/flowvault/internal/vault/controller/convert.go new file mode 100644 index 0000000..6704f64 --- /dev/null +++ b/flowvault/internal/vault/controller/convert.go @@ -0,0 +1,326 @@ +package controller + +import ( + "encoding/json" + "errors" + "net/http" + "strings" + + api "github.com/skyflowapi/skyflow-go/flowvault/internal/generated" + "github.com/skyflowapi/skyflow-go/flowvault/internal/generated/core" + fvhelpers "github.com/skyflowapi/skyflow-go/flowvault/internal/helpers" + fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + fverrors "github.com/skyflowapi/skyflow-go/flowvault/utils/error" +) + +// result returns the response body and headers. A non-2xx response whose body still +// carries per-record results (hasRecords) is returned as a body, not an error, so callers +// see which records failed. +func result[T any](resp *core.Response[*T], err error, hasRecords func(*T) bool) (*T, http.Header, *fverrors.SkyflowError) { + if err == nil { + body, header := new(T), http.Header{} + if resp != nil { + if resp.Body != nil { + body = resp.Body + } + if resp.Header != nil { + header = resp.Header + } + } + return body, header, nil + } + var apiErr *core.APIError + if !errors.As(err, &apiErr) { + return nil, nil, fverrors.NewSkyflowError(fverrors.SERVER, err.Error()) + } + if hasRecords != nil && apiErr.Unwrap() != nil { + body := new(T) + if json.Unmarshal([]byte(apiErr.Unwrap().Error()), body) == nil && hasRecords(body) { + header := apiErr.Header + if header == nil { + header = http.Header{} + } + return body, header, nil + } + } + header := apiErr.Header + if header == nil { + header = http.Header{} + } + return nil, nil, fverrors.SkyflowErrorApi(apiErr, header) +} + +func deref[T any](value *T) T { + var zero T + if value == nil { + return zero + } + return *value +} + +func optionalString(value string) *string { + if strings.TrimSpace(value) == "" { + return nil + } + return &value +} + +func optionalInt(value int) *int { + if value == 0 { + return nil + } + return &value +} + +func toWireUniqueValues(values []map[string]interface{}) []*api.UniqueValue { + if values == nil { + return nil + } + wire := make([]*api.UniqueValue, 0, len(values)) + for _, value := range values { + wire = append(wire, &api.UniqueValue{Data: value}) + } + return wire +} + +func toWireColumnRedactions(redactions []fvcommon.ColumnRedaction) []*api.ColumnRedactions { + if redactions == nil { + return nil + } + wire := make([]*api.ColumnRedactions, 0, len(redactions)) + for _, redaction := range redactions { + wire = append(wire, &api.ColumnRedactions{ColumnName: redaction.ColumnName, Redaction: redaction.Redaction}) + } + return wire +} + +func toWireUpsert(upsert *fvcommon.UpsertOptions) *api.Upsert { + if upsert == nil { + return nil + } + wire := &api.Upsert{UniqueColumns: upsert.UniqueColumns} + if upsert.UpdateType != "" { + wire.UpdateType = api.UpsertUpdateType(upsert.UpdateType).Ptr() + } + return wire +} + +func toWireInsert(vaultID string, request fvcommon.InsertRequest) *api.InsertRequest { + wire := &api.InsertRequest{ + VaultID: vaultID, + TableName: request.TableName, + Upsert: toWireUpsert(request.Upsert), + Records: make([]*api.InsertRecordData, 0, len(request.Records)), + } + for _, record := range request.Records { + wire.Records = append(wire.Records, &api.InsertRecordData{ + Data: record.Data, + Tokens: record.Tokens, + TableName: optionalString(record.TableName), + Upsert: toWireUpsert(record.Upsert), + }) + } + return wire +} + +func toWireUpdate(vaultID string, request fvcommon.UpdateRequest) *api.UpdateRequest { + wire := &api.UpdateRequest{ + VaultID: vaultID, + TableName: request.TableName, + Records: make([]*api.UpdateRecordData, 0, len(request.Records)), + } + if request.UpdateType != "" { + wire.UpdateType = api.UpdateRequestUpdateType(request.UpdateType).Ptr() + } + for _, record := range request.Records { + wire.Records = append(wire.Records, &api.UpdateRecordData{ + SkyflowID: record.SkyflowId, + Data: record.Data, + Tokens: record.Tokens, + TableName: optionalString(record.TableName), + }) + } + return wire +} + +func toWireGet(vaultID string, request fvcommon.GetRequest) *api.GetRequest { + wire := &api.GetRequest{VaultID: vaultID} + if request.Records != nil { + wire.Records = make([]*api.GetRequestData, 0, len(request.Records)) + for _, record := range request.Records { + wire.Records = append(wire.Records, &api.GetRequestData{ + TableName: record.TableName, + SkyflowIDs: record.SkyflowIds, + Columns: record.Columns, + ColumnRedactions: toWireColumnRedactions(record.ColumnRedactions), + UniqueValues: toWireUniqueValues(record.UniqueValues), + }) + } + return wire + } + wire.TableName = optionalString(request.TableName) + wire.SkyflowIDs = request.SkyflowIds + wire.UniqueValues = toWireUniqueValues(request.UniqueValues) + wire.Columns = request.Columns + wire.ColumnRedactions = toWireColumnRedactions(request.ColumnRedactions) + wire.Limit = optionalInt(request.Limit) + wire.Offset = optionalInt(request.Offset) + return wire +} + +func toWireDelete(vaultID string, request fvcommon.DeleteRequest) *api.DeleteRequest { + return &api.DeleteRequest{ + VaultID: vaultID, + TableName: optionalString(request.TableName), + SkyflowIDs: request.SkyflowIds, + UniqueValues: toWireUniqueValues(request.UniqueValues), + } +} + +func toWireDetokenize(vaultID string, request fvcommon.DetokenizeRequest) *api.DetokenizeRequest { + wire := &api.DetokenizeRequest{VaultID: vaultID, Tokens: request.Tokens} + for _, redaction := range request.TokenGroupRedactions { + wire.TokenGroupRedactions = append(wire.TokenGroupRedactions, &api.TokenGroupRedactions{ + TokenGroupName: optionalString(redaction.TokenGroupName), + Redaction: optionalString(redaction.Redaction), + }) + } + return wire +} + +func toWireGetTokens(vaultID string, request fvcommon.GetTokensRequest) *api.GetTokensFromValuesRequest { + wire := &api.GetTokensFromValuesRequest{ + VaultID: vaultID, + Records: make([]*api.GetTokensFromValuesRequestObject, 0, len(request.Records)), + } + for _, record := range request.Records { + wire.Records = append(wire.Records, &api.GetTokensFromValuesRequestObject{Value: record.Value, TokenGroupName: record.TokenGroupName}) + } + return wire +} + +func toWireUploadFiles(vaultID string, request fvcommon.UploadFilesRequest, files [][]fvhelpers.ResolvedFile) *api.FileUploadRequest { + wire := &api.FileUploadRequest{VaultID: vaultID, Records: make([]*api.FileUploadRecord, 0, len(request.Records))} + for i, record := range request.Records { + columns := make([]*api.FileUploadColumn, 0, len(record.Columns)) + for j, column := range record.Columns { + columns = append(columns, &api.FileUploadColumn{Column: column.Column, FileName: optionalString(files[i][j].Name)}) + } + wire.Records = append(wire.Records, &api.FileUploadRecord{ + TableName: record.TableName, + SkyflowID: optionalString(record.SkyflowId), + Columns: columns, + }) + } + return wire +} + +func toWireDeleteFiles(vaultID string, request fvcommon.DeleteFilesRequest) *api.FileDeleteRequest { + wire := &api.FileDeleteRequest{VaultID: vaultID, Records: make([]*api.FileDeleteRecord, 0, len(request.Records))} + for _, record := range request.Records { + wire.Records = append(wire.Records, &api.FileDeleteRecord{ + TableName: record.TableName, + SkyflowID: optionalString(record.SkyflowId), + Columns: record.Columns, + UniqueValues: toWireUniqueValues(record.UniqueValues), + }) + } + return wire +} + +// toRecordResults converts insert, update and get records. +func toRecordResults(records []*api.RecordResponseObject, header http.Header) []fvcommon.InsertResponseRecord { + results := make([]fvcommon.InsertResponseRecord, 0, len(records)) + for _, record := range records { + if record == nil { + continue + } + recordErr := deref(record.Error) + results = append(results, fvcommon.InsertResponseRecord{ + TableName: deref(record.TableName), + SkyflowId: deref(record.SkyflowID), + Tokens: fvhelpers.ParseTokens(record.Tokens), + Data: record.Data, + HashedData: fvhelpers.ParseHashedData(record.HashedData), + HttpCode: record.HTTPCode, + Error: recordErr, + RequestId: fvhelpers.RequestIdFor(header, recordErr), + }) + } + return results +} + +func toDeleteResults(records []*api.DeleteResponseObject, header http.Header) []fvcommon.DeleteResponseRecord { + results := make([]fvcommon.DeleteResponseRecord, 0, len(records)) + for _, record := range records { + if record == nil { + continue + } + recordErr := deref(record.Error) + results = append(results, fvcommon.DeleteResponseRecord{ + SkyflowId: record.SkyflowID, + HttpCode: record.HTTPCode, + Error: recordErr, + RequestId: fvhelpers.RequestIdFor(header, recordErr), + }) + } + return results +} + +func toDetokenizeResults(records []*api.DetokenizeResponseObject, header http.Header) []fvcommon.DetokenizeResponseRecord { + results := make([]fvcommon.DetokenizeResponseRecord, 0, len(records)) + for _, record := range records { + if record == nil { + continue + } + recordErr := deref(record.Error) + results = append(results, fvcommon.DetokenizeResponseRecord{ + Token: record.Token, + Value: deref(record.Value), + TokenGroupName: deref(record.TokenGroupName), + Metadata: fvhelpers.ParseDetokenizeMetadata(record.Metadata), + HttpCode: deref(record.HTTPCode), + Error: recordErr, + RequestId: fvhelpers.RequestIdFor(header, recordErr), + }) + } + return results +} + +func toGetTokensResults(records []*api.TokenizeResponseObject, header http.Header) []fvcommon.GetTokensResponseRecord { + results := make([]fvcommon.GetTokensResponseRecord, 0, len(records)) + for _, record := range records { + if record == nil { + continue + } + recordErr := deref(record.Error) + results = append(results, fvcommon.GetTokensResponseRecord{ + Value: deref(record.Value), + TokenGroupName: deref(record.TokenGroupName), + Token: record.Token, + HttpCode: deref(record.HTTPCode), + Error: recordErr, + RequestId: fvhelpers.RequestIdFor(header, recordErr), + }) + } + return results +} + +func toDeleteFilesResults(records []*api.FileDeleteResponseObject, header http.Header) []fvcommon.DeleteFilesResponseRecord { + results := make([]fvcommon.DeleteFilesResponseRecord, 0, len(records)) + for _, record := range records { + if record == nil { + continue + } + recordErr := deref(record.Error) + results = append(results, fvcommon.DeleteFilesResponseRecord{ + SkyflowId: deref(record.SkyflowID), + TableName: record.TableName, + Columns: fvhelpers.ParseDeleteFilesColumns(record.Data), + HttpCode: record.HTTPCode, + Error: recordErr, + RequestId: fvhelpers.RequestIdFor(header, recordErr), + }) + } + return results +} diff --git a/flowvault/internal/vault/controller/vault_controller.go b/flowvault/internal/vault/controller/vault_controller.go index 9a09d51..34f08cf 100644 --- a/flowvault/internal/vault/controller/vault_controller.go +++ b/flowvault/internal/vault/controller/vault_controller.go @@ -2,72 +2,267 @@ package controller import ( "context" + "fmt" + fvconsts "github.com/skyflowapi/skyflow-go/flowvault/internal/constants" + api "github.com/skyflowapi/skyflow-go/flowvault/internal/generated" + fvhelpers "github.com/skyflowapi/skyflow-go/flowvault/internal/helpers" + fvvalidation "github.com/skyflowapi/skyflow-go/flowvault/internal/validation" fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" fverrors "github.com/skyflowapi/skyflow-go/flowvault/utils/error" fvlogger "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" + logs "github.com/skyflowapi/skyflow-go/flowvault/utils/messages" ) // VaultController is flowvault's vault controller. // Embeds BaseController for shared auth/HTTP logic — same as Java's // VaultController extends AbstractBaseVaultController. +// +// Every operation validates its request, prepares auth, headers and the HTTP client, then +// calls the FlowService /v2 API through the generated client in flowvault/internal/generated. type VaultController struct { BaseController } -func NewVaultController(config fvcommon.VaultConfig, commonCreds *fvcommon.Credentials) *VaultController { +func NewVaultController(config fvcommon.VaultConfig, client ClientSettings) *VaultController { return &VaultController{ BaseController: BaseController{ - Config: &config, - CommonCreds: commonCreds, - CustomHeaders: make(map[fvcommon.CustomHeaderKey]string), + config: config, + client: client, }, } } -// Insert handles flowvault-specific InsertRequest (ContinueOnError, flowvault Upsert). +// prepare logs the call, validates the request and resolves auth, headers and HTTP settings. +func (v *VaultController) prepare(method, operation string, interceptor fvcommon.Interceptor, validate func() *fverrors.SkyflowError) (*RequestSetup, *fverrors.SkyflowError) { + fvlogger.Info(fmt.Sprintf(logs.METHOD_TRIGGERED, method)) + fvlogger.Info(fmt.Sprintf(logs.VALIDATING_REQUEST, method)) + if err := validate(); err != nil { + return nil, err + } + return v.SetupRequest(operation, interceptor) +} + +// finish logs how the call ended and passes its error through. +func finish(method string, err *fverrors.SkyflowError) *fverrors.SkyflowError { + if err != nil { + fvlogger.Error(fmt.Sprintf(logs.REQUEST_REJECTED, method)) + return err + } + fvlogger.Info(fmt.Sprintf(logs.REQUEST_RESOLVED, method)) + return nil +} + +func hasRecords[T any](records func(*T) int) func(*T) bool { + return func(body *T) bool { return records(body) > 0 } +} + +// Insert inserts records, optionally upserting on unique columns. func (v *VaultController) Insert(ctx context.Context, request fvcommon.InsertRequest, options fvcommon.InsertOptions) (*fvcommon.InsertResponse, *fverrors.SkyflowError) { - fvlogger.Info("flowvault Insert triggered for vault: " + v.Config.VaultId) - fvlogger.Debug("Insert request: ", request) - if request.Table == "" { - return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, "table must not be empty") + setup, err := v.prepare(fvconsts.METHOD_INSERT, fvconsts.OPERATION_INSERT, options.Interceptor, func() *fverrors.SkyflowError { + return fvvalidation.ValidateInsert(request) + }) + if err != nil { + return nil, err } - if len(request.Values) == 0 { - return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, "values must not be empty") + resp, callErr := setup.API.Records.WithRawResponse.InsertRecords(ctx, toWireInsert(setup.VaultID, request)) + body, header, err := result(resp, callErr, hasRecords(func(b *api.InsertResponse) int { return len(b.Records) })) + if err := finish(fvconsts.METHOD_INSERT, err); err != nil { + return nil, err } - if err := v.SetupClient(options.CustomHeaders); err != nil { + return &fvcommon.InsertResponse{Records: toRecordResults(body.Records, header)}, nil +} + +// Update updates existing records by Skyflow ID. +func (v *VaultController) Update(ctx context.Context, request fvcommon.UpdateRequest, options fvcommon.UpdateOptions) (*fvcommon.UpdateResponse, *fverrors.SkyflowError) { + setup, err := v.prepare(fvconsts.METHOD_UPDATE, fvconsts.OPERATION_UPDATE, options.Interceptor, func() *fverrors.SkyflowError { + return fvvalidation.ValidateUpdate(request) + }) + if err != nil { return nil, err } - // TODO: translate flowvault InsertRequest → REST API call via v.HTTPClient - return &fvcommon.InsertResponse{}, nil + resp, callErr := setup.API.Records.WithRawResponse.UpdateRecords(ctx, toWireUpdate(setup.VaultID, request)) + body, header, err := result(resp, callErr, hasRecords(func(b *api.UpdateResponse) int { return len(b.Records) })) + if err := finish(fvconsts.METHOD_UPDATE, err); err != nil { + return nil, err + } + return &fvcommon.UpdateResponse{Records: toRecordResults(body.Records, header)}, nil } -// Get — BaseController handles auth, VaultController translates flowvault request type. +// Get fetches records by Skyflow IDs or unique values, from one table or several. func (v *VaultController) Get(ctx context.Context, request fvcommon.GetRequest, options fvcommon.GetOptions) (*fvcommon.GetResponse, *fverrors.SkyflowError) { - fvlogger.Info("flowvault Get triggered for vault: " + v.Config.VaultId) - if err := v.SetupClient(toStringHeaders(options.CustomHeaders)); err != nil { + setup, err := v.prepare(fvconsts.METHOD_GET, fvconsts.OPERATION_GET, options.Interceptor, func() *fverrors.SkyflowError { + return fvvalidation.ValidateGet(request) + }) + if err != nil { + return nil, err + } + resp, callErr := setup.API.Records.WithRawResponse.GetRecords(ctx, toWireGet(setup.VaultID, request)) + body, header, err := result(resp, callErr, hasRecords(func(b *api.GetResponse) int { return len(b.Records) })) + if err := finish(fvconsts.METHOD_GET, err); err != nil { return nil, err } - // TODO: call Skyflow REST API via v.HTTPClient - return &fvcommon.GetResponse{}, nil + return &fvcommon.GetResponse{Records: toRecordResults(body.Records, header)}, nil } -// Delete — same pattern +// Delete deletes records by Skyflow IDs or unique values. func (v *VaultController) Delete(ctx context.Context, request fvcommon.DeleteRequest, options fvcommon.DeleteOptions) (*fvcommon.DeleteResponse, *fverrors.SkyflowError) { - fvlogger.Info("flowvault Delete triggered for vault: " + v.Config.VaultId) - if err := v.SetupClient(toStringHeaders(options.CustomHeaders)); err != nil { + setup, err := v.prepare(fvconsts.METHOD_DELETE, fvconsts.OPERATION_DELETE, options.Interceptor, func() *fverrors.SkyflowError { + return fvvalidation.ValidateDelete(request) + }) + if err != nil { + return nil, err + } + resp, callErr := setup.API.Records.WithRawResponse.DeleteRecords(ctx, toWireDelete(setup.VaultID, request)) + body, header, err := result(resp, callErr, hasRecords(func(b *api.DeleteResponse) int { return len(b.Records) })) + if err := finish(fvconsts.METHOD_DELETE, err); err != nil { + return nil, err + } + return &fvcommon.DeleteResponse{Records: toDeleteResults(body.Records, header)}, nil +} + +// Detokenize returns the values behind tokens, applying per-token-group redaction. +func (v *VaultController) Detokenize(ctx context.Context, request fvcommon.DetokenizeRequest, options fvcommon.DetokenizeOptions) (*fvcommon.DetokenizeResponse, *fverrors.SkyflowError) { + setup, err := v.prepare(fvconsts.METHOD_DETOKENIZE, fvconsts.OPERATION_DETOKENIZE, options.Interceptor, func() *fverrors.SkyflowError { + return fvvalidation.ValidateDetokenize(request) + }) + if err != nil { + return nil, err + } + resp, callErr := setup.API.Tokens.WithRawResponse.Detokenize(ctx, toWireDetokenize(setup.VaultID, request)) + body, header, err := result(resp, callErr, hasRecords(func(b *api.DetokenizeResponse) int { return len(b.Response) })) + if err := finish(fvconsts.METHOD_DETOKENIZE, err); err != nil { return nil, err } - // TODO: call Skyflow REST API via v.HTTPClient - return &fvcommon.DeleteResponse{}, nil + return &fvcommon.DetokenizeResponse{Records: toDetokenizeResults(body.Response, header)}, nil } -// toStringHeaders converts map[CustomHeaderKey]string to map[string]string. -// Needed because CustomHeaderKey is a named type, not a raw string. -func toStringHeaders(h map[fvcommon.CustomHeaderKey]string) map[string]string { - out := make(map[string]string, len(h)) - for k, v := range h { - out[string(k)] = v +// GetTokens returns existing tokens for values in deterministic token groups. +func (v *VaultController) GetTokens(ctx context.Context, request fvcommon.GetTokensRequest, options fvcommon.GetTokensOptions) (*fvcommon.GetTokensResponse, *fverrors.SkyflowError) { + setup, err := v.prepare(fvconsts.METHOD_GET_TOKENS, fvconsts.OPERATION_GET_TOKENS, options.Interceptor, func() *fverrors.SkyflowError { + return fvvalidation.ValidateGetTokens(request) + }) + if err != nil { + return nil, err + } + resp, callErr := setup.API.Tokens.WithRawResponse.GetTokens(ctx, toWireGetTokens(setup.VaultID, request)) + body, header, err := result(resp, callErr, hasRecords(func(b *api.GetTokensFromValuesResponse) int { return len(b.Records) })) + if err := finish(fvconsts.METHOD_GET_TOKENS, err); err != nil { + return nil, err + } + return &fvcommon.GetTokensResponse{Records: toGetTokensResults(body.Records, header)}, nil +} + +// Query runs a SELECT query. It returns at most 25 rows per call. +func (v *VaultController) Query(ctx context.Context, request fvcommon.QueryRequest, options fvcommon.QueryOptions) (*fvcommon.QueryResponse, *fverrors.SkyflowError) { + setup, err := v.prepare(fvconsts.METHOD_QUERY, fvconsts.OPERATION_QUERY, options.Interceptor, func() *fverrors.SkyflowError { + return fvvalidation.ValidateQuery(request) + }) + if err != nil { + return nil, err + } + resp, callErr := setup.API.Query.WithRawResponse.ExecuteQuery(ctx, &api.ExecuteQueryRequest{VaultID: setup.VaultID, Query: request.Query}) + body, header, err := result(resp, callErr, nil) + if err := finish(fvconsts.METHOD_QUERY, err); err != nil { + return nil, err + } + fields := make([]map[string]interface{}, 0, len(body.Records)) + for _, record := range body.Records { + if record != nil { + fields = append(fields, record.Data) + } + } + response := &fvcommon.QueryResponse{Fields: fields, RequestId: header.Get(fvconsts.RESPONSE_HEADER_REQUEST_ID)} + if body.Metadata != nil { + response.Metadata.Columns = body.Metadata.Columns + } + return response, nil +} + +// UploadFiles uploads files into file columns. It requests a signed upload URL per column, +// then PUTs each file to its URL, at most MAX_CONCURRENT_UPLOADS at once. The PUTs use +// fvhelpers.UploadHTTPClient: no retries, no overall timeout and no SDK headers. A column +// that can't be uploaded is reported as FAILED or SKIPPED; the other columns are unaffected. +func (v *VaultController) UploadFiles(ctx context.Context, request fvcommon.UploadFilesRequest, options fvcommon.UploadFilesOptions) (*fvcommon.UploadFilesResponse, *fverrors.SkyflowError) { + files := make([][]fvhelpers.ResolvedFile, len(request.Records)) + setup, err := v.prepare(fvconsts.METHOD_UPLOAD_FILES, fvconsts.OPERATION_UPLOAD_FILES, options.Interceptor, func() *fverrors.SkyflowError { + if err := fvvalidation.ValidateUploadFiles(request); err != nil { + return err + } + // Read every file up front so a bad file fails the call before anything is uploaded. + for i, record := range request.Records { + for _, column := range record.Columns { + file, err := fvhelpers.ResolveFile(column) + if err != nil { + return err + } + files[i] = append(files[i], file) + } + } + return nil + }) + if err != nil { + return nil, err + } + resp, callErr := setup.API.Files.WithRawResponse.UploadFiles(ctx, toWireUploadFiles(setup.VaultID, request, files)) + body, header, err := result(resp, callErr, hasRecords(func(b *api.FileUploadResponse) int { return len(b.Records) })) + if err := finish(fvconsts.METHOD_UPLOAD_FILES, err); err != nil { + return nil, err + } + + // Records come back in request order, so record i holds the signed URLs for request record i. + records := make([]fvcommon.UploadFilesResponseRecord, len(body.Records)) + var uploads []func() + for i, wireRecord := range body.Records { + if wireRecord == nil { + continue + } + recordErr := deref(wireRecord.Error) + records[i] = fvcommon.UploadFilesResponseRecord{ + SkyflowId: wireRecord.SkyflowID, + TableName: wireRecord.TableName, + HttpCode: wireRecord.HTTPCode, + Error: recordErr, + RequestId: fvhelpers.RequestIdFor(header, recordErr), + } + if i >= len(request.Records) { + continue + } + columns := request.Records[i].Columns + records[i].Columns = make([]fvcommon.UploadFilesColumnResult, len(columns)) + for j, column := range columns { + file := files[i][j] + columnResult := &records[i].Columns[j] + *columnResult = fvcommon.UploadFilesColumnResult{Column: column.Column, FileName: file.Name, UploadStatus: fvcommon.UploadStatusSkipped} + if recordErr != "" { + columnResult.Error = recordErr + continue + } + signedURL, _ := wireRecord.Data[column.Column].(string) + if signedURL == "" { + continue + } + uploads = append(uploads, func() { + columnResult.UploadStatus, columnResult.Error = fvhelpers.PutFile(ctx, fvhelpers.UploadHTTPClient(), signedURL, file) + }) + } + } + fvhelpers.RunConcurrently(uploads, fvconsts.MAX_CONCURRENT_UPLOADS) + return &fvcommon.UploadFilesResponse{Records: records}, nil +} + +// DeleteFiles deletes files from file columns of existing records, including the stored objects. +func (v *VaultController) DeleteFiles(ctx context.Context, request fvcommon.DeleteFilesRequest, options fvcommon.DeleteFilesOptions) (*fvcommon.DeleteFilesResponse, *fverrors.SkyflowError) { + setup, err := v.prepare(fvconsts.METHOD_DELETE_FILES, fvconsts.OPERATION_DELETE_FILES, options.Interceptor, func() *fverrors.SkyflowError { + return fvvalidation.ValidateDeleteFiles(request) + }) + if err != nil { + return nil, err + } + resp, callErr := setup.API.Files.WithRawResponse.DeleteFiles(ctx, toWireDeleteFiles(setup.VaultID, request)) + body, header, err := result(resp, callErr, hasRecords(func(b *api.FileDeleteResponse) int { return len(b.Records) })) + if err := finish(fvconsts.METHOD_DELETE_FILES, err); err != nil { + return nil, err } - return out + return &fvcommon.DeleteFilesResponse{Records: toDeleteFilesResults(body.Records, header)}, nil } diff --git a/flowvault/test/resources/client_api_key.txt b/flowvault/test/resources/client_api_key.txt new file mode 100644 index 0000000..6f45c7d --- /dev/null +++ b/flowvault/test/resources/client_api_key.txt @@ -0,0 +1 @@ +sky-ab123-abcd1234cdef1234abcd4321cdef4321 diff --git a/flowvault/test/resources/vault_api_key.txt b/flowvault/test/resources/vault_api_key.txt new file mode 100644 index 0000000..849659b --- /dev/null +++ b/flowvault/test/resources/vault_api_key.txt @@ -0,0 +1 @@ +sky-zz999-abcd1234cdef1234abcd4321cdef4321 diff --git a/flowvault/utils/common/common.go b/flowvault/utils/common/common.go index c608856..9d29da1 100644 --- a/flowvault/utils/common/common.go +++ b/flowvault/utils/common/common.go @@ -22,6 +22,7 @@ const ( type ( Credentials = cfg.Credentials VaultConfig = cfg.VaultConfig + HttpConfig = cfg.HttpConfig ConnectionConfig = cfg.ConnectionConfig CustomHeaderKey = cfg.CustomHeaderKey ) @@ -39,156 +40,5 @@ type ( SignedDataTokensResponse = sa.SignedDataTokensResponse ) -// --------------------------------------------------------------------------- -// FlowVault vault API types. These belong to flowvault alone and evolve with -// the FlowVault API, independently of the v2 (SkyVault) SDK. -// --------------------------------------------------------------------------- - -type RedactionType string - -const ( - PLAIN_TEXT RedactionType = "PLAIN_TEXT" - DEFAULT RedactionType = "DEFAULT" - MASKED RedactionType = "MASKED" - REDACTED RedactionType = "REDACTED" -) - -type OrderByEnum string - -const ( - ASCENDING OrderByEnum = "ASCENDING" - DESCENDING OrderByEnum = "DESCENDING" - NONE OrderByEnum = "NONE" -) - -type BYOT string - -const ( - DISABLE BYOT = "DISABLE" - ENABLE BYOT = "ENABLE" - ENABLE_STRICT BYOT = "ENABLE_STRICT" -) - -// Get - -type GetRequest struct { - Table string - Ids []string -} - -type GetOptions struct { - RedactionType RedactionType - ReturnTokens bool - Fields []string - Offset string - Limit string - DownloadUrl bool - ColumnName string - ColumnValues []string - OrderBy OrderByEnum - CustomHeaders map[CustomHeaderKey]string -} - -type GetResponse struct { - Data []map[string]interface{} - Errors []map[string]interface{} -} - -// Detokenize - -type DetokenizeRequest struct { - DetokenizeData []DetokenizeData -} - -type DetokenizeData struct { - Token string - RedactionType RedactionType -} - -type DetokenizeOptions struct { - ContinueOnError bool - DownloadUrl bool - CustomHeaders map[CustomHeaderKey]string -} - -type DetokenizeRecordResponse struct { - Token string - Value string - Type string - Error string - RequestId string -} - -type DetokenizeResponse struct { - DetokenizedFields []DetokenizeRecordResponse - Errors []DetokenizeRecordResponse -} - -// Delete - -type DeleteRequest struct { - Table string - Ids []string -} - -type DeleteOptions struct { - CustomHeaders map[CustomHeaderKey]string -} - -type DeleteResponse struct { - DeletedIds []string - Errors []map[string]interface{} -} - -// Update - -type UpdateRequest struct { - Table string - Data map[string]interface{} - Tokens map[string]interface{} -} - -type UpdateOptions struct { - ReturnTokens bool - TokenMode BYOT - CustomHeaders map[CustomHeaderKey]string -} - -type UpdateResponse struct { - UpdatedField map[string]interface{} - Errors []map[string]interface{} -} - -// Query - -type QueryRequest struct { - Query string -} - -type QueryOptions struct { - CustomHeaders map[CustomHeaderKey]string -} - -type QueryResponse struct { - Fields []map[string]interface{} - Errors []map[string]interface{} -} - -// Tokenize - -type TokenizeRequest struct { - ColumnGroup string - Value string -} - -type TokenizeOptions struct { - CustomHeaders map[CustomHeaderKey]string -} - -type TokenizeResponse struct { - Tokens []string - Errors []map[string]interface{} -} - // IsCredentialsEmpty reports whether no credential source is set. func IsCredentialsEmpty(c Credentials) bool { return cfg.IsCredentialsEmpty(c) } diff --git a/flowvault/utils/common/delete.go b/flowvault/utils/common/delete.go new file mode 100644 index 0000000..b2e98a6 --- /dev/null +++ b/flowvault/utils/common/delete.go @@ -0,0 +1,22 @@ +package common + +type DeleteRequest struct { + TableName string // required + SkyflowIds []string + UniqueValues []map[string]interface{} +} + +type DeleteResponseRecord struct { + SkyflowId string `json:"skyflowId"` + HttpCode int `json:"httpCode"` + Error string `json:"error"` + RequestId string `json:"requestId"` +} + +type DeleteResponse struct { + Records []DeleteResponseRecord `json:"records"` +} + +type DeleteOptions struct { + Interceptor Interceptor +} diff --git a/flowvault/utils/common/delete_files.go b/flowvault/utils/common/delete_files.go new file mode 100644 index 0000000..28960ab --- /dev/null +++ b/flowvault/utils/common/delete_files.go @@ -0,0 +1,35 @@ +package common + +// DeleteFilesRequestRecord takes exactly one of SkyflowId or UniqueValues. +type DeleteFilesRequestRecord struct { + TableName string // required + Columns []string // required + SkyflowId string + UniqueValues []map[string]interface{} +} + +type DeleteFilesRequest struct { + Records []DeleteFilesRequestRecord +} + +type DeleteFilesColumnResult struct { + Column string `json:"column"` + Status string `json:"status"` +} + +type DeleteFilesResponseRecord struct { + SkyflowId string `json:"skyflowId"` + TableName string `json:"tableName"` + Columns []DeleteFilesColumnResult `json:"columns"` // nil on a failed record + HttpCode int `json:"httpCode"` + Error string `json:"error"` + RequestId string `json:"requestId"` +} + +type DeleteFilesResponse struct { + Records []DeleteFilesResponseRecord `json:"records"` +} + +type DeleteFilesOptions struct { + Interceptor Interceptor +} diff --git a/flowvault/utils/common/detokenize.go b/flowvault/utils/common/detokenize.go new file mode 100644 index 0000000..19d4dab --- /dev/null +++ b/flowvault/utils/common/detokenize.go @@ -0,0 +1,29 @@ +package common + +type DetokenizeRequest struct { + Tokens []string // required + TokenGroupRedactions []TokenGroupRedaction +} + +type DetokenizeRecordMetadata struct { + SkyflowId string `json:"skyflowId"` + TableName string `json:"tableName"` +} + +type DetokenizeResponseRecord struct { + Token string `json:"token"` + Value interface{} `json:"value"` + TokenGroupName string `json:"tokenGroupName"` + Metadata *DetokenizeRecordMetadata `json:"metadata"` + HttpCode int `json:"httpCode"` + Error string `json:"error"` + RequestId string `json:"requestId"` +} + +type DetokenizeResponse struct { + Records []DetokenizeResponseRecord `json:"records"` +} + +type DetokenizeOptions struct { + Interceptor Interceptor +} diff --git a/flowvault/utils/common/get.go b/flowvault/utils/common/get.go new file mode 100644 index 0000000..ad0c5aa --- /dev/null +++ b/flowvault/utils/common/get.go @@ -0,0 +1,29 @@ +package common + +type GetRequest struct { + TableName string + SkyflowIds []string + UniqueValues []map[string]interface{} + Columns []string + ColumnRedactions []ColumnRedaction + Limit, Offset int // 0 = not sent + Records []GetRequestRecord // multi-table mode; excludes the fields above +} + +type GetRequestRecord struct { + TableName string + SkyflowIds []string + UniqueValues []map[string]interface{} + Columns []string + ColumnRedactions []ColumnRedaction +} + +type GetResponseRecord = InsertResponseRecord + +type GetResponse struct { + Records []GetResponseRecord `json:"records"` +} + +type GetOptions struct { + Interceptor Interceptor +} diff --git a/flowvault/utils/common/get_tokens.go b/flowvault/utils/common/get_tokens.go new file mode 100644 index 0000000..bb964d0 --- /dev/null +++ b/flowvault/utils/common/get_tokens.go @@ -0,0 +1,27 @@ +package common + +type GetTokensRequest struct { + Records []GetTokensRequestRecord +} + +type GetTokensRequestRecord struct { + Value interface{} // required + TokenGroupName string // required, deterministic group +} + +type GetTokensResponseRecord struct { + Value interface{} `json:"value"` + TokenGroupName string `json:"tokenGroupName"` + Token string `json:"token"` + HttpCode int `json:"httpCode"` + Error string `json:"error"` + RequestId string `json:"requestId"` +} + +type GetTokensResponse struct { + Records []GetTokensResponseRecord `json:"records"` +} + +type GetTokensOptions struct { + Interceptor Interceptor +} diff --git a/flowvault/utils/common/insert.go b/flowvault/utils/common/insert.go index 7e932fa..cf96e13 100644 --- a/flowvault/utils/common/insert.go +++ b/flowvault/utils/common/insert.go @@ -1,23 +1,39 @@ package common -// InsertRequest is the FlowVault insert request. Unlike v2, ContinueOnError -// is part of the request rather than the options. +type UpsertOptions struct { + UniqueColumns []string // required + UpdateType UpdateType +} + type InsertRequest struct { - Table string - Values []map[string]interface{} - ContinueOnError bool + TableName string // set here OR on every record + Records []InsertRequestRecord + Upsert *UpsertOptions // only with request-level TableName } -// InsertOptions are the FlowVault insert options. CustomHeaders uses plain -// string keys (v2 uses a typed CustomHeaderKey). -type InsertOptions struct { - Upsert string - CustomHeaders map[string]string +type InsertRequestRecord struct { + Data map[string]interface{} // required + Tokens map[string]interface{} + TableName string // only if not set on the request + Upsert *UpsertOptions // only with record-level TableName +} + +// InsertResponseRecord is the per-record result of Insert, Update and Get. +type InsertResponseRecord struct { + TableName string `json:"tableName"` + SkyflowId string `json:"skyflowId"` + Tokens map[string][]Token `json:"tokens"` + Data map[string]interface{} `json:"data"` + HashedData map[string][]HashedValue `json:"hashedData"` + HttpCode int `json:"httpCode"` + Error string `json:"error"` + RequestId string `json:"requestId"` } -// InsertResponse is the FlowVault insert response. type InsertResponse struct { - InsertedFields []map[string]interface{} - Tokens []map[string]interface{} - Errors []map[string]interface{} + Records []InsertResponseRecord `json:"records"` +} + +type InsertOptions struct { + Interceptor Interceptor } diff --git a/flowvault/utils/common/query.go b/flowvault/utils/common/query.go new file mode 100644 index 0000000..1d37311 --- /dev/null +++ b/flowvault/utils/common/query.go @@ -0,0 +1,19 @@ +package common + +type QueryRequest struct { + Query string +} + +type QueryResponseMetadata struct { + Columns []string `json:"columns"` +} + +type QueryResponse struct { + Fields []map[string]interface{} `json:"fields"` // one map per row + Metadata QueryResponseMetadata `json:"metadata"` + RequestId string `json:"requestId"` // call-level, always populated +} + +type QueryOptions struct { + Interceptor Interceptor +} diff --git a/flowvault/utils/common/types.go b/flowvault/utils/common/types.go new file mode 100644 index 0000000..ca509ac --- /dev/null +++ b/flowvault/utils/common/types.go @@ -0,0 +1,60 @@ +package common + +// UpdateType controls how an update or upsert applies data to an existing record. +type UpdateType string + +const ( + // UpdateTypeUpdate merges the given columns into the record (server default). + UpdateTypeUpdate UpdateType = "UPDATE" + // UpdateTypeReplace overwrites the record with the given columns. + UpdateTypeReplace UpdateType = "REPLACE" +) + +// RequestContext is passed to an options Interceptor before each API call. +type RequestContext struct { + Operation string + headers map[CustomHeaderKey]string +} + +// AddHeader adds a custom header to the outgoing request. +func (rc *RequestContext) AddHeader(key CustomHeaderKey, value string) { + if rc.headers == nil { + rc.headers = make(map[CustomHeaderKey]string) + } + rc.headers[key] = value +} + +// Headers returns a copy of the headers added so far. +func (rc *RequestContext) Headers() map[CustomHeaderKey]string { + headers := make(map[CustomHeaderKey]string, len(rc.headers)) + for key, value := range rc.headers { + headers[key] = value + } + return headers +} + +// Interceptor customises a single call, e.g. by adding headers. +type Interceptor func(rc *RequestContext) + +// Token is one token issued for a column value. +type Token struct { + Token string `json:"token"` + TokenGroupName string `json:"tokenGroupName,omitempty"` + Path string `json:"path,omitempty"` +} + +// HashedValue is one hash computed for a column value. +type HashedValue struct { + Data string `json:"data"` + HashName string `json:"hashName,omitempty"` +} + +type ColumnRedaction struct { + ColumnName string + Redaction string +} + +type TokenGroupRedaction struct { + TokenGroupName string + Redaction string +} diff --git a/flowvault/utils/common/update.go b/flowvault/utils/common/update.go new file mode 100644 index 0000000..56dbb9a --- /dev/null +++ b/flowvault/utils/common/update.go @@ -0,0 +1,24 @@ +package common + +type UpdateRequest struct { + TableName string // set here OR on every record + Records []UpdateRequestRecord + UpdateType UpdateType // request level only +} + +type UpdateRequestRecord struct { + SkyflowId string // required + Data map[string]interface{} // required + Tokens map[string]interface{} + TableName string // only if not set on the request +} + +type UpdateResponseRecord = InsertResponseRecord + +type UpdateResponse struct { + Records []UpdateResponseRecord `json:"records"` +} + +type UpdateOptions struct { + Interceptor Interceptor +} diff --git a/flowvault/utils/common/upload_files.go b/flowvault/utils/common/upload_files.go new file mode 100644 index 0000000..1e5fe0a --- /dev/null +++ b/flowvault/utils/common/upload_files.go @@ -0,0 +1,55 @@ +package common + +import "os" + +// UploadStatus is the outcome of uploading one file column. +type UploadStatus string + +const ( + UploadStatusUploaded UploadStatus = "UPLOADED" + UploadStatusFailed UploadStatus = "FAILED" + UploadStatusSkipped UploadStatus = "SKIPPED" +) + +type UploadFilesRequest struct { + Records []UploadFilesRequestRecord +} + +type UploadFilesRequestRecord struct { + TableName string // required + SkyflowId string // empty -> new record; set -> update existing record + Columns []UploadFilesRequestColumn +} + +// UploadFilesRequestColumn takes exactly one of FilePath, Base64 or FileObject. +type UploadFilesRequestColumn struct { + Column string // required + FilePath string + Base64 string // requires FileName + FileObject *os.File // caller closes + FileName string // derived from path/object if empty; server UUID if none +} + +type UploadFilesColumnResult struct { + Column string `json:"column"` + FileName string `json:"fileName"` + UploadStatus UploadStatus `json:"uploadStatus"` + Error string `json:"error"` +} + +type UploadFilesResponseRecord struct { + SkyflowId string `json:"skyflowId"` + TableName string `json:"tableName"` + Columns []UploadFilesColumnResult `json:"columns"` + HttpCode int `json:"httpCode"` + Error string `json:"error"` + RequestId string `json:"requestId"` +} + +type UploadFilesResponse struct { + Records []UploadFilesResponseRecord `json:"records"` +} + +type UploadFilesOptions struct { + Interceptor Interceptor +} diff --git a/flowvault/utils/error/messages.go b/flowvault/utils/error/messages.go index 221afd9..487df5a 100644 --- a/flowvault/utils/error/messages.go +++ b/flowvault/utils/error/messages.go @@ -9,3 +9,44 @@ const ( VAULT_ID_EXISTS_IN_CONFIG_LIST string = internal.SDK_PREFIX + " Validation error. %s already exists in the config list. Specify a new vaultId." EMPTY_VAULT_ID string = internal.SDK_PREFIX + " Validation error. Invalid vault ID. Specify a valid vault ID." ) + +// Request validation messages. Messages containing %s / %d are formatted with fmt.Sprintf. +const ( + EMPTY_RECORDS string = internal.SDK_PREFIX + " Validation error. %s failed. Records must not be empty." + TABLE_IN_BOTH_PLACES string = internal.SDK_PREFIX + " Validation error. %s failed. Table name is set on the request and on a record. Set it on the request or on every record, not both." + TABLE_MISSING string = internal.SDK_PREFIX + " Validation error. %s failed. Table name is missing. Set it on the request or on every record." + RECORD_UPSERT_NOT_ALLOWED string = internal.SDK_PREFIX + " Validation error. Insert failed. Upsert is set on a record while the table name is set on the request. Set upsert on the request instead." + REQUEST_UPSERT_NOT_ALLOWED string = internal.SDK_PREFIX + " Validation error. Insert failed. Upsert is set on the request while the table name is set on the records. Set upsert on each record instead." + EMPTY_UNIQUE_COLUMNS string = internal.SDK_PREFIX + " Validation error. Insert failed. Upsert unique columns must not be empty." + INVALID_UPDATE_TYPE string = internal.SDK_PREFIX + " Validation error. %s failed. Invalid update type %q. Use UPDATE or REPLACE." + EMPTY_DATA string = internal.SDK_PREFIX + " Validation error. %s failed. Data must not be empty for record at index %d." + EMPTY_KEY_IN_DATA string = internal.SDK_PREFIX + " Validation error. %s failed. Data contains an empty column name for record at index %d." + MISSING_SKYFLOW_ID string = internal.SDK_PREFIX + " Validation error. Update failed. Skyflow ID is missing for record at index %d." + GET_MODE_CONFLICT string = internal.SDK_PREFIX + " Validation error. Get failed. Set either the single-table fields or Records, not both." + NEGATIVE_LIMIT_OFFSET string = internal.SDK_PREFIX + " Validation error. Get failed. Limit and Offset must not be negative." + EMPTY_TABLE_NAME string = internal.SDK_PREFIX + " Validation error. %s failed. Table name must not be empty." + IDS_OR_UNIQUE_VALUES string = internal.SDK_PREFIX + " Validation error. %s failed. Set exactly one of SkyflowIds or UniqueValues." + EMPTY_ID string = internal.SDK_PREFIX + " Validation error. %s failed. Skyflow IDs must not contain empty values." + EMPTY_UNIQUE_VALUE string = internal.SDK_PREFIX + " Validation error. %s failed. Unique values must not contain empty entries." + INVALID_COLUMN_REDACTION string = internal.SDK_PREFIX + " Validation error. %s failed. Column redactions need a column name and a redaction." + EMPTY_TOKENS string = internal.SDK_PREFIX + " Validation error. Detokenize failed. Tokens must not be empty." + EMPTY_TOKEN string = internal.SDK_PREFIX + " Validation error. Detokenize failed. Tokens must not contain empty values." + INVALID_TOKEN_GROUP_REDACTION string = internal.SDK_PREFIX + " Validation error. Detokenize failed. Token group redactions need a token group name and a redaction." + MISSING_VALUE string = internal.SDK_PREFIX + " Validation error. GetTokens failed. Value is missing for record at index %d." + MISSING_TOKEN_GROUP_NAME string = internal.SDK_PREFIX + " Validation error. GetTokens failed. Token group name is missing for record at index %d." + EMPTY_QUERY string = internal.SDK_PREFIX + " Validation error. Query failed. Query must not be empty." + EMPTY_COLUMNS string = internal.SDK_PREFIX + " Validation error. %s failed. Columns must not be empty for record at index %d." + EMPTY_COLUMN_NAME string = internal.SDK_PREFIX + " Validation error. %s failed. Column name must not be empty for record at index %d." + FILE_SOURCE_COUNT string = internal.SDK_PREFIX + " Validation error. UploadFiles failed. Column %q needs exactly one of FilePath, Base64 or FileObject." + BASE64_NEEDS_FILE_NAME string = internal.SDK_PREFIX + " Validation error. UploadFiles failed. Column %q uses Base64, so FileName is required." + INVALID_BASE64 string = internal.SDK_PREFIX + " Validation error. UploadFiles failed. Column %q has invalid base64 content." + FILE_NOT_READABLE string = internal.SDK_PREFIX + " Validation error. UploadFiles failed. Could not read file for column %q: %s" + DELETE_FILES_TARGET string = internal.SDK_PREFIX + " Validation error. DeleteFiles failed. Set exactly one of SkyflowId or UniqueValues for record at index %d." + DELETE_FILES_LIMIT string = internal.SDK_PREFIX + " Validation error. DeleteFiles failed. At most %d records are allowed per request." + DUPLICATE_COLUMN string = internal.SDK_PREFIX + " Validation error. %s failed. Column %q is listed more than once for record at index %d." + EMPTY_CREDENTIALS string = internal.SDK_PREFIX + " Validation error. Invalid credentials. Specify a valid credentials." + BEARER_TOKEN_EXPIRED string = internal.SDK_PREFIX + " Validation error. Bearer token is expired. Specify a valid bearer token." + MULTIPLE_VAULT_CONFIGS string = internal.SDK_PREFIX + " Validation error. Multiple vault configurations found. Specify a vaultId." + INVALID_HEADER_KEY string = internal.SDK_PREFIX + " Validation error. %s: header key %q is not supported. Use x-skyflow-account-id, x-skyflow-account-name or x-request-id." + EMPTY_HEADER_VALUE string = internal.SDK_PREFIX + " Validation error. %s: header %q must have a value." +) diff --git a/flowvault/utils/messages/logs.go b/flowvault/utils/messages/logs.go index 07e88c4..8796c62 100644 --- a/flowvault/utils/messages/logs.go +++ b/flowvault/utils/messages/logs.go @@ -3,12 +3,20 @@ package logs import . "github.com/skyflowapi/skyflow-go/flowvault/internal/constants" const ( - CLIENT_INITIALIZED = SDK_LOG_PREFIX + "Initialized skyflow client successfully." - VALIDATING_VAULT_CONFIG = SDK_LOG_PREFIX + "Validating vault config." - VALIDATING_CRED = SDK_LOG_PREFIX + "Validating credentials." - VAULT_CONTROLLER_INITIALIZED = SDK_LOG_PREFIX + "Initialized vault controller with vault ID %s." - CURRENT_LOG_LEVEL = SDK_LOG_PREFIX + "Current log level is %v." - EMPTY_VAULT_ARRAY = SDK_LOG_PREFIX + "Vault config array is empty." - VAULT_CONFIG_EXISTS = SDK_LOG_PREFIX + "Vault config with vault ID %s already exists." - VAULT_CONFIG_DOES_NOT_EXIST = SDK_LOG_PREFIX + "Vault config with vault ID %s doesn't exist." + CLIENT_INITIALIZED = SDK_LOG_PREFIX + "Initialized skyflow client successfully." + VALIDATING_VAULT_CONFIG = SDK_LOG_PREFIX + "Validating vault config." + VALIDATING_CRED = SDK_LOG_PREFIX + "Validating credentials." + VAULT_CONTROLLER_INITIALIZED = SDK_LOG_PREFIX + "Initialized vault controller with vault ID %s." + CURRENT_LOG_LEVEL = SDK_LOG_PREFIX + "Current log level is %v." + EMPTY_VAULT_ARRAY = SDK_LOG_PREFIX + "Vault config array is empty." + VAULT_CONFIG_EXISTS = SDK_LOG_PREFIX + "Vault config with vault ID %s already exists." + VAULT_CONFIG_DOES_NOT_EXIST = SDK_LOG_PREFIX + "Vault config with vault ID %s doesn't exist." + BEARER_TOKEN_EXPIRED = SDK_LOG_PREFIX + "Bearer token is expired." + GENERATE_BEARER_TOKEN_TRIGGERED = SDK_LOG_PREFIX + "Generating a new bearer token." + REUSE_BEARER_TOKEN = SDK_LOG_PREFIX + "Reusing the cached bearer token." + EMPTY_CREDENTIALS = SDK_LOG_PREFIX + "No credentials found in the vault config, client config or SKYFLOW_CREDENTIALS." + METHOD_TRIGGERED = SDK_LOG_PREFIX + "%s method triggered." + VALIDATING_REQUEST = SDK_LOG_PREFIX + "Validating %s request." + REQUEST_RESOLVED = SDK_LOG_PREFIX + "%s request resolved." + REQUEST_REJECTED = SDK_LOG_PREFIX + "%s request resulted in failure." ) diff --git a/samples/flowvault/README.md b/samples/flowvault/README.md new file mode 100644 index 0000000..049edc1 --- /dev/null +++ b/samples/flowvault/README.md @@ -0,0 +1,62 @@ +# FlowVault Go samples + +Runnable examples for the Skyflow FlowVault Go SDK (`github.com/skyflowapi/skyflow-go/flowvault`), one file per operation. + +## Prerequisites + +- Go 1.25+ +- A FlowVault vault and Skyflow credentials: a service-account `credentials.json`, an API key, or a bearer token. + +## Configure + +Each sample has placeholders near the top. Replace them with your own values: + +```go +credentials := common.Credentials{Path: ""} // or ApiKey / Token / CredentialsString +vaultConfig := common.VaultConfig{ + VaultId: "", + ClusterId: "", // from the vault URL: https://{clusterId}.skyvault.skyflowapis.com + Env: common.PROD, // DEV, STAGE, SANDBOX or PROD + Credentials: credentials, +} +``` + +## Run + +Each file is its own program. Run one from this folder: + +```bash +cd samples/flowvault +go run vaultapi/insert_records.go +``` + +## Vault operations + +| Sample | Demonstrates | +|---|---| +| [insert_records.go](vaultapi/insert_records.go) | Insert records (table name and upsert on the request, or on each record) | +| [get_records.go](vaultapi/get_records.go) | Get records by Skyflow ID, by unique value, or from several tables | +| [update_record.go](vaultapi/update_record.go) | Update a record (`UPDATE` or `REPLACE`) | +| [delete_records.go](vaultapi/delete_records.go) | Delete records | +| [detokenize_records.go](vaultapi/detokenize_records.go) | Detokenize tokens | +| [query_records.go](vaultapi/query_records.go) | Run a read-only SQL `SELECT` query | +| [get_tokens.go](vaultapi/get_tokens.go) | Look up existing tokens for values (deterministic token groups) | +| [upload_files.go](vaultapi/upload_files.go) | Upload files to file columns (`FilePath` / `Base64` / `FileObject`) | +| [delete_files.go](vaultapi/delete_files.go) | Delete files from file columns (by `SkyflowId` or `UniqueValues`) | + +## Custom headers and HTTP config + +| Sample | Demonstrates | +|---|---| +| [custom_header_example.go](vaultapi/custom_header_example.go) | Add headers to one call with an `InsertOptions` interceptor | +| [timeout_and_retry_config_example.go](vaultapi/timeout_and_retry_config_example.go) | Client-wide `WithHttpConfig` and per-vault `Timeout` / `MaxRetries` | + +## Service account (token generation) + +| Sample | Demonstrates | +|---|---| +| [bearer_token_generation_example.go](serviceaccount/bearer_token_generation_example.go) | Bearer token from a credentials file path or a credentials string | +| [bearer_token_generation_with_context_example.go](serviceaccount/bearer_token_generation_with_context_example.go) | Context-aware bearer tokens (string context and object context) | +| [bearer_token_generation_using_goroutines_example.go](serviceaccount/bearer_token_generation_using_goroutines_example.go) | Concurrent token generation from goroutines with a shared context | +| [scoped_token_generation_example.go](serviceaccount/scoped_token_generation_example.go) | Scoped tokens via `RoleIds` | +| [signed_token_generation_example.go](serviceaccount/signed_token_generation_example.go) | Signed data tokens (`GenerateSignedDataTokens`) | diff --git a/samples/flowvault/go.mod b/samples/flowvault/go.mod new file mode 100644 index 0000000..a973bee --- /dev/null +++ b/samples/flowvault/go.mod @@ -0,0 +1,14 @@ +module github.com/skyflowapi/skyflow-go/samples/flowvault + +go 1.25.0 + +require github.com/skyflowapi/skyflow-go/flowvault v0.0.0 + +require ( + github.com/golang-jwt/jwt/v4 v4.5.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/skyflowapi/skyflow-go/common v0.0.0-20261019104456-6115e81fcef9 // indirect +) + +// FlowVault isn't published yet; build the samples against the code in this repository. +replace github.com/skyflowapi/skyflow-go/flowvault => ../../flowvault diff --git a/samples/flowvault/go.sum b/samples/flowvault/go.sum new file mode 100644 index 0000000..ce8cae7 --- /dev/null +++ b/samples/flowvault/go.sum @@ -0,0 +1,42 @@ +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oXJCETMO23COyaKGP6fHVpkpWpg= +github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/onsi/ginkgo/v2 v2.33.1 h1:62XfWJbEIjPSCSDYrsCPP/SHcmH7O0M8ne8zfqQ2iz4= +github.com/onsi/ginkgo/v2 v2.33.1/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= +github.com/onsi/gomega v1.44.0 h1:eAiGl3Pw5jz5GQdDff0BcxYpAX1JxW8xD7mFUuwNfZQ= +github.com/onsi/gomega v1.44.0/go.mod h1:e/C2HwaZ1DhvjzXXuFhcR7hY7Sh9pl7MmoWKEjzwcdA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/skyflowapi/skyflow-go/common v0.0.0-20261019104456-6115e81fcef9 h1:0SNuvxkS+mr+sy7lDZvwmgjgAIC+HrqEr81lsTTXEgQ= +github.com/skyflowapi/skyflow-go/common v0.0.0-20261019104456-6115e81fcef9/go.mod h1:T9roW4Qpv1WFo6tgLjbfLrdOCOz20lVE8ENf2bGb7zI= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/samples/flowvault/serviceaccount/bearer_token_generation_example.go b/samples/flowvault/serviceaccount/bearer_token_generation_example.go new file mode 100644 index 0000000..ca2cdca --- /dev/null +++ b/samples/flowvault/serviceaccount/bearer_token_generation_example.go @@ -0,0 +1,66 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "encoding/json" + "fmt" + + "github.com/skyflowapi/skyflow-go/flowvault/serviceaccount" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" +) + +/** + * This example generates a bearer token in two ways, and reuses it until it expires: + * 1. From the path to a credentials.json file. + * 2. From the credentials as a JSON string. + */ + +var ( + filePath = "" + bearerToken = "" +) + +// credentialsString returns the credentials as a JSON string. +func credentialsString() string { + credentials, _ := json.Marshal(map[string]string{ + "clientID": "", + "clientName": "", + "tokenURI": "", + "keyID": "", + "privateKey": "", + }) + return string(credentials) +} + +func bearerTokenFromFilePath() string { + if bearerToken != "" && !serviceaccount.IsExpired(bearerToken) { + return bearerToken + } + response, err := serviceaccount.GenerateBearerToken(filePath, common.BearerTokenOptions{}) + if err != nil { + fmt.Println("Error generating token from file path:", err.GetMessage()) + return "" + } + bearerToken = response.AccessToken + return bearerToken +} + +func bearerTokenFromCredentialsString() string { + if bearerToken != "" && !serviceaccount.IsExpired(bearerToken) { + return bearerToken + } + response, err := serviceaccount.GenerateBearerTokenFromCreds(credentialsString(), common.BearerTokenOptions{}) + if err != nil { + fmt.Println("Error generating token from credentials string:", err.GetMessage()) + return "" + } + bearerToken = response.AccessToken + return bearerToken +} + +func main() { + fmt.Println("Generated Bearer Token (from file):", bearerTokenFromFilePath()) + fmt.Println("Generated Bearer Token (from string):", bearerTokenFromCredentialsString()) +} diff --git a/samples/flowvault/serviceaccount/bearer_token_generation_using_goroutines_example.go b/samples/flowvault/serviceaccount/bearer_token_generation_using_goroutines_example.go new file mode 100644 index 0000000..76f3e40 --- /dev/null +++ b/samples/flowvault/serviceaccount/bearer_token_generation_using_goroutines_example.go @@ -0,0 +1,73 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "encoding/json" + "fmt" + "sync" + + "github.com/skyflowapi/skyflow-go/flowvault/serviceaccount" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" +) + +/** + * This example generates bearer tokens concurrently from two goroutines, one using a credentials + * file path and one using a credentials string, with a shared context. Each goroutine generates + * and prints a token five times. + */ + +var ( + filePath = "" + options = common.BearerTokenOptions{Ctx: "abc"} +) + +// credentialsString returns the credentials as a JSON string. +func credentialsString() string { + credentials, _ := json.Marshal(map[string]string{ + "clientID": "", + "clientName": "", + "tokenURI": "", + "keyID": "", + "privateKey": "", + }) + return string(credentials) +} + +func generateFromFilePath() { + for i := 0; i < 5; i++ { + response, err := serviceaccount.GenerateBearerToken(filePath, options) + if err != nil { + fmt.Println("Error generating token from file path:", err.GetMessage()) + continue + } + fmt.Println(response.AccessToken) + } +} + +func generateFromCredentialsString() { + credentials := credentialsString() + for i := 0; i < 5; i++ { + response, err := serviceaccount.GenerateBearerTokenFromCreds(credentials, options) + if err != nil { + fmt.Println("Error generating token from credentials string:", err.GetMessage()) + continue + } + fmt.Println(response.AccessToken) + } +} + +func main() { + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + generateFromFilePath() + }() + go func() { + defer wg.Done() + generateFromCredentialsString() + }() + wg.Wait() +} diff --git a/samples/flowvault/serviceaccount/bearer_token_generation_with_context_example.go b/samples/flowvault/serviceaccount/bearer_token_generation_with_context_example.go new file mode 100644 index 0000000..df8624f --- /dev/null +++ b/samples/flowvault/serviceaccount/bearer_token_generation_with_context_example.go @@ -0,0 +1,55 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "fmt" + + "github.com/skyflowapi/skyflow-go/flowvault/serviceaccount" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" +) + +/** + * This example generates bearer tokens that carry a context, for context-aware access policies. + * 1. A string context, referenced in a Skyflow policy as request.context. + * 2. An object context, whose keys are referenced as request.context.. + */ + +var filePath = "" + +// Approach 1: a string context. +// Use a simple string identifier when your policy references a single context value. +func bearerTokenWithStringContext() string { + response, err := serviceaccount.GenerateBearerToken(filePath, common.BearerTokenOptions{Ctx: "user_12345"}) + if err != nil { + fmt.Println("Error generating token:", err.GetMessage()) + return "" + } + return response.AccessToken +} + +// Approach 2: an object context. +// Use a map when your policy needs several context values. Each key maps to a Skyflow CEL policy +// variable under request.context.*, for example: +// +// request.context.role == "admin" && request.context.department == "finance" +func bearerTokenWithObjectContext() string { + response, err := serviceaccount.GenerateBearerToken(filePath, common.BearerTokenOptions{ + Ctx: map[string]interface{}{ + "role": "admin", + "department": "finance", + "user_id": "user_12345", + }, + }) + if err != nil { + fmt.Println("Error generating token:", err.GetMessage()) + return "" + } + return response.AccessToken +} + +func main() { + fmt.Println("Bearer token (string context):", bearerTokenWithStringContext()) + fmt.Println("Bearer token (object context):", bearerTokenWithObjectContext()) +} diff --git a/samples/flowvault/serviceaccount/scoped_token_generation_example.go b/samples/flowvault/serviceaccount/scoped_token_generation_example.go new file mode 100644 index 0000000..1e28918 --- /dev/null +++ b/samples/flowvault/serviceaccount/scoped_token_generation_example.go @@ -0,0 +1,51 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "encoding/json" + "fmt" + + "github.com/skyflowapi/skyflow-go/flowvault/serviceaccount" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" +) + +/** + * This example generates scoped bearer tokens, limited to specific roles, from a credentials + * file path and from a credentials string. + */ + +var ( + filePath = "" + // Scoped tokens grant only the access of the roles listed here. + options = common.BearerTokenOptions{RoleIds: []string{"", ""}} +) + +// credentialsString returns the credentials as a JSON string. +func credentialsString() string { + credentials, _ := json.Marshal(map[string]string{ + "clientID": "", + "clientName": "", + "tokenURI": "", + "keyID": "", + "privateKey": "", + }) + return string(credentials) +} + +func main() { + fromFile, err := serviceaccount.GenerateBearerToken(filePath, options) + if err != nil { + fmt.Println("Error generating token from file path:", err.GetMessage()) + } else { + fmt.Println("Scoped Token (using file path):", fromFile.AccessToken) + } + + fromString, err := serviceaccount.GenerateBearerTokenFromCreds(credentialsString(), options) + if err != nil { + fmt.Println("Error generating token from credentials string:", err.GetMessage()) + } else { + fmt.Println("Scoped Token (using credentials string):", fromString.AccessToken) + } +} diff --git a/samples/flowvault/serviceaccount/signed_token_generation_example.go b/samples/flowvault/serviceaccount/signed_token_generation_example.go new file mode 100644 index 0000000..e48b3af --- /dev/null +++ b/samples/flowvault/serviceaccount/signed_token_generation_example.go @@ -0,0 +1,78 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "encoding/json" + "fmt" + + "github.com/skyflowapi/skyflow-go/flowvault/serviceaccount" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" +) + +/** + * This example generates signed data tokens, with a string context and with an object context. + * Each result pairs a data token with its signed token. + */ + +var filePath = "" + +// credentialsString returns the credentials as a JSON string. +func credentialsString() string { + credentials, _ := json.Marshal(map[string]string{ + "clientID": "", + "clientName": "", + "tokenURI": "", + "keyID": "", + "privateKey": "", + }) + return string(credentials) +} + +func printResults(label string, results []common.SignedDataTokensResponse) { + fmt.Println(label) + for _, result := range results { + fmt.Printf(" Token: %s, Signed Token: %s\n", result.Token, result.SignedToken) + } +} + +// Approach 1: a string context, from a credentials file path. +func signedTokensWithStringContext() { + results, err := serviceaccount.GenerateSignedDataTokens(filePath, common.SignedDataTokensOptions{ + Ctx: "user_12345", + DataTokens: []string{"", ""}, + TimeToLive: 30, // seconds + }) + if err != nil { + fmt.Println("Error:", err.GetMessage()) + return + } + printResults("Signed tokens (string context):", results) +} + +// Approach 2: an object context, from a credentials string. Each key maps to a Skyflow CEL +// policy variable under request.context.*, for example: +// +// request.context.role == "analyst" && request.context.department == "research" +func signedTokensWithObjectContext() { + results, err := serviceaccount.GenerateSignedDataTokensFromCreds(credentialsString(), common.SignedDataTokensOptions{ + Ctx: map[string]interface{}{ + "role": "analyst", + "department": "research", + "user_id": "user_67890", + }, + DataTokens: []string{"", ""}, + TimeToLive: 30, + }) + if err != nil { + fmt.Println("Error:", err.GetMessage()) + return + } + printResults("Signed tokens (object context):", results) +} + +func main() { + signedTokensWithStringContext() + signedTokensWithObjectContext() +} diff --git a/samples/flowvault/vaultapi/custom_header_example.go b/samples/flowvault/vaultapi/custom_header_example.go new file mode 100644 index 0000000..013e644 --- /dev/null +++ b/samples/flowvault/vaultapi/custom_header_example.go @@ -0,0 +1,79 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + + "github.com/skyflowapi/skyflow-go/flowvault/client" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" +) + +/** + * This example sends custom headers with one call, using the Interceptor option. + * + * Every operation's options take an Interceptor. It runs just before the request is sent and + * can add headers with rc.AddHeader. The supported keys are common.SkyflowAccountId, + * common.SkyflowAccountName and common.RequestIdHeader. + * + * Steps: + * 1. Set up credentials and the vault config. + * 2. Configure the Skyflow client. + * 3. Insert a record with an interceptor that adds a request ID header. + * 4. Handle the response and errors. + */ +func main() { + // Step 1: Set up credentials and the vault config + vaultConfig := common.VaultConfig{ + VaultId: "", + ClusterId: "", + Env: common.DEV, + Credentials: common.Credentials{Token: ""}, + } + + // Step 2: Configure the Skyflow client + skyflowClient, err := client.NewSkyflow(client.WithVaults(vaultConfig), client.WithLogLevel(logger.DEBUG)) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + vault, err := skyflowClient.Vault(vaultConfig.VaultId) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + + // Step 3: Insert a record with an interceptor that adds a request ID header + request := common.InsertRequest{ + TableName: "", + Upsert: &common.UpsertOptions{UniqueColumns: []string{""}}, + Records: []common.InsertRequestRecord{{Data: map[string]interface{}{"": ""}}}, + } + options := common.InsertOptions{ + Interceptor: func(rc *common.RequestContext) { + requestID := makeRequestID() + fmt.Println("request id =>", requestID, "for", rc.Operation) + rc.AddHeader(common.RequestIdHeader, requestID) + }, + } + response, err := vault.Insert(context.Background(), request, options) + + // Step 4: Handle the response and errors + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + fmt.Println("Records:", response.Records) +} + +// makeRequestID returns a random ID to trace this call. +func makeRequestID() string { + id := make([]byte, 16) + _, _ = rand.Read(id) + return hex.EncodeToString(id) +} diff --git a/samples/flowvault/vaultapi/delete_files.go b/samples/flowvault/vaultapi/delete_files.go new file mode 100644 index 0000000..fda1690 --- /dev/null +++ b/samples/flowvault/vaultapi/delete_files.go @@ -0,0 +1,69 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "context" + "fmt" + + "github.com/skyflowapi/skyflow-go/flowvault/client" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" +) + +/** + * This example deletes files from the file columns of records in a FlowVault vault, including + * the stored files. + * + * Steps: + * 1. Set up credentials and the vault config. + * 2. Configure the Skyflow client. + * 3. Delete the files. + * 4. Handle the response and errors. + */ +func main() { + // Step 1: Set up credentials and the vault config + credentials := common.Credentials{Path: ""} // or ApiKey / Token / CredentialsString + vaultConfig := common.VaultConfig{ + VaultId: "", + ClusterId: "", // from the vault URL: https://{clusterId}.skyvault.skyflowapis.com + Env: common.PROD, // DEV, STAGE, SANDBOX or PROD + Credentials: credentials, + } + + // Step 2: Configure the Skyflow client + skyflowClient, err := client.NewSkyflow(client.WithVaults(vaultConfig), client.WithLogLevel(logger.ERROR)) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + vault, err := skyflowClient.Vault(vaultConfig.VaultId) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + + // Step 3: Delete the files. Each record sets exactly one of SkyflowId or UniqueValues. + request := common.DeleteFilesRequest{Records: []common.DeleteFilesRequestRecord{ + {TableName: "", Columns: []string{""}, SkyflowId: ""}, + // {TableName: "", Columns: []string{""}, + // UniqueValues: []map[string]interface{}{{"email": "john@example.com"}}}, + }} + response, err := vault.DeleteFiles(context.Background(), request, common.DeleteFilesOptions{}) + + // Step 4: Handle the response and errors. + // One record per matched record. Each has SkyflowId, TableName, Columns (each with Column + // and Status "DELETED"; empty on a failed record), HttpCode, Error and RequestId. + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + for _, record := range response.Records { + if record.Error == "" { + fmt.Printf("%s -> %v\n", record.SkyflowId, record.Columns) + } else { + fmt.Printf("%s failed (%d): %s [requestId=%s]\n", record.SkyflowId, record.HttpCode, record.Error, record.RequestId) + } + } +} diff --git a/samples/flowvault/vaultapi/delete_records.go b/samples/flowvault/vaultapi/delete_records.go new file mode 100644 index 0000000..5a8666d --- /dev/null +++ b/samples/flowvault/vaultapi/delete_records.go @@ -0,0 +1,67 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "context" + "fmt" + + "github.com/skyflowapi/skyflow-go/flowvault/client" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" +) + +/** + * This example deletes records from a FlowVault vault by Skyflow ID. + * + * Steps: + * 1. Set up credentials and the vault config. + * 2. Configure the Skyflow client. + * 3. Delete records. + * 4. Handle the response and errors. + */ +func main() { + // Step 1: Set up credentials and the vault config + credentials := common.Credentials{Path: ""} // or ApiKey / Token / CredentialsString + vaultConfig := common.VaultConfig{ + VaultId: "", + ClusterId: "", // from the vault URL: https://{clusterId}.skyvault.skyflowapis.com + Env: common.PROD, // DEV, STAGE, SANDBOX or PROD + Credentials: credentials, + } + + // Step 2: Configure the Skyflow client + skyflowClient, err := client.NewSkyflow(client.WithVaults(vaultConfig), client.WithLogLevel(logger.ERROR)) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + vault, err := skyflowClient.Vault(vaultConfig.VaultId) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + + // Step 3: Delete records. Set exactly one of SkyflowIds or UniqueValues. + request := common.DeleteRequest{ + TableName: "", + SkyflowIds: []string{"", ""}, + // UniqueValues: []map[string]interface{}{{"email": "john@example.com"}}, + } + response, err := vault.Delete(context.Background(), request, common.DeleteOptions{}) + + // Step 4: Handle the response and errors. + // One record per input, in order. Each has SkyflowId, HttpCode, Error and RequestId. + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + for _, record := range response.Records { + if record.Error == "" { + fmt.Println(record.SkyflowId, "deleted") + } else { + fmt.Printf("%s failed (%d): %s [requestId=%s]\n", record.SkyflowId, record.HttpCode, record.Error, record.RequestId) + } + } +} diff --git a/samples/flowvault/vaultapi/detokenize_records.go b/samples/flowvault/vaultapi/detokenize_records.go new file mode 100644 index 0000000..3cc7201 --- /dev/null +++ b/samples/flowvault/vaultapi/detokenize_records.go @@ -0,0 +1,67 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "context" + "fmt" + + "github.com/skyflowapi/skyflow-go/flowvault/client" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" +) + +/** + * This example returns the values behind tokens in a FlowVault vault. + * + * Steps: + * 1. Set up credentials and the vault config. + * 2. Configure the Skyflow client. + * 3. Detokenize tokens. + * 4. Handle the response and errors. + */ +func main() { + // Step 1: Set up credentials and the vault config + credentials := common.Credentials{Path: ""} // or ApiKey / Token / CredentialsString + vaultConfig := common.VaultConfig{ + VaultId: "", + ClusterId: "", // from the vault URL: https://{clusterId}.skyvault.skyflowapis.com + Env: common.PROD, // DEV, STAGE, SANDBOX or PROD + Credentials: credentials, + } + + // Step 2: Configure the Skyflow client + skyflowClient, err := client.NewSkyflow(client.WithVaults(vaultConfig), client.WithLogLevel(logger.ERROR)) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + vault, err := skyflowClient.Vault(vaultConfig.VaultId) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + + // Step 3: Detokenize tokens. Redactions are set per token group. + request := common.DetokenizeRequest{ + Tokens: []string{"", ""}, + // TokenGroupRedactions: []common.TokenGroupRedaction{{TokenGroupName: "", Redaction: ""}}, + } + response, err := vault.Detokenize(context.Background(), request, common.DetokenizeOptions{}) + + // Step 4: Handle the response and errors. + // One record per input token, in order. Each has Token, Value, TokenGroupName, Metadata + // (SkyflowId and TableName), HttpCode, Error and RequestId. + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + for _, record := range response.Records { + if record.Error == "" { + fmt.Printf("%s -> %v (%s)\n", record.Token, record.Value, record.TokenGroupName) + } else { + fmt.Printf("%s failed (%d): %s [requestId=%s]\n", record.Token, record.HttpCode, record.Error, record.RequestId) + } + } +} diff --git a/samples/flowvault/vaultapi/get_records.go b/samples/flowvault/vaultapi/get_records.go new file mode 100644 index 0000000..e83beb1 --- /dev/null +++ b/samples/flowvault/vaultapi/get_records.go @@ -0,0 +1,78 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "context" + "fmt" + + "github.com/skyflowapi/skyflow-go/flowvault/client" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" +) + +/** + * This example retrieves records from a FlowVault vault by Skyflow ID. + * + * Steps: + * 1. Set up credentials and the vault config. + * 2. Configure the Skyflow client. + * 3. Get records. + * 4. Handle the response and errors. + */ +func main() { + // Step 1: Set up credentials and the vault config + credentials := common.Credentials{Path: ""} // or ApiKey / Token / CredentialsString + vaultConfig := common.VaultConfig{ + VaultId: "", + ClusterId: "", // from the vault URL: https://{clusterId}.skyvault.skyflowapis.com + Env: common.PROD, // DEV, STAGE, SANDBOX or PROD + Credentials: credentials, + } + + // Step 2: Configure the Skyflow client + skyflowClient, err := client.NewSkyflow(client.WithVaults(vaultConfig), client.WithLogLevel(logger.ERROR)) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + vault, err := skyflowClient.Vault(vaultConfig.VaultId) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + + // Step 3: Get records by Skyflow ID. Set exactly one of SkyflowIds or UniqueValues. + request := common.GetRequest{ + TableName: "", + SkyflowIds: []string{"", ""}, + // Columns: []string{"name", "email"}, // return only these columns + // ColumnRedactions: []common.ColumnRedaction{{ColumnName: "email", Redaction: ""}}, + // Limit: 25, Offset: 0, + } + // To look records up by a unique column instead of by Skyflow ID: + // request := common.GetRequest{TableName: "", UniqueValues: []map[string]interface{}{{"email": "john@example.com"}}} + // + // To get from several tables in one call, use Records instead of the fields above: + // request := common.GetRequest{Records: []common.GetRequestRecord{ + // {TableName: "", SkyflowIds: []string{""}}, + // {TableName: "", UniqueValues: []map[string]interface{}{{"email": "john@example.com"}}}, + // }} + response, err := vault.Get(context.Background(), request, common.GetOptions{}) + + // Step 4: Handle the response and errors. + // One record per input, in order. Each has SkyflowId, TableName, Data, Tokens, HashedData, + // HttpCode, Error and RequestId. + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + for _, record := range response.Records { + if record.Error == "" { + fmt.Printf("%s -> data=%v\n", record.SkyflowId, record.Data) + } else { + fmt.Printf("failed (%d): %s [requestId=%s]\n", record.HttpCode, record.Error, record.RequestId) + } + } +} diff --git a/samples/flowvault/vaultapi/get_tokens.go b/samples/flowvault/vaultapi/get_tokens.go new file mode 100644 index 0000000..7b226d3 --- /dev/null +++ b/samples/flowvault/vaultapi/get_tokens.go @@ -0,0 +1,69 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "context" + "fmt" + + "github.com/skyflowapi/skyflow-go/flowvault/client" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" +) + +/** + * This example looks up the existing tokens for values in a FlowVault vault. + * + * Only deterministic token groups are supported, and no new tokens are created. + * + * Steps: + * 1. Set up credentials and the vault config. + * 2. Configure the Skyflow client. + * 3. Get tokens for values. + * 4. Handle the response and errors. + */ +func main() { + // Step 1: Set up credentials and the vault config + credentials := common.Credentials{Path: ""} // or ApiKey / Token / CredentialsString + vaultConfig := common.VaultConfig{ + VaultId: "", + ClusterId: "", // from the vault URL: https://{clusterId}.skyvault.skyflowapis.com + Env: common.PROD, // DEV, STAGE, SANDBOX or PROD + Credentials: credentials, + } + + // Step 2: Configure the Skyflow client + skyflowClient, err := client.NewSkyflow(client.WithVaults(vaultConfig), client.WithLogLevel(logger.ERROR)) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + vault, err := skyflowClient.Vault(vaultConfig.VaultId) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + + // Step 3: Get tokens for values + request := common.GetTokensRequest{Records: []common.GetTokensRequestRecord{ + {Value: "john@example.com", TokenGroupName: ""}, + {Value: "jane@example.com", TokenGroupName: ""}, + }} + response, err := vault.GetTokens(context.Background(), request, common.GetTokensOptions{}) + + // Step 4: Handle the response and errors. + // One record per input, in order. Each has Value, TokenGroupName, Token, HttpCode, Error + // and RequestId. A value with no token has Error set. + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + for _, record := range response.Records { + if record.Error == "" { + fmt.Printf("%v -> %s\n", record.Value, record.Token) + } else { + fmt.Printf("%v failed (%d): %s [requestId=%s]\n", record.Value, record.HttpCode, record.Error, record.RequestId) + } + } +} diff --git a/samples/flowvault/vaultapi/insert_records.go b/samples/flowvault/vaultapi/insert_records.go new file mode 100644 index 0000000..ec101e1 --- /dev/null +++ b/samples/flowvault/vaultapi/insert_records.go @@ -0,0 +1,83 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "context" + "fmt" + + "github.com/skyflowapi/skyflow-go/flowvault/client" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" +) + +/** + * This example inserts records into a FlowVault vault and prints the tokens returned for each. + * + * The table name and upsert are set at exactly ONE level: on the request (applies to every + * record) or on every record. Setting them on both, or mixing the two, is a validation error. + * + * Steps: + * 1. Set up credentials and the vault config. + * 2. Configure the Skyflow client. + * 3. Insert records. + * 4. Handle the response and errors. + */ +func main() { + // Step 1: Set up credentials and the vault config + credentials := common.Credentials{Path: ""} // or ApiKey / Token / CredentialsString + vaultConfig := common.VaultConfig{ + VaultId: "", + ClusterId: "", // from the vault URL: https://{clusterId}.skyvault.skyflowapis.com + Env: common.PROD, // DEV, STAGE, SANDBOX or PROD + Credentials: credentials, + } + + // Step 2: Configure the Skyflow client + skyflowClient, err := client.NewSkyflow(client.WithVaults(vaultConfig), client.WithLogLevel(logger.ERROR)) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + vault, err := skyflowClient.Vault(vaultConfig.VaultId) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + + // Step 3: Insert records. The table name and upsert here apply to every record. + request := common.InsertRequest{ + TableName: "", + Upsert: &common.UpsertOptions{UniqueColumns: []string{"email"}, UpdateType: common.UpdateTypeUpdate}, + Records: []common.InsertRequestRecord{ + {Data: map[string]interface{}{"name": "John Doe", "email": "john@example.com"}}, + }, + } + // To set the table (and upsert) per record instead, for example to insert into several + // tables in one call, leave TableName and Upsert empty on the request: + // + // request := common.InsertRequest{ + // Records: []common.InsertRequestRecord{ + // {TableName: "", Data: map[string]interface{}{"name": "Jane Doe"}}, + // {TableName: "", Data: map[string]interface{}{"email": "jane@example.com"}, + // Upsert: &common.UpsertOptions{UniqueColumns: []string{"email"}, UpdateType: common.UpdateTypeReplace}}, + // }, + // } + response, err := vault.Insert(context.Background(), request, common.InsertOptions{}) + + // Step 4: Handle the response and errors. + // Validation errors and whole-call failures come back as err. Otherwise there is one record + // per input, in order; a record that failed has Error, HttpCode and RequestId set. + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + for _, record := range response.Records { + if record.Error == "" { + fmt.Printf("%s -> tokens=%v\n", record.SkyflowId, record.Tokens) + } else { + fmt.Printf("failed (%d): %s [requestId=%s]\n", record.HttpCode, record.Error, record.RequestId) + } + } +} diff --git a/samples/flowvault/vaultapi/query_records.go b/samples/flowvault/vaultapi/query_records.go new file mode 100644 index 0000000..dd4e58f --- /dev/null +++ b/samples/flowvault/vaultapi/query_records.go @@ -0,0 +1,63 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "context" + "fmt" + + "github.com/skyflowapi/skyflow-go/flowvault/client" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" +) + +/** + * This example runs a read-only SQL query against a FlowVault vault. + * + * Only SELECT is supported, and a call returns at most 25 rows. Page with OFFSET. + * + * Steps: + * 1. Set up credentials and the vault config. + * 2. Configure the Skyflow client. + * 3. Run the query. + * 4. Handle the response and errors. + */ +func main() { + // Step 1: Set up credentials and the vault config + credentials := common.Credentials{Path: ""} // or ApiKey / Token / CredentialsString + vaultConfig := common.VaultConfig{ + VaultId: "", + ClusterId: "", // from the vault URL: https://{clusterId}.skyvault.skyflowapis.com + Env: common.PROD, // DEV, STAGE, SANDBOX or PROD + Credentials: credentials, + } + + // Step 2: Configure the Skyflow client + skyflowClient, err := client.NewSkyflow(client.WithVaults(vaultConfig), client.WithLogLevel(logger.ERROR)) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + vault, err := skyflowClient.Vault(vaultConfig.VaultId) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + + // Step 3: Run the query + request := common.QueryRequest{Query: "SELECT name, email FROM LIMIT 25"} + response, err := vault.Query(context.Background(), request, common.QueryOptions{}) + + // Step 4: Handle the response and errors. + // Query has no per-record status: any failure comes back as err. + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + fmt.Println("Request ID:", response.RequestId) + fmt.Println("Columns:", response.Metadata.Columns) + for _, row := range response.Fields { + fmt.Println(row) + } +} diff --git a/samples/flowvault/vaultapi/timeout_and_retry_config_example.go b/samples/flowvault/vaultapi/timeout_and_retry_config_example.go new file mode 100644 index 0000000..cf93f17 --- /dev/null +++ b/samples/flowvault/vaultapi/timeout_and_retry_config_example.go @@ -0,0 +1,69 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "fmt" + "time" + + "github.com/skyflowapi/skyflow-go/flowvault/client" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" +) + +/** + * This example sets HTTP timeout and retry settings, for the whole client and for one vault. + * + * Each setting is resolved separately: the vault's value, then the client's, then the SDK + * default. + * + * Timeout overall limit for one call, including retries and backoff. Default 60s. + * MaxRetries retries after the first attempt, for 408, 429 and 5xx. Default 0 (off). + * InitialRetryDelay wait before the first retry. Default 500ms. + * MaxRetryDelay the most the wait can grow to; it doubles each retry. Default 2000ms. + * + * A context deadline still applies when it is shorter. File uploads in UploadFiles are not + * retried and have no overall timeout, so large files aren't cut off. + * + * Steps: + * 1. Set up credentials and the vault config, with per-vault overrides. + * 2. Configure the Skyflow client with client-wide defaults. + */ +func main() { + // Step 1: Set up credentials and the vault config, with per-vault overrides + vaultRetries := 2 + vaultConfig := common.VaultConfig{ + VaultId: "", + ClusterId: "", + Env: common.PROD, + Credentials: common.Credentials{Path: ""}, + // These win over the client-wide values below, for this vault only. + Timeout: 30 * time.Second, + MaxRetries: &vaultRetries, + } + + // Step 2: Configure the Skyflow client with client-wide defaults. + // Any vault that doesn't set a field uses these. + clientRetries := 3 + skyflowClient, err := client.NewSkyflow( + client.WithHttpConfig(common.HttpConfig{ + Timeout: 60 * time.Second, + MaxRetries: &clientRetries, + InitialRetryDelay: 500 * time.Millisecond, + MaxRetryDelay: 4 * time.Second, + }), + client.WithVaults(vaultConfig), + client.WithLogLevel(logger.ERROR), + ) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + fmt.Println("Skyflow client configured with custom timeout and retry settings:", skyflowClient != nil) + + // Use the client as usual; calls now fail at the configured timeouts and retry 408, 429 and + // 5xx responses with exponential backoff and jitter. + // vault, _ := skyflowClient.Vault(vaultConfig.VaultId) + // response, err := vault.Detokenize(context.Background(), detokenizeRequest, common.DetokenizeOptions{}) +} diff --git a/samples/flowvault/vaultapi/update_record.go b/samples/flowvault/vaultapi/update_record.go new file mode 100644 index 0000000..e4299cf --- /dev/null +++ b/samples/flowvault/vaultapi/update_record.go @@ -0,0 +1,71 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "context" + "fmt" + + "github.com/skyflowapi/skyflow-go/flowvault/client" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" +) + +/** + * This example updates a record in a FlowVault vault. + * + * UpdateType is set on the request: UPDATE (the default) changes only the given columns, + * REPLACE overwrites the whole record. + * + * Steps: + * 1. Set up credentials and the vault config. + * 2. Configure the Skyflow client. + * 3. Update the record. + * 4. Handle the response and errors. + */ +func main() { + // Step 1: Set up credentials and the vault config + credentials := common.Credentials{Path: ""} // or ApiKey / Token / CredentialsString + vaultConfig := common.VaultConfig{ + VaultId: "", + ClusterId: "", // from the vault URL: https://{clusterId}.skyvault.skyflowapis.com + Env: common.PROD, // DEV, STAGE, SANDBOX or PROD + Credentials: credentials, + } + + // Step 2: Configure the Skyflow client + skyflowClient, err := client.NewSkyflow(client.WithVaults(vaultConfig), client.WithLogLevel(logger.ERROR)) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + vault, err := skyflowClient.Vault(vaultConfig.VaultId) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + + // Step 3: Update the record. Set the table name on the request or on every record, not both. + request := common.UpdateRequest{ + TableName: "", + UpdateType: common.UpdateTypeUpdate, + Records: []common.UpdateRequestRecord{ + {SkyflowId: "", Data: map[string]interface{}{"name": "Jane Doe"}}, + }, + } + response, err := vault.Update(context.Background(), request, common.UpdateOptions{}) + + // Step 4: Handle the response and errors + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + for _, record := range response.Records { + if record.Error == "" { + fmt.Printf("%s updated -> tokens=%v\n", record.SkyflowId, record.Tokens) + } else { + fmt.Printf("%s failed (%d): %s [requestId=%s]\n", record.SkyflowId, record.HttpCode, record.Error, record.RequestId) + } + } +} diff --git a/samples/flowvault/vaultapi/upload_files.go b/samples/flowvault/vaultapi/upload_files.go new file mode 100644 index 0000000..78ab8d7 --- /dev/null +++ b/samples/flowvault/vaultapi/upload_files.go @@ -0,0 +1,89 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "context" + "fmt" + "os" + + "github.com/skyflowapi/skyflow-go/flowvault/client" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" +) + +/** + * This example uploads files into the file columns of a FlowVault vault. + * + * The SDK requests a signed upload URL for each column, then uploads each file to its URL. + * + * Steps: + * 1. Set up credentials and the vault config. + * 2. Configure the Skyflow client. + * 3. Upload the files. + * 4. Handle the response and errors. + */ +func main() { + // Step 1: Set up credentials and the vault config + credentials := common.Credentials{Path: ""} // or ApiKey / Token / CredentialsString + vaultConfig := common.VaultConfig{ + VaultId: "", + ClusterId: "", // from the vault URL: https://{clusterId}.skyvault.skyflowapis.com + Env: common.PROD, // DEV, STAGE, SANDBOX or PROD + Credentials: credentials, + } + + // Step 2: Configure the Skyflow client + skyflowClient, err := client.NewSkyflow(client.WithVaults(vaultConfig), client.WithLogLevel(logger.ERROR)) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + vault, err := skyflowClient.Vault(vaultConfig.VaultId) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + + // Step 3: Upload the files. Each column takes exactly one of FilePath, Base64 (needs + // FileName) or FileObject. Leave SkyflowId empty to create a new record, or set it to upload + // into an existing record. + file, openErr := os.Open("") + if openErr != nil { + fmt.Println("Could not open file:", openErr) + return + } + defer file.Close() // the caller closes a FileObject + + request := common.UploadFilesRequest{Records: []common.UploadFilesRequestRecord{ + { + TableName: "", + Columns: []common.UploadFilesRequestColumn{ + {Column: "", FilePath: ""}, + {Column: "", Base64: "", FileName: ""}, + {Column: "", FileObject: file}, + }, + }, + // {TableName: "", SkyflowId: "", + // Columns: []common.UploadFilesRequestColumn{{Column: "", FilePath: ""}}}, + }} + response, err := vault.UploadFiles(context.Background(), request, common.UploadFilesOptions{}) + + // Step 4: Handle the response and errors. + // One record per input record, in order. Each has SkyflowId, TableName, Columns, HttpCode, + // Error and RequestId. Each column has Column, FileName, UploadStatus (UPLOADED, FAILED or + // SKIPPED) and Error. + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + for _, record := range response.Records { + if record.Error != "" { + fmt.Printf("record failed (%d): %s [requestId=%s]\n", record.HttpCode, record.Error, record.RequestId) + } + for _, column := range record.Columns { + fmt.Printf("%s %s: %s %s\n", record.SkyflowId, column.Column, column.UploadStatus, column.Error) + } + } +} diff --git a/v2/client/client.go b/v2/client/client.go index 627ba25..fc3e014 100644 --- a/v2/client/client.go +++ b/v2/client/client.go @@ -310,7 +310,9 @@ func (s *Skyflow) UpdateVaultConfig(updatedConfig vaultutils.VaultConfig) *error if updatedConfig.ClusterId != "" { vaultSvc.controller.Config.ClusterId = updatedConfig.ClusterId } - vaultSvc.controller.Config.Env = updatedConfig.Env + if updatedConfig.Env.IsSet() { + vaultSvc.controller.Config.Env = updatedConfig.Env + } } // Update the credentials in the detect controller if provided if detectExists && detectSvc.controller != nil { @@ -322,7 +324,9 @@ func (s *Skyflow) UpdateVaultConfig(updatedConfig vaultutils.VaultConfig) *error if updatedConfig.ClusterId != "" { detectSvc.controller.Config.ClusterId = updatedConfig.ClusterId } - detectSvc.controller.Config.Env = updatedConfig.Env + if updatedConfig.Env.IsSet() { + detectSvc.controller.Config.Env = updatedConfig.Env + } } // Update the config in the vaultapi service diff --git a/v2/client/operations.go b/v2/client/operations.go new file mode 100644 index 0000000..7351c42 --- /dev/null +++ b/v2/client/operations.go @@ -0,0 +1,17 @@ +package client + +import ( + "github.com/skyflowapi/skyflow-go/common/vault" + "github.com/skyflowapi/skyflow-go/v2/utils/common" +) + +// Compile-time checks that the v2 vault service implements the operations shared with +// the other Skyflow Go SDKs. +var ( + _ vault.Inserter[common.InsertRequest, common.InsertOptions, common.InsertResponse] = (*vaultService)(nil) + _ vault.Updater[common.UpdateRequest, common.UpdateOptions, common.UpdateResponse] = (*vaultService)(nil) + _ vault.Getter[common.GetRequest, common.GetOptions, common.GetResponse] = (*vaultService)(nil) + _ vault.Deleter[common.DeleteRequest, common.DeleteOptions, common.DeleteResponse] = (*vaultService)(nil) + _ vault.Querier[common.QueryRequest, common.QueryOptions, common.QueryResponse] = (*vaultService)(nil) + _ vault.Detokenizer[common.DetokenizeRequest, common.DetokenizeOptions, common.DetokenizeResponse] = (*vaultService)(nil) +) diff --git a/v2/go.mod b/v2/go.mod index dddfeae..ed9499d 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -7,7 +7,7 @@ require ( github.com/google/uuid v1.6.0 github.com/onsi/ginkgo/v2 v2.23.4 github.com/onsi/gomega v1.38.0 - github.com/skyflowapi/skyflow-go/common v0.0.0-20261014151608-a64b46a2705c + github.com/skyflowapi/skyflow-go/common v0.0.0-20261019104456-6115e81fcef9 github.com/stretchr/testify v1.8.4 ) diff --git a/v2/go.sum b/v2/go.sum index 63e5a48..3c0319c 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -24,8 +24,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= -github.com/skyflowapi/skyflow-go/common v0.0.0-20261014151608-a64b46a2705c h1:oWxTNYJ++iBGbcVpJ2oNhyB36WNNVMyBpusEAcSL7pw= -github.com/skyflowapi/skyflow-go/common v0.0.0-20261014151608-a64b46a2705c/go.mod h1:T9roW4Qpv1WFo6tgLjbfLrdOCOz20lVE8ENf2bGb7zI= +github.com/skyflowapi/skyflow-go/common v0.0.0-20261019104456-6115e81fcef9 h1:0SNuvxkS+mr+sy7lDZvwmgjgAIC+HrqEr81lsTTXEgQ= +github.com/skyflowapi/skyflow-go/common v0.0.0-20261019104456-6115e81fcef9/go.mod h1:T9roW4Qpv1WFo6tgLjbfLrdOCOz20lVE8ENf2bGb7zI= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= diff --git a/v2/utils/error/skyflow_exception_test.go b/v2/utils/error/skyflow_exception_test.go new file mode 100644 index 0000000..ddc3268 --- /dev/null +++ b/v2/utils/error/skyflow_exception_test.go @@ -0,0 +1,58 @@ +package errors_test + +import ( + "errors" + "io" + "net/http" + "strings" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + skyflowError "github.com/skyflowapi/skyflow-go/v2/utils/error" +) + +// The error logic lives in skyflow-go/common/errors and is tested there. These specs check +// that v2's wrappers forward to it and keep v2's error codes. + +func TestSkyflowError(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "SkyflowError Suite") +} + +const apiErrorBody = `{"error":{"grpc_code":3,"http_code":400,"http_status":"Bad Request","message":"Invalid request","details":[]}}` + +var _ = Describe("SkyflowError", func() { + It("creates an error with a code and message", func() { + err := skyflowError.NewSkyflowError(skyflowError.INVALID_INPUT_CODE, "bad input") + Expect(err.GetHttpCode()).To(Equal(string(skyflowError.INVALID_INPUT_CODE))) + Expect(err.GetMessage()).To(ContainSubstring("bad input")) + }) + + It("keeps v2's error codes", func() { + Expect(string(skyflowError.SERVER)).To(Equal("500")) + Expect(string(skyflowError.INVALID_INPUT_CODE)).To(Equal("400")) + Expect(string(skyflowError.PARTIAL)).To(Equal("404")) + }) + + It("builds an error from an API response", func() { + response := http.Response{ + StatusCode: http.StatusBadRequest, + Header: http.Header{"Content-Type": {"application/json"}, "X-Request-Id": {"req-1"}}, + Body: io.NopCloser(strings.NewReader(apiErrorBody)), + } + err := skyflowError.SkyflowApiError(response) + Expect(err.GetHttpCode()).To(Equal("400")) + Expect(err.GetMessage()).To(ContainSubstring("Invalid request")) + Expect(err.GetGrpcCode()).To(Equal("3")) + Expect(err.GetRequestId()).To(Equal("req-1")) + }) + + It("builds an error from a generated client error", func() { + header := http.Header{"X-Request-Id": {"req-2"}} + err := skyflowError.SkyflowErrorApi(errors.New("400: "+apiErrorBody), header) + Expect(err.GetHttpCode()).To(Equal("400")) + Expect(err.GetMessage()).To(ContainSubstring("Invalid request")) + Expect(err.GetRequestId()).To(Equal("req-2")) + }) +})