From 6115e81fcef9e1499653f4622cf0404fdf733c9c Mon Sep 17 00:00:00 2001 From: Himanshu Pal Date: Mon, 19 Oct 2026 16:14:56 +0530 Subject: [PATCH 1/5] SK-3223 add shared vault interfaces, HTTP config and unset Env to common - common/vault: generic per-operation interfaces (Inserter, Updater, Getter, Deleter, Querier, Detokenizer, TokenGetter, FilesUploader, FilesDeleter) that the v2 and flowvault vault services both implement. - config: HttpConfig and the Timeout, MaxRetries, InitialRetryDelay and MaxRetryDelay fields on VaultConfig, rejected when negative. - config: Env now starts at 1 so the zero value means "not set" (resolves to PROD); Env.IsSet() lets updates keep the current Env. Co-Authored-By: Claude Opus 5.5 --- common/config/config.go | 38 ++++++++++++++++++++++- common/config/messages.go | 2 ++ common/config/validate.go | 9 ++++++ common/vault/operations.go | 63 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 common/vault/operations.go diff --git a/common/config/config.go b/common/config/config.go index 7da6197..ef65416 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -7,15 +7,25 @@ // its own, because the vault APIs differ between products. package config +import "time" + +// Env is the Skyflow environment a vault runs in. The zero value means "not set": it +// resolves to PROD when the vault URL is built, and UpdateVaultConfig leaves the current +// Env unchanged when the update doesn't set one. type Env int const ( - PROD Env = iota + PROD Env = iota + 1 STAGE SANDBOX DEV ) +// IsSet reports whether the Env was set explicitly. +func (e Env) IsSet() bool { + return e != 0 +} + type Credentials struct { Path string Roles []string @@ -33,6 +43,32 @@ type VaultConfig struct { BaseVaultURL string Env Env Credentials Credentials + + // Optional HTTP settings for this vault. Zero values (and a nil MaxRetries) + // mean "not set": the client-level HttpConfig applies, then the SDK default. + Timeout time.Duration + MaxRetries *int + InitialRetryDelay time.Duration + MaxRetryDelay time.Duration +} + +// HttpConfig holds client-level call timeout and retry settings. A value set on +// VaultConfig wins over the one here. +type HttpConfig struct { + Timeout time.Duration + MaxRetries *int + InitialRetryDelay time.Duration + MaxRetryDelay time.Duration +} + +// HttpConfig returns the vault's own HTTP settings. +func (c VaultConfig) HttpConfig() HttpConfig { + return HttpConfig{ + Timeout: c.Timeout, + MaxRetries: c.MaxRetries, + InitialRetryDelay: c.InitialRetryDelay, + MaxRetryDelay: c.MaxRetryDelay, + } } type ConnectionConfig struct { diff --git a/common/config/messages.go b/common/config/messages.go index e7cbedf..f1c285c 100644 --- a/common/config/messages.go +++ b/common/config/messages.go @@ -12,6 +12,7 @@ const ( errInvalidApiKey = " Initialization failed. Invalid credentials. Specify valid api key." errEmptyRoles = " Initialization failed. Invalid roles. Specify at least one role." errEmptyRoleInRoles = " Initialization failed. Invalid role. Specify a valid role." + errNegativeHttpConfig = " Initialization failed. Invalid HTTP config. Timeout, retry count and retry delays must not be negative." ) const ( @@ -24,4 +25,5 @@ const ( logInvalidApiKey = "Invalid credentials. Api key is invalid." logEmptyRoles = "Invalid credentials. Roles can not be empty." logEmptyOrNullRoleInRoles = "Invalid credentials. Role can not be null or empty in roles at index %v." + logNegativeHttpConfig = "Invalid HTTP config. Timeout, retry count and retry delays must not be negative." ) diff --git a/common/config/validate.go b/common/config/validate.go index e562fe7..7f34626 100644 --- a/common/config/validate.go +++ b/common/config/validate.go @@ -55,6 +55,15 @@ func (v Validator) ValidateVaultConfig(vaultConfig VaultConfig) *skyflowError.Sk } else if !isValidHTTPURL(baseVaultUrl) { return v.invalid(logVaultUrlIsInvalid, errInvalidVaultUrl) } + return v.ValidateHttpConfig(vaultConfig.HttpConfig()) +} + +// ValidateHttpConfig checks that timeout, retry count and retry delays are not negative. +func (v Validator) ValidateHttpConfig(httpConfig HttpConfig) *skyflowError.SkyflowError { + if httpConfig.Timeout < 0 || httpConfig.InitialRetryDelay < 0 || httpConfig.MaxRetryDelay < 0 || + (httpConfig.MaxRetries != nil && *httpConfig.MaxRetries < 0) { + return v.invalid(logNegativeHttpConfig, errNegativeHttpConfig) + } return nil } diff --git a/common/vault/operations.go b/common/vault/operations.go new file mode 100644 index 0000000..8e8204a --- /dev/null +++ b/common/vault/operations.go @@ -0,0 +1,63 @@ +// Package vault defines the vault operations shared by the Skyflow Go SDKs. +// +// The SDKs (v2 for SkyVault, flowvault for FlowVault) expose the same methods but with +// their own request, options and response types, because the vault APIs differ. Each +// interface here is generic over those types, so both SDKs' vault services satisfy the +// same contract: +// +// var _ vault.Inserter[common.InsertRequest, common.InsertOptions, common.InsertResponse] = (*vaultService)(nil) +// +// Code that works with either SDK can depend on these interfaces instead of a concrete +// vault service. +package vault + +import ( + "context" + + skyflowError "github.com/skyflowapi/skyflow-go/common/errors" +) + +// Inserter inserts records into a vault. +type Inserter[Req, Opts, Resp any] interface { + Insert(ctx context.Context, request Req, options Opts) (*Resp, *skyflowError.SkyflowError) +} + +// Updater updates existing records in a vault. +type Updater[Req, Opts, Resp any] interface { + Update(ctx context.Context, request Req, options Opts) (*Resp, *skyflowError.SkyflowError) +} + +// Getter retrieves records from a vault. +type Getter[Req, Opts, Resp any] interface { + Get(ctx context.Context, request Req, options Opts) (*Resp, *skyflowError.SkyflowError) +} + +// Deleter deletes records from a vault. +type Deleter[Req, Opts, Resp any] interface { + Delete(ctx context.Context, request Req, options Opts) (*Resp, *skyflowError.SkyflowError) +} + +// Querier runs a query against a vault. +type Querier[Req, Opts, Resp any] interface { + Query(ctx context.Context, request Req, options Opts) (*Resp, *skyflowError.SkyflowError) +} + +// Detokenizer returns the values behind tokens. +type Detokenizer[Req, Opts, Resp any] interface { + Detokenize(ctx context.Context, request Req, options Opts) (*Resp, *skyflowError.SkyflowError) +} + +// TokenGetter returns existing tokens for values. +type TokenGetter[Req, Opts, Resp any] interface { + GetTokens(ctx context.Context, request Req, options Opts) (*Resp, *skyflowError.SkyflowError) +} + +// FilesUploader uploads files into file columns. +type FilesUploader[Req, Opts, Resp any] interface { + UploadFiles(ctx context.Context, request Req, options Opts) (*Resp, *skyflowError.SkyflowError) +} + +// FilesDeleter deletes files from file columns. +type FilesDeleter[Req, Opts, Resp any] interface { + DeleteFiles(ctx context.Context, request Req, options Opts) (*Resp, *skyflowError.SkyflowError) +} From 5789398fc8e90efee0369301e3deb3ffb5e07346 Mon Sep 17 00:00:00 2001 From: Himanshu Pal Date: Mon, 19 Oct 2026 16:15:37 +0530 Subject: [PATCH 2/5] SK-3223 implement shared vault interfaces in v2 and keep Env on update - v2/client/operations.go: compile-time checks that the v2 vault service implements the common/vault Insert, Update, Get, Delete, Query and Detokenize interfaces. No behaviour change. - UpdateVaultConfig only changes Env when the update sets one, instead of resetting it to PROD. Co-Authored-By: Claude Opus 5.5 --- v2/client/client.go | 8 ++++++-- v2/client/operations.go | 17 +++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 v2/client/operations.go diff --git a/v2/client/client.go b/v2/client/client.go index 627ba25..fc3e014 100644 --- a/v2/client/client.go +++ b/v2/client/client.go @@ -310,7 +310,9 @@ func (s *Skyflow) UpdateVaultConfig(updatedConfig vaultutils.VaultConfig) *error if updatedConfig.ClusterId != "" { vaultSvc.controller.Config.ClusterId = updatedConfig.ClusterId } - vaultSvc.controller.Config.Env = updatedConfig.Env + if updatedConfig.Env.IsSet() { + vaultSvc.controller.Config.Env = updatedConfig.Env + } } // Update the credentials in the detect controller if provided if detectExists && detectSvc.controller != nil { @@ -322,7 +324,9 @@ func (s *Skyflow) UpdateVaultConfig(updatedConfig vaultutils.VaultConfig) *error if updatedConfig.ClusterId != "" { detectSvc.controller.Config.ClusterId = updatedConfig.ClusterId } - detectSvc.controller.Config.Env = updatedConfig.Env + if updatedConfig.Env.IsSet() { + detectSvc.controller.Config.Env = updatedConfig.Env + } } // Update the config in the vaultapi service diff --git a/v2/client/operations.go b/v2/client/operations.go new file mode 100644 index 0000000..7351c42 --- /dev/null +++ b/v2/client/operations.go @@ -0,0 +1,17 @@ +package client + +import ( + "github.com/skyflowapi/skyflow-go/common/vault" + "github.com/skyflowapi/skyflow-go/v2/utils/common" +) + +// Compile-time checks that the v2 vault service implements the operations shared with +// the other Skyflow Go SDKs. +var ( + _ vault.Inserter[common.InsertRequest, common.InsertOptions, common.InsertResponse] = (*vaultService)(nil) + _ vault.Updater[common.UpdateRequest, common.UpdateOptions, common.UpdateResponse] = (*vaultService)(nil) + _ vault.Getter[common.GetRequest, common.GetOptions, common.GetResponse] = (*vaultService)(nil) + _ vault.Deleter[common.DeleteRequest, common.DeleteOptions, common.DeleteResponse] = (*vaultService)(nil) + _ vault.Querier[common.QueryRequest, common.QueryOptions, common.QueryResponse] = (*vaultService)(nil) + _ vault.Detokenizer[common.DetokenizeRequest, common.DetokenizeOptions, common.DetokenizeResponse] = (*vaultService)(nil) +) From c3bb0cd76638ee3a2d5a96abaf0a0d3d3c2f18da Mon Sep 17 00:00:00 2001 From: Himanshu Pal Date: Mon, 19 Oct 2026 16:15:38 +0530 Subject: [PATCH 3/5] SK-3223 implement unary flowvault interfaces and add samples Implements Insert, Update, Get, Delete, Detokenize, GetTokens, Query, UploadFiles and DeleteFiles on the generated FlowService /v2 client. - Validation matching the Java and Python FlowVault SDKs: table name and upsert at exactly one level, request-level UpdateType, unique column checks, DeleteFiles record limit and duplicate columns. - Per-call headers through an Interceptor on every operation's options. - Timeout and retries (408, 429, 5xx with backoff and jitter) configured per client (WithHttpConfig) or per vault; off by default. - UploadFiles requests signed URLs, then uploads at most 5 files at once with a separate client: no retries, no overall timeout, no SDK headers. - Per-record results for partial failures, including non-2xx responses that carry records; whole-call failures map to SkyflowError. - Client is safe for concurrent use; credential changes drop cached tokens. - FlowVault vaults are reached on the skyvault subdomain. - SDK name skyflow-flowvault-go in the metadata header. - Removes the unwired client-level WithCustomHeaders. - Unit tests against a fake FlowService server; fake keys live in flowvault/test/resources. - samples/flowvault: one sample per operation, custom headers, timeout/retry config and service-account token generation. Co-Authored-By: Claude Opus 5.5 --- flowvault/client/client.go | 122 +++++-- flowvault/client/client_test.go | 210 +++++++++++ flowvault/client/operations.go | 20 ++ flowvault/client/vault_service.go | 114 ++---- flowvault/go.mod | 24 +- flowvault/go.sum | 63 +++- flowvault/internal/constants/constants.go | 59 +++- flowvault/internal/helpers/files.go | 104 ++++++ flowvault/internal/helpers/helpers.go | 2 +- flowvault/internal/helpers/helpers_test.go | 224 ++++++++++++ flowvault/internal/helpers/http.go | 147 ++++++++ flowvault/internal/helpers/parse.go | 123 +++++++ flowvault/internal/testutil/resources.go | 22 ++ flowvault/internal/validation/validations.go | 307 +++++++++++++++++ .../internal/validation/validations_test.go | 124 +++++++ .../internal/vault/controller/api_test.go | 305 ++++++++++++++++ .../vault/controller/base_controller.go | 255 ++++++++++---- .../vault/controller/controller_test.go | 87 +++++ .../internal/vault/controller/convert.go | 326 ++++++++++++++++++ .../vault/controller/vault_controller.go | 257 ++++++++++++-- flowvault/test/resources/client_api_key.txt | 1 + flowvault/test/resources/vault_api_key.txt | 1 + flowvault/utils/common/common.go | 152 +------- flowvault/utils/common/delete.go | 22 ++ flowvault/utils/common/delete_files.go | 35 ++ flowvault/utils/common/detokenize.go | 29 ++ flowvault/utils/common/get.go | 29 ++ flowvault/utils/common/get_tokens.go | 27 ++ flowvault/utils/common/insert.go | 44 ++- flowvault/utils/common/query.go | 19 + flowvault/utils/common/types.go | 60 ++++ flowvault/utils/common/update.go | 24 ++ flowvault/utils/common/upload_files.go | 55 +++ flowvault/utils/error/messages.go | 41 +++ flowvault/utils/messages/logs.go | 24 +- samples/flowvault/README.md | 62 ++++ samples/flowvault/go.mod | 17 + samples/flowvault/go.sum | 40 +++ .../bearer_token_generation_example.go | 66 ++++ ...ken_generation_using_goroutines_example.go | 73 ++++ ...r_token_generation_with_context_example.go | 55 +++ .../scoped_token_generation_example.go | 51 +++ .../signed_token_generation_example.go | 78 +++++ .../vaultapi/custom_header_example.go | 79 +++++ samples/flowvault/vaultapi/delete_files.go | 69 ++++ samples/flowvault/vaultapi/delete_records.go | 67 ++++ .../flowvault/vaultapi/detokenize_records.go | 67 ++++ samples/flowvault/vaultapi/get_records.go | 78 +++++ samples/flowvault/vaultapi/get_tokens.go | 69 ++++ samples/flowvault/vaultapi/insert_records.go | 83 +++++ samples/flowvault/vaultapi/query_records.go | 63 ++++ .../timeout_and_retry_config_example.go | 69 ++++ samples/flowvault/vaultapi/update_record.go | 71 ++++ samples/flowvault/vaultapi/upload_files.go | 89 +++++ 54 files changed, 4311 insertions(+), 393 deletions(-) create mode 100644 flowvault/client/client_test.go create mode 100644 flowvault/client/operations.go create mode 100644 flowvault/internal/helpers/files.go create mode 100644 flowvault/internal/helpers/helpers_test.go create mode 100644 flowvault/internal/helpers/http.go create mode 100644 flowvault/internal/helpers/parse.go create mode 100644 flowvault/internal/testutil/resources.go create mode 100644 flowvault/internal/validation/validations.go create mode 100644 flowvault/internal/validation/validations_test.go create mode 100644 flowvault/internal/vault/controller/api_test.go create mode 100644 flowvault/internal/vault/controller/controller_test.go create mode 100644 flowvault/internal/vault/controller/convert.go create mode 100644 flowvault/test/resources/client_api_key.txt create mode 100644 flowvault/test/resources/vault_api_key.txt create mode 100644 flowvault/utils/common/delete.go create mode 100644 flowvault/utils/common/delete_files.go create mode 100644 flowvault/utils/common/detokenize.go create mode 100644 flowvault/utils/common/get.go create mode 100644 flowvault/utils/common/get_tokens.go create mode 100644 flowvault/utils/common/query.go create mode 100644 flowvault/utils/common/types.go create mode 100644 flowvault/utils/common/update.go create mode 100644 flowvault/utils/common/upload_files.go create mode 100644 samples/flowvault/README.md create mode 100644 samples/flowvault/go.mod create mode 100644 samples/flowvault/go.sum create mode 100644 samples/flowvault/serviceaccount/bearer_token_generation_example.go create mode 100644 samples/flowvault/serviceaccount/bearer_token_generation_using_goroutines_example.go create mode 100644 samples/flowvault/serviceaccount/bearer_token_generation_with_context_example.go create mode 100644 samples/flowvault/serviceaccount/scoped_token_generation_example.go create mode 100644 samples/flowvault/serviceaccount/signed_token_generation_example.go create mode 100644 samples/flowvault/vaultapi/custom_header_example.go create mode 100644 samples/flowvault/vaultapi/delete_files.go create mode 100644 samples/flowvault/vaultapi/delete_records.go create mode 100644 samples/flowvault/vaultapi/detokenize_records.go create mode 100644 samples/flowvault/vaultapi/get_records.go create mode 100644 samples/flowvault/vaultapi/get_tokens.go create mode 100644 samples/flowvault/vaultapi/insert_records.go create mode 100644 samples/flowvault/vaultapi/query_records.go create mode 100644 samples/flowvault/vaultapi/timeout_and_retry_config_example.go create mode 100644 samples/flowvault/vaultapi/update_record.go create mode 100644 samples/flowvault/vaultapi/upload_files.go diff --git a/flowvault/client/client.go b/flowvault/client/client.go index de774d1..60bc336 100644 --- a/flowvault/client/client.go +++ b/flowvault/client/client.go @@ -2,20 +2,24 @@ package client import ( "fmt" + "sync" fvhelpers "github.com/skyflowapi/skyflow-go/flowvault/internal/helpers" + "github.com/skyflowapi/skyflow-go/flowvault/internal/vault/controller" fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" fverrors "github.com/skyflowapi/skyflow-go/flowvault/utils/error" fvlogger "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" logs "github.com/skyflowapi/skyflow-go/flowvault/utils/messages" ) -// Skyflow is the flowvault client entry point. +// Skyflow is the flowvault client entry point. It is safe for concurrent use: vaults and +// client settings can be changed while calls are running. type Skyflow struct { + mu sync.RWMutex vaultServices map[string]*VaultService credentials *fvcommon.Credentials logLevel fvlogger.LogLevel - customHeaders map[fvcommon.CustomHeaderKey]string + httpConfig fvcommon.HttpConfig } type Option func(*Skyflow) *fverrors.SkyflowError @@ -60,22 +64,55 @@ func WithCredentials(credentials fvcommon.Credentials) Option { // WithLogLevel sets the logging level. func WithLogLevel(logLevel fvlogger.LogLevel) Option { return func(s *Skyflow) *fverrors.SkyflowError { + s.mu.Lock() + defer s.mu.Unlock() s.logLevel = logLevel fvlogger.SetLogLevel(logLevel) return nil } } -// WithCustomHeaders sets custom headers sent with every request from this client. -func WithCustomHeaders(headers map[fvcommon.CustomHeaderKey]string) Option { +// WithHttpConfig sets client-level timeout and retry defaults. A value set on a +// VaultConfig wins over the one set here. +func WithHttpConfig(httpConfig fvcommon.HttpConfig) Option { return func(s *Skyflow) *fverrors.SkyflowError { - s.customHeaders = headers + if err := fvhelpers.ConfigValidator().ValidateHttpConfig(httpConfig); err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + s.httpConfig = copyHttpConfig(httpConfig) + s.syncVaults() return nil } } +// clientSettings returns the settings vaults fall back to. The caller holds s.mu. +func (s *Skyflow) clientSettings() controller.ClientSettings { + return controller.ClientSettings{Credentials: s.credentials, HttpConfig: s.httpConfig} +} + +// syncVaults pushes client-level settings to every registered vault. The caller holds s.mu. +func (s *Skyflow) syncVaults() { + settings := s.clientSettings() + for _, svc := range s.vaultServices { + svc.controller.SetClientSettings(settings) + } +} + +// copyHttpConfig copies MaxRetries so later changes to the caller's value don't leak in. +func copyHttpConfig(config fvcommon.HttpConfig) fvcommon.HttpConfig { + if config.MaxRetries != nil { + retries := *config.MaxRetries + config.MaxRetries = &retries + } + return config +} + // Vault returns the vault service registered under vaultId. func (s *Skyflow) Vault(vaultId string) (*VaultService, *fverrors.SkyflowError) { + s.mu.RLock() + defer s.mu.RUnlock() svc, ok := s.vaultServices[vaultId] if !ok { return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, fverrors.VAULT_ID_NOT_IN_CONFIG_LIST) @@ -94,11 +131,17 @@ func (s *Skyflow) AddVaultConfig(cfg fvcommon.VaultConfig) *fverrors.SkyflowErro if err := fvhelpers.ConfigValidator().ValidateVaultConfig(cfg); err != nil { return err } + s.mu.Lock() + defer s.mu.Unlock() if _, exists := s.vaultServices[cfg.VaultId]; exists { fvlogger.Error(fmt.Sprintf(logs.VAULT_CONFIG_EXISTS, cfg.VaultId)) return fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, fmt.Sprintf(fverrors.VAULT_ID_EXISTS_IN_CONFIG_LIST, cfg.VaultId)) } - s.vaultServices[cfg.VaultId] = newVaultService(s, &cfg) + if cfg.MaxRetries != nil { + retries := *cfg.MaxRetries + cfg.MaxRetries = &retries + } + s.vaultServices[cfg.VaultId] = newVaultService(controller.NewVaultController(cfg, s.clientSettings())) fvlogger.Info(fmt.Sprintf(logs.VAULT_CONTROLLER_INITIALIZED, cfg.VaultId)) return nil } @@ -110,30 +153,54 @@ func (s *Skyflow) UpdateVaultConfig(updatedConfig fvcommon.VaultConfig) *fverror if err := fvhelpers.ConfigValidator().ValidateUpdateVaultConfig(updatedConfig); err != nil { return err } + if err := fvhelpers.ConfigValidator().ValidateHttpConfig(updatedConfig.HttpConfig()); err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() svc, exists := s.vaultServices[updatedConfig.VaultId] if !exists { fvlogger.Error(fmt.Sprintf(logs.VAULT_CONFIG_DOES_NOT_EXIST, updatedConfig.VaultId)) return fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, fverrors.VAULT_ID_NOT_IN_CONFIG_LIST) } - // The service holds a pointer to its config, so in-place updates reach it - // and any controller it has built; cached tokens are dropped on change. - if !fvcommon.IsCredentialsEmpty(updatedConfig.Credentials) { - svc.config.Credentials = updatedConfig.Credentials - svc.resetAuth() + // Only the fields set on updatedConfig change. The merged config replaces the old one in + // a single step, so a call that is running keeps the config it started with. + config := svc.controller.Config() + credentialsChanged := !fvcommon.IsCredentialsEmpty(updatedConfig.Credentials) + if credentialsChanged { + config.Credentials = updatedConfig.Credentials } if updatedConfig.ClusterId != "" { - svc.config.ClusterId = updatedConfig.ClusterId + config.ClusterId = updatedConfig.ClusterId } if updatedConfig.BaseVaultUrl != "" { - svc.config.BaseVaultUrl = updatedConfig.BaseVaultUrl + config.BaseVaultUrl = updatedConfig.BaseVaultUrl + } + if updatedConfig.Env.IsSet() { + config.Env = updatedConfig.Env + } + if updatedConfig.Timeout != 0 { + config.Timeout = updatedConfig.Timeout + } + if updatedConfig.MaxRetries != nil { + retries := *updatedConfig.MaxRetries + config.MaxRetries = &retries + } + if updatedConfig.InitialRetryDelay != 0 { + config.InitialRetryDelay = updatedConfig.InitialRetryDelay } - svc.config.Env = updatedConfig.Env + if updatedConfig.MaxRetryDelay != 0 { + config.MaxRetryDelay = updatedConfig.MaxRetryDelay + } + svc.controller.SetConfig(config, credentialsChanged) return nil } // RemoveVaultConfig removes the vault config registered under vaultId. // Returns an error if the vault ID is not registered. func (s *Skyflow) RemoveVaultConfig(vaultId string) *fverrors.SkyflowError { + s.mu.Lock() + defer s.mu.Unlock() if _, exists := s.vaultServices[vaultId]; !exists { fvlogger.Error(fmt.Sprintf(logs.VAULT_CONFIG_DOES_NOT_EXIST, vaultId)) return fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, fverrors.VAULT_ID_NOT_IN_CONFIG_LIST) @@ -142,13 +209,17 @@ func (s *Skyflow) RemoveVaultConfig(vaultId string) *fverrors.SkyflowError { return nil } -// GetVaultConfig returns the VaultConfig registered under vaultId, or an error if it is not found. +// GetVaultConfig returns a copy of the VaultConfig registered under vaultId, or an error if +// it is not found. Use UpdateVaultConfig to change it. func (s *Skyflow) GetVaultConfig(vaultId string) (*fvcommon.VaultConfig, *fverrors.SkyflowError) { + s.mu.RLock() + defer s.mu.RUnlock() svc, exists := s.vaultServices[vaultId] if !exists { return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, fverrors.VAULT_ID_NOT_IN_CONFIG_LIST) } - return svc.config, nil + config := svc.controller.Config() + return &config, nil } // --------------------------------------------------------------------------- @@ -161,10 +232,11 @@ func (s *Skyflow) AddSkyflowCredentials(credentials fvcommon.Credentials) *fverr if err := fvhelpers.ConfigValidator().ValidateCredentials(credentials); err != nil { return err } + s.mu.Lock() + defer s.mu.Unlock() + // A new pointer, so every vault drops its cached token. s.credentials = &credentials - for _, svc := range s.vaultServices { - svc.resetAuth() - } + s.syncVaults() return nil } @@ -175,7 +247,13 @@ func (s *Skyflow) UpdateSkyflowCredentials(credentials fvcommon.Credentials) *fv // GetSkyflowCredentials returns the client-level credentials, or nil if none are set. func (s *Skyflow) GetSkyflowCredentials() *fvcommon.Credentials { - return s.credentials + s.mu.RLock() + defer s.mu.RUnlock() + if s.credentials == nil { + return nil + } + credentials := *s.credentials + return &credentials } // --------------------------------------------------------------------------- @@ -184,6 +262,8 @@ func (s *Skyflow) GetSkyflowCredentials() *fvcommon.Credentials { // UpdateLogLevel changes the log level for the client. func (s *Skyflow) UpdateLogLevel(logLevel fvlogger.LogLevel) { + s.mu.Lock() + defer s.mu.Unlock() fvlogger.Info(fmt.Sprintf(logs.CURRENT_LOG_LEVEL, s.logLevel)) s.logLevel = logLevel fvlogger.SetLogLevel(logLevel) @@ -191,6 +271,8 @@ func (s *Skyflow) UpdateLogLevel(logLevel fvlogger.LogLevel) { // GetLoglevel returns the current log level of the client. func (s *Skyflow) GetLoglevel() *fvlogger.LogLevel { + s.mu.RLock() + defer s.mu.RUnlock() logLevel := s.logLevel return &logLevel } diff --git a/flowvault/client/client_test.go b/flowvault/client/client_test.go new file mode 100644 index 0000000..593e65e --- /dev/null +++ b/flowvault/client/client_test.go @@ -0,0 +1,210 @@ +package client + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/skyflowapi/skyflow-go/flowvault/internal/testutil" + fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" +) + +func TestClient(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Flowvault Client Suite") +} + +const testVaultID = "vault123" + +var testApiKey = testutil.Resource("client_api_key.txt") + +func newTestClient(opts ...Option) *Skyflow { + options := append([]Option{ + WithVaults(fvcommon.VaultConfig{VaultId: testVaultID, ClusterId: "cluster", Env: fvcommon.DEV}), + WithCredentials(fvcommon.Credentials{ApiKey: testApiKey}), + }, opts...) + client, err := NewSkyflow(options...) + Expect(err).To(BeNil()) + return client +} + +var _ = Describe("Skyflow client", func() { + It("rejects an empty vault list and duplicate vault IDs", func() { + _, err := NewSkyflow(WithVaults()) + Expect(err.GetMessage()).To(ContainSubstring("No vault configurations available")) + + config := fvcommon.VaultConfig{VaultId: testVaultID, ClusterId: "cluster"} + _, err = NewSkyflow(WithVaults(config, config)) + Expect(err.GetMessage()).To(ContainSubstring("already exists")) + }) + + It("rejects negative HTTP settings on the client and on a vault", func() { + _, err := NewSkyflow(WithHttpConfig(fvcommon.HttpConfig{Timeout: -time.Second})) + Expect(err.GetMessage()).To(ContainSubstring("must not be negative")) + + retries := -1 + _, err = NewSkyflow(WithVaults(fvcommon.VaultConfig{VaultId: testVaultID, ClusterId: "cluster", MaxRetries: &retries})) + Expect(err.GetMessage()).To(ContainSubstring("must not be negative")) + }) + + It("passes client settings to vaults registered before or after them", func() { + retries := 2 + client := newTestClient( + WithHttpConfig(fvcommon.HttpConfig{MaxRetries: &retries}), + ) + vault, err := client.Vault(testVaultID) + Expect(err).To(BeNil()) + settings := vault.controller.ClientSettings() + Expect(settings.Credentials.ApiKey).To(Equal(testApiKey)) + Expect(*settings.HttpConfig.MaxRetries).To(Equal(2)) + + Expect(client.AddVaultConfig(fvcommon.VaultConfig{VaultId: "other", ClusterId: "cluster"})).To(BeNil()) + other, _ := client.Vault("other") + Expect(*other.controller.ClientSettings().HttpConfig.MaxRetries).To(Equal(2)) + }) + + It("keeps the vault's Env when an update doesn't set one", func() { + client := newTestClient() + Expect(client.UpdateVaultConfig(fvcommon.VaultConfig{VaultId: testVaultID, ClusterId: "other"})).To(BeNil()) + config, _ := client.GetVaultConfig(testVaultID) + Expect(config.Env).To(Equal(fvcommon.DEV)) + Expect(config.ClusterId).To(Equal("other")) + + Expect(client.UpdateVaultConfig(fvcommon.VaultConfig{VaultId: testVaultID, Env: fvcommon.STAGE})).To(BeNil()) + config, _ = client.GetVaultConfig(testVaultID) + Expect(config.Env).To(Equal(fvcommon.STAGE)) + }) + + It("returns copies, so changing them doesn't change the client", func() { + client := newTestClient() + config, _ := client.GetVaultConfig(testVaultID) + config.ClusterId = "changed" + again, _ := client.GetVaultConfig(testVaultID) + Expect(again.ClusterId).To(Equal("cluster")) + + creds := client.GetSkyflowCredentials() + creds.ApiKey = "changed" + Expect(client.GetSkyflowCredentials().ApiKey).To(Equal(testApiKey)) + }) + + It("can be changed while calls are running", func() { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{}`)) + })) + defer server.Close() + client := newTestClient(WithVaults(fvcommon.VaultConfig{VaultId: "local", BaseVaultUrl: server.URL})) + vault, err := client.Vault("local") + Expect(err).To(BeNil()) + + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(2) + go func() { + defer wg.Done() + defer GinkgoRecover() + for j := 0; j < 20; j++ { + _, err := vault.Query(context.Background(), fvcommon.QueryRequest{Query: "SELECT 1"}, fvcommon.QueryOptions{}) + Expect(err).To(BeNil()) + } + }() + go func(i int) { + defer wg.Done() + defer GinkgoRecover() + id := fmt.Sprintf("extra-%d", i) + for j := 0; j < 20; j++ { + Expect(client.AddVaultConfig(fvcommon.VaultConfig{VaultId: id, ClusterId: "c"})).To(BeNil()) + Expect(client.UpdateVaultConfig(fvcommon.VaultConfig{VaultId: "local", Timeout: time.Duration(j+1) * time.Second})).To(BeNil()) + Expect(client.UpdateSkyflowCredentials(fvcommon.Credentials{ApiKey: testApiKey})).To(BeNil()) + _, _ = client.GetVaultConfig("local") + Expect(client.RemoveVaultConfig(id)).To(BeNil()) + } + }(i) + } + wg.Wait() + }) + + It("updates a vault's HTTP settings in place", func() { + client := newTestClient() + retries := 3 + Expect(client.UpdateVaultConfig(fvcommon.VaultConfig{VaultId: testVaultID, Timeout: 5 * time.Second, MaxRetries: &retries})).To(BeNil()) + config, _ := client.GetVaultConfig(testVaultID) + Expect(config.Timeout).To(Equal(5 * time.Second)) + Expect(*config.MaxRetries).To(Equal(3)) + }) +}) + +var _ = Describe("VaultService", func() { + var ( + vault *VaultService + ctx = context.Background() + ) + BeforeEach(func() { + var err error + vault, err = newTestClient().Vault(testVaultID) + Expect(err).To(BeNil()) + }) + + It("validates before anything else", func() { + _, err := vault.Insert(ctx, fvcommon.InsertRequest{}, fvcommon.InsertOptions{}) + Expect(err.GetMessage()).To(ContainSubstring("Records must not be empty")) + _, err = vault.Query(ctx, fvcommon.QueryRequest{}, fvcommon.QueryOptions{}) + Expect(err.GetMessage()).To(ContainSubstring("Query must not be empty")) + }) + + It("rejects interceptor headers that aren't supported", func() { + _, err := vault.Query(ctx, fvcommon.QueryRequest{Query: "SELECT 1"}, fvcommon.QueryOptions{ + Interceptor: func(rc *fvcommon.RequestContext) { rc.AddHeader("x-other", "v") }, + }) + Expect(err.GetMessage()).To(ContainSubstring("is not supported")) + }) + + It("sends valid requests to the vault", func() { + var paths []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + paths = append(paths, r.URL.Path) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{}`)) + })) + defer server.Close() + client := newTestClient(WithVaults(fvcommon.VaultConfig{VaultId: "local", BaseVaultUrl: server.URL})) + vault, err := client.Vault("local") + Expect(err).To(BeNil()) + + data := map[string]interface{}{"a": 1} + _, err = vault.Insert(ctx, fvcommon.InsertRequest{TableName: "t", Records: []fvcommon.InsertRequestRecord{{Data: data}}}, fvcommon.InsertOptions{}) + Expect(err).To(BeNil()) + _, err = vault.Update(ctx, fvcommon.UpdateRequest{TableName: "t", Records: []fvcommon.UpdateRequestRecord{{SkyflowId: "id", Data: data}}}, fvcommon.UpdateOptions{}) + Expect(err).To(BeNil()) + _, err = vault.Get(ctx, fvcommon.GetRequest{TableName: "t", SkyflowIds: []string{"id"}}, fvcommon.GetOptions{}) + Expect(err).To(BeNil()) + _, err = vault.Delete(ctx, fvcommon.DeleteRequest{TableName: "t", SkyflowIds: []string{"id"}}, fvcommon.DeleteOptions{}) + Expect(err).To(BeNil()) + _, err = vault.Detokenize(ctx, fvcommon.DetokenizeRequest{Tokens: []string{"tok"}}, fvcommon.DetokenizeOptions{}) + Expect(err).To(BeNil()) + _, err = vault.GetTokens(ctx, fvcommon.GetTokensRequest{Records: []fvcommon.GetTokensRequestRecord{{Value: "v", TokenGroupName: "g"}}}, fvcommon.GetTokensOptions{}) + Expect(err).To(BeNil()) + _, err = vault.Query(ctx, fvcommon.QueryRequest{Query: "SELECT 1"}, fvcommon.QueryOptions{}) + Expect(err).To(BeNil()) + _, err = vault.UploadFiles(ctx, fvcommon.UploadFilesRequest{Records: []fvcommon.UploadFilesRequestRecord{ + {TableName: "t", Columns: []fvcommon.UploadFilesRequestColumn{{Column: "c", Base64: "YQ==", FileName: "a.txt"}}}, + }}, fvcommon.UploadFilesOptions{}) + Expect(err).To(BeNil()) + _, err = vault.DeleteFiles(ctx, fvcommon.DeleteFilesRequest{Records: []fvcommon.DeleteFilesRequestRecord{ + {TableName: "t", Columns: []string{"c"}, SkyflowId: "id"}, + }}, fvcommon.DeleteFilesOptions{}) + Expect(err).To(BeNil()) + + Expect(paths).To(Equal([]string{ + "/v2/records/insert", "/v2/records/update", "/v2/records/get", "/v2/records/delete", + "/v2/tokens/detokenize", "/v2/records/getTokens", "/v2/query", "/v2/files/upload", "/v2/files/delete", + })) + }) +}) diff --git a/flowvault/client/operations.go b/flowvault/client/operations.go new file mode 100644 index 0000000..9264e6c --- /dev/null +++ b/flowvault/client/operations.go @@ -0,0 +1,20 @@ +package client + +import ( + "github.com/skyflowapi/skyflow-go/common/vault" + fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" +) + +// Compile-time checks that the flowvault vault service implements the operations shared +// with the other Skyflow Go SDKs. +var ( + _ vault.Inserter[fvcommon.InsertRequest, fvcommon.InsertOptions, fvcommon.InsertResponse] = (*VaultService)(nil) + _ vault.Updater[fvcommon.UpdateRequest, fvcommon.UpdateOptions, fvcommon.UpdateResponse] = (*VaultService)(nil) + _ vault.Getter[fvcommon.GetRequest, fvcommon.GetOptions, fvcommon.GetResponse] = (*VaultService)(nil) + _ vault.Deleter[fvcommon.DeleteRequest, fvcommon.DeleteOptions, fvcommon.DeleteResponse] = (*VaultService)(nil) + _ vault.Querier[fvcommon.QueryRequest, fvcommon.QueryOptions, fvcommon.QueryResponse] = (*VaultService)(nil) + _ vault.Detokenizer[fvcommon.DetokenizeRequest, fvcommon.DetokenizeOptions, fvcommon.DetokenizeResponse] = (*VaultService)(nil) + _ vault.TokenGetter[fvcommon.GetTokensRequest, fvcommon.GetTokensOptions, fvcommon.GetTokensResponse] = (*VaultService)(nil) + _ vault.FilesUploader[fvcommon.UploadFilesRequest, fvcommon.UploadFilesOptions, fvcommon.UploadFilesResponse] = (*VaultService)(nil) + _ vault.FilesDeleter[fvcommon.DeleteFilesRequest, fvcommon.DeleteFilesOptions, fvcommon.DeleteFilesResponse] = (*VaultService)(nil) +) diff --git a/flowvault/client/vault_service.go b/flowvault/client/vault_service.go index 13b6703..7a83c4c 100644 --- a/flowvault/client/vault_service.go +++ b/flowvault/client/vault_service.go @@ -1,105 +1,67 @@ package client import ( + "context" + + "github.com/skyflowapi/skyflow-go/flowvault/internal/vault/controller" fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" fverrors "github.com/skyflowapi/skyflow-go/flowvault/utils/error" - fvlogger "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" ) -// VaultService handles all vault operations for a single vault. +// VaultService handles all vault operations for a single vault. Every method is unary: +// one request, one response. // -// It keeps a pointer to its config and a back-reference to the client, so -// UpdateVaultConfig and credential updates on the client are visible to it -// without any copying. +// The client updates the controller's settings in place, so a VaultService obtained +// earlier sees UpdateVaultConfig and credential changes on its next call. type VaultService struct { - client *Skyflow - config *fvcommon.VaultConfig + controller *controller.VaultController } -func newVaultService(client *Skyflow, config *fvcommon.VaultConfig) *VaultService { - return &VaultService{client: client, config: config} +func newVaultService(controller *controller.VaultController) *VaultService { + return &VaultService{controller: controller} } -// credentials returns the client-level credentials (nil if none are set). -func (v *VaultService) credentials() *fvcommon.Credentials { - return v.client.credentials +// Insert inserts records into the vault, optionally upserting on unique columns. +func (v *VaultService) Insert(ctx context.Context, request fvcommon.InsertRequest, options fvcommon.InsertOptions) (*fvcommon.InsertResponse, *fverrors.SkyflowError) { + return v.controller.Insert(ctx, request, options) } -// resetAuth drops any cached token so the next call re-authenticates with -// the current credentials. The vault controller is not wired yet; once it is, -// this is where its Token/ApiKey are cleared. -func (v *VaultService) resetAuth() {} +// Update updates existing records by Skyflow ID. +func (v *VaultService) Update(ctx context.Context, request fvcommon.UpdateRequest, options fvcommon.UpdateOptions) (*fvcommon.UpdateResponse, *fverrors.SkyflowError) { + return v.controller.Update(ctx, request, options) +} -// Insert inserts records into the vault. -func (v *VaultService) Insert(request fvcommon.InsertRequest, options fvcommon.InsertOptions) (*fvcommon.InsertResponse, *fverrors.SkyflowError) { - fvlogger.Info("Insert triggered for vault: " + v.config.VaultId + ", table: " + request.Table) - fvlogger.Debug("Insert request: ", request) - if request.Table == "" { - return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, "table must not be empty") - } - if len(request.Values) == 0 { - return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, "values must not be empty") - } - // TODO: implement actual HTTP call to Skyflow vault API - return &fvcommon.InsertResponse{}, nil +// Get retrieves records by Skyflow IDs or unique values. +func (v *VaultService) Get(ctx context.Context, request fvcommon.GetRequest, options fvcommon.GetOptions) (*fvcommon.GetResponse, *fverrors.SkyflowError) { + return v.controller.Get(ctx, request, options) } -// Detokenize detokenizes the given tokens. -func (v *VaultService) Detokenize(request fvcommon.DetokenizeRequest, options fvcommon.DetokenizeOptions) (*fvcommon.DetokenizeResponse, *fverrors.SkyflowError) { - fvlogger.Info("Detokenize triggered for vault: " + v.config.VaultId) - if len(request.DetokenizeData) == 0 { - return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, "detokenize data must not be empty") - } - // TODO: implement actual HTTP call - return &fvcommon.DetokenizeResponse{}, nil +// Delete deletes records by Skyflow IDs or unique values. +func (v *VaultService) Delete(ctx context.Context, request fvcommon.DeleteRequest, options fvcommon.DeleteOptions) (*fvcommon.DeleteResponse, *fverrors.SkyflowError) { + return v.controller.Delete(ctx, request, options) } -// Get retrieves records from the vault. -func (v *VaultService) Get(request fvcommon.GetRequest, options fvcommon.GetOptions) (*fvcommon.GetResponse, *fverrors.SkyflowError) { - fvlogger.Info("Get triggered for vault: " + v.config.VaultId + ", table: " + request.Table) - if request.Table == "" { - return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, "table must not be empty") - } - // TODO: implement actual HTTP call - return &fvcommon.GetResponse{}, nil +// Detokenize returns the values behind tokens. +func (v *VaultService) Detokenize(ctx context.Context, request fvcommon.DetokenizeRequest, options fvcommon.DetokenizeOptions) (*fvcommon.DetokenizeResponse, *fverrors.SkyflowError) { + return v.controller.Detokenize(ctx, request, options) } -// Delete deletes records from the vault. -func (v *VaultService) Delete(request fvcommon.DeleteRequest, options fvcommon.DeleteOptions) (*fvcommon.DeleteResponse, *fverrors.SkyflowError) { - fvlogger.Info("Delete triggered for vault: " + v.config.VaultId + ", table: " + request.Table) - if request.Table == "" { - return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, "table must not be empty") - } - // TODO: implement actual HTTP call - return &fvcommon.DeleteResponse{}, nil +// GetTokens returns existing tokens for values in deterministic token groups. +func (v *VaultService) GetTokens(ctx context.Context, request fvcommon.GetTokensRequest, options fvcommon.GetTokensOptions) (*fvcommon.GetTokensResponse, *fverrors.SkyflowError) { + return v.controller.GetTokens(ctx, request, options) } -// Update updates records in the vault. -func (v *VaultService) Update(request fvcommon.UpdateRequest, options fvcommon.UpdateOptions) (*fvcommon.UpdateResponse, *fverrors.SkyflowError) { - fvlogger.Info("Update triggered for vault: " + v.config.VaultId + ", table: " + request.Table) - if request.Table == "" { - return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, "table must not be empty") - } - // TODO: implement actual HTTP call - return &fvcommon.UpdateResponse{}, nil +// Query runs a SELECT query against the vault. +func (v *VaultService) Query(ctx context.Context, request fvcommon.QueryRequest, options fvcommon.QueryOptions) (*fvcommon.QueryResponse, *fverrors.SkyflowError) { + return v.controller.Query(ctx, request, options) } -// Query executes a query against the vault. -func (v *VaultService) Query(request fvcommon.QueryRequest, options fvcommon.QueryOptions) (*fvcommon.QueryResponse, *fverrors.SkyflowError) { - fvlogger.Info("Query triggered for vault: " + v.config.VaultId) - if request.Query == "" { - return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, "query must not be empty") - } - // TODO: implement actual HTTP call - return &fvcommon.QueryResponse{}, nil +// UploadFiles uploads files into file columns. +func (v *VaultService) UploadFiles(ctx context.Context, request fvcommon.UploadFilesRequest, options fvcommon.UploadFilesOptions) (*fvcommon.UploadFilesResponse, *fverrors.SkyflowError) { + return v.controller.UploadFiles(ctx, request, options) } -// Tokenize tokenizes values using the vault. -func (v *VaultService) Tokenize(requests []fvcommon.TokenizeRequest, options fvcommon.TokenizeOptions) (*fvcommon.TokenizeResponse, *fverrors.SkyflowError) { - fvlogger.Info("Tokenize triggered for vault: " + v.config.VaultId) - if len(requests) == 0 { - return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, "tokenize requests must not be empty") - } - // TODO: implement actual HTTP call - return &fvcommon.TokenizeResponse{}, nil +// DeleteFiles deletes files from file columns of existing records. +func (v *VaultService) DeleteFiles(ctx context.Context, request fvcommon.DeleteFilesRequest, options fvcommon.DeleteFilesOptions) (*fvcommon.DeleteFilesResponse, *fverrors.SkyflowError) { + return v.controller.DeleteFiles(ctx, request, options) } diff --git a/flowvault/go.mod b/flowvault/go.mod index def0414..ebfa281 100644 --- a/flowvault/go.mod +++ b/flowvault/go.mod @@ -8,7 +8,27 @@ require ( ) require ( + github.com/onsi/ginkgo/v2 v2.33.1 + github.com/onsi/gomega v1.44.0 + github.com/stretchr/testify v1.8.4 +) + +require ( + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/onsi/ginkgo/v2 v2.33.1 // indirect - github.com/onsi/gomega v1.44.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/objx v0.5.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/mod v0.36.0 // indirect + golang.org/x/net v0.56.0 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/text v0.38.0 // indirect + golang.org/x/tools v0.45.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/flowvault/go.sum b/flowvault/go.sum index 11985f8..4a73ca5 100644 --- a/flowvault/go.sum +++ b/flowvault/go.sum @@ -1,9 +1,20 @@ github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= +github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= +github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -12,23 +23,59 @@ github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oX github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= github.com/onsi/ginkgo/v2 v2.33.1 h1:62XfWJbEIjPSCSDYrsCPP/SHcmH7O0M8ne8zfqQ2iz4= github.com/onsi/ginkgo/v2 v2.33.1/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= github.com/onsi/gomega v1.44.0 h1:eAiGl3Pw5jz5GQdDff0BcxYpAX1JxW8xD7mFUuwNfZQ= github.com/onsi/gomega v1.44.0/go.mod h1:e/C2HwaZ1DhvjzXXuFhcR7hY7Sh9pl7MmoWKEjzwcdA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/skyflowapi/skyflow-go/common v0.0.0-20261014151608-a64b46a2705c h1:oWxTNYJ++iBGbcVpJ2oNhyB36WNNVMyBpusEAcSL7pw= github.com/skyflowapi/skyflow-go/common v0.0.0-20261014151608-a64b46a2705c/go.mod h1:T9roW4Qpv1WFo6tgLjbfLrdOCOz20lVE8ENf2bGb7zI= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= -golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= -golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= -golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= -golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= +google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/flowvault/internal/constants/constants.go b/flowvault/internal/constants/constants.go index 322e304..38a1210 100644 --- a/flowvault/internal/constants/constants.go +++ b/flowvault/internal/constants/constants.go @@ -1,16 +1,18 @@ package internal -// internal private constants — flowvault copy (cannot import v2's internal) +// internal private constants. FlowVault (v3) vaults are served from the skyvault subdomain, +// not v2's vault subdomain. const ( SDK_METRICS_HEADER_KEY = "sky-metadata" SECURE_PROTOCOL = "https://" - DEV_DOMAIN = ".vault.skyflowapis.dev" - STAGE_DOMAIN = ".vault.skyflowapis.tech" - SANDBOX_DOMAIN = ".vault.skyflowapis-preview.com" - PROD_DOMAIN = ".vault.skyflowapis.com" + DEV_DOMAIN = ".skyvault.skyflowapis.dev" + STAGE_DOMAIN = ".skyvault.skyflowapis.tech" + SANDBOX_DOMAIN = ".skyvault.skyflowapis-preview.com" + PROD_DOMAIN = ".skyvault.skyflowapis.com" GRANT_TYPE = "urn:ietf:params:oauth:grant-type:jwt-bearer" - SDK_NAME = "Skyflow Go SDK " - SDK_VERSION = "v1.0.0" + SDK_NAME = "Skyflow FlowVault Go SDK " + SDK_VERSION = "v0.1.0" // flowvault module version; bump with each flowvault/vX.Y.Z tag + METRICS_SDK_NAME = "skyflow-flowvault-go" SDK_PREFIX = SDK_NAME + SDK_VERSION SDK_LOG_PREFIX = "[ " + SDK_PREFIX + " ] " ERROR_FROM_CLIENT = "error-from-client" @@ -34,3 +36,46 @@ const ( ERROR_KEY_DETAILS = "details" ERROR_KEY_FROM_CLIENT = "from_client" ) + +// Method names used in logs and error messages, and operation names passed to interceptors. +const ( + METHOD_INSERT = "Insert" + METHOD_UPDATE = "Update" + METHOD_GET = "Get" + METHOD_DELETE = "Delete" + METHOD_DETOKENIZE = "Detokenize" + METHOD_GET_TOKENS = "GetTokens" + METHOD_QUERY = "Query" + METHOD_UPLOAD_FILES = "UploadFiles" + METHOD_DELETE_FILES = "DeleteFiles" + + OPERATION_INSERT = "INSERT" + OPERATION_UPDATE = "UPDATE" + OPERATION_GET = "GET" + OPERATION_DELETE = "DELETE" + OPERATION_DETOKENIZE = "DETOKENIZE" + OPERATION_GET_TOKENS = "GET_TOKENS" + OPERATION_QUERY = "QUERY" + OPERATION_UPLOAD_FILES = "UPLOAD_FILES" + OPERATION_DELETE_FILES = "DELETE_FILES" + + MAX_DELETE_FILES_RECORDS = 20000 + MAX_CONCURRENT_UPLOADS = 5 + RESPONSE_HEADER_REQUEST_ID = "x-request-id" + SKYFLOW_CREDENTIALS_ENV = "SKYFLOW_CREDENTIALS" +) + +// HTTP defaults, used when neither the vault nor the client sets a value. +const ( + DEFAULT_TIMEOUT_SECONDS = 60 + DEFAULT_MAX_RETRIES = 0 + DEFAULT_INITIAL_RETRY_DELAY_MILLIS = 500 + DEFAULT_MAX_RETRY_DELAY_MILLIS = 2000 + RETRY_JITTER_FACTOR = 0.2 + + // Signed-URL uploads: bounds for connecting and for the storage service's response, + // matching the Java and Python defaults. There is no overall upload timeout. + UPLOAD_CONNECT_TIMEOUT_SECONDS = 10 + UPLOAD_RESPONSE_HEADER_TIMEOUT_SECONDS = 10 + UPLOAD_KEEP_ALIVE_SECONDS = 30 +) diff --git a/flowvault/internal/helpers/files.go b/flowvault/internal/helpers/files.go new file mode 100644 index 0000000..f8777aa --- /dev/null +++ b/flowvault/internal/helpers/files.go @@ -0,0 +1,104 @@ +package helpers + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + "io" + "mime" + "net/http" + "os" + "path/filepath" + "sync" + + fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + fverrors "github.com/skyflowapi/skyflow-go/flowvault/utils/error" +) + +const ( + putFailedStatus = "PUT failed: %d" + putFailedRequest = "PUT failed: %s" + contentType = "Content-Type" +) + +// ResolvedFile is a file column's content plus the name and content type to upload it with. +type ResolvedFile struct { + Name string + ContentType string + Content []byte +} + +// ResolveFile reads the column's file from FilePath, Base64 or FileObject and works out +// its name and content type. +func ResolveFile(column fvcommon.UploadFilesRequestColumn) (ResolvedFile, *fverrors.SkyflowError) { + file := ResolvedFile{Name: column.FileName} + switch { + case column.FilePath != "": + content, err := os.ReadFile(column.FilePath) + if err != nil { + return file, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, fmt.Sprintf(fverrors.FILE_NOT_READABLE, column.Column, err.Error())) + } + file.Content = content + if file.Name == "" { + file.Name = filepath.Base(column.FilePath) + } + case column.Base64 != "": + content, err := base64.StdEncoding.DecodeString(column.Base64) + if err != nil { + return file, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, fmt.Sprintf(fverrors.INVALID_BASE64, column.Column)) + } + file.Content = content + default: + content, err := io.ReadAll(column.FileObject) + if err != nil { + return file, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, fmt.Sprintf(fverrors.FILE_NOT_READABLE, column.Column, err.Error())) + } + file.Content = content + if file.Name == "" && column.FileObject.Name() != "" { + file.Name = filepath.Base(column.FileObject.Name()) + } + } + if file.Name != "" { + file.ContentType = mime.TypeByExtension(filepath.Ext(file.Name)) + } + return file, nil +} + +// PutFile uploads one file to its signed URL. The signature is in the URL, so no +// Authorization header is sent. +func PutFile(ctx context.Context, client *http.Client, signedURL string, file ResolvedFile) (fvcommon.UploadStatus, string) { + req, err := http.NewRequestWithContext(ctx, http.MethodPut, signedURL, bytes.NewReader(file.Content)) + if err != nil { + return fvcommon.UploadStatusFailed, fmt.Sprintf(putFailedRequest, err.Error()) + } + if file.ContentType != "" { + req.Header.Set(contentType, file.ContentType) + } + resp, err := client.Do(req) + if err != nil { + return fvcommon.UploadStatusFailed, fmt.Sprintf(putFailedRequest, err.Error()) + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, resp.Body) + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return fvcommon.UploadStatusFailed, fmt.Sprintf(putFailedStatus, resp.StatusCode) + } + return fvcommon.UploadStatusUploaded, "" +} + +// RunConcurrently runs tasks with at most limit running at once, and waits for all of them. +func RunConcurrently(tasks []func(), limit int) { + var wg sync.WaitGroup + slots := make(chan struct{}, limit) + for _, task := range tasks { + wg.Add(1) + slots <- struct{}{} + go func(run func()) { + defer wg.Done() + defer func() { <-slots }() + run() + }(task) + } + wg.Wait() +} diff --git a/flowvault/internal/helpers/helpers.go b/flowvault/internal/helpers/helpers.go index 7fc53cd..427f7e3 100644 --- a/flowvault/internal/helpers/helpers.go +++ b/flowvault/internal/helpers/helpers.go @@ -30,7 +30,7 @@ func GetURLWithEnv(env fvcommon.Env, clusterId string) string { // CreateSDKMetadata builds the sky-metadata header value. func CreateSDKMetadata() string { data := map[string]string{ - "sdk_name_version": fmt.Sprintf("skyflow-go@%s", fvconsts.SDK_VERSION), + "sdk_name_version": fmt.Sprintf("%s@%s", fvconsts.METRICS_SDK_NAME, fvconsts.SDK_VERSION), "device_model": runtime.GOOS, "os_details": fmt.Sprintf("%s %s", runtime.GOOS, runtime.GOARCH), "runtime_details": runtime.Version(), diff --git a/flowvault/internal/helpers/helpers_test.go b/flowvault/internal/helpers/helpers_test.go new file mode 100644 index 0000000..fc80eed --- /dev/null +++ b/flowvault/internal/helpers/helpers_test.go @@ -0,0 +1,224 @@ +package helpers + +import ( + "context" + "encoding/base64" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" +) + +func TestHelpers(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Flowvault Helpers Suite") +} + +var _ = Describe("HTTP config", func() { + It("prefers vault, then client, then defaults", func() { + vaultRetries, clientRetries := 2, 5 + resolved := ResolveHttpConfig( + fvcommon.HttpConfig{Timeout: 5 * time.Second, MaxRetries: &vaultRetries}, + fvcommon.HttpConfig{Timeout: 9 * time.Second, MaxRetries: &clientRetries, MaxRetryDelay: time.Second}, + ) + Expect(resolved).To(Equal(ResolvedHttpConfig{ + Timeout: 5 * time.Second, + MaxRetries: 2, + InitialRetryDelay: 500 * time.Millisecond, + MaxRetryDelay: time.Second, + })) + + defaults := ResolveHttpConfig(fvcommon.HttpConfig{}, fvcommon.HttpConfig{}) + Expect(defaults.Timeout).To(Equal(60 * time.Second)) + Expect(defaults.MaxRetries).To(Equal(0)) + Expect(defaults.MaxRetryDelay).To(Equal(2000 * time.Millisecond)) + }) + + It("keeps backoff within the jittered bounds", func() { + for attempt := 1; attempt <= 5; attempt++ { + delay := backoff(attempt, 500*time.Millisecond, 2*time.Second) + Expect(delay).To(BeNumerically("<=", 2400*time.Millisecond)) + Expect(delay).To(BeNumerically(">=", 400*time.Millisecond)) + } + }) + + It("retries retryable statuses up to MaxRetries and resends the body", func() { + var calls int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + Expect(string(body)).To(Equal("payload")) + if atomic.AddInt32(&calls, 1) < 3 { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + transport := &retryTransport{ + base: http.DefaultTransport, + config: ResolvedHttpConfig{MaxRetries: 3, InitialRetryDelay: time.Millisecond, MaxRetryDelay: time.Millisecond}, + sleep: func(time.Duration) {}, + } + req, _ := http.NewRequest(http.MethodPost, server.URL, strings.NewReader("payload")) + resp, err := transport.RoundTrip(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + Expect(atomic.LoadInt32(&calls)).To(Equal(int32(3))) + }) + + It("does not retry when retries are off", func() { + var calls int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&calls, 1) + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + resp, err := NewHTTPClient(ResolveHttpConfig(fvcommon.HttpConfig{}, fvcommon.HttpConfig{})).Get(server.URL) + Expect(err).To(BeNil()) + Expect(resp.StatusCode).To(Equal(http.StatusTooManyRequests)) + Expect(atomic.LoadInt32(&calls)).To(Equal(int32(1))) + }) +}) + +var _ = Describe("Files", func() { + It("resolves files from a path, base64 and an *os.File", func() { + dir := GinkgoT().TempDir() + path := filepath.Join(dir, "resume.pdf") + Expect(os.WriteFile(path, []byte("pdf"), 0o600)).To(Succeed()) + + file, err := ResolveFile(fvcommon.UploadFilesRequestColumn{Column: "c", FilePath: path}) + Expect(err).To(BeNil()) + Expect(file).To(Equal(ResolvedFile{Name: "resume.pdf", ContentType: "application/pdf", Content: []byte("pdf")})) + + file, err = ResolveFile(fvcommon.UploadFilesRequestColumn{Column: "c", Base64: base64.StdEncoding.EncodeToString([]byte("txt")), FileName: "notes.txt"}) + Expect(err).To(BeNil()) + Expect(file.Content).To(Equal([]byte("txt"))) + Expect(file.ContentType).To(HavePrefix("text/plain")) + + handle, _ := os.Open(path) + defer handle.Close() + file, err = ResolveFile(fvcommon.UploadFilesRequestColumn{Column: "c", FileObject: handle}) + Expect(err).To(BeNil()) + Expect(file.Name).To(Equal("resume.pdf")) + }) + + It("rejects an unreadable path and invalid base64", func() { + _, err := ResolveFile(fvcommon.UploadFilesRequestColumn{Column: "c", FilePath: "/does/not/exist"}) + Expect(err.GetMessage()).To(ContainSubstring("Could not read file")) + _, err = ResolveFile(fvcommon.UploadFilesRequestColumn{Column: "c", Base64: "%%%", FileName: "a.txt"}) + Expect(err.GetMessage()).To(ContainSubstring("invalid base64")) + }) + + It("PUTs a file without an Authorization header and reports failures", func() { + var got *http.Request + var body string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + raw, _ := io.ReadAll(r.Body) + got, body = r, string(raw) + if r.URL.Path == "/forbidden" { + w.WriteHeader(http.StatusForbidden) + return + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + file := ResolvedFile{Name: "a.pdf", ContentType: "application/pdf", Content: []byte("pdf")} + + status, msg := PutFile(context.Background(), http.DefaultClient, server.URL+"/ok", file) + Expect(status).To(Equal(fvcommon.UploadStatusUploaded)) + Expect(msg).To(BeEmpty()) + Expect(got.Method).To(Equal(http.MethodPut)) + Expect(got.Header.Get("Content-Type")).To(Equal("application/pdf")) + Expect(got.Header.Get("Authorization")).To(BeEmpty()) + Expect(body).To(Equal("pdf")) + + status, msg = PutFile(context.Background(), http.DefaultClient, server.URL+"/forbidden", file) + Expect(status).To(Equal(fvcommon.UploadStatusFailed)) + Expect(msg).To(Equal("PUT failed: 403")) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + status, _ = PutFile(ctx, http.DefaultClient, server.URL+"/ok", file) + Expect(status).To(Equal(fvcommon.UploadStatusFailed)) + }) + + It("uploads with no retries and no overall timeout, but bounded connect and response waits", func() { + client := UploadHTTPClient() + Expect(client).To(BeIdenticalTo(UploadHTTPClient())) + Expect(client.Timeout).To(BeZero()) + transport, ok := client.Transport.(*http.Transport) + Expect(ok).To(BeTrue()) + Expect(transport.TLSHandshakeTimeout).To(Equal(10 * time.Second)) + Expect(transport.ResponseHeaderTimeout).To(Equal(10 * time.Second)) + }) + + It("runs at most limit tasks at once", func() { + var running, peak int32 + tasks := make([]func(), 20) + for i := range tasks { + tasks[i] = func() { + now := atomic.AddInt32(&running, 1) + for { + old := atomic.LoadInt32(&peak) + if now <= old || atomic.CompareAndSwapInt32(&peak, old, now) { + break + } + } + time.Sleep(2 * time.Millisecond) + atomic.AddInt32(&running, -1) + } + } + RunConcurrently(tasks, 5) + Expect(atomic.LoadInt32(&peak)).To(BeNumerically("<=", 5)) + Expect(atomic.LoadInt32(&running)).To(Equal(int32(0))) + }) +}) + +var _ = Describe("Parsing", func() { + It("parses tokens and hashed data in object, list and string form", func() { + tokens := ParseTokens(map[string]interface{}{ + "email": []interface{}{map[string]interface{}{"token": "tok", "tokenGroupName": "det", "path": "$.a"}}, + "ssn": "plain-token", + }) + Expect(tokens["email"]).To(Equal([]fvcommon.Token{{Token: "tok", TokenGroupName: "det", Path: "$.a"}})) + Expect(tokens["ssn"]).To(Equal([]fvcommon.Token{{Token: "plain-token"}})) + Expect(ParseTokens(nil)).To(BeNil()) + + hashes := ParseHashedData(map[string]interface{}{"email": map[string]interface{}{"data": "abc", "hashName": "h1"}}) + Expect(hashes["email"]).To(Equal([]fvcommon.HashedValue{{Data: "abc", HashName: "h1"}})) + }) + + It("accepts table or tableName in detokenize metadata", func() { + Expect(ParseDetokenizeMetadata(map[string]interface{}{"skyflowID": "id", "table": "t1"})). + To(Equal(&fvcommon.DetokenizeRecordMetadata{SkyflowId: "id", TableName: "t1"})) + Expect(ParseDetokenizeMetadata(map[string]interface{}{"skyflowId": "id", "tableName": "t2"})). + To(Equal(&fvcommon.DetokenizeRecordMetadata{SkyflowId: "id", TableName: "t2"})) + Expect(ParseDetokenizeMetadata(nil)).To(BeNil()) + }) + + It("parses delete-files columns sorted by name", func() { + Expect(ParseDeleteFilesColumns(map[string]interface{}{ + "resumePDF": map[string]interface{}{"status": "DELETED"}, + "photoID": "DELETED", + })).To(Equal([]fvcommon.DeleteFilesColumnResult{{Column: "photoID", Status: "DELETED"}, {Column: "resumePDF", Status: "DELETED"}})) + Expect(ParseDeleteFilesColumns(nil)).To(BeNil()) + }) + + It("sets the request ID only for failed records", func() { + header := http.Header{} + header.Set("x-request-id", "req-1") + Expect(RequestIdFor(header, "boom")).To(Equal("req-1")) + Expect(RequestIdFor(header, "")).To(BeEmpty()) + }) +}) diff --git a/flowvault/internal/helpers/http.go b/flowvault/internal/helpers/http.go new file mode 100644 index 0000000..a521886 --- /dev/null +++ b/flowvault/internal/helpers/http.go @@ -0,0 +1,147 @@ +package helpers + +import ( + "io" + "math/rand" + "net" + "net/http" + "time" + + fvconsts "github.com/skyflowapi/skyflow-go/flowvault/internal/constants" + fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" +) + +// ResolvedHttpConfig is the effective timeout and retry policy for a call. +type ResolvedHttpConfig struct { + Timeout time.Duration + MaxRetries int + InitialRetryDelay time.Duration + MaxRetryDelay time.Duration +} + +// ResolveHttpConfig picks each setting from the vault config, then the client config, +// then the SDK default. +func ResolveHttpConfig(vault, client fvcommon.HttpConfig) ResolvedHttpConfig { + resolved := ResolvedHttpConfig{ + Timeout: firstDuration(vault.Timeout, client.Timeout, fvconsts.DEFAULT_TIMEOUT_SECONDS*time.Second), + MaxRetries: fvconsts.DEFAULT_MAX_RETRIES, + InitialRetryDelay: firstDuration(vault.InitialRetryDelay, client.InitialRetryDelay, fvconsts.DEFAULT_INITIAL_RETRY_DELAY_MILLIS*time.Millisecond), + MaxRetryDelay: firstDuration(vault.MaxRetryDelay, client.MaxRetryDelay, fvconsts.DEFAULT_MAX_RETRY_DELAY_MILLIS*time.Millisecond), + } + if vault.MaxRetries != nil { + resolved.MaxRetries = *vault.MaxRetries + } else if client.MaxRetries != nil { + resolved.MaxRetries = *client.MaxRetries + } + return resolved +} + +func firstDuration(values ...time.Duration) time.Duration { + for _, value := range values { + if value > 0 { + return value + } + } + return 0 +} + +// NewHTTPClient builds the client used for API calls. Timeout covers the whole call, +// including retries. +func NewHTTPClient(config ResolvedHttpConfig) *http.Client { + return &http.Client{ + Timeout: config.Timeout, + Transport: &retryTransport{ + base: http.DefaultTransport, + config: config, + }, + } +} + +// uploadClient is shared by all uploads so they reuse one connection pool. +var uploadClient = newUploadHTTPClient() + +// UploadHTTPClient returns the client that PUTs files to signed URLs. Like the Java and +// Python SDKs, it doesn't retry and has no overall timeout, because a large file can take +// longer than one API call's timeout while still uploading. Connecting and waiting for the +// storage service's response are bounded, so a stalled connection still fails, and the +// caller's context can cancel an upload at any time. +func UploadHTTPClient() *http.Client { + return uploadClient +} + +func newUploadHTTPClient() *http.Client { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.DialContext = (&net.Dialer{ + Timeout: fvconsts.UPLOAD_CONNECT_TIMEOUT_SECONDS * time.Second, + KeepAlive: fvconsts.UPLOAD_KEEP_ALIVE_SECONDS * time.Second, + }).DialContext + transport.TLSHandshakeTimeout = fvconsts.UPLOAD_CONNECT_TIMEOUT_SECONDS * time.Second + transport.ResponseHeaderTimeout = fvconsts.UPLOAD_RESPONSE_HEADER_TIMEOUT_SECONDS * time.Second + return &http.Client{Transport: transport} +} + +// retryTransport retries 408, 429 and 5xx responses with exponential backoff and jitter. +type retryTransport struct { + base http.RoundTripper + config ResolvedHttpConfig + sleep func(time.Duration) // overridden in tests +} + +func (t *retryTransport) RoundTrip(req *http.Request) (*http.Response, error) { + resp, err := t.base.RoundTrip(req) + for retry := 0; err == nil && shouldRetry(resp.StatusCode) && retry < t.config.MaxRetries; retry++ { + if req.Body != nil && req.GetBody == nil { + break + } + retryReq := req.Clone(req.Context()) + if req.GetBody != nil { + body, bodyErr := req.GetBody() + if bodyErr != nil { + break + } + retryReq.Body = body + } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + if waitErr := t.wait(req, backoff(retry+1, t.config.InitialRetryDelay, t.config.MaxRetryDelay)); waitErr != nil { + return nil, waitErr + } + resp, err = t.base.RoundTrip(retryReq) + } + return resp, err +} + +func (t *retryTransport) wait(req *http.Request, delay time.Duration) error { + if t.sleep != nil { + t.sleep(delay) + return nil + } + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-req.Context().Done(): + return req.Context().Err() + case <-timer.C: + return nil + } +} + +func shouldRetry(status int) bool { + return status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= http.StatusInternalServerError +} + +// backoff doubles the delay per attempt up to max, then applies +/-20% jitter. +func backoff(attempt int, initial, max time.Duration) time.Duration { + delay := initial + for step := 1; step < attempt && delay < max; step++ { + delay *= 2 + } + if delay > max { + delay = max + } + jitter := time.Duration(float64(delay) * fvconsts.RETRY_JITTER_FACTOR) + if jitter <= 0 { + return delay + } + return delay - jitter + time.Duration(rand.Int63n(int64(2*jitter)+1)) //nolint:gosec // jitter, not security +} diff --git a/flowvault/internal/helpers/parse.go b/flowvault/internal/helpers/parse.go new file mode 100644 index 0000000..e051a8c --- /dev/null +++ b/flowvault/internal/helpers/parse.go @@ -0,0 +1,123 @@ +package helpers + +import ( + "fmt" + "net/http" + "sort" + + fvconsts "github.com/skyflowapi/skyflow-go/flowvault/internal/constants" + fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" +) + +const statusKey = "status" + +// ParseTokens accepts a column value that is a token string, a token object or a list of either. +func ParseTokens(raw map[string]interface{}) map[string][]fvcommon.Token { + if raw == nil { + return nil + } + parsed := make(map[string][]fvcommon.Token, len(raw)) + for column, value := range raw { + var tokens []fvcommon.Token + for _, item := range asList(value) { + switch entry := item.(type) { + case map[string]interface{}: + tokens = append(tokens, fvcommon.Token{ + Token: stringField(entry, "token"), + TokenGroupName: stringField(entry, "tokenGroupName"), + Path: stringField(entry, "path"), + }) + case nil: + default: + tokens = append(tokens, fvcommon.Token{Token: fmt.Sprint(entry)}) + } + } + if tokens != nil { + parsed[column] = tokens + } + } + return parsed +} + +// ParseHashedData accepts a column value that is a hash string, a {data, hashName} object or a list of either. +func ParseHashedData(raw map[string]interface{}) map[string][]fvcommon.HashedValue { + if raw == nil { + return nil + } + parsed := make(map[string][]fvcommon.HashedValue, len(raw)) + for column, value := range raw { + var hashes []fvcommon.HashedValue + for _, item := range asList(value) { + switch entry := item.(type) { + case map[string]interface{}: + hashes = append(hashes, fvcommon.HashedValue{Data: stringField(entry, "data"), HashName: stringField(entry, "hashName")}) + case nil: + default: + hashes = append(hashes, fvcommon.HashedValue{Data: fmt.Sprint(entry)}) + } + } + if hashes != nil { + parsed[column] = hashes + } + } + return parsed +} + +// ParseDetokenizeMetadata accepts both "table" and "tableName" keys. +func ParseDetokenizeMetadata(raw map[string]interface{}) *fvcommon.DetokenizeRecordMetadata { + if raw == nil { + return nil + } + return &fvcommon.DetokenizeRecordMetadata{ + SkyflowId: stringField(raw, "skyflowID", "skyflowId"), + TableName: stringField(raw, "tableName", "table"), + } +} + +func asList(value interface{}) []interface{} { + if list, ok := value.([]interface{}); ok { + return list + } + return []interface{}{value} +} + +// stringField returns the first of keys present in entry as a string. +func stringField(entry map[string]interface{}, keys ...string) string { + for _, key := range keys { + if value, ok := entry[key]; ok && value != nil { + if text, isString := value.(string); isString { + return text + } + return fmt.Sprint(value) + } + } + return "" +} + +// ParseDeleteFilesColumns converts {column: {"status": "DELETED"}} into results sorted by column. +func ParseDeleteFilesColumns(data map[string]interface{}) []fvcommon.DeleteFilesColumnResult { + if data == nil { + return nil + } + columns := make([]fvcommon.DeleteFilesColumnResult, 0, len(data)) + for name, value := range data { + status := "" + switch entry := value.(type) { + case map[string]interface{}: + status = stringField(entry, statusKey) + case string: + status = entry + } + columns = append(columns, fvcommon.DeleteFilesColumnResult{Column: name, Status: status}) + } + sort.Slice(columns, func(i, j int) bool { return columns[i].Column < columns[j].Column }) + return columns +} + +// RequestIdFor returns the call's request ID for a record that failed, and "" otherwise. +func RequestIdFor(header http.Header, recordErr string) string { + if recordErr == "" || header == nil { + return "" + } + return header.Get(fvconsts.RESPONSE_HEADER_REQUEST_ID) +} diff --git a/flowvault/internal/testutil/resources.go b/flowvault/internal/testutil/resources.go new file mode 100644 index 0000000..08eb9f1 --- /dev/null +++ b/flowvault/internal/testutil/resources.go @@ -0,0 +1,22 @@ +// Package testutil loads test fixtures from flowvault/test/resources. Fake credentials live +// there instead of in test source, so secret scanners don't flag the tests. +package testutil + +import ( + "os" + "path/filepath" + "runtime" + "strings" +) + +// Resource returns the trimmed contents of a file in flowvault/test/resources. It panics if +// the file can't be read, which fails the test that asked for it. +func Resource(name string) string { + _, thisFile, _, _ := runtime.Caller(0) + path := filepath.Join(filepath.Dir(thisFile), "..", "..", "test", "resources", name) + content, err := os.ReadFile(path) + if err != nil { + panic(err) + } + return strings.TrimSpace(string(content)) +} diff --git a/flowvault/internal/validation/validations.go b/flowvault/internal/validation/validations.go new file mode 100644 index 0000000..d684fca --- /dev/null +++ b/flowvault/internal/validation/validations.go @@ -0,0 +1,307 @@ +package validation + +import ( + "fmt" + "strings" + + fvconsts "github.com/skyflowapi/skyflow-go/flowvault/internal/constants" + fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + fverrors "github.com/skyflowapi/skyflow-go/flowvault/utils/error" +) + +func invalid(format string, args ...interface{}) *fverrors.SkyflowError { + message := format + if len(args) > 0 { + message = fmt.Sprintf(format, args...) + } + return fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, message) +} + +func blank(value string) bool { + return strings.TrimSpace(value) == "" +} + +func ValidateInsert(req fvcommon.InsertRequest) *fverrors.SkyflowError { + name := fvconsts.METHOD_INSERT + if len(req.Records) == 0 { + return invalid(fverrors.EMPTY_RECORDS, name) + } + tableAtRequest := !blank(req.TableName) + for i, record := range req.Records { + if err := validateData(name, i, record.Data); err != nil { + return err + } + tableAtRecord := !blank(record.TableName) + if tableAtRequest && tableAtRecord { + return invalid(fverrors.TABLE_IN_BOTH_PLACES, name) + } + if !tableAtRequest && !tableAtRecord { + return invalid(fverrors.TABLE_MISSING, name) + } + if tableAtRequest && record.Upsert != nil { + return invalid(fverrors.RECORD_UPSERT_NOT_ALLOWED) + } + if err := validateUpsert(record.Upsert); err != nil { + return err + } + } + if !tableAtRequest && req.Upsert != nil { + return invalid(fverrors.REQUEST_UPSERT_NOT_ALLOWED) + } + return validateUpsert(req.Upsert) +} + +func validateUpsert(upsert *fvcommon.UpsertOptions) *fverrors.SkyflowError { + if upsert == nil { + return nil + } + if len(upsert.UniqueColumns) == 0 { + return invalid(fverrors.EMPTY_UNIQUE_COLUMNS) + } + for _, column := range upsert.UniqueColumns { + if blank(column) { + return invalid(fverrors.EMPTY_UNIQUE_COLUMNS) + } + } + return validateUpdateType(fvconsts.METHOD_INSERT, upsert.UpdateType) +} + +func validateUpdateType(name string, updateType fvcommon.UpdateType) *fverrors.SkyflowError { + switch updateType { + case "", fvcommon.UpdateTypeUpdate, fvcommon.UpdateTypeReplace: + return nil + } + return invalid(fverrors.INVALID_UPDATE_TYPE, name, string(updateType)) +} + +func validateData(name string, index int, data map[string]interface{}) *fverrors.SkyflowError { + if len(data) == 0 { + return invalid(fverrors.EMPTY_DATA, name, index) + } + for key := range data { + if blank(key) { + return invalid(fverrors.EMPTY_KEY_IN_DATA, name, index) + } + } + return nil +} + +func ValidateUpdate(req fvcommon.UpdateRequest) *fverrors.SkyflowError { + name := fvconsts.METHOD_UPDATE + if len(req.Records) == 0 { + return invalid(fverrors.EMPTY_RECORDS, name) + } + if err := validateUpdateType(name, req.UpdateType); err != nil { + return err + } + tableAtRequest := !blank(req.TableName) + for i, record := range req.Records { + if blank(record.SkyflowId) { + return invalid(fverrors.MISSING_SKYFLOW_ID, i) + } + if err := validateData(name, i, record.Data); err != nil { + return err + } + tableAtRecord := !blank(record.TableName) + if tableAtRequest && tableAtRecord { + return invalid(fverrors.TABLE_IN_BOTH_PLACES, name) + } + if !tableAtRequest && !tableAtRecord { + return invalid(fverrors.TABLE_MISSING, name) + } + } + return nil +} + +func ValidateGet(req fvcommon.GetRequest) *fverrors.SkyflowError { + name := fvconsts.METHOD_GET + if req.Limit < 0 || req.Offset < 0 { + return invalid(fverrors.NEGATIVE_LIMIT_OFFSET) + } + if req.Records != nil { + if req.TableName != "" || req.SkyflowIds != nil || req.UniqueValues != nil || req.Columns != nil || + req.ColumnRedactions != nil || req.Limit != 0 || req.Offset != 0 { + return invalid(fverrors.GET_MODE_CONFLICT) + } + if len(req.Records) == 0 { + return invalid(fverrors.EMPTY_RECORDS, name) + } + for _, record := range req.Records { + if err := validateTableTarget(name, record.TableName, record.SkyflowIds, record.UniqueValues); err != nil { + return err + } + if err := validateColumnRedactions(name, record.ColumnRedactions); err != nil { + return err + } + } + return nil + } + if err := validateTableTarget(name, req.TableName, req.SkyflowIds, req.UniqueValues); err != nil { + return err + } + return validateColumnRedactions(name, req.ColumnRedactions) +} + +func ValidateDelete(req fvcommon.DeleteRequest) *fverrors.SkyflowError { + return validateTableTarget(fvconsts.METHOD_DELETE, req.TableName, req.SkyflowIds, req.UniqueValues) +} + +// validateTableTarget checks a table name plus exactly one of IDs / unique values. +func validateTableTarget(name, table string, ids []string, uniqueValues []map[string]interface{}) *fverrors.SkyflowError { + if blank(table) { + return invalid(fverrors.EMPTY_TABLE_NAME, name) + } + if (len(ids) > 0) == (len(uniqueValues) > 0) { + return invalid(fverrors.IDS_OR_UNIQUE_VALUES, name) + } + for _, id := range ids { + if blank(id) { + return invalid(fverrors.EMPTY_ID, name) + } + } + return validateUniqueValues(name, uniqueValues) +} + +func validateUniqueValues(name string, uniqueValues []map[string]interface{}) *fverrors.SkyflowError { + for _, value := range uniqueValues { + if len(value) == 0 { + return invalid(fverrors.EMPTY_UNIQUE_VALUE, name) + } + } + return nil +} + +func validateColumnRedactions(name string, redactions []fvcommon.ColumnRedaction) *fverrors.SkyflowError { + for _, redaction := range redactions { + if blank(redaction.ColumnName) || blank(redaction.Redaction) { + return invalid(fverrors.INVALID_COLUMN_REDACTION, name) + } + } + return nil +} + +func ValidateDetokenize(req fvcommon.DetokenizeRequest) *fverrors.SkyflowError { + if len(req.Tokens) == 0 { + return invalid(fverrors.EMPTY_TOKENS) + } + for _, token := range req.Tokens { + if blank(token) { + return invalid(fverrors.EMPTY_TOKEN) + } + } + for _, redaction := range req.TokenGroupRedactions { + if blank(redaction.TokenGroupName) || blank(redaction.Redaction) { + return invalid(fverrors.INVALID_TOKEN_GROUP_REDACTION) + } + } + return nil +} + +func ValidateGetTokens(req fvcommon.GetTokensRequest) *fverrors.SkyflowError { + if len(req.Records) == 0 { + return invalid(fverrors.EMPTY_RECORDS, fvconsts.METHOD_GET_TOKENS) + } + for i, record := range req.Records { + if record.Value == nil { + return invalid(fverrors.MISSING_VALUE, i) + } + if blank(record.TokenGroupName) { + return invalid(fverrors.MISSING_TOKEN_GROUP_NAME, i) + } + } + return nil +} + +func ValidateQuery(req fvcommon.QueryRequest) *fverrors.SkyflowError { + if blank(req.Query) { + return invalid(fverrors.EMPTY_QUERY) + } + return nil +} + +func ValidateUploadFiles(req fvcommon.UploadFilesRequest) *fverrors.SkyflowError { + name := fvconsts.METHOD_UPLOAD_FILES + if len(req.Records) == 0 { + return invalid(fverrors.EMPTY_RECORDS, name) + } + for i, record := range req.Records { + if blank(record.TableName) { + return invalid(fverrors.EMPTY_TABLE_NAME, name) + } + if len(record.Columns) == 0 { + return invalid(fverrors.EMPTY_COLUMNS, name, i) + } + for _, column := range record.Columns { + if blank(column.Column) { + return invalid(fverrors.EMPTY_COLUMN_NAME, name, i) + } + sources := 0 + if column.FilePath != "" { + sources++ + } + if column.Base64 != "" { + sources++ + } + if column.FileObject != nil { + sources++ + } + if sources != 1 { + return invalid(fverrors.FILE_SOURCE_COUNT, column.Column) + } + if column.Base64 != "" && blank(column.FileName) { + return invalid(fverrors.BASE64_NEEDS_FILE_NAME, column.Column) + } + } + } + return nil +} + +func ValidateDeleteFiles(req fvcommon.DeleteFilesRequest) *fverrors.SkyflowError { + name := fvconsts.METHOD_DELETE_FILES + if len(req.Records) == 0 { + return invalid(fverrors.EMPTY_RECORDS, name) + } + if len(req.Records) > fvconsts.MAX_DELETE_FILES_RECORDS { + return invalid(fverrors.DELETE_FILES_LIMIT, fvconsts.MAX_DELETE_FILES_RECORDS) + } + for i, record := range req.Records { + if blank(record.TableName) { + return invalid(fverrors.EMPTY_TABLE_NAME, name) + } + if len(record.Columns) == 0 { + return invalid(fverrors.EMPTY_COLUMNS, name, i) + } + seen := make(map[string]bool, len(record.Columns)) + for _, column := range record.Columns { + if blank(column) { + return invalid(fverrors.EMPTY_COLUMN_NAME, name, i) + } + if seen[column] { + return invalid(fverrors.DUPLICATE_COLUMN, name, column, i) + } + seen[column] = true + } + if blank(record.SkyflowId) == (len(record.UniqueValues) == 0) { + return invalid(fverrors.DELETE_FILES_TARGET, i) + } + if err := validateUniqueValues(name, record.UniqueValues); err != nil { + return err + } + } + return nil +} + +// ValidateCustomHeaders checks that only supported header keys are set, each with a value. +func ValidateCustomHeaders(headers map[fvcommon.CustomHeaderKey]string, tag string) *fverrors.SkyflowError { + for key, value := range headers { + switch key { + case fvcommon.SkyflowAccountId, fvcommon.SkyflowAccountName, fvcommon.RequestIdHeader: + default: + return invalid(fverrors.INVALID_HEADER_KEY, tag, string(key)) + } + if blank(value) { + return invalid(fverrors.EMPTY_HEADER_VALUE, tag, string(key)) + } + } + return nil +} diff --git a/flowvault/internal/validation/validations_test.go b/flowvault/internal/validation/validations_test.go new file mode 100644 index 0000000..ed35b8c --- /dev/null +++ b/flowvault/internal/validation/validations_test.go @@ -0,0 +1,124 @@ +package validation_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + . "github.com/skyflowapi/skyflow-go/flowvault/internal/validation" + fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + fverrors "github.com/skyflowapi/skyflow-go/flowvault/utils/error" +) + +func TestValidation(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Flowvault Validation Suite") +} + +func expectError(err *fverrors.SkyflowError, message string) { + if message == "" { + Expect(err).To(BeNil()) + return + } + Expect(err).ToNot(BeNil()) + Expect(err.GetMessage()).To(ContainSubstring(message)) +} + +var data = map[string]interface{}{"name": "john"} + +var _ = Describe("ValidateInsert", func() { + DescribeTable("table and upsert placement", + func(req fvcommon.InsertRequest, message string) { expectError(ValidateInsert(req), message) }, + Entry("request-level table", fvcommon.InsertRequest{TableName: "t", Records: []fvcommon.InsertRequestRecord{{Data: data}}}, ""), + Entry("record-level table and upsert", fvcommon.InsertRequest{Records: []fvcommon.InsertRequestRecord{{Data: data, TableName: "t", Upsert: &fvcommon.UpsertOptions{UniqueColumns: []string{"name"}}}}}, ""), + Entry("no records", fvcommon.InsertRequest{TableName: "t"}, "Records must not be empty"), + Entry("empty data", fvcommon.InsertRequest{TableName: "t", Records: []fvcommon.InsertRequestRecord{{}}}, "Data must not be empty"), + Entry("empty column name", fvcommon.InsertRequest{TableName: "t", Records: []fvcommon.InsertRequestRecord{{Data: map[string]interface{}{" ": 1}}}}, "empty column name"), + Entry("table in both places", fvcommon.InsertRequest{TableName: "t", Records: []fvcommon.InsertRequestRecord{{Data: data, TableName: "t"}}}, "not both"), + Entry("table missing on a record", fvcommon.InsertRequest{Records: []fvcommon.InsertRequestRecord{{Data: data, TableName: "t"}, {Data: data}}}, "Table name is missing"), + Entry("record upsert with request table", fvcommon.InsertRequest{TableName: "t", Records: []fvcommon.InsertRequestRecord{{Data: data, Upsert: &fvcommon.UpsertOptions{UniqueColumns: []string{"a"}}}}}, "Set upsert on the request"), + Entry("request upsert with record tables", fvcommon.InsertRequest{Upsert: &fvcommon.UpsertOptions{UniqueColumns: []string{"a"}}, Records: []fvcommon.InsertRequestRecord{{Data: data, TableName: "t"}}}, "Set upsert on each record"), + Entry("upsert without unique columns", fvcommon.InsertRequest{TableName: "t", Upsert: &fvcommon.UpsertOptions{}, Records: []fvcommon.InsertRequestRecord{{Data: data}}}, "unique columns"), + Entry("invalid upsert update type", fvcommon.InsertRequest{TableName: "t", Upsert: &fvcommon.UpsertOptions{UniqueColumns: []string{"a"}, UpdateType: "MERGE"}, Records: []fvcommon.InsertRequestRecord{{Data: data}}}, "Invalid update type"), + ) +}) + +var _ = Describe("ValidateUpdate", func() { + DescribeTable("requests", + func(req fvcommon.UpdateRequest, message string) { expectError(ValidateUpdate(req), message) }, + Entry("valid", fvcommon.UpdateRequest{TableName: "t", UpdateType: fvcommon.UpdateTypeReplace, Records: []fvcommon.UpdateRequestRecord{{SkyflowId: "id", Data: data}}}, ""), + Entry("missing skyflow ID", fvcommon.UpdateRequest{TableName: "t", Records: []fvcommon.UpdateRequestRecord{{Data: data}}}, "Skyflow ID is missing"), + Entry("table in both places", fvcommon.UpdateRequest{TableName: "t", Records: []fvcommon.UpdateRequestRecord{{SkyflowId: "id", Data: data, TableName: "t"}}}, "not both"), + Entry("table missing", fvcommon.UpdateRequest{Records: []fvcommon.UpdateRequestRecord{{SkyflowId: "id", Data: data}}}, "Table name is missing"), + Entry("invalid update type", fvcommon.UpdateRequest{TableName: "t", UpdateType: "MERGE", Records: []fvcommon.UpdateRequestRecord{{SkyflowId: "id", Data: data}}}, "Invalid update type"), + ) +}) + +var _ = Describe("ValidateGet", func() { + DescribeTable("requests", + func(req fvcommon.GetRequest, message string) { expectError(ValidateGet(req), message) }, + Entry("ids", fvcommon.GetRequest{TableName: "t", SkyflowIds: []string{"id"}}, ""), + Entry("multi-table", fvcommon.GetRequest{Records: []fvcommon.GetRequestRecord{{TableName: "t", UniqueValues: []map[string]interface{}{data}}}}, ""), + Entry("both modes", fvcommon.GetRequest{TableName: "t", Records: []fvcommon.GetRequestRecord{{TableName: "t", SkyflowIds: []string{"id"}}}}, "not both"), + Entry("empty multi-table records", fvcommon.GetRequest{Records: []fvcommon.GetRequestRecord{}}, "Records must not be empty"), + Entry("ids and unique values", fvcommon.GetRequest{TableName: "t", SkyflowIds: []string{"id"}, UniqueValues: []map[string]interface{}{data}}, "exactly one"), + Entry("neither ids nor unique values", fvcommon.GetRequest{TableName: "t"}, "exactly one"), + Entry("empty unique value", fvcommon.GetRequest{TableName: "t", UniqueValues: []map[string]interface{}{{}}}, "empty entries"), + Entry("negative limit", fvcommon.GetRequest{TableName: "t", SkyflowIds: []string{"id"}, Limit: -1}, "must not be negative"), + Entry("bad redaction", fvcommon.GetRequest{TableName: "t", SkyflowIds: []string{"id"}, ColumnRedactions: []fvcommon.ColumnRedaction{{ColumnName: "a"}}}, "Column redactions"), + ) +}) + +var _ = Describe("Other validators", func() { + It("validates Delete", func() { + expectError(ValidateDelete(fvcommon.DeleteRequest{SkyflowIds: []string{"id"}}), "Table name must not be empty") + expectError(ValidateDelete(fvcommon.DeleteRequest{TableName: "t", SkyflowIds: []string{""}}), "empty values") + expectError(ValidateDelete(fvcommon.DeleteRequest{TableName: "t", UniqueValues: []map[string]interface{}{data}}), "") + }) + + It("validates Detokenize", func() { + expectError(ValidateDetokenize(fvcommon.DetokenizeRequest{}), "Tokens must not be empty") + expectError(ValidateDetokenize(fvcommon.DetokenizeRequest{Tokens: []string{" "}}), "empty values") + expectError(ValidateDetokenize(fvcommon.DetokenizeRequest{Tokens: []string{"tok"}, TokenGroupRedactions: []fvcommon.TokenGroupRedaction{{TokenGroupName: "g"}}}), "Token group redactions") + expectError(ValidateDetokenize(fvcommon.DetokenizeRequest{Tokens: []string{"tok"}}), "") + }) + + It("validates GetTokens and Query", func() { + expectError(ValidateGetTokens(fvcommon.GetTokensRequest{}), "Records must not be empty") + expectError(ValidateGetTokens(fvcommon.GetTokensRequest{Records: []fvcommon.GetTokensRequestRecord{{TokenGroupName: "g"}}}), "Value is missing") + expectError(ValidateGetTokens(fvcommon.GetTokensRequest{Records: []fvcommon.GetTokensRequestRecord{{Value: "v"}}}), "Token group name is missing") + expectError(ValidateQuery(fvcommon.QueryRequest{Query: " "}), "Query must not be empty") + expectError(ValidateQuery(fvcommon.QueryRequest{Query: "SELECT 1"}), "") + }) + + It("validates UploadFiles", func() { + column := func(c fvcommon.UploadFilesRequestColumn) fvcommon.UploadFilesRequest { + return fvcommon.UploadFilesRequest{Records: []fvcommon.UploadFilesRequestRecord{{TableName: "t", Columns: []fvcommon.UploadFilesRequestColumn{c}}}} + } + expectError(ValidateUploadFiles(fvcommon.UploadFilesRequest{Records: []fvcommon.UploadFilesRequestRecord{{TableName: "t"}}}), "Columns must not be empty") + expectError(ValidateUploadFiles(column(fvcommon.UploadFilesRequestColumn{FilePath: "a"})), "Column name must not be empty") + expectError(ValidateUploadFiles(column(fvcommon.UploadFilesRequestColumn{Column: "c"})), "exactly one of FilePath") + expectError(ValidateUploadFiles(column(fvcommon.UploadFilesRequestColumn{Column: "c", FilePath: "a", Base64: "b"})), "exactly one of FilePath") + expectError(ValidateUploadFiles(column(fvcommon.UploadFilesRequestColumn{Column: "c", Base64: "b"})), "FileName is required") + expectError(ValidateUploadFiles(column(fvcommon.UploadFilesRequestColumn{Column: "c", Base64: "b", FileName: "a.txt"})), "") + }) + + It("validates DeleteFiles", func() { + record := func(r fvcommon.DeleteFilesRequestRecord) fvcommon.DeleteFilesRequest { + return fvcommon.DeleteFilesRequest{Records: []fvcommon.DeleteFilesRequestRecord{r}} + } + expectError(ValidateDeleteFiles(record(fvcommon.DeleteFilesRequestRecord{TableName: "t", Columns: []string{"c"}})), "exactly one of SkyflowId") + expectError(ValidateDeleteFiles(record(fvcommon.DeleteFilesRequestRecord{TableName: "t", Columns: []string{"c"}, SkyflowId: "id", UniqueValues: []map[string]interface{}{data}})), "exactly one of SkyflowId") + expectError(ValidateDeleteFiles(record(fvcommon.DeleteFilesRequestRecord{TableName: "t", Columns: []string{""}, SkyflowId: "id"})), "Column name must not be empty") + expectError(ValidateDeleteFiles(record(fvcommon.DeleteFilesRequestRecord{TableName: "t", Columns: []string{"c", "d", "c"}, SkyflowId: "id"})), `Column "c" is listed more than once for record at index 0`) + expectError(ValidateDeleteFiles(fvcommon.DeleteFilesRequest{Records: make([]fvcommon.DeleteFilesRequestRecord, 20001)}), "At most 20000") + expectError(ValidateDeleteFiles(record(fvcommon.DeleteFilesRequestRecord{TableName: "t", Columns: []string{"c"}, SkyflowId: "id"})), "") + }) + + It("validates custom headers", func() { + expectError(ValidateCustomHeaders(map[fvcommon.CustomHeaderKey]string{"x-other": "v"}, "Insert"), "is not supported") + expectError(ValidateCustomHeaders(map[fvcommon.CustomHeaderKey]string{fvcommon.RequestIdHeader: " "}, "Insert"), "must have a value") + expectError(ValidateCustomHeaders(map[fvcommon.CustomHeaderKey]string{fvcommon.SkyflowAccountId: "acc"}, "Insert"), "") + }) +}) diff --git a/flowvault/internal/vault/controller/api_test.go b/flowvault/internal/vault/controller/api_test.go new file mode 100644 index 0000000..b9eee4d --- /dev/null +++ b/flowvault/internal/vault/controller/api_test.go @@ -0,0 +1,305 @@ +package controller + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" +) + +// fakeFlowService records each request and answers with the response set for its path. +type fakeFlowService struct { + mu sync.Mutex + server *httptest.Server + requests map[string]map[string]interface{} + headers map[string]http.Header + responses map[string]fakeResponse + puts map[string]string + putCalls map[string]int +} + +type fakeResponse struct { + status int + body string +} + +func newFakeFlowService() *fakeFlowService { + f := &fakeFlowService{ + requests: map[string]map[string]interface{}{}, + headers: map[string]http.Header{}, + responses: map[string]fakeResponse{}, + puts: map[string]string{}, + putCalls: map[string]int{}, + } + f.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + raw, _ := io.ReadAll(r.Body) + f.mu.Lock() + defer f.mu.Unlock() + if r.Method == http.MethodPut { + f.puts[r.URL.Path] = string(raw) + f.putCalls[r.URL.Path]++ + f.headers[r.URL.Path] = r.Header.Clone() + if strings.HasPrefix(r.URL.Path, "/signed/unavailable") { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + return + } + var body map[string]interface{} + _ = json.Unmarshal(raw, &body) + f.requests[r.URL.Path] = body + f.headers[r.URL.Path] = r.Header.Clone() + response, ok := f.responses[r.URL.Path] + if !ok { + response = fakeResponse{status: http.StatusOK, body: `{}`} + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("x-request-id", "req-123") + w.WriteHeader(response.status) + _, _ = io.WriteString(w, response.body) + })) + return f +} + +func (f *fakeFlowService) respond(path string, status int, body string) { + f.responses[path] = fakeResponse{status: status, body: body} +} + +var _ = Describe("VaultController API calls", func() { + var ( + fake *fakeFlowService + v *VaultController + ctx = context.Background() + ) + BeforeEach(func() { + fake = newFakeFlowService() + DeferCleanup(fake.server.Close) + v = NewVaultController(fvcommon.VaultConfig{VaultId: "vault1", BaseVaultUrl: fake.server.URL}, ClientSettings{Credentials: &fvcommon.Credentials{ApiKey: testApiKey}}) + }) + + It("inserts with request-level table and upsert, and parses tokens and per-record errors", func() { + fake.respond("/v2/records/insert", http.StatusOK, `{"records":[ + {"skyflowID":"id1","tableName":"users","httpCode":200,"tokens":{"email":[{"token":"tok1","tokenGroupName":"det"}]},"hashedData":{"email":[{"data":"h","hashName":"sha"}]}}, + {"httpCode":400,"error":"bad record"}]}`) + resp, err := v.Insert(ctx, fvcommon.InsertRequest{ + TableName: "users", + Upsert: &fvcommon.UpsertOptions{UniqueColumns: []string{"email"}, UpdateType: fvcommon.UpdateTypeReplace}, + Records: []fvcommon.InsertRequestRecord{{Data: map[string]interface{}{"email": "a@b.c"}}, {Data: map[string]interface{}{"email": "x"}}}, + }, fvcommon.InsertOptions{}) + Expect(err).To(BeNil()) + + sent := fake.requests["/v2/records/insert"] + Expect(sent["vaultID"]).To(Equal("vault1")) + Expect(sent["tableName"]).To(Equal("users")) + Expect(sent["upsert"]).To(Equal(map[string]interface{}{"uniqueColumns": []interface{}{"email"}, "updateType": "REPLACE"})) + Expect(sent["records"]).To(HaveLen(2)) + Expect(fake.headers["/v2/records/insert"].Get("Authorization")).To(Equal("Bearer " + testApiKey)) + Expect(fake.headers["/v2/records/insert"].Get("sky-metadata")).ToNot(BeEmpty()) + + Expect(resp.Records).To(HaveLen(2)) + Expect(resp.Records[0].SkyflowId).To(Equal("id1")) + Expect(resp.Records[0].Tokens["email"]).To(Equal([]fvcommon.Token{{Token: "tok1", TokenGroupName: "det"}})) + Expect(resp.Records[0].HashedData["email"]).To(Equal([]fvcommon.HashedValue{{Data: "h", HashName: "sha"}})) + Expect(resp.Records[0].RequestId).To(BeEmpty()) + Expect(resp.Records[1].Error).To(Equal("bad record")) + Expect(resp.Records[1].HttpCode).To(Equal(400)) + Expect(resp.Records[1].RequestId).To(Equal("req-123")) + }) + + It("sends record-level tables and upserts", func() { + _, err := v.Insert(ctx, fvcommon.InsertRequest{Records: []fvcommon.InsertRequestRecord{ + {Data: map[string]interface{}{"a": 1}, TableName: "t1", Upsert: &fvcommon.UpsertOptions{UniqueColumns: []string{"a"}}}, + }}, fvcommon.InsertOptions{}) + Expect(err).To(BeNil()) + record := fake.requests["/v2/records/insert"]["records"].([]interface{})[0].(map[string]interface{}) + Expect(record["tableName"]).To(Equal("t1")) + Expect(record["upsert"]).To(Equal(map[string]interface{}{"uniqueColumns": []interface{}{"a"}})) + }) + + It("returns per-record results from a non-2xx response that carries records", func() { + fake.respond("/v2/records/update", http.StatusBadRequest, `{"records":[{"skyflowID":"id1","httpCode":400,"error":"column missing"}]}`) + resp, err := v.Update(ctx, fvcommon.UpdateRequest{ + TableName: "users", UpdateType: fvcommon.UpdateTypeUpdate, + Records: []fvcommon.UpdateRequestRecord{{SkyflowId: "id1", Data: map[string]interface{}{"a": 1}}}, + }, fvcommon.UpdateOptions{}) + Expect(err).To(BeNil()) + Expect(fake.requests["/v2/records/update"]["updateType"]).To(Equal("UPDATE")) + Expect(resp.Records[0].Error).To(Equal("column missing")) + Expect(resp.Records[0].RequestId).To(Equal("req-123")) + }) + + It("maps a whole-call API error to a SkyflowError", func() { + fake.respond("/v2/query", http.StatusBadRequest, `{"error":{"grpc_code":3,"http_code":400,"http_status":"Bad Request","message":"Invalid query"}}`) + _, err := v.Query(ctx, fvcommon.QueryRequest{Query: "SELECT 1"}, fvcommon.QueryOptions{}) + Expect(err).ToNot(BeNil()) + Expect(err.GetMessage()).To(ContainSubstring("Invalid query")) + Expect(err.GetHttpCode()).To(Equal("400")) + Expect(err.GetRequestId()).To(Equal("req-123")) + }) + + It("maps an error with an unexpected status code", func() { + fake.respond("/v2/records/get", http.StatusBadGateway, `{"error":{"http_code":502,"message":"upstream down"}}`) + _, err := v.Get(ctx, fvcommon.GetRequest{TableName: "t", SkyflowIds: []string{"id"}}, fvcommon.GetOptions{}) + Expect(err).ToNot(BeNil()) + Expect(err.GetMessage()).To(ContainSubstring("upstream down")) + }) + + It("gets from one table with limit, offset and redactions", func() { + fake.respond("/v2/records/get", http.StatusOK, `{"records":[{"skyflowID":"id1","data":{"name":"j"},"httpCode":200}]}`) + resp, err := v.Get(ctx, fvcommon.GetRequest{ + TableName: "t", SkyflowIds: []string{"id1"}, Columns: []string{"name"}, Limit: 10, Offset: 5, + ColumnRedactions: []fvcommon.ColumnRedaction{{ColumnName: "name", Redaction: "MASKED"}}, + }, fvcommon.GetOptions{}) + Expect(err).To(BeNil()) + sent := fake.requests["/v2/records/get"] + Expect(sent["tableName"]).To(Equal("t")) + Expect(sent["limit"]).To(BeEquivalentTo(10)) + Expect(sent["offset"]).To(BeEquivalentTo(5)) + Expect(sent["columnRedactions"]).To(Equal([]interface{}{map[string]interface{}{"columnName": "name", "redaction": "MASKED"}})) + Expect(resp.Records[0].Data).To(Equal(map[string]interface{}{"name": "j"})) + }) + + It("gets from several tables", func() { + _, err := v.Get(ctx, fvcommon.GetRequest{Records: []fvcommon.GetRequestRecord{ + {TableName: "t1", SkyflowIds: []string{"a"}}, + {TableName: "t2", UniqueValues: []map[string]interface{}{{"email": "x"}}}, + }}, fvcommon.GetOptions{}) + Expect(err).To(BeNil()) + sent := fake.requests["/v2/records/get"] + Expect(sent).ToNot(HaveKey("tableName")) + records := sent["records"].([]interface{}) + Expect(records[1].(map[string]interface{})["uniqueValues"]).To(Equal([]interface{}{map[string]interface{}{"data": map[string]interface{}{"email": "x"}}})) + }) + + It("deletes records", func() { + fake.respond("/v2/records/delete", http.StatusOK, `{"records":[{"skyflowID":"id1","httpCode":200}]}`) + resp, err := v.Delete(ctx, fvcommon.DeleteRequest{TableName: "t", SkyflowIds: []string{"id1"}}, fvcommon.DeleteOptions{}) + Expect(err).To(BeNil()) + Expect(fake.requests["/v2/records/delete"]["skyflowIDs"]).To(Equal([]interface{}{"id1"})) + Expect(resp.Records).To(Equal([]fvcommon.DeleteResponseRecord{{SkyflowId: "id1", HttpCode: 200}})) + }) + + It("detokenizes with redactions and metadata", func() { + fake.respond("/v2/tokens/detokenize", http.StatusOK, `{"response":[ + {"token":"tok1","value":"john","tokenGroupName":"g","httpCode":200,"metadata":{"skyflowID":"id1","tableName":"users"}}, + {"token":"tok2","httpCode":404,"error":"not found"}]}`) + resp, err := v.Detokenize(ctx, fvcommon.DetokenizeRequest{ + Tokens: []string{"tok1", "tok2"}, + TokenGroupRedactions: []fvcommon.TokenGroupRedaction{{TokenGroupName: "g", Redaction: "PLAIN_TEXT"}}, + }, fvcommon.DetokenizeOptions{}) + Expect(err).To(BeNil()) + Expect(fake.requests["/v2/tokens/detokenize"]["tokenGroupRedactions"]).To(Equal([]interface{}{map[string]interface{}{"tokenGroupName": "g", "redaction": "PLAIN_TEXT"}})) + Expect(resp.Records[0].Value).To(Equal("john")) + Expect(resp.Records[0].Metadata).To(Equal(&fvcommon.DetokenizeRecordMetadata{SkyflowId: "id1", TableName: "users"})) + Expect(resp.Records[1].HttpCode).To(Equal(404)) + Expect(resp.Records[1].RequestId).To(Equal("req-123")) + }) + + It("gets tokens for values", func() { + fake.respond("/v2/records/getTokens", http.StatusOK, `{"records":[{"token":"tok1","value":"a@b.c","tokenGroupName":"det","httpCode":200}]}`) + resp, err := v.GetTokens(ctx, fvcommon.GetTokensRequest{Records: []fvcommon.GetTokensRequestRecord{{Value: "a@b.c", TokenGroupName: "det"}}}, fvcommon.GetTokensOptions{}) + Expect(err).To(BeNil()) + Expect(fake.requests["/v2/records/getTokens"]["records"]).To(Equal([]interface{}{map[string]interface{}{"value": "a@b.c", "tokenGroupName": "det"}})) + Expect(resp.Records).To(Equal([]fvcommon.GetTokensResponseRecord{{Value: "a@b.c", TokenGroupName: "det", Token: "tok1", HttpCode: 200}})) + }) + + It("runs a query and returns rows, columns and the request ID", func() { + fake.respond("/v2/query", http.StatusOK, `{"records":[{"data":{"name":"j"}}],"metadata":{"columns":["name"]}}`) + resp, err := v.Query(ctx, fvcommon.QueryRequest{Query: "SELECT name FROM users"}, fvcommon.QueryOptions{}) + Expect(err).To(BeNil()) + Expect(fake.requests["/v2/query"]["query"]).To(Equal("SELECT name FROM users")) + Expect(resp.Fields).To(Equal([]map[string]interface{}{{"name": "j"}})) + Expect(resp.Metadata.Columns).To(Equal([]string{"name"})) + Expect(resp.RequestId).To(Equal("req-123")) + }) + + It("uploads files to their signed URLs and reports each column", func() { + fake.respond("/v2/files/upload", http.StatusOK, `{"records":[ + {"skyflowID":"new1","tableName":"docs","httpCode":200,"data":{"resume":"`+fake.server.URL+`/signed/resume"}}, + {"skyflowID":"id2","tableName":"docs","httpCode":404,"error":"record not found"}]}`) + resp, err := v.UploadFiles(ctx, fvcommon.UploadFilesRequest{Records: []fvcommon.UploadFilesRequestRecord{ + {TableName: "docs", Columns: []fvcommon.UploadFilesRequestColumn{ + {Column: "resume", Base64: "cGRm", FileName: "resume.pdf"}, + {Column: "photo", Base64: "aW1n", FileName: "photo.png"}, + }}, + {TableName: "docs", SkyflowId: "id2", Columns: []fvcommon.UploadFilesRequestColumn{{Column: "resume", Base64: "cGRm", FileName: "b.pdf"}}}, + }}, fvcommon.UploadFilesOptions{}) + Expect(err).To(BeNil()) + + sent := fake.requests["/v2/files/upload"]["records"].([]interface{}) + Expect(sent[0].(map[string]interface{})).ToNot(HaveKey("skyflowID")) + Expect(sent[0].(map[string]interface{})["columns"]).To(Equal([]interface{}{ + map[string]interface{}{"column": "resume", "fileName": "resume.pdf"}, + map[string]interface{}{"column": "photo", "fileName": "photo.png"}, + })) + Expect(sent[1].(map[string]interface{})["skyflowID"]).To(Equal("id2")) + + Expect(fake.puts["/signed/resume"]).To(Equal("pdf")) + Expect(fake.headers["/signed/resume"].Get("Authorization")).To(BeEmpty()) + Expect(resp.Records[0].SkyflowId).To(Equal("new1")) + Expect(resp.Records[0].Columns).To(Equal([]fvcommon.UploadFilesColumnResult{ + {Column: "resume", FileName: "resume.pdf", UploadStatus: fvcommon.UploadStatusUploaded}, + {Column: "photo", FileName: "photo.png", UploadStatus: fvcommon.UploadStatusSkipped}, + })) + Expect(resp.Records[1].Error).To(Equal("record not found")) + Expect(resp.Records[1].RequestId).To(Equal("req-123")) + Expect(resp.Records[1].Columns[0].UploadStatus).To(Equal(fvcommon.UploadStatusSkipped)) + Expect(resp.Records[1].Columns[0].Error).To(Equal("record not found")) + }) + + It("doesn't retry a failed file upload, even with retries on", func() { + retries := 3 + v.SetConfig(fvcommon.VaultConfig{VaultId: "vault1", BaseVaultUrl: fake.server.URL, MaxRetries: &retries, InitialRetryDelay: time.Millisecond}, false) + fake.respond("/v2/files/upload", http.StatusOK, `{"records":[ + {"skyflowID":"id1","tableName":"docs","httpCode":200,"data":{"resume":"`+fake.server.URL+`/signed/unavailable"}}]}`) + resp, err := v.UploadFiles(ctx, fvcommon.UploadFilesRequest{Records: []fvcommon.UploadFilesRequestRecord{ + {TableName: "docs", Columns: []fvcommon.UploadFilesRequestColumn{{Column: "resume", Base64: "cGRm", FileName: "resume.pdf"}}}, + }}, fvcommon.UploadFilesOptions{ + Interceptor: func(rc *fvcommon.RequestContext) { rc.AddHeader(fvcommon.RequestIdHeader, "mine") }, + }) + Expect(err).To(BeNil()) + Expect(fake.putCalls["/signed/unavailable"]).To(Equal(1)) + Expect(fake.headers["/signed/unavailable"].Get("x-request-id")).To(BeEmpty()) + Expect(fake.headers["/signed/unavailable"].Get("sky-metadata")).To(BeEmpty()) + Expect(resp.Records[0].Columns[0]).To(Equal(fvcommon.UploadFilesColumnResult{ + Column: "resume", FileName: "resume.pdf", UploadStatus: fvcommon.UploadStatusFailed, Error: "PUT failed: 503", + })) + }) + + It("deletes files", func() { + fake.respond("/v2/files/delete", http.StatusOK, `{"records":[{"skyflowID":"id1","tableName":"docs","httpCode":200,"data":{"resume":{"status":"DELETED"}}}]}`) + resp, err := v.DeleteFiles(ctx, fvcommon.DeleteFilesRequest{Records: []fvcommon.DeleteFilesRequestRecord{ + {TableName: "docs", Columns: []string{"resume"}, UniqueValues: []map[string]interface{}{{"email": "x"}}}, + }}, fvcommon.DeleteFilesOptions{}) + Expect(err).To(BeNil()) + record := fake.requests["/v2/files/delete"]["records"].([]interface{})[0].(map[string]interface{}) + Expect(record).ToNot(HaveKey("skyflowID")) + Expect(record["columns"]).To(Equal([]interface{}{"resume"})) + Expect(resp.Records[0].Columns).To(Equal([]fvcommon.DeleteFilesColumnResult{{Column: "resume", Status: "DELETED"}})) + }) + + It("sends interceptor headers", func() { + _, err := v.Query(ctx, fvcommon.QueryRequest{Query: "SELECT 1"}, fvcommon.QueryOptions{ + Interceptor: func(rc *fvcommon.RequestContext) { rc.AddHeader(fvcommon.RequestIdHeader, "mine") }, + }) + Expect(err).To(BeNil()) + Expect(fake.headers["/v2/query"].Get("x-request-id")).To(Equal("mine")) + }) + + It("reports a network failure as a SkyflowError", func() { + fake.server.Close() + _, err := v.Query(ctx, fvcommon.QueryRequest{Query: "SELECT 1"}, fvcommon.QueryOptions{}) + Expect(err).ToNot(BeNil()) + Expect(err.GetCode()).To(ContainSubstring("500")) + }) +}) diff --git a/flowvault/internal/vault/controller/base_controller.go b/flowvault/internal/vault/controller/base_controller.go index 9ea2088..74f4274 100644 --- a/flowvault/internal/vault/controller/base_controller.go +++ b/flowvault/internal/vault/controller/base_controller.go @@ -3,28 +3,60 @@ package controller import ( "net/http" "os" + "sync" fvconsts "github.com/skyflowapi/skyflow-go/flowvault/internal/constants" + apiclient "github.com/skyflowapi/skyflow-go/flowvault/internal/generated/client" + "github.com/skyflowapi/skyflow-go/flowvault/internal/generated/option" fvhelpers "github.com/skyflowapi/skyflow-go/flowvault/internal/helpers" + fvvalidation "github.com/skyflowapi/skyflow-go/flowvault/internal/validation" fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" fverrors "github.com/skyflowapi/skyflow-go/flowvault/utils/error" fvlogger "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" + logs "github.com/skyflowapi/skyflow-go/flowvault/utils/messages" ) // BaseController holds shared auth state and HTTP client logic for flowvault vault operations. // flowvault's VaultController embeds this — equivalent to Java's AbstractBaseVaultController. // -// Can only use v2's EXPORTED packages (serviceaccount, utils/*). -// v2's internal/* are module-private and cannot be imported from flowvault. +// It is safe for concurrent use: calls read a snapshot of the settings, and the client can +// change them (SetConfig, SetClientSettings) while calls are running. type BaseController struct { - Config *fvcommon.VaultConfig - CommonCreds *fvcommon.Credentials - Token string - ApiKey string - HTTPClient *http.Client - BaseURL string - AuthHeader string - CustomHeaders map[fvcommon.CustomHeaderKey]string + settingsMu sync.RWMutex + config fvcommon.VaultConfig + client ClientSettings + // authVersion changes whenever the credentials may have changed, so a token generated + // from older credentials is never reused. + authVersion uint64 + + tokenMu sync.Mutex + token string + tokenVersion uint64 +} + +// ClientSettings are the client-level settings every vault falls back to. +type ClientSettings struct { + Credentials *fvcommon.Credentials + HttpConfig fvcommon.HttpConfig +} + +// settings is a consistent copy of the controller's settings for one call. +type settings struct { + config fvcommon.VaultConfig + client ClientSettings + authVersion uint64 +} + +// RequestSetup is everything one API call needs: the vault ID, where to send it, how to +// authenticate, which headers to add, the HTTP client (timeout and retries applied) and +// the generated API client built from all of these. +type RequestSetup struct { + VaultID string + BaseURL string + AuthToken string + Headers http.Header + HTTPClient *http.Client + API *apiclient.Client } // GenerateToken resolves a bearer token from credentials. @@ -56,93 +88,166 @@ func GenerateToken(creds fvcommon.Credentials) (*string, *fverrors.SkyflowError) } token = resp.AccessToken default: - return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, "no valid credentials provided") + return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, fverrors.EMPTY_CREDENTIALS) } return &token, nil } -// RefreshToken checks token expiry and regenerates if needed. -// fvhelpers.Auth().IsExpired is exported — reused directly. -func (b *BaseController) RefreshToken() *fverrors.SkyflowError { - creds, err := b.resolveCredentials() - if err != nil { - return err - } - if b.Token == "" || fvhelpers.Auth().IsExpired(b.Token) { - fvlogger.Info("generating new bearer token") - token, err := GenerateToken(*creds) - if err != nil { - return err - } - b.Token = *token - } else { - fvlogger.Info("reusing existing bearer token") - } - return nil +// Config returns a copy of the vault config. +func (b *BaseController) Config() fvcommon.VaultConfig { + b.settingsMu.RLock() + defer b.settingsMu.RUnlock() + return b.config } -// SetupClient prepares the authenticated HTTP client for a vault request. -func (b *BaseController) SetupClient(customHeaders map[string]string) *fverrors.SkyflowError { - // 1. resolve auth token - if b.Config.Credentials.ApiKey != "" { - b.ApiKey = b.Config.Credentials.ApiKey - } else if b.Config.Credentials.Token != "" { - if fvhelpers.Auth().IsExpired(b.Config.Credentials.Token) { - return fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, "bearer token is expired") - } - b.Token = b.Config.Credentials.Token - } else { - if err := b.RefreshToken(); err != nil { - return err - } +// SetConfig replaces the vault config. Set resetAuth when the credentials changed. +func (b *BaseController) SetConfig(config fvcommon.VaultConfig, resetAuth bool) { + b.settingsMu.Lock() + defer b.settingsMu.Unlock() + b.config = config + if resetAuth { + b.authVersion++ } +} - b.AuthHeader = b.Token - if b.ApiKey != "" { - b.AuthHeader = b.ApiKey - } +// ClientSettings returns the client-level settings this vault uses. +func (b *BaseController) ClientSettings() ClientSettings { + b.settingsMu.RLock() + defer b.settingsMu.RUnlock() + return b.client +} - // 2. build base URL - if b.Config.BaseVaultUrl != "" { - b.BaseURL = b.Config.BaseVaultUrl - } else { - b.BaseURL = fvhelpers.GetURLWithEnv(b.Config.Env, b.Config.ClusterId) +// SetClientSettings replaces the client-level settings. A change of client credentials +// drops the cached token. +func (b *BaseController) SetClientSettings(client ClientSettings) { + b.settingsMu.Lock() + defer b.settingsMu.Unlock() + if client.Credentials != b.client.Credentials { + b.authVersion++ } + b.client = client +} + +// ResetAuth drops the cached bearer token so the next call re-authenticates. +func (b *BaseController) ResetAuth() { + b.settingsMu.Lock() + defer b.settingsMu.Unlock() + b.authVersion++ +} - // 3. build default headers applied to every request - b.HTTPClient = &http.Client{} - return nil +func (b *BaseController) snapshot() settings { + b.settingsMu.RLock() + defer b.settingsMu.RUnlock() + return settings{config: b.config, client: b.client, authVersion: b.authVersion} } -// NewRequest builds an authenticated *http.Request for a vault API call. -func (b *BaseController) NewRequest(method, path string, body interface{}) (*http.Request, error) { - req, err := http.NewRequest(method, b.BaseURL+path, nil) +// SetupRequest resolves auth, headers and HTTP settings for one call. The interceptor, +// when set, can add per-call headers. +func (b *BaseController) SetupRequest(operation string, interceptor fvcommon.Interceptor) (*RequestSetup, *fverrors.SkyflowError) { + current := b.snapshot() + headers, err := buildHeaders(operation, interceptor) if err != nil { return nil, err } - req.Header.Set("Authorization", "Bearer "+b.AuthHeader) - req.Header.Set("Content-Type", "application/json") - req.Header.Set(fvconsts.SDK_METRICS_HEADER_KEY, fvhelpers.CreateSDKMetadata()) - for k, val := range b.CustomHeaders { - req.Header.Set(string(k), val) + authToken, err := b.authToken(current) + if err != nil { + return nil, err + } + httpConfig := fvhelpers.ResolveHttpConfig(current.config.HttpConfig(), current.client.HttpConfig) + setup := &RequestSetup{ + VaultID: current.config.VaultId, + BaseURL: baseURL(current.config), + AuthToken: authToken, + Headers: headers, + HTTPClient: fvhelpers.NewHTTPClient(httpConfig), + } + // Retries happen in setup.HTTPClient, so the generated client makes a single attempt. + setup.API = apiclient.NewClient( + option.WithBaseURL(setup.BaseURL), + option.WithHTTPClient(setup.HTTPClient), + option.WithHTTPHeader(setup.Headers), + option.WithToken(setup.AuthToken), + option.WithMaxAttempts(1), + ) + return setup, nil +} + +// buildHeaders merges the interceptor's per-call headers and the SDK metadata header. +func buildHeaders(operation string, interceptor fvcommon.Interceptor) (http.Header, *fverrors.SkyflowError) { + headers := http.Header{} + if interceptor != nil { + rc := &fvcommon.RequestContext{Operation: operation} + interceptor(rc) + if added := rc.Headers(); len(added) > 0 { + if err := fvvalidation.ValidateCustomHeaders(added, operation); err != nil { + return nil, err + } + for key, value := range added { + headers.Set(string(key), value) + } + } } - return req, nil + headers.Set(fvconsts.SDK_METRICS_HEADER_KEY, fvhelpers.CreateSDKMetadata()) + return headers, nil } -func (b *BaseController) resolveCredentials() (*fvcommon.Credentials, *fverrors.SkyflowError) { - if b.Config != nil && !isEmpty(b.Config.Credentials) { - return &b.Config.Credentials, nil +// authToken returns the API key or bearer token to send, generating and caching a +// bearer token from service-account credentials when needed. +func (b *BaseController) authToken(current settings) (string, *fverrors.SkyflowError) { + creds, err := resolveCredentials(current) + if err != nil { + return "", err + } + if creds.ApiKey != "" { + return creds.ApiKey, nil + } + if creds.Token != "" { + if fvhelpers.Auth().IsExpired(creds.Token) { + fvlogger.Error(logs.BEARER_TOKEN_EXPIRED) + return "", fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, fverrors.BEARER_TOKEN_EXPIRED) + } + return creds.Token, nil + } + b.tokenMu.Lock() + defer b.tokenMu.Unlock() + if b.token != "" && b.tokenVersion == current.authVersion && !fvhelpers.Auth().IsExpired(b.token) { + fvlogger.Info(logs.REUSE_BEARER_TOKEN) + return b.token, nil + } + fvlogger.Info(logs.GENERATE_BEARER_TOKEN_TRIGGERED) + token, err := GenerateToken(*creds) + if err != nil { + return "", err + } + // A call that started before a credentials change must not replace a newer token. + if current.authVersion >= b.tokenVersion { + b.token, b.tokenVersion = *token, current.authVersion } - if b.CommonCreds != nil && !isEmpty(*b.CommonCreds) { - return b.CommonCreds, nil + return *token, nil +} + +func baseURL(config fvcommon.VaultConfig) string { + if config.BaseVaultUrl != "" { + return config.BaseVaultUrl } - if envCreds := os.Getenv("SKYFLOW_CREDENTIALS"); envCreds != "" { - c := fvcommon.Credentials{CredentialsString: envCreds} - return &c, nil + if config.BaseVaultURL != "" { + return config.BaseVaultURL } - return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, "no credentials found") + return fvhelpers.GetURLWithEnv(config.Env, config.ClusterId) } -func isEmpty(c fvcommon.Credentials) bool { - return c.Path == "" && c.CredentialsString == "" && c.Token == "" && c.ApiKey == "" +// resolveCredentials picks vault credentials, then client credentials, then SKYFLOW_CREDENTIALS. +func resolveCredentials(current settings) (*fvcommon.Credentials, *fverrors.SkyflowError) { + if !fvcommon.IsCredentialsEmpty(current.config.Credentials) { + creds := current.config.Credentials + return &creds, nil + } + if current.client.Credentials != nil && !fvcommon.IsCredentialsEmpty(*current.client.Credentials) { + return current.client.Credentials, nil + } + if envCreds := os.Getenv(fvconsts.SKYFLOW_CREDENTIALS_ENV); envCreds != "" { + return &fvcommon.Credentials{CredentialsString: envCreds}, nil + } + fvlogger.Error(logs.EMPTY_CREDENTIALS) + return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, fverrors.EMPTY_CREDENTIALS) } diff --git a/flowvault/internal/vault/controller/controller_test.go b/flowvault/internal/vault/controller/controller_test.go new file mode 100644 index 0000000..02e69fb --- /dev/null +++ b/flowvault/internal/vault/controller/controller_test.go @@ -0,0 +1,87 @@ +package controller + +import ( + "os" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/skyflowapi/skyflow-go/flowvault/internal/testutil" + fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" +) + +func TestController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Flowvault Controller Suite") +} + +var ( + testApiKey = testutil.Resource("client_api_key.txt") + vaultApiKey = testutil.Resource("vault_api_key.txt") +) + +var _ = Describe("SetupRequest", func() { + It("uses the client API key and the interceptor's headers", func() { + config := fvcommon.VaultConfig{VaultId: "v", ClusterId: "cluster", Env: fvcommon.DEV, Timeout: 5 * time.Second} + v := NewVaultController(config, ClientSettings{Credentials: &fvcommon.Credentials{ApiKey: testApiKey}}) + + var operation string + setup, err := v.SetupRequest("QUERY", func(rc *fvcommon.RequestContext) { + operation = rc.Operation + rc.AddHeader(fvcommon.RequestIdHeader, "custom-id") + rc.AddHeader(fvcommon.SkyflowAccountId, "acc") + }) + Expect(err).To(BeNil()) + Expect(operation).To(Equal("QUERY")) + Expect(setup.AuthToken).To(Equal(testApiKey)) + Expect(setup.BaseURL).To(Equal("https://cluster.skyvault.skyflowapis.dev")) + Expect(setup.Headers.Get("x-skyflow-account-id")).To(Equal("acc")) + Expect(setup.Headers.Get("x-request-id")).To(Equal("custom-id")) + Expect(setup.Headers.Get("sky-metadata")).To(ContainSubstring(`"sdk_name_version":"skyflow-flowvault-go@`)) + Expect(setup.HTTPClient.Timeout).To(Equal(5 * time.Second)) + }) + + It("prefers vault credentials and BaseVaultUrl", func() { + config := fvcommon.VaultConfig{ + VaultId: "v", BaseVaultUrl: "https://example.com", + Credentials: fvcommon.Credentials{ApiKey: vaultApiKey}, + } + setup, err := NewVaultController(config, ClientSettings{Credentials: &fvcommon.Credentials{ApiKey: testApiKey}}).SetupRequest("GET", nil) + Expect(err).To(BeNil()) + Expect(setup.AuthToken).To(Equal(vaultApiKey)) + Expect(setup.BaseURL).To(Equal("https://example.com")) + }) + + It("uses PROD when Env isn't set", func() { + setup, err := NewVaultController(fvcommon.VaultConfig{VaultId: "v", ClusterId: "cluster"}, ClientSettings{Credentials: &fvcommon.Credentials{ApiKey: testApiKey}}).SetupRequest("GET", nil) + Expect(err).To(BeNil()) + Expect(setup.BaseURL).To(Equal("https://cluster.skyvault.skyflowapis.com")) + }) + + It("drops the cached token when credentials change", func() { + v := NewVaultController(fvcommon.VaultConfig{VaultId: "v"}, ClientSettings{}) + version := v.snapshot().authVersion + v.SetClientSettings(ClientSettings{Credentials: &fvcommon.Credentials{ApiKey: testApiKey}}) + Expect(v.snapshot().authVersion).To(Equal(version + 1)) + v.SetClientSettings(ClientSettings{Credentials: v.ClientSettings().Credentials, HttpConfig: fvcommon.HttpConfig{Timeout: time.Second}}) + Expect(v.snapshot().authVersion).To(Equal(version + 1)) + v.SetConfig(v.Config(), true) + Expect(v.snapshot().authVersion).To(Equal(version + 2)) + v.ResetAuth() + Expect(v.snapshot().authVersion).To(Equal(version + 3)) + }) + + It("fails when no credentials are available", func() { + Expect(os.Unsetenv("SKYFLOW_CREDENTIALS")).To(Succeed()) + _, err := NewVaultController(fvcommon.VaultConfig{VaultId: "v", ClusterId: "c"}, ClientSettings{}).SetupRequest("GET", nil) + Expect(err.GetMessage()).To(ContainSubstring("Invalid credentials")) + }) + + It("rejects an expired bearer token", func() { + config := fvcommon.VaultConfig{VaultId: "v", ClusterId: "c", Credentials: fvcommon.Credentials{Token: "not-a-jwt"}} + _, err := NewVaultController(config, ClientSettings{}).SetupRequest("GET", nil) + Expect(err.GetMessage()).To(ContainSubstring("expired")) + }) +}) diff --git a/flowvault/internal/vault/controller/convert.go b/flowvault/internal/vault/controller/convert.go new file mode 100644 index 0000000..6704f64 --- /dev/null +++ b/flowvault/internal/vault/controller/convert.go @@ -0,0 +1,326 @@ +package controller + +import ( + "encoding/json" + "errors" + "net/http" + "strings" + + api "github.com/skyflowapi/skyflow-go/flowvault/internal/generated" + "github.com/skyflowapi/skyflow-go/flowvault/internal/generated/core" + fvhelpers "github.com/skyflowapi/skyflow-go/flowvault/internal/helpers" + fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + fverrors "github.com/skyflowapi/skyflow-go/flowvault/utils/error" +) + +// result returns the response body and headers. A non-2xx response whose body still +// carries per-record results (hasRecords) is returned as a body, not an error, so callers +// see which records failed. +func result[T any](resp *core.Response[*T], err error, hasRecords func(*T) bool) (*T, http.Header, *fverrors.SkyflowError) { + if err == nil { + body, header := new(T), http.Header{} + if resp != nil { + if resp.Body != nil { + body = resp.Body + } + if resp.Header != nil { + header = resp.Header + } + } + return body, header, nil + } + var apiErr *core.APIError + if !errors.As(err, &apiErr) { + return nil, nil, fverrors.NewSkyflowError(fverrors.SERVER, err.Error()) + } + if hasRecords != nil && apiErr.Unwrap() != nil { + body := new(T) + if json.Unmarshal([]byte(apiErr.Unwrap().Error()), body) == nil && hasRecords(body) { + header := apiErr.Header + if header == nil { + header = http.Header{} + } + return body, header, nil + } + } + header := apiErr.Header + if header == nil { + header = http.Header{} + } + return nil, nil, fverrors.SkyflowErrorApi(apiErr, header) +} + +func deref[T any](value *T) T { + var zero T + if value == nil { + return zero + } + return *value +} + +func optionalString(value string) *string { + if strings.TrimSpace(value) == "" { + return nil + } + return &value +} + +func optionalInt(value int) *int { + if value == 0 { + return nil + } + return &value +} + +func toWireUniqueValues(values []map[string]interface{}) []*api.UniqueValue { + if values == nil { + return nil + } + wire := make([]*api.UniqueValue, 0, len(values)) + for _, value := range values { + wire = append(wire, &api.UniqueValue{Data: value}) + } + return wire +} + +func toWireColumnRedactions(redactions []fvcommon.ColumnRedaction) []*api.ColumnRedactions { + if redactions == nil { + return nil + } + wire := make([]*api.ColumnRedactions, 0, len(redactions)) + for _, redaction := range redactions { + wire = append(wire, &api.ColumnRedactions{ColumnName: redaction.ColumnName, Redaction: redaction.Redaction}) + } + return wire +} + +func toWireUpsert(upsert *fvcommon.UpsertOptions) *api.Upsert { + if upsert == nil { + return nil + } + wire := &api.Upsert{UniqueColumns: upsert.UniqueColumns} + if upsert.UpdateType != "" { + wire.UpdateType = api.UpsertUpdateType(upsert.UpdateType).Ptr() + } + return wire +} + +func toWireInsert(vaultID string, request fvcommon.InsertRequest) *api.InsertRequest { + wire := &api.InsertRequest{ + VaultID: vaultID, + TableName: request.TableName, + Upsert: toWireUpsert(request.Upsert), + Records: make([]*api.InsertRecordData, 0, len(request.Records)), + } + for _, record := range request.Records { + wire.Records = append(wire.Records, &api.InsertRecordData{ + Data: record.Data, + Tokens: record.Tokens, + TableName: optionalString(record.TableName), + Upsert: toWireUpsert(record.Upsert), + }) + } + return wire +} + +func toWireUpdate(vaultID string, request fvcommon.UpdateRequest) *api.UpdateRequest { + wire := &api.UpdateRequest{ + VaultID: vaultID, + TableName: request.TableName, + Records: make([]*api.UpdateRecordData, 0, len(request.Records)), + } + if request.UpdateType != "" { + wire.UpdateType = api.UpdateRequestUpdateType(request.UpdateType).Ptr() + } + for _, record := range request.Records { + wire.Records = append(wire.Records, &api.UpdateRecordData{ + SkyflowID: record.SkyflowId, + Data: record.Data, + Tokens: record.Tokens, + TableName: optionalString(record.TableName), + }) + } + return wire +} + +func toWireGet(vaultID string, request fvcommon.GetRequest) *api.GetRequest { + wire := &api.GetRequest{VaultID: vaultID} + if request.Records != nil { + wire.Records = make([]*api.GetRequestData, 0, len(request.Records)) + for _, record := range request.Records { + wire.Records = append(wire.Records, &api.GetRequestData{ + TableName: record.TableName, + SkyflowIDs: record.SkyflowIds, + Columns: record.Columns, + ColumnRedactions: toWireColumnRedactions(record.ColumnRedactions), + UniqueValues: toWireUniqueValues(record.UniqueValues), + }) + } + return wire + } + wire.TableName = optionalString(request.TableName) + wire.SkyflowIDs = request.SkyflowIds + wire.UniqueValues = toWireUniqueValues(request.UniqueValues) + wire.Columns = request.Columns + wire.ColumnRedactions = toWireColumnRedactions(request.ColumnRedactions) + wire.Limit = optionalInt(request.Limit) + wire.Offset = optionalInt(request.Offset) + return wire +} + +func toWireDelete(vaultID string, request fvcommon.DeleteRequest) *api.DeleteRequest { + return &api.DeleteRequest{ + VaultID: vaultID, + TableName: optionalString(request.TableName), + SkyflowIDs: request.SkyflowIds, + UniqueValues: toWireUniqueValues(request.UniqueValues), + } +} + +func toWireDetokenize(vaultID string, request fvcommon.DetokenizeRequest) *api.DetokenizeRequest { + wire := &api.DetokenizeRequest{VaultID: vaultID, Tokens: request.Tokens} + for _, redaction := range request.TokenGroupRedactions { + wire.TokenGroupRedactions = append(wire.TokenGroupRedactions, &api.TokenGroupRedactions{ + TokenGroupName: optionalString(redaction.TokenGroupName), + Redaction: optionalString(redaction.Redaction), + }) + } + return wire +} + +func toWireGetTokens(vaultID string, request fvcommon.GetTokensRequest) *api.GetTokensFromValuesRequest { + wire := &api.GetTokensFromValuesRequest{ + VaultID: vaultID, + Records: make([]*api.GetTokensFromValuesRequestObject, 0, len(request.Records)), + } + for _, record := range request.Records { + wire.Records = append(wire.Records, &api.GetTokensFromValuesRequestObject{Value: record.Value, TokenGroupName: record.TokenGroupName}) + } + return wire +} + +func toWireUploadFiles(vaultID string, request fvcommon.UploadFilesRequest, files [][]fvhelpers.ResolvedFile) *api.FileUploadRequest { + wire := &api.FileUploadRequest{VaultID: vaultID, Records: make([]*api.FileUploadRecord, 0, len(request.Records))} + for i, record := range request.Records { + columns := make([]*api.FileUploadColumn, 0, len(record.Columns)) + for j, column := range record.Columns { + columns = append(columns, &api.FileUploadColumn{Column: column.Column, FileName: optionalString(files[i][j].Name)}) + } + wire.Records = append(wire.Records, &api.FileUploadRecord{ + TableName: record.TableName, + SkyflowID: optionalString(record.SkyflowId), + Columns: columns, + }) + } + return wire +} + +func toWireDeleteFiles(vaultID string, request fvcommon.DeleteFilesRequest) *api.FileDeleteRequest { + wire := &api.FileDeleteRequest{VaultID: vaultID, Records: make([]*api.FileDeleteRecord, 0, len(request.Records))} + for _, record := range request.Records { + wire.Records = append(wire.Records, &api.FileDeleteRecord{ + TableName: record.TableName, + SkyflowID: optionalString(record.SkyflowId), + Columns: record.Columns, + UniqueValues: toWireUniqueValues(record.UniqueValues), + }) + } + return wire +} + +// toRecordResults converts insert, update and get records. +func toRecordResults(records []*api.RecordResponseObject, header http.Header) []fvcommon.InsertResponseRecord { + results := make([]fvcommon.InsertResponseRecord, 0, len(records)) + for _, record := range records { + if record == nil { + continue + } + recordErr := deref(record.Error) + results = append(results, fvcommon.InsertResponseRecord{ + TableName: deref(record.TableName), + SkyflowId: deref(record.SkyflowID), + Tokens: fvhelpers.ParseTokens(record.Tokens), + Data: record.Data, + HashedData: fvhelpers.ParseHashedData(record.HashedData), + HttpCode: record.HTTPCode, + Error: recordErr, + RequestId: fvhelpers.RequestIdFor(header, recordErr), + }) + } + return results +} + +func toDeleteResults(records []*api.DeleteResponseObject, header http.Header) []fvcommon.DeleteResponseRecord { + results := make([]fvcommon.DeleteResponseRecord, 0, len(records)) + for _, record := range records { + if record == nil { + continue + } + recordErr := deref(record.Error) + results = append(results, fvcommon.DeleteResponseRecord{ + SkyflowId: record.SkyflowID, + HttpCode: record.HTTPCode, + Error: recordErr, + RequestId: fvhelpers.RequestIdFor(header, recordErr), + }) + } + return results +} + +func toDetokenizeResults(records []*api.DetokenizeResponseObject, header http.Header) []fvcommon.DetokenizeResponseRecord { + results := make([]fvcommon.DetokenizeResponseRecord, 0, len(records)) + for _, record := range records { + if record == nil { + continue + } + recordErr := deref(record.Error) + results = append(results, fvcommon.DetokenizeResponseRecord{ + Token: record.Token, + Value: deref(record.Value), + TokenGroupName: deref(record.TokenGroupName), + Metadata: fvhelpers.ParseDetokenizeMetadata(record.Metadata), + HttpCode: deref(record.HTTPCode), + Error: recordErr, + RequestId: fvhelpers.RequestIdFor(header, recordErr), + }) + } + return results +} + +func toGetTokensResults(records []*api.TokenizeResponseObject, header http.Header) []fvcommon.GetTokensResponseRecord { + results := make([]fvcommon.GetTokensResponseRecord, 0, len(records)) + for _, record := range records { + if record == nil { + continue + } + recordErr := deref(record.Error) + results = append(results, fvcommon.GetTokensResponseRecord{ + Value: deref(record.Value), + TokenGroupName: deref(record.TokenGroupName), + Token: record.Token, + HttpCode: deref(record.HTTPCode), + Error: recordErr, + RequestId: fvhelpers.RequestIdFor(header, recordErr), + }) + } + return results +} + +func toDeleteFilesResults(records []*api.FileDeleteResponseObject, header http.Header) []fvcommon.DeleteFilesResponseRecord { + results := make([]fvcommon.DeleteFilesResponseRecord, 0, len(records)) + for _, record := range records { + if record == nil { + continue + } + recordErr := deref(record.Error) + results = append(results, fvcommon.DeleteFilesResponseRecord{ + SkyflowId: deref(record.SkyflowID), + TableName: record.TableName, + Columns: fvhelpers.ParseDeleteFilesColumns(record.Data), + HttpCode: record.HTTPCode, + Error: recordErr, + RequestId: fvhelpers.RequestIdFor(header, recordErr), + }) + } + return results +} diff --git a/flowvault/internal/vault/controller/vault_controller.go b/flowvault/internal/vault/controller/vault_controller.go index 9a09d51..34f08cf 100644 --- a/flowvault/internal/vault/controller/vault_controller.go +++ b/flowvault/internal/vault/controller/vault_controller.go @@ -2,72 +2,267 @@ package controller import ( "context" + "fmt" + fvconsts "github.com/skyflowapi/skyflow-go/flowvault/internal/constants" + api "github.com/skyflowapi/skyflow-go/flowvault/internal/generated" + fvhelpers "github.com/skyflowapi/skyflow-go/flowvault/internal/helpers" + fvvalidation "github.com/skyflowapi/skyflow-go/flowvault/internal/validation" fvcommon "github.com/skyflowapi/skyflow-go/flowvault/utils/common" fverrors "github.com/skyflowapi/skyflow-go/flowvault/utils/error" fvlogger "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" + logs "github.com/skyflowapi/skyflow-go/flowvault/utils/messages" ) // VaultController is flowvault's vault controller. // Embeds BaseController for shared auth/HTTP logic — same as Java's // VaultController extends AbstractBaseVaultController. +// +// Every operation validates its request, prepares auth, headers and the HTTP client, then +// calls the FlowService /v2 API through the generated client in flowvault/internal/generated. type VaultController struct { BaseController } -func NewVaultController(config fvcommon.VaultConfig, commonCreds *fvcommon.Credentials) *VaultController { +func NewVaultController(config fvcommon.VaultConfig, client ClientSettings) *VaultController { return &VaultController{ BaseController: BaseController{ - Config: &config, - CommonCreds: commonCreds, - CustomHeaders: make(map[fvcommon.CustomHeaderKey]string), + config: config, + client: client, }, } } -// Insert handles flowvault-specific InsertRequest (ContinueOnError, flowvault Upsert). +// prepare logs the call, validates the request and resolves auth, headers and HTTP settings. +func (v *VaultController) prepare(method, operation string, interceptor fvcommon.Interceptor, validate func() *fverrors.SkyflowError) (*RequestSetup, *fverrors.SkyflowError) { + fvlogger.Info(fmt.Sprintf(logs.METHOD_TRIGGERED, method)) + fvlogger.Info(fmt.Sprintf(logs.VALIDATING_REQUEST, method)) + if err := validate(); err != nil { + return nil, err + } + return v.SetupRequest(operation, interceptor) +} + +// finish logs how the call ended and passes its error through. +func finish(method string, err *fverrors.SkyflowError) *fverrors.SkyflowError { + if err != nil { + fvlogger.Error(fmt.Sprintf(logs.REQUEST_REJECTED, method)) + return err + } + fvlogger.Info(fmt.Sprintf(logs.REQUEST_RESOLVED, method)) + return nil +} + +func hasRecords[T any](records func(*T) int) func(*T) bool { + return func(body *T) bool { return records(body) > 0 } +} + +// Insert inserts records, optionally upserting on unique columns. func (v *VaultController) Insert(ctx context.Context, request fvcommon.InsertRequest, options fvcommon.InsertOptions) (*fvcommon.InsertResponse, *fverrors.SkyflowError) { - fvlogger.Info("flowvault Insert triggered for vault: " + v.Config.VaultId) - fvlogger.Debug("Insert request: ", request) - if request.Table == "" { - return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, "table must not be empty") + setup, err := v.prepare(fvconsts.METHOD_INSERT, fvconsts.OPERATION_INSERT, options.Interceptor, func() *fverrors.SkyflowError { + return fvvalidation.ValidateInsert(request) + }) + if err != nil { + return nil, err } - if len(request.Values) == 0 { - return nil, fverrors.NewSkyflowError(fverrors.INVALID_INPUT_CODE, "values must not be empty") + resp, callErr := setup.API.Records.WithRawResponse.InsertRecords(ctx, toWireInsert(setup.VaultID, request)) + body, header, err := result(resp, callErr, hasRecords(func(b *api.InsertResponse) int { return len(b.Records) })) + if err := finish(fvconsts.METHOD_INSERT, err); err != nil { + return nil, err } - if err := v.SetupClient(options.CustomHeaders); err != nil { + return &fvcommon.InsertResponse{Records: toRecordResults(body.Records, header)}, nil +} + +// Update updates existing records by Skyflow ID. +func (v *VaultController) Update(ctx context.Context, request fvcommon.UpdateRequest, options fvcommon.UpdateOptions) (*fvcommon.UpdateResponse, *fverrors.SkyflowError) { + setup, err := v.prepare(fvconsts.METHOD_UPDATE, fvconsts.OPERATION_UPDATE, options.Interceptor, func() *fverrors.SkyflowError { + return fvvalidation.ValidateUpdate(request) + }) + if err != nil { return nil, err } - // TODO: translate flowvault InsertRequest → REST API call via v.HTTPClient - return &fvcommon.InsertResponse{}, nil + resp, callErr := setup.API.Records.WithRawResponse.UpdateRecords(ctx, toWireUpdate(setup.VaultID, request)) + body, header, err := result(resp, callErr, hasRecords(func(b *api.UpdateResponse) int { return len(b.Records) })) + if err := finish(fvconsts.METHOD_UPDATE, err); err != nil { + return nil, err + } + return &fvcommon.UpdateResponse{Records: toRecordResults(body.Records, header)}, nil } -// Get — BaseController handles auth, VaultController translates flowvault request type. +// Get fetches records by Skyflow IDs or unique values, from one table or several. func (v *VaultController) Get(ctx context.Context, request fvcommon.GetRequest, options fvcommon.GetOptions) (*fvcommon.GetResponse, *fverrors.SkyflowError) { - fvlogger.Info("flowvault Get triggered for vault: " + v.Config.VaultId) - if err := v.SetupClient(toStringHeaders(options.CustomHeaders)); err != nil { + setup, err := v.prepare(fvconsts.METHOD_GET, fvconsts.OPERATION_GET, options.Interceptor, func() *fverrors.SkyflowError { + return fvvalidation.ValidateGet(request) + }) + if err != nil { + return nil, err + } + resp, callErr := setup.API.Records.WithRawResponse.GetRecords(ctx, toWireGet(setup.VaultID, request)) + body, header, err := result(resp, callErr, hasRecords(func(b *api.GetResponse) int { return len(b.Records) })) + if err := finish(fvconsts.METHOD_GET, err); err != nil { return nil, err } - // TODO: call Skyflow REST API via v.HTTPClient - return &fvcommon.GetResponse{}, nil + return &fvcommon.GetResponse{Records: toRecordResults(body.Records, header)}, nil } -// Delete — same pattern +// Delete deletes records by Skyflow IDs or unique values. func (v *VaultController) Delete(ctx context.Context, request fvcommon.DeleteRequest, options fvcommon.DeleteOptions) (*fvcommon.DeleteResponse, *fverrors.SkyflowError) { - fvlogger.Info("flowvault Delete triggered for vault: " + v.Config.VaultId) - if err := v.SetupClient(toStringHeaders(options.CustomHeaders)); err != nil { + setup, err := v.prepare(fvconsts.METHOD_DELETE, fvconsts.OPERATION_DELETE, options.Interceptor, func() *fverrors.SkyflowError { + return fvvalidation.ValidateDelete(request) + }) + if err != nil { + return nil, err + } + resp, callErr := setup.API.Records.WithRawResponse.DeleteRecords(ctx, toWireDelete(setup.VaultID, request)) + body, header, err := result(resp, callErr, hasRecords(func(b *api.DeleteResponse) int { return len(b.Records) })) + if err := finish(fvconsts.METHOD_DELETE, err); err != nil { + return nil, err + } + return &fvcommon.DeleteResponse{Records: toDeleteResults(body.Records, header)}, nil +} + +// Detokenize returns the values behind tokens, applying per-token-group redaction. +func (v *VaultController) Detokenize(ctx context.Context, request fvcommon.DetokenizeRequest, options fvcommon.DetokenizeOptions) (*fvcommon.DetokenizeResponse, *fverrors.SkyflowError) { + setup, err := v.prepare(fvconsts.METHOD_DETOKENIZE, fvconsts.OPERATION_DETOKENIZE, options.Interceptor, func() *fverrors.SkyflowError { + return fvvalidation.ValidateDetokenize(request) + }) + if err != nil { + return nil, err + } + resp, callErr := setup.API.Tokens.WithRawResponse.Detokenize(ctx, toWireDetokenize(setup.VaultID, request)) + body, header, err := result(resp, callErr, hasRecords(func(b *api.DetokenizeResponse) int { return len(b.Response) })) + if err := finish(fvconsts.METHOD_DETOKENIZE, err); err != nil { return nil, err } - // TODO: call Skyflow REST API via v.HTTPClient - return &fvcommon.DeleteResponse{}, nil + return &fvcommon.DetokenizeResponse{Records: toDetokenizeResults(body.Response, header)}, nil } -// toStringHeaders converts map[CustomHeaderKey]string to map[string]string. -// Needed because CustomHeaderKey is a named type, not a raw string. -func toStringHeaders(h map[fvcommon.CustomHeaderKey]string) map[string]string { - out := make(map[string]string, len(h)) - for k, v := range h { - out[string(k)] = v +// GetTokens returns existing tokens for values in deterministic token groups. +func (v *VaultController) GetTokens(ctx context.Context, request fvcommon.GetTokensRequest, options fvcommon.GetTokensOptions) (*fvcommon.GetTokensResponse, *fverrors.SkyflowError) { + setup, err := v.prepare(fvconsts.METHOD_GET_TOKENS, fvconsts.OPERATION_GET_TOKENS, options.Interceptor, func() *fverrors.SkyflowError { + return fvvalidation.ValidateGetTokens(request) + }) + if err != nil { + return nil, err + } + resp, callErr := setup.API.Tokens.WithRawResponse.GetTokens(ctx, toWireGetTokens(setup.VaultID, request)) + body, header, err := result(resp, callErr, hasRecords(func(b *api.GetTokensFromValuesResponse) int { return len(b.Records) })) + if err := finish(fvconsts.METHOD_GET_TOKENS, err); err != nil { + return nil, err + } + return &fvcommon.GetTokensResponse{Records: toGetTokensResults(body.Records, header)}, nil +} + +// Query runs a SELECT query. It returns at most 25 rows per call. +func (v *VaultController) Query(ctx context.Context, request fvcommon.QueryRequest, options fvcommon.QueryOptions) (*fvcommon.QueryResponse, *fverrors.SkyflowError) { + setup, err := v.prepare(fvconsts.METHOD_QUERY, fvconsts.OPERATION_QUERY, options.Interceptor, func() *fverrors.SkyflowError { + return fvvalidation.ValidateQuery(request) + }) + if err != nil { + return nil, err + } + resp, callErr := setup.API.Query.WithRawResponse.ExecuteQuery(ctx, &api.ExecuteQueryRequest{VaultID: setup.VaultID, Query: request.Query}) + body, header, err := result(resp, callErr, nil) + if err := finish(fvconsts.METHOD_QUERY, err); err != nil { + return nil, err + } + fields := make([]map[string]interface{}, 0, len(body.Records)) + for _, record := range body.Records { + if record != nil { + fields = append(fields, record.Data) + } + } + response := &fvcommon.QueryResponse{Fields: fields, RequestId: header.Get(fvconsts.RESPONSE_HEADER_REQUEST_ID)} + if body.Metadata != nil { + response.Metadata.Columns = body.Metadata.Columns + } + return response, nil +} + +// UploadFiles uploads files into file columns. It requests a signed upload URL per column, +// then PUTs each file to its URL, at most MAX_CONCURRENT_UPLOADS at once. The PUTs use +// fvhelpers.UploadHTTPClient: no retries, no overall timeout and no SDK headers. A column +// that can't be uploaded is reported as FAILED or SKIPPED; the other columns are unaffected. +func (v *VaultController) UploadFiles(ctx context.Context, request fvcommon.UploadFilesRequest, options fvcommon.UploadFilesOptions) (*fvcommon.UploadFilesResponse, *fverrors.SkyflowError) { + files := make([][]fvhelpers.ResolvedFile, len(request.Records)) + setup, err := v.prepare(fvconsts.METHOD_UPLOAD_FILES, fvconsts.OPERATION_UPLOAD_FILES, options.Interceptor, func() *fverrors.SkyflowError { + if err := fvvalidation.ValidateUploadFiles(request); err != nil { + return err + } + // Read every file up front so a bad file fails the call before anything is uploaded. + for i, record := range request.Records { + for _, column := range record.Columns { + file, err := fvhelpers.ResolveFile(column) + if err != nil { + return err + } + files[i] = append(files[i], file) + } + } + return nil + }) + if err != nil { + return nil, err + } + resp, callErr := setup.API.Files.WithRawResponse.UploadFiles(ctx, toWireUploadFiles(setup.VaultID, request, files)) + body, header, err := result(resp, callErr, hasRecords(func(b *api.FileUploadResponse) int { return len(b.Records) })) + if err := finish(fvconsts.METHOD_UPLOAD_FILES, err); err != nil { + return nil, err + } + + // Records come back in request order, so record i holds the signed URLs for request record i. + records := make([]fvcommon.UploadFilesResponseRecord, len(body.Records)) + var uploads []func() + for i, wireRecord := range body.Records { + if wireRecord == nil { + continue + } + recordErr := deref(wireRecord.Error) + records[i] = fvcommon.UploadFilesResponseRecord{ + SkyflowId: wireRecord.SkyflowID, + TableName: wireRecord.TableName, + HttpCode: wireRecord.HTTPCode, + Error: recordErr, + RequestId: fvhelpers.RequestIdFor(header, recordErr), + } + if i >= len(request.Records) { + continue + } + columns := request.Records[i].Columns + records[i].Columns = make([]fvcommon.UploadFilesColumnResult, len(columns)) + for j, column := range columns { + file := files[i][j] + columnResult := &records[i].Columns[j] + *columnResult = fvcommon.UploadFilesColumnResult{Column: column.Column, FileName: file.Name, UploadStatus: fvcommon.UploadStatusSkipped} + if recordErr != "" { + columnResult.Error = recordErr + continue + } + signedURL, _ := wireRecord.Data[column.Column].(string) + if signedURL == "" { + continue + } + uploads = append(uploads, func() { + columnResult.UploadStatus, columnResult.Error = fvhelpers.PutFile(ctx, fvhelpers.UploadHTTPClient(), signedURL, file) + }) + } + } + fvhelpers.RunConcurrently(uploads, fvconsts.MAX_CONCURRENT_UPLOADS) + return &fvcommon.UploadFilesResponse{Records: records}, nil +} + +// DeleteFiles deletes files from file columns of existing records, including the stored objects. +func (v *VaultController) DeleteFiles(ctx context.Context, request fvcommon.DeleteFilesRequest, options fvcommon.DeleteFilesOptions) (*fvcommon.DeleteFilesResponse, *fverrors.SkyflowError) { + setup, err := v.prepare(fvconsts.METHOD_DELETE_FILES, fvconsts.OPERATION_DELETE_FILES, options.Interceptor, func() *fverrors.SkyflowError { + return fvvalidation.ValidateDeleteFiles(request) + }) + if err != nil { + return nil, err + } + resp, callErr := setup.API.Files.WithRawResponse.DeleteFiles(ctx, toWireDeleteFiles(setup.VaultID, request)) + body, header, err := result(resp, callErr, hasRecords(func(b *api.FileDeleteResponse) int { return len(b.Records) })) + if err := finish(fvconsts.METHOD_DELETE_FILES, err); err != nil { + return nil, err } - return out + return &fvcommon.DeleteFilesResponse{Records: toDeleteFilesResults(body.Records, header)}, nil } diff --git a/flowvault/test/resources/client_api_key.txt b/flowvault/test/resources/client_api_key.txt new file mode 100644 index 0000000..6f45c7d --- /dev/null +++ b/flowvault/test/resources/client_api_key.txt @@ -0,0 +1 @@ +sky-ab123-abcd1234cdef1234abcd4321cdef4321 diff --git a/flowvault/test/resources/vault_api_key.txt b/flowvault/test/resources/vault_api_key.txt new file mode 100644 index 0000000..849659b --- /dev/null +++ b/flowvault/test/resources/vault_api_key.txt @@ -0,0 +1 @@ +sky-zz999-abcd1234cdef1234abcd4321cdef4321 diff --git a/flowvault/utils/common/common.go b/flowvault/utils/common/common.go index c608856..9d29da1 100644 --- a/flowvault/utils/common/common.go +++ b/flowvault/utils/common/common.go @@ -22,6 +22,7 @@ const ( type ( Credentials = cfg.Credentials VaultConfig = cfg.VaultConfig + HttpConfig = cfg.HttpConfig ConnectionConfig = cfg.ConnectionConfig CustomHeaderKey = cfg.CustomHeaderKey ) @@ -39,156 +40,5 @@ type ( SignedDataTokensResponse = sa.SignedDataTokensResponse ) -// --------------------------------------------------------------------------- -// FlowVault vault API types. These belong to flowvault alone and evolve with -// the FlowVault API, independently of the v2 (SkyVault) SDK. -// --------------------------------------------------------------------------- - -type RedactionType string - -const ( - PLAIN_TEXT RedactionType = "PLAIN_TEXT" - DEFAULT RedactionType = "DEFAULT" - MASKED RedactionType = "MASKED" - REDACTED RedactionType = "REDACTED" -) - -type OrderByEnum string - -const ( - ASCENDING OrderByEnum = "ASCENDING" - DESCENDING OrderByEnum = "DESCENDING" - NONE OrderByEnum = "NONE" -) - -type BYOT string - -const ( - DISABLE BYOT = "DISABLE" - ENABLE BYOT = "ENABLE" - ENABLE_STRICT BYOT = "ENABLE_STRICT" -) - -// Get - -type GetRequest struct { - Table string - Ids []string -} - -type GetOptions struct { - RedactionType RedactionType - ReturnTokens bool - Fields []string - Offset string - Limit string - DownloadUrl bool - ColumnName string - ColumnValues []string - OrderBy OrderByEnum - CustomHeaders map[CustomHeaderKey]string -} - -type GetResponse struct { - Data []map[string]interface{} - Errors []map[string]interface{} -} - -// Detokenize - -type DetokenizeRequest struct { - DetokenizeData []DetokenizeData -} - -type DetokenizeData struct { - Token string - RedactionType RedactionType -} - -type DetokenizeOptions struct { - ContinueOnError bool - DownloadUrl bool - CustomHeaders map[CustomHeaderKey]string -} - -type DetokenizeRecordResponse struct { - Token string - Value string - Type string - Error string - RequestId string -} - -type DetokenizeResponse struct { - DetokenizedFields []DetokenizeRecordResponse - Errors []DetokenizeRecordResponse -} - -// Delete - -type DeleteRequest struct { - Table string - Ids []string -} - -type DeleteOptions struct { - CustomHeaders map[CustomHeaderKey]string -} - -type DeleteResponse struct { - DeletedIds []string - Errors []map[string]interface{} -} - -// Update - -type UpdateRequest struct { - Table string - Data map[string]interface{} - Tokens map[string]interface{} -} - -type UpdateOptions struct { - ReturnTokens bool - TokenMode BYOT - CustomHeaders map[CustomHeaderKey]string -} - -type UpdateResponse struct { - UpdatedField map[string]interface{} - Errors []map[string]interface{} -} - -// Query - -type QueryRequest struct { - Query string -} - -type QueryOptions struct { - CustomHeaders map[CustomHeaderKey]string -} - -type QueryResponse struct { - Fields []map[string]interface{} - Errors []map[string]interface{} -} - -// Tokenize - -type TokenizeRequest struct { - ColumnGroup string - Value string -} - -type TokenizeOptions struct { - CustomHeaders map[CustomHeaderKey]string -} - -type TokenizeResponse struct { - Tokens []string - Errors []map[string]interface{} -} - // IsCredentialsEmpty reports whether no credential source is set. func IsCredentialsEmpty(c Credentials) bool { return cfg.IsCredentialsEmpty(c) } diff --git a/flowvault/utils/common/delete.go b/flowvault/utils/common/delete.go new file mode 100644 index 0000000..b2e98a6 --- /dev/null +++ b/flowvault/utils/common/delete.go @@ -0,0 +1,22 @@ +package common + +type DeleteRequest struct { + TableName string // required + SkyflowIds []string + UniqueValues []map[string]interface{} +} + +type DeleteResponseRecord struct { + SkyflowId string `json:"skyflowId"` + HttpCode int `json:"httpCode"` + Error string `json:"error"` + RequestId string `json:"requestId"` +} + +type DeleteResponse struct { + Records []DeleteResponseRecord `json:"records"` +} + +type DeleteOptions struct { + Interceptor Interceptor +} diff --git a/flowvault/utils/common/delete_files.go b/flowvault/utils/common/delete_files.go new file mode 100644 index 0000000..28960ab --- /dev/null +++ b/flowvault/utils/common/delete_files.go @@ -0,0 +1,35 @@ +package common + +// DeleteFilesRequestRecord takes exactly one of SkyflowId or UniqueValues. +type DeleteFilesRequestRecord struct { + TableName string // required + Columns []string // required + SkyflowId string + UniqueValues []map[string]interface{} +} + +type DeleteFilesRequest struct { + Records []DeleteFilesRequestRecord +} + +type DeleteFilesColumnResult struct { + Column string `json:"column"` + Status string `json:"status"` +} + +type DeleteFilesResponseRecord struct { + SkyflowId string `json:"skyflowId"` + TableName string `json:"tableName"` + Columns []DeleteFilesColumnResult `json:"columns"` // nil on a failed record + HttpCode int `json:"httpCode"` + Error string `json:"error"` + RequestId string `json:"requestId"` +} + +type DeleteFilesResponse struct { + Records []DeleteFilesResponseRecord `json:"records"` +} + +type DeleteFilesOptions struct { + Interceptor Interceptor +} diff --git a/flowvault/utils/common/detokenize.go b/flowvault/utils/common/detokenize.go new file mode 100644 index 0000000..19d4dab --- /dev/null +++ b/flowvault/utils/common/detokenize.go @@ -0,0 +1,29 @@ +package common + +type DetokenizeRequest struct { + Tokens []string // required + TokenGroupRedactions []TokenGroupRedaction +} + +type DetokenizeRecordMetadata struct { + SkyflowId string `json:"skyflowId"` + TableName string `json:"tableName"` +} + +type DetokenizeResponseRecord struct { + Token string `json:"token"` + Value interface{} `json:"value"` + TokenGroupName string `json:"tokenGroupName"` + Metadata *DetokenizeRecordMetadata `json:"metadata"` + HttpCode int `json:"httpCode"` + Error string `json:"error"` + RequestId string `json:"requestId"` +} + +type DetokenizeResponse struct { + Records []DetokenizeResponseRecord `json:"records"` +} + +type DetokenizeOptions struct { + Interceptor Interceptor +} diff --git a/flowvault/utils/common/get.go b/flowvault/utils/common/get.go new file mode 100644 index 0000000..ad0c5aa --- /dev/null +++ b/flowvault/utils/common/get.go @@ -0,0 +1,29 @@ +package common + +type GetRequest struct { + TableName string + SkyflowIds []string + UniqueValues []map[string]interface{} + Columns []string + ColumnRedactions []ColumnRedaction + Limit, Offset int // 0 = not sent + Records []GetRequestRecord // multi-table mode; excludes the fields above +} + +type GetRequestRecord struct { + TableName string + SkyflowIds []string + UniqueValues []map[string]interface{} + Columns []string + ColumnRedactions []ColumnRedaction +} + +type GetResponseRecord = InsertResponseRecord + +type GetResponse struct { + Records []GetResponseRecord `json:"records"` +} + +type GetOptions struct { + Interceptor Interceptor +} diff --git a/flowvault/utils/common/get_tokens.go b/flowvault/utils/common/get_tokens.go new file mode 100644 index 0000000..bb964d0 --- /dev/null +++ b/flowvault/utils/common/get_tokens.go @@ -0,0 +1,27 @@ +package common + +type GetTokensRequest struct { + Records []GetTokensRequestRecord +} + +type GetTokensRequestRecord struct { + Value interface{} // required + TokenGroupName string // required, deterministic group +} + +type GetTokensResponseRecord struct { + Value interface{} `json:"value"` + TokenGroupName string `json:"tokenGroupName"` + Token string `json:"token"` + HttpCode int `json:"httpCode"` + Error string `json:"error"` + RequestId string `json:"requestId"` +} + +type GetTokensResponse struct { + Records []GetTokensResponseRecord `json:"records"` +} + +type GetTokensOptions struct { + Interceptor Interceptor +} diff --git a/flowvault/utils/common/insert.go b/flowvault/utils/common/insert.go index 7e932fa..cf96e13 100644 --- a/flowvault/utils/common/insert.go +++ b/flowvault/utils/common/insert.go @@ -1,23 +1,39 @@ package common -// InsertRequest is the FlowVault insert request. Unlike v2, ContinueOnError -// is part of the request rather than the options. +type UpsertOptions struct { + UniqueColumns []string // required + UpdateType UpdateType +} + type InsertRequest struct { - Table string - Values []map[string]interface{} - ContinueOnError bool + TableName string // set here OR on every record + Records []InsertRequestRecord + Upsert *UpsertOptions // only with request-level TableName } -// InsertOptions are the FlowVault insert options. CustomHeaders uses plain -// string keys (v2 uses a typed CustomHeaderKey). -type InsertOptions struct { - Upsert string - CustomHeaders map[string]string +type InsertRequestRecord struct { + Data map[string]interface{} // required + Tokens map[string]interface{} + TableName string // only if not set on the request + Upsert *UpsertOptions // only with record-level TableName +} + +// InsertResponseRecord is the per-record result of Insert, Update and Get. +type InsertResponseRecord struct { + TableName string `json:"tableName"` + SkyflowId string `json:"skyflowId"` + Tokens map[string][]Token `json:"tokens"` + Data map[string]interface{} `json:"data"` + HashedData map[string][]HashedValue `json:"hashedData"` + HttpCode int `json:"httpCode"` + Error string `json:"error"` + RequestId string `json:"requestId"` } -// InsertResponse is the FlowVault insert response. type InsertResponse struct { - InsertedFields []map[string]interface{} - Tokens []map[string]interface{} - Errors []map[string]interface{} + Records []InsertResponseRecord `json:"records"` +} + +type InsertOptions struct { + Interceptor Interceptor } diff --git a/flowvault/utils/common/query.go b/flowvault/utils/common/query.go new file mode 100644 index 0000000..1d37311 --- /dev/null +++ b/flowvault/utils/common/query.go @@ -0,0 +1,19 @@ +package common + +type QueryRequest struct { + Query string +} + +type QueryResponseMetadata struct { + Columns []string `json:"columns"` +} + +type QueryResponse struct { + Fields []map[string]interface{} `json:"fields"` // one map per row + Metadata QueryResponseMetadata `json:"metadata"` + RequestId string `json:"requestId"` // call-level, always populated +} + +type QueryOptions struct { + Interceptor Interceptor +} diff --git a/flowvault/utils/common/types.go b/flowvault/utils/common/types.go new file mode 100644 index 0000000..ca509ac --- /dev/null +++ b/flowvault/utils/common/types.go @@ -0,0 +1,60 @@ +package common + +// UpdateType controls how an update or upsert applies data to an existing record. +type UpdateType string + +const ( + // UpdateTypeUpdate merges the given columns into the record (server default). + UpdateTypeUpdate UpdateType = "UPDATE" + // UpdateTypeReplace overwrites the record with the given columns. + UpdateTypeReplace UpdateType = "REPLACE" +) + +// RequestContext is passed to an options Interceptor before each API call. +type RequestContext struct { + Operation string + headers map[CustomHeaderKey]string +} + +// AddHeader adds a custom header to the outgoing request. +func (rc *RequestContext) AddHeader(key CustomHeaderKey, value string) { + if rc.headers == nil { + rc.headers = make(map[CustomHeaderKey]string) + } + rc.headers[key] = value +} + +// Headers returns a copy of the headers added so far. +func (rc *RequestContext) Headers() map[CustomHeaderKey]string { + headers := make(map[CustomHeaderKey]string, len(rc.headers)) + for key, value := range rc.headers { + headers[key] = value + } + return headers +} + +// Interceptor customises a single call, e.g. by adding headers. +type Interceptor func(rc *RequestContext) + +// Token is one token issued for a column value. +type Token struct { + Token string `json:"token"` + TokenGroupName string `json:"tokenGroupName,omitempty"` + Path string `json:"path,omitempty"` +} + +// HashedValue is one hash computed for a column value. +type HashedValue struct { + Data string `json:"data"` + HashName string `json:"hashName,omitempty"` +} + +type ColumnRedaction struct { + ColumnName string + Redaction string +} + +type TokenGroupRedaction struct { + TokenGroupName string + Redaction string +} diff --git a/flowvault/utils/common/update.go b/flowvault/utils/common/update.go new file mode 100644 index 0000000..56dbb9a --- /dev/null +++ b/flowvault/utils/common/update.go @@ -0,0 +1,24 @@ +package common + +type UpdateRequest struct { + TableName string // set here OR on every record + Records []UpdateRequestRecord + UpdateType UpdateType // request level only +} + +type UpdateRequestRecord struct { + SkyflowId string // required + Data map[string]interface{} // required + Tokens map[string]interface{} + TableName string // only if not set on the request +} + +type UpdateResponseRecord = InsertResponseRecord + +type UpdateResponse struct { + Records []UpdateResponseRecord `json:"records"` +} + +type UpdateOptions struct { + Interceptor Interceptor +} diff --git a/flowvault/utils/common/upload_files.go b/flowvault/utils/common/upload_files.go new file mode 100644 index 0000000..1e5fe0a --- /dev/null +++ b/flowvault/utils/common/upload_files.go @@ -0,0 +1,55 @@ +package common + +import "os" + +// UploadStatus is the outcome of uploading one file column. +type UploadStatus string + +const ( + UploadStatusUploaded UploadStatus = "UPLOADED" + UploadStatusFailed UploadStatus = "FAILED" + UploadStatusSkipped UploadStatus = "SKIPPED" +) + +type UploadFilesRequest struct { + Records []UploadFilesRequestRecord +} + +type UploadFilesRequestRecord struct { + TableName string // required + SkyflowId string // empty -> new record; set -> update existing record + Columns []UploadFilesRequestColumn +} + +// UploadFilesRequestColumn takes exactly one of FilePath, Base64 or FileObject. +type UploadFilesRequestColumn struct { + Column string // required + FilePath string + Base64 string // requires FileName + FileObject *os.File // caller closes + FileName string // derived from path/object if empty; server UUID if none +} + +type UploadFilesColumnResult struct { + Column string `json:"column"` + FileName string `json:"fileName"` + UploadStatus UploadStatus `json:"uploadStatus"` + Error string `json:"error"` +} + +type UploadFilesResponseRecord struct { + SkyflowId string `json:"skyflowId"` + TableName string `json:"tableName"` + Columns []UploadFilesColumnResult `json:"columns"` + HttpCode int `json:"httpCode"` + Error string `json:"error"` + RequestId string `json:"requestId"` +} + +type UploadFilesResponse struct { + Records []UploadFilesResponseRecord `json:"records"` +} + +type UploadFilesOptions struct { + Interceptor Interceptor +} diff --git a/flowvault/utils/error/messages.go b/flowvault/utils/error/messages.go index 221afd9..487df5a 100644 --- a/flowvault/utils/error/messages.go +++ b/flowvault/utils/error/messages.go @@ -9,3 +9,44 @@ const ( VAULT_ID_EXISTS_IN_CONFIG_LIST string = internal.SDK_PREFIX + " Validation error. %s already exists in the config list. Specify a new vaultId." EMPTY_VAULT_ID string = internal.SDK_PREFIX + " Validation error. Invalid vault ID. Specify a valid vault ID." ) + +// Request validation messages. Messages containing %s / %d are formatted with fmt.Sprintf. +const ( + EMPTY_RECORDS string = internal.SDK_PREFIX + " Validation error. %s failed. Records must not be empty." + TABLE_IN_BOTH_PLACES string = internal.SDK_PREFIX + " Validation error. %s failed. Table name is set on the request and on a record. Set it on the request or on every record, not both." + TABLE_MISSING string = internal.SDK_PREFIX + " Validation error. %s failed. Table name is missing. Set it on the request or on every record." + RECORD_UPSERT_NOT_ALLOWED string = internal.SDK_PREFIX + " Validation error. Insert failed. Upsert is set on a record while the table name is set on the request. Set upsert on the request instead." + REQUEST_UPSERT_NOT_ALLOWED string = internal.SDK_PREFIX + " Validation error. Insert failed. Upsert is set on the request while the table name is set on the records. Set upsert on each record instead." + EMPTY_UNIQUE_COLUMNS string = internal.SDK_PREFIX + " Validation error. Insert failed. Upsert unique columns must not be empty." + INVALID_UPDATE_TYPE string = internal.SDK_PREFIX + " Validation error. %s failed. Invalid update type %q. Use UPDATE or REPLACE." + EMPTY_DATA string = internal.SDK_PREFIX + " Validation error. %s failed. Data must not be empty for record at index %d." + EMPTY_KEY_IN_DATA string = internal.SDK_PREFIX + " Validation error. %s failed. Data contains an empty column name for record at index %d." + MISSING_SKYFLOW_ID string = internal.SDK_PREFIX + " Validation error. Update failed. Skyflow ID is missing for record at index %d." + GET_MODE_CONFLICT string = internal.SDK_PREFIX + " Validation error. Get failed. Set either the single-table fields or Records, not both." + NEGATIVE_LIMIT_OFFSET string = internal.SDK_PREFIX + " Validation error. Get failed. Limit and Offset must not be negative." + EMPTY_TABLE_NAME string = internal.SDK_PREFIX + " Validation error. %s failed. Table name must not be empty." + IDS_OR_UNIQUE_VALUES string = internal.SDK_PREFIX + " Validation error. %s failed. Set exactly one of SkyflowIds or UniqueValues." + EMPTY_ID string = internal.SDK_PREFIX + " Validation error. %s failed. Skyflow IDs must not contain empty values." + EMPTY_UNIQUE_VALUE string = internal.SDK_PREFIX + " Validation error. %s failed. Unique values must not contain empty entries." + INVALID_COLUMN_REDACTION string = internal.SDK_PREFIX + " Validation error. %s failed. Column redactions need a column name and a redaction." + EMPTY_TOKENS string = internal.SDK_PREFIX + " Validation error. Detokenize failed. Tokens must not be empty." + EMPTY_TOKEN string = internal.SDK_PREFIX + " Validation error. Detokenize failed. Tokens must not contain empty values." + INVALID_TOKEN_GROUP_REDACTION string = internal.SDK_PREFIX + " Validation error. Detokenize failed. Token group redactions need a token group name and a redaction." + MISSING_VALUE string = internal.SDK_PREFIX + " Validation error. GetTokens failed. Value is missing for record at index %d." + MISSING_TOKEN_GROUP_NAME string = internal.SDK_PREFIX + " Validation error. GetTokens failed. Token group name is missing for record at index %d." + EMPTY_QUERY string = internal.SDK_PREFIX + " Validation error. Query failed. Query must not be empty." + EMPTY_COLUMNS string = internal.SDK_PREFIX + " Validation error. %s failed. Columns must not be empty for record at index %d." + EMPTY_COLUMN_NAME string = internal.SDK_PREFIX + " Validation error. %s failed. Column name must not be empty for record at index %d." + FILE_SOURCE_COUNT string = internal.SDK_PREFIX + " Validation error. UploadFiles failed. Column %q needs exactly one of FilePath, Base64 or FileObject." + BASE64_NEEDS_FILE_NAME string = internal.SDK_PREFIX + " Validation error. UploadFiles failed. Column %q uses Base64, so FileName is required." + INVALID_BASE64 string = internal.SDK_PREFIX + " Validation error. UploadFiles failed. Column %q has invalid base64 content." + FILE_NOT_READABLE string = internal.SDK_PREFIX + " Validation error. UploadFiles failed. Could not read file for column %q: %s" + DELETE_FILES_TARGET string = internal.SDK_PREFIX + " Validation error. DeleteFiles failed. Set exactly one of SkyflowId or UniqueValues for record at index %d." + DELETE_FILES_LIMIT string = internal.SDK_PREFIX + " Validation error. DeleteFiles failed. At most %d records are allowed per request." + DUPLICATE_COLUMN string = internal.SDK_PREFIX + " Validation error. %s failed. Column %q is listed more than once for record at index %d." + EMPTY_CREDENTIALS string = internal.SDK_PREFIX + " Validation error. Invalid credentials. Specify a valid credentials." + BEARER_TOKEN_EXPIRED string = internal.SDK_PREFIX + " Validation error. Bearer token is expired. Specify a valid bearer token." + MULTIPLE_VAULT_CONFIGS string = internal.SDK_PREFIX + " Validation error. Multiple vault configurations found. Specify a vaultId." + INVALID_HEADER_KEY string = internal.SDK_PREFIX + " Validation error. %s: header key %q is not supported. Use x-skyflow-account-id, x-skyflow-account-name or x-request-id." + EMPTY_HEADER_VALUE string = internal.SDK_PREFIX + " Validation error. %s: header %q must have a value." +) diff --git a/flowvault/utils/messages/logs.go b/flowvault/utils/messages/logs.go index 07e88c4..8796c62 100644 --- a/flowvault/utils/messages/logs.go +++ b/flowvault/utils/messages/logs.go @@ -3,12 +3,20 @@ package logs import . "github.com/skyflowapi/skyflow-go/flowvault/internal/constants" const ( - CLIENT_INITIALIZED = SDK_LOG_PREFIX + "Initialized skyflow client successfully." - VALIDATING_VAULT_CONFIG = SDK_LOG_PREFIX + "Validating vault config." - VALIDATING_CRED = SDK_LOG_PREFIX + "Validating credentials." - VAULT_CONTROLLER_INITIALIZED = SDK_LOG_PREFIX + "Initialized vault controller with vault ID %s." - CURRENT_LOG_LEVEL = SDK_LOG_PREFIX + "Current log level is %v." - EMPTY_VAULT_ARRAY = SDK_LOG_PREFIX + "Vault config array is empty." - VAULT_CONFIG_EXISTS = SDK_LOG_PREFIX + "Vault config with vault ID %s already exists." - VAULT_CONFIG_DOES_NOT_EXIST = SDK_LOG_PREFIX + "Vault config with vault ID %s doesn't exist." + CLIENT_INITIALIZED = SDK_LOG_PREFIX + "Initialized skyflow client successfully." + VALIDATING_VAULT_CONFIG = SDK_LOG_PREFIX + "Validating vault config." + VALIDATING_CRED = SDK_LOG_PREFIX + "Validating credentials." + VAULT_CONTROLLER_INITIALIZED = SDK_LOG_PREFIX + "Initialized vault controller with vault ID %s." + CURRENT_LOG_LEVEL = SDK_LOG_PREFIX + "Current log level is %v." + EMPTY_VAULT_ARRAY = SDK_LOG_PREFIX + "Vault config array is empty." + VAULT_CONFIG_EXISTS = SDK_LOG_PREFIX + "Vault config with vault ID %s already exists." + VAULT_CONFIG_DOES_NOT_EXIST = SDK_LOG_PREFIX + "Vault config with vault ID %s doesn't exist." + BEARER_TOKEN_EXPIRED = SDK_LOG_PREFIX + "Bearer token is expired." + GENERATE_BEARER_TOKEN_TRIGGERED = SDK_LOG_PREFIX + "Generating a new bearer token." + REUSE_BEARER_TOKEN = SDK_LOG_PREFIX + "Reusing the cached bearer token." + EMPTY_CREDENTIALS = SDK_LOG_PREFIX + "No credentials found in the vault config, client config or SKYFLOW_CREDENTIALS." + METHOD_TRIGGERED = SDK_LOG_PREFIX + "%s method triggered." + VALIDATING_REQUEST = SDK_LOG_PREFIX + "Validating %s request." + REQUEST_RESOLVED = SDK_LOG_PREFIX + "%s request resolved." + REQUEST_REJECTED = SDK_LOG_PREFIX + "%s request resulted in failure." ) diff --git a/samples/flowvault/README.md b/samples/flowvault/README.md new file mode 100644 index 0000000..049edc1 --- /dev/null +++ b/samples/flowvault/README.md @@ -0,0 +1,62 @@ +# FlowVault Go samples + +Runnable examples for the Skyflow FlowVault Go SDK (`github.com/skyflowapi/skyflow-go/flowvault`), one file per operation. + +## Prerequisites + +- Go 1.25+ +- A FlowVault vault and Skyflow credentials: a service-account `credentials.json`, an API key, or a bearer token. + +## Configure + +Each sample has placeholders near the top. Replace them with your own values: + +```go +credentials := common.Credentials{Path: ""} // or ApiKey / Token / CredentialsString +vaultConfig := common.VaultConfig{ + VaultId: "", + ClusterId: "", // from the vault URL: https://{clusterId}.skyvault.skyflowapis.com + Env: common.PROD, // DEV, STAGE, SANDBOX or PROD + Credentials: credentials, +} +``` + +## Run + +Each file is its own program. Run one from this folder: + +```bash +cd samples/flowvault +go run vaultapi/insert_records.go +``` + +## Vault operations + +| Sample | Demonstrates | +|---|---| +| [insert_records.go](vaultapi/insert_records.go) | Insert records (table name and upsert on the request, or on each record) | +| [get_records.go](vaultapi/get_records.go) | Get records by Skyflow ID, by unique value, or from several tables | +| [update_record.go](vaultapi/update_record.go) | Update a record (`UPDATE` or `REPLACE`) | +| [delete_records.go](vaultapi/delete_records.go) | Delete records | +| [detokenize_records.go](vaultapi/detokenize_records.go) | Detokenize tokens | +| [query_records.go](vaultapi/query_records.go) | Run a read-only SQL `SELECT` query | +| [get_tokens.go](vaultapi/get_tokens.go) | Look up existing tokens for values (deterministic token groups) | +| [upload_files.go](vaultapi/upload_files.go) | Upload files to file columns (`FilePath` / `Base64` / `FileObject`) | +| [delete_files.go](vaultapi/delete_files.go) | Delete files from file columns (by `SkyflowId` or `UniqueValues`) | + +## Custom headers and HTTP config + +| Sample | Demonstrates | +|---|---| +| [custom_header_example.go](vaultapi/custom_header_example.go) | Add headers to one call with an `InsertOptions` interceptor | +| [timeout_and_retry_config_example.go](vaultapi/timeout_and_retry_config_example.go) | Client-wide `WithHttpConfig` and per-vault `Timeout` / `MaxRetries` | + +## Service account (token generation) + +| Sample | Demonstrates | +|---|---| +| [bearer_token_generation_example.go](serviceaccount/bearer_token_generation_example.go) | Bearer token from a credentials file path or a credentials string | +| [bearer_token_generation_with_context_example.go](serviceaccount/bearer_token_generation_with_context_example.go) | Context-aware bearer tokens (string context and object context) | +| [bearer_token_generation_using_goroutines_example.go](serviceaccount/bearer_token_generation_using_goroutines_example.go) | Concurrent token generation from goroutines with a shared context | +| [scoped_token_generation_example.go](serviceaccount/scoped_token_generation_example.go) | Scoped tokens via `RoleIds` | +| [signed_token_generation_example.go](serviceaccount/signed_token_generation_example.go) | Signed data tokens (`GenerateSignedDataTokens`) | diff --git a/samples/flowvault/go.mod b/samples/flowvault/go.mod new file mode 100644 index 0000000..1aeec54 --- /dev/null +++ b/samples/flowvault/go.mod @@ -0,0 +1,17 @@ +module github.com/skyflowapi/skyflow-go/samples/flowvault + +go 1.25.0 + +require github.com/skyflowapi/skyflow-go/flowvault v0.0.0 + +require ( + github.com/golang-jwt/jwt/v4 v4.5.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/skyflowapi/skyflow-go/common v0.0.0-20261014151608-a64b46a2705c // indirect +) + +// FlowVault isn't published yet; build the samples against the code in this repository. +replace ( + github.com/skyflowapi/skyflow-go/common => ../../common + github.com/skyflowapi/skyflow-go/flowvault => ../../flowvault +) diff --git a/samples/flowvault/go.sum b/samples/flowvault/go.sum new file mode 100644 index 0000000..5582f4d --- /dev/null +++ b/samples/flowvault/go.sum @@ -0,0 +1,40 @@ +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oXJCETMO23COyaKGP6fHVpkpWpg= +github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/onsi/ginkgo/v2 v2.33.1 h1:62XfWJbEIjPSCSDYrsCPP/SHcmH7O0M8ne8zfqQ2iz4= +github.com/onsi/ginkgo/v2 v2.33.1/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= +github.com/onsi/gomega v1.44.0 h1:eAiGl3Pw5jz5GQdDff0BcxYpAX1JxW8xD7mFUuwNfZQ= +github.com/onsi/gomega v1.44.0/go.mod h1:e/C2HwaZ1DhvjzXXuFhcR7hY7Sh9pl7MmoWKEjzwcdA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/samples/flowvault/serviceaccount/bearer_token_generation_example.go b/samples/flowvault/serviceaccount/bearer_token_generation_example.go new file mode 100644 index 0000000..ca2cdca --- /dev/null +++ b/samples/flowvault/serviceaccount/bearer_token_generation_example.go @@ -0,0 +1,66 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "encoding/json" + "fmt" + + "github.com/skyflowapi/skyflow-go/flowvault/serviceaccount" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" +) + +/** + * This example generates a bearer token in two ways, and reuses it until it expires: + * 1. From the path to a credentials.json file. + * 2. From the credentials as a JSON string. + */ + +var ( + filePath = "" + bearerToken = "" +) + +// credentialsString returns the credentials as a JSON string. +func credentialsString() string { + credentials, _ := json.Marshal(map[string]string{ + "clientID": "", + "clientName": "", + "tokenURI": "", + "keyID": "", + "privateKey": "", + }) + return string(credentials) +} + +func bearerTokenFromFilePath() string { + if bearerToken != "" && !serviceaccount.IsExpired(bearerToken) { + return bearerToken + } + response, err := serviceaccount.GenerateBearerToken(filePath, common.BearerTokenOptions{}) + if err != nil { + fmt.Println("Error generating token from file path:", err.GetMessage()) + return "" + } + bearerToken = response.AccessToken + return bearerToken +} + +func bearerTokenFromCredentialsString() string { + if bearerToken != "" && !serviceaccount.IsExpired(bearerToken) { + return bearerToken + } + response, err := serviceaccount.GenerateBearerTokenFromCreds(credentialsString(), common.BearerTokenOptions{}) + if err != nil { + fmt.Println("Error generating token from credentials string:", err.GetMessage()) + return "" + } + bearerToken = response.AccessToken + return bearerToken +} + +func main() { + fmt.Println("Generated Bearer Token (from file):", bearerTokenFromFilePath()) + fmt.Println("Generated Bearer Token (from string):", bearerTokenFromCredentialsString()) +} diff --git a/samples/flowvault/serviceaccount/bearer_token_generation_using_goroutines_example.go b/samples/flowvault/serviceaccount/bearer_token_generation_using_goroutines_example.go new file mode 100644 index 0000000..76f3e40 --- /dev/null +++ b/samples/flowvault/serviceaccount/bearer_token_generation_using_goroutines_example.go @@ -0,0 +1,73 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "encoding/json" + "fmt" + "sync" + + "github.com/skyflowapi/skyflow-go/flowvault/serviceaccount" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" +) + +/** + * This example generates bearer tokens concurrently from two goroutines, one using a credentials + * file path and one using a credentials string, with a shared context. Each goroutine generates + * and prints a token five times. + */ + +var ( + filePath = "" + options = common.BearerTokenOptions{Ctx: "abc"} +) + +// credentialsString returns the credentials as a JSON string. +func credentialsString() string { + credentials, _ := json.Marshal(map[string]string{ + "clientID": "", + "clientName": "", + "tokenURI": "", + "keyID": "", + "privateKey": "", + }) + return string(credentials) +} + +func generateFromFilePath() { + for i := 0; i < 5; i++ { + response, err := serviceaccount.GenerateBearerToken(filePath, options) + if err != nil { + fmt.Println("Error generating token from file path:", err.GetMessage()) + continue + } + fmt.Println(response.AccessToken) + } +} + +func generateFromCredentialsString() { + credentials := credentialsString() + for i := 0; i < 5; i++ { + response, err := serviceaccount.GenerateBearerTokenFromCreds(credentials, options) + if err != nil { + fmt.Println("Error generating token from credentials string:", err.GetMessage()) + continue + } + fmt.Println(response.AccessToken) + } +} + +func main() { + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + generateFromFilePath() + }() + go func() { + defer wg.Done() + generateFromCredentialsString() + }() + wg.Wait() +} diff --git a/samples/flowvault/serviceaccount/bearer_token_generation_with_context_example.go b/samples/flowvault/serviceaccount/bearer_token_generation_with_context_example.go new file mode 100644 index 0000000..df8624f --- /dev/null +++ b/samples/flowvault/serviceaccount/bearer_token_generation_with_context_example.go @@ -0,0 +1,55 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "fmt" + + "github.com/skyflowapi/skyflow-go/flowvault/serviceaccount" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" +) + +/** + * This example generates bearer tokens that carry a context, for context-aware access policies. + * 1. A string context, referenced in a Skyflow policy as request.context. + * 2. An object context, whose keys are referenced as request.context.. + */ + +var filePath = "" + +// Approach 1: a string context. +// Use a simple string identifier when your policy references a single context value. +func bearerTokenWithStringContext() string { + response, err := serviceaccount.GenerateBearerToken(filePath, common.BearerTokenOptions{Ctx: "user_12345"}) + if err != nil { + fmt.Println("Error generating token:", err.GetMessage()) + return "" + } + return response.AccessToken +} + +// Approach 2: an object context. +// Use a map when your policy needs several context values. Each key maps to a Skyflow CEL policy +// variable under request.context.*, for example: +// +// request.context.role == "admin" && request.context.department == "finance" +func bearerTokenWithObjectContext() string { + response, err := serviceaccount.GenerateBearerToken(filePath, common.BearerTokenOptions{ + Ctx: map[string]interface{}{ + "role": "admin", + "department": "finance", + "user_id": "user_12345", + }, + }) + if err != nil { + fmt.Println("Error generating token:", err.GetMessage()) + return "" + } + return response.AccessToken +} + +func main() { + fmt.Println("Bearer token (string context):", bearerTokenWithStringContext()) + fmt.Println("Bearer token (object context):", bearerTokenWithObjectContext()) +} diff --git a/samples/flowvault/serviceaccount/scoped_token_generation_example.go b/samples/flowvault/serviceaccount/scoped_token_generation_example.go new file mode 100644 index 0000000..1e28918 --- /dev/null +++ b/samples/flowvault/serviceaccount/scoped_token_generation_example.go @@ -0,0 +1,51 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "encoding/json" + "fmt" + + "github.com/skyflowapi/skyflow-go/flowvault/serviceaccount" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" +) + +/** + * This example generates scoped bearer tokens, limited to specific roles, from a credentials + * file path and from a credentials string. + */ + +var ( + filePath = "" + // Scoped tokens grant only the access of the roles listed here. + options = common.BearerTokenOptions{RoleIds: []string{"", ""}} +) + +// credentialsString returns the credentials as a JSON string. +func credentialsString() string { + credentials, _ := json.Marshal(map[string]string{ + "clientID": "", + "clientName": "", + "tokenURI": "", + "keyID": "", + "privateKey": "", + }) + return string(credentials) +} + +func main() { + fromFile, err := serviceaccount.GenerateBearerToken(filePath, options) + if err != nil { + fmt.Println("Error generating token from file path:", err.GetMessage()) + } else { + fmt.Println("Scoped Token (using file path):", fromFile.AccessToken) + } + + fromString, err := serviceaccount.GenerateBearerTokenFromCreds(credentialsString(), options) + if err != nil { + fmt.Println("Error generating token from credentials string:", err.GetMessage()) + } else { + fmt.Println("Scoped Token (using credentials string):", fromString.AccessToken) + } +} diff --git a/samples/flowvault/serviceaccount/signed_token_generation_example.go b/samples/flowvault/serviceaccount/signed_token_generation_example.go new file mode 100644 index 0000000..e48b3af --- /dev/null +++ b/samples/flowvault/serviceaccount/signed_token_generation_example.go @@ -0,0 +1,78 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "encoding/json" + "fmt" + + "github.com/skyflowapi/skyflow-go/flowvault/serviceaccount" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" +) + +/** + * This example generates signed data tokens, with a string context and with an object context. + * Each result pairs a data token with its signed token. + */ + +var filePath = "" + +// credentialsString returns the credentials as a JSON string. +func credentialsString() string { + credentials, _ := json.Marshal(map[string]string{ + "clientID": "", + "clientName": "", + "tokenURI": "", + "keyID": "", + "privateKey": "", + }) + return string(credentials) +} + +func printResults(label string, results []common.SignedDataTokensResponse) { + fmt.Println(label) + for _, result := range results { + fmt.Printf(" Token: %s, Signed Token: %s\n", result.Token, result.SignedToken) + } +} + +// Approach 1: a string context, from a credentials file path. +func signedTokensWithStringContext() { + results, err := serviceaccount.GenerateSignedDataTokens(filePath, common.SignedDataTokensOptions{ + Ctx: "user_12345", + DataTokens: []string{"", ""}, + TimeToLive: 30, // seconds + }) + if err != nil { + fmt.Println("Error:", err.GetMessage()) + return + } + printResults("Signed tokens (string context):", results) +} + +// Approach 2: an object context, from a credentials string. Each key maps to a Skyflow CEL +// policy variable under request.context.*, for example: +// +// request.context.role == "analyst" && request.context.department == "research" +func signedTokensWithObjectContext() { + results, err := serviceaccount.GenerateSignedDataTokensFromCreds(credentialsString(), common.SignedDataTokensOptions{ + Ctx: map[string]interface{}{ + "role": "analyst", + "department": "research", + "user_id": "user_67890", + }, + DataTokens: []string{"", ""}, + TimeToLive: 30, + }) + if err != nil { + fmt.Println("Error:", err.GetMessage()) + return + } + printResults("Signed tokens (object context):", results) +} + +func main() { + signedTokensWithStringContext() + signedTokensWithObjectContext() +} diff --git a/samples/flowvault/vaultapi/custom_header_example.go b/samples/flowvault/vaultapi/custom_header_example.go new file mode 100644 index 0000000..013e644 --- /dev/null +++ b/samples/flowvault/vaultapi/custom_header_example.go @@ -0,0 +1,79 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + + "github.com/skyflowapi/skyflow-go/flowvault/client" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" +) + +/** + * This example sends custom headers with one call, using the Interceptor option. + * + * Every operation's options take an Interceptor. It runs just before the request is sent and + * can add headers with rc.AddHeader. The supported keys are common.SkyflowAccountId, + * common.SkyflowAccountName and common.RequestIdHeader. + * + * Steps: + * 1. Set up credentials and the vault config. + * 2. Configure the Skyflow client. + * 3. Insert a record with an interceptor that adds a request ID header. + * 4. Handle the response and errors. + */ +func main() { + // Step 1: Set up credentials and the vault config + vaultConfig := common.VaultConfig{ + VaultId: "", + ClusterId: "", + Env: common.DEV, + Credentials: common.Credentials{Token: ""}, + } + + // Step 2: Configure the Skyflow client + skyflowClient, err := client.NewSkyflow(client.WithVaults(vaultConfig), client.WithLogLevel(logger.DEBUG)) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + vault, err := skyflowClient.Vault(vaultConfig.VaultId) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + + // Step 3: Insert a record with an interceptor that adds a request ID header + request := common.InsertRequest{ + TableName: "", + Upsert: &common.UpsertOptions{UniqueColumns: []string{""}}, + Records: []common.InsertRequestRecord{{Data: map[string]interface{}{"": ""}}}, + } + options := common.InsertOptions{ + Interceptor: func(rc *common.RequestContext) { + requestID := makeRequestID() + fmt.Println("request id =>", requestID, "for", rc.Operation) + rc.AddHeader(common.RequestIdHeader, requestID) + }, + } + response, err := vault.Insert(context.Background(), request, options) + + // Step 4: Handle the response and errors + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + fmt.Println("Records:", response.Records) +} + +// makeRequestID returns a random ID to trace this call. +func makeRequestID() string { + id := make([]byte, 16) + _, _ = rand.Read(id) + return hex.EncodeToString(id) +} diff --git a/samples/flowvault/vaultapi/delete_files.go b/samples/flowvault/vaultapi/delete_files.go new file mode 100644 index 0000000..fda1690 --- /dev/null +++ b/samples/flowvault/vaultapi/delete_files.go @@ -0,0 +1,69 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "context" + "fmt" + + "github.com/skyflowapi/skyflow-go/flowvault/client" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" +) + +/** + * This example deletes files from the file columns of records in a FlowVault vault, including + * the stored files. + * + * Steps: + * 1. Set up credentials and the vault config. + * 2. Configure the Skyflow client. + * 3. Delete the files. + * 4. Handle the response and errors. + */ +func main() { + // Step 1: Set up credentials and the vault config + credentials := common.Credentials{Path: ""} // or ApiKey / Token / CredentialsString + vaultConfig := common.VaultConfig{ + VaultId: "", + ClusterId: "", // from the vault URL: https://{clusterId}.skyvault.skyflowapis.com + Env: common.PROD, // DEV, STAGE, SANDBOX or PROD + Credentials: credentials, + } + + // Step 2: Configure the Skyflow client + skyflowClient, err := client.NewSkyflow(client.WithVaults(vaultConfig), client.WithLogLevel(logger.ERROR)) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + vault, err := skyflowClient.Vault(vaultConfig.VaultId) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + + // Step 3: Delete the files. Each record sets exactly one of SkyflowId or UniqueValues. + request := common.DeleteFilesRequest{Records: []common.DeleteFilesRequestRecord{ + {TableName: "", Columns: []string{""}, SkyflowId: ""}, + // {TableName: "", Columns: []string{""}, + // UniqueValues: []map[string]interface{}{{"email": "john@example.com"}}}, + }} + response, err := vault.DeleteFiles(context.Background(), request, common.DeleteFilesOptions{}) + + // Step 4: Handle the response and errors. + // One record per matched record. Each has SkyflowId, TableName, Columns (each with Column + // and Status "DELETED"; empty on a failed record), HttpCode, Error and RequestId. + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + for _, record := range response.Records { + if record.Error == "" { + fmt.Printf("%s -> %v\n", record.SkyflowId, record.Columns) + } else { + fmt.Printf("%s failed (%d): %s [requestId=%s]\n", record.SkyflowId, record.HttpCode, record.Error, record.RequestId) + } + } +} diff --git a/samples/flowvault/vaultapi/delete_records.go b/samples/flowvault/vaultapi/delete_records.go new file mode 100644 index 0000000..5a8666d --- /dev/null +++ b/samples/flowvault/vaultapi/delete_records.go @@ -0,0 +1,67 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "context" + "fmt" + + "github.com/skyflowapi/skyflow-go/flowvault/client" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" +) + +/** + * This example deletes records from a FlowVault vault by Skyflow ID. + * + * Steps: + * 1. Set up credentials and the vault config. + * 2. Configure the Skyflow client. + * 3. Delete records. + * 4. Handle the response and errors. + */ +func main() { + // Step 1: Set up credentials and the vault config + credentials := common.Credentials{Path: ""} // or ApiKey / Token / CredentialsString + vaultConfig := common.VaultConfig{ + VaultId: "", + ClusterId: "", // from the vault URL: https://{clusterId}.skyvault.skyflowapis.com + Env: common.PROD, // DEV, STAGE, SANDBOX or PROD + Credentials: credentials, + } + + // Step 2: Configure the Skyflow client + skyflowClient, err := client.NewSkyflow(client.WithVaults(vaultConfig), client.WithLogLevel(logger.ERROR)) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + vault, err := skyflowClient.Vault(vaultConfig.VaultId) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + + // Step 3: Delete records. Set exactly one of SkyflowIds or UniqueValues. + request := common.DeleteRequest{ + TableName: "", + SkyflowIds: []string{"", ""}, + // UniqueValues: []map[string]interface{}{{"email": "john@example.com"}}, + } + response, err := vault.Delete(context.Background(), request, common.DeleteOptions{}) + + // Step 4: Handle the response and errors. + // One record per input, in order. Each has SkyflowId, HttpCode, Error and RequestId. + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + for _, record := range response.Records { + if record.Error == "" { + fmt.Println(record.SkyflowId, "deleted") + } else { + fmt.Printf("%s failed (%d): %s [requestId=%s]\n", record.SkyflowId, record.HttpCode, record.Error, record.RequestId) + } + } +} diff --git a/samples/flowvault/vaultapi/detokenize_records.go b/samples/flowvault/vaultapi/detokenize_records.go new file mode 100644 index 0000000..3cc7201 --- /dev/null +++ b/samples/flowvault/vaultapi/detokenize_records.go @@ -0,0 +1,67 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "context" + "fmt" + + "github.com/skyflowapi/skyflow-go/flowvault/client" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" +) + +/** + * This example returns the values behind tokens in a FlowVault vault. + * + * Steps: + * 1. Set up credentials and the vault config. + * 2. Configure the Skyflow client. + * 3. Detokenize tokens. + * 4. Handle the response and errors. + */ +func main() { + // Step 1: Set up credentials and the vault config + credentials := common.Credentials{Path: ""} // or ApiKey / Token / CredentialsString + vaultConfig := common.VaultConfig{ + VaultId: "", + ClusterId: "", // from the vault URL: https://{clusterId}.skyvault.skyflowapis.com + Env: common.PROD, // DEV, STAGE, SANDBOX or PROD + Credentials: credentials, + } + + // Step 2: Configure the Skyflow client + skyflowClient, err := client.NewSkyflow(client.WithVaults(vaultConfig), client.WithLogLevel(logger.ERROR)) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + vault, err := skyflowClient.Vault(vaultConfig.VaultId) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + + // Step 3: Detokenize tokens. Redactions are set per token group. + request := common.DetokenizeRequest{ + Tokens: []string{"", ""}, + // TokenGroupRedactions: []common.TokenGroupRedaction{{TokenGroupName: "", Redaction: ""}}, + } + response, err := vault.Detokenize(context.Background(), request, common.DetokenizeOptions{}) + + // Step 4: Handle the response and errors. + // One record per input token, in order. Each has Token, Value, TokenGroupName, Metadata + // (SkyflowId and TableName), HttpCode, Error and RequestId. + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + for _, record := range response.Records { + if record.Error == "" { + fmt.Printf("%s -> %v (%s)\n", record.Token, record.Value, record.TokenGroupName) + } else { + fmt.Printf("%s failed (%d): %s [requestId=%s]\n", record.Token, record.HttpCode, record.Error, record.RequestId) + } + } +} diff --git a/samples/flowvault/vaultapi/get_records.go b/samples/flowvault/vaultapi/get_records.go new file mode 100644 index 0000000..e83beb1 --- /dev/null +++ b/samples/flowvault/vaultapi/get_records.go @@ -0,0 +1,78 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "context" + "fmt" + + "github.com/skyflowapi/skyflow-go/flowvault/client" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" +) + +/** + * This example retrieves records from a FlowVault vault by Skyflow ID. + * + * Steps: + * 1. Set up credentials and the vault config. + * 2. Configure the Skyflow client. + * 3. Get records. + * 4. Handle the response and errors. + */ +func main() { + // Step 1: Set up credentials and the vault config + credentials := common.Credentials{Path: ""} // or ApiKey / Token / CredentialsString + vaultConfig := common.VaultConfig{ + VaultId: "", + ClusterId: "", // from the vault URL: https://{clusterId}.skyvault.skyflowapis.com + Env: common.PROD, // DEV, STAGE, SANDBOX or PROD + Credentials: credentials, + } + + // Step 2: Configure the Skyflow client + skyflowClient, err := client.NewSkyflow(client.WithVaults(vaultConfig), client.WithLogLevel(logger.ERROR)) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + vault, err := skyflowClient.Vault(vaultConfig.VaultId) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + + // Step 3: Get records by Skyflow ID. Set exactly one of SkyflowIds or UniqueValues. + request := common.GetRequest{ + TableName: "", + SkyflowIds: []string{"", ""}, + // Columns: []string{"name", "email"}, // return only these columns + // ColumnRedactions: []common.ColumnRedaction{{ColumnName: "email", Redaction: ""}}, + // Limit: 25, Offset: 0, + } + // To look records up by a unique column instead of by Skyflow ID: + // request := common.GetRequest{TableName: "", UniqueValues: []map[string]interface{}{{"email": "john@example.com"}}} + // + // To get from several tables in one call, use Records instead of the fields above: + // request := common.GetRequest{Records: []common.GetRequestRecord{ + // {TableName: "", SkyflowIds: []string{""}}, + // {TableName: "", UniqueValues: []map[string]interface{}{{"email": "john@example.com"}}}, + // }} + response, err := vault.Get(context.Background(), request, common.GetOptions{}) + + // Step 4: Handle the response and errors. + // One record per input, in order. Each has SkyflowId, TableName, Data, Tokens, HashedData, + // HttpCode, Error and RequestId. + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + for _, record := range response.Records { + if record.Error == "" { + fmt.Printf("%s -> data=%v\n", record.SkyflowId, record.Data) + } else { + fmt.Printf("failed (%d): %s [requestId=%s]\n", record.HttpCode, record.Error, record.RequestId) + } + } +} diff --git a/samples/flowvault/vaultapi/get_tokens.go b/samples/flowvault/vaultapi/get_tokens.go new file mode 100644 index 0000000..7b226d3 --- /dev/null +++ b/samples/flowvault/vaultapi/get_tokens.go @@ -0,0 +1,69 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "context" + "fmt" + + "github.com/skyflowapi/skyflow-go/flowvault/client" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" +) + +/** + * This example looks up the existing tokens for values in a FlowVault vault. + * + * Only deterministic token groups are supported, and no new tokens are created. + * + * Steps: + * 1. Set up credentials and the vault config. + * 2. Configure the Skyflow client. + * 3. Get tokens for values. + * 4. Handle the response and errors. + */ +func main() { + // Step 1: Set up credentials and the vault config + credentials := common.Credentials{Path: ""} // or ApiKey / Token / CredentialsString + vaultConfig := common.VaultConfig{ + VaultId: "", + ClusterId: "", // from the vault URL: https://{clusterId}.skyvault.skyflowapis.com + Env: common.PROD, // DEV, STAGE, SANDBOX or PROD + Credentials: credentials, + } + + // Step 2: Configure the Skyflow client + skyflowClient, err := client.NewSkyflow(client.WithVaults(vaultConfig), client.WithLogLevel(logger.ERROR)) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + vault, err := skyflowClient.Vault(vaultConfig.VaultId) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + + // Step 3: Get tokens for values + request := common.GetTokensRequest{Records: []common.GetTokensRequestRecord{ + {Value: "john@example.com", TokenGroupName: ""}, + {Value: "jane@example.com", TokenGroupName: ""}, + }} + response, err := vault.GetTokens(context.Background(), request, common.GetTokensOptions{}) + + // Step 4: Handle the response and errors. + // One record per input, in order. Each has Value, TokenGroupName, Token, HttpCode, Error + // and RequestId. A value with no token has Error set. + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + for _, record := range response.Records { + if record.Error == "" { + fmt.Printf("%v -> %s\n", record.Value, record.Token) + } else { + fmt.Printf("%v failed (%d): %s [requestId=%s]\n", record.Value, record.HttpCode, record.Error, record.RequestId) + } + } +} diff --git a/samples/flowvault/vaultapi/insert_records.go b/samples/flowvault/vaultapi/insert_records.go new file mode 100644 index 0000000..ec101e1 --- /dev/null +++ b/samples/flowvault/vaultapi/insert_records.go @@ -0,0 +1,83 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "context" + "fmt" + + "github.com/skyflowapi/skyflow-go/flowvault/client" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" +) + +/** + * This example inserts records into a FlowVault vault and prints the tokens returned for each. + * + * The table name and upsert are set at exactly ONE level: on the request (applies to every + * record) or on every record. Setting them on both, or mixing the two, is a validation error. + * + * Steps: + * 1. Set up credentials and the vault config. + * 2. Configure the Skyflow client. + * 3. Insert records. + * 4. Handle the response and errors. + */ +func main() { + // Step 1: Set up credentials and the vault config + credentials := common.Credentials{Path: ""} // or ApiKey / Token / CredentialsString + vaultConfig := common.VaultConfig{ + VaultId: "", + ClusterId: "", // from the vault URL: https://{clusterId}.skyvault.skyflowapis.com + Env: common.PROD, // DEV, STAGE, SANDBOX or PROD + Credentials: credentials, + } + + // Step 2: Configure the Skyflow client + skyflowClient, err := client.NewSkyflow(client.WithVaults(vaultConfig), client.WithLogLevel(logger.ERROR)) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + vault, err := skyflowClient.Vault(vaultConfig.VaultId) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + + // Step 3: Insert records. The table name and upsert here apply to every record. + request := common.InsertRequest{ + TableName: "", + Upsert: &common.UpsertOptions{UniqueColumns: []string{"email"}, UpdateType: common.UpdateTypeUpdate}, + Records: []common.InsertRequestRecord{ + {Data: map[string]interface{}{"name": "John Doe", "email": "john@example.com"}}, + }, + } + // To set the table (and upsert) per record instead, for example to insert into several + // tables in one call, leave TableName and Upsert empty on the request: + // + // request := common.InsertRequest{ + // Records: []common.InsertRequestRecord{ + // {TableName: "", Data: map[string]interface{}{"name": "Jane Doe"}}, + // {TableName: "", Data: map[string]interface{}{"email": "jane@example.com"}, + // Upsert: &common.UpsertOptions{UniqueColumns: []string{"email"}, UpdateType: common.UpdateTypeReplace}}, + // }, + // } + response, err := vault.Insert(context.Background(), request, common.InsertOptions{}) + + // Step 4: Handle the response and errors. + // Validation errors and whole-call failures come back as err. Otherwise there is one record + // per input, in order; a record that failed has Error, HttpCode and RequestId set. + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + for _, record := range response.Records { + if record.Error == "" { + fmt.Printf("%s -> tokens=%v\n", record.SkyflowId, record.Tokens) + } else { + fmt.Printf("failed (%d): %s [requestId=%s]\n", record.HttpCode, record.Error, record.RequestId) + } + } +} diff --git a/samples/flowvault/vaultapi/query_records.go b/samples/flowvault/vaultapi/query_records.go new file mode 100644 index 0000000..dd4e58f --- /dev/null +++ b/samples/flowvault/vaultapi/query_records.go @@ -0,0 +1,63 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "context" + "fmt" + + "github.com/skyflowapi/skyflow-go/flowvault/client" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" +) + +/** + * This example runs a read-only SQL query against a FlowVault vault. + * + * Only SELECT is supported, and a call returns at most 25 rows. Page with OFFSET. + * + * Steps: + * 1. Set up credentials and the vault config. + * 2. Configure the Skyflow client. + * 3. Run the query. + * 4. Handle the response and errors. + */ +func main() { + // Step 1: Set up credentials and the vault config + credentials := common.Credentials{Path: ""} // or ApiKey / Token / CredentialsString + vaultConfig := common.VaultConfig{ + VaultId: "", + ClusterId: "", // from the vault URL: https://{clusterId}.skyvault.skyflowapis.com + Env: common.PROD, // DEV, STAGE, SANDBOX or PROD + Credentials: credentials, + } + + // Step 2: Configure the Skyflow client + skyflowClient, err := client.NewSkyflow(client.WithVaults(vaultConfig), client.WithLogLevel(logger.ERROR)) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + vault, err := skyflowClient.Vault(vaultConfig.VaultId) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + + // Step 3: Run the query + request := common.QueryRequest{Query: "SELECT name, email FROM LIMIT 25"} + response, err := vault.Query(context.Background(), request, common.QueryOptions{}) + + // Step 4: Handle the response and errors. + // Query has no per-record status: any failure comes back as err. + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + fmt.Println("Request ID:", response.RequestId) + fmt.Println("Columns:", response.Metadata.Columns) + for _, row := range response.Fields { + fmt.Println(row) + } +} diff --git a/samples/flowvault/vaultapi/timeout_and_retry_config_example.go b/samples/flowvault/vaultapi/timeout_and_retry_config_example.go new file mode 100644 index 0000000..cf93f17 --- /dev/null +++ b/samples/flowvault/vaultapi/timeout_and_retry_config_example.go @@ -0,0 +1,69 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "fmt" + "time" + + "github.com/skyflowapi/skyflow-go/flowvault/client" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" +) + +/** + * This example sets HTTP timeout and retry settings, for the whole client and for one vault. + * + * Each setting is resolved separately: the vault's value, then the client's, then the SDK + * default. + * + * Timeout overall limit for one call, including retries and backoff. Default 60s. + * MaxRetries retries after the first attempt, for 408, 429 and 5xx. Default 0 (off). + * InitialRetryDelay wait before the first retry. Default 500ms. + * MaxRetryDelay the most the wait can grow to; it doubles each retry. Default 2000ms. + * + * A context deadline still applies when it is shorter. File uploads in UploadFiles are not + * retried and have no overall timeout, so large files aren't cut off. + * + * Steps: + * 1. Set up credentials and the vault config, with per-vault overrides. + * 2. Configure the Skyflow client with client-wide defaults. + */ +func main() { + // Step 1: Set up credentials and the vault config, with per-vault overrides + vaultRetries := 2 + vaultConfig := common.VaultConfig{ + VaultId: "", + ClusterId: "", + Env: common.PROD, + Credentials: common.Credentials{Path: ""}, + // These win over the client-wide values below, for this vault only. + Timeout: 30 * time.Second, + MaxRetries: &vaultRetries, + } + + // Step 2: Configure the Skyflow client with client-wide defaults. + // Any vault that doesn't set a field uses these. + clientRetries := 3 + skyflowClient, err := client.NewSkyflow( + client.WithHttpConfig(common.HttpConfig{ + Timeout: 60 * time.Second, + MaxRetries: &clientRetries, + InitialRetryDelay: 500 * time.Millisecond, + MaxRetryDelay: 4 * time.Second, + }), + client.WithVaults(vaultConfig), + client.WithLogLevel(logger.ERROR), + ) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + fmt.Println("Skyflow client configured with custom timeout and retry settings:", skyflowClient != nil) + + // Use the client as usual; calls now fail at the configured timeouts and retry 408, 429 and + // 5xx responses with exponential backoff and jitter. + // vault, _ := skyflowClient.Vault(vaultConfig.VaultId) + // response, err := vault.Detokenize(context.Background(), detokenizeRequest, common.DetokenizeOptions{}) +} diff --git a/samples/flowvault/vaultapi/update_record.go b/samples/flowvault/vaultapi/update_record.go new file mode 100644 index 0000000..e4299cf --- /dev/null +++ b/samples/flowvault/vaultapi/update_record.go @@ -0,0 +1,71 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "context" + "fmt" + + "github.com/skyflowapi/skyflow-go/flowvault/client" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" +) + +/** + * This example updates a record in a FlowVault vault. + * + * UpdateType is set on the request: UPDATE (the default) changes only the given columns, + * REPLACE overwrites the whole record. + * + * Steps: + * 1. Set up credentials and the vault config. + * 2. Configure the Skyflow client. + * 3. Update the record. + * 4. Handle the response and errors. + */ +func main() { + // Step 1: Set up credentials and the vault config + credentials := common.Credentials{Path: ""} // or ApiKey / Token / CredentialsString + vaultConfig := common.VaultConfig{ + VaultId: "", + ClusterId: "", // from the vault URL: https://{clusterId}.skyvault.skyflowapis.com + Env: common.PROD, // DEV, STAGE, SANDBOX or PROD + Credentials: credentials, + } + + // Step 2: Configure the Skyflow client + skyflowClient, err := client.NewSkyflow(client.WithVaults(vaultConfig), client.WithLogLevel(logger.ERROR)) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + vault, err := skyflowClient.Vault(vaultConfig.VaultId) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + + // Step 3: Update the record. Set the table name on the request or on every record, not both. + request := common.UpdateRequest{ + TableName: "", + UpdateType: common.UpdateTypeUpdate, + Records: []common.UpdateRequestRecord{ + {SkyflowId: "", Data: map[string]interface{}{"name": "Jane Doe"}}, + }, + } + response, err := vault.Update(context.Background(), request, common.UpdateOptions{}) + + // Step 4: Handle the response and errors + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + for _, record := range response.Records { + if record.Error == "" { + fmt.Printf("%s updated -> tokens=%v\n", record.SkyflowId, record.Tokens) + } else { + fmt.Printf("%s failed (%d): %s [requestId=%s]\n", record.SkyflowId, record.HttpCode, record.Error, record.RequestId) + } + } +} diff --git a/samples/flowvault/vaultapi/upload_files.go b/samples/flowvault/vaultapi/upload_files.go new file mode 100644 index 0000000..78ab8d7 --- /dev/null +++ b/samples/flowvault/vaultapi/upload_files.go @@ -0,0 +1,89 @@ +/* +Copyright (c) 2026 Skyflow, Inc. +*/ +package main + +import ( + "context" + "fmt" + "os" + + "github.com/skyflowapi/skyflow-go/flowvault/client" + "github.com/skyflowapi/skyflow-go/flowvault/utils/common" + "github.com/skyflowapi/skyflow-go/flowvault/utils/logger" +) + +/** + * This example uploads files into the file columns of a FlowVault vault. + * + * The SDK requests a signed upload URL for each column, then uploads each file to its URL. + * + * Steps: + * 1. Set up credentials and the vault config. + * 2. Configure the Skyflow client. + * 3. Upload the files. + * 4. Handle the response and errors. + */ +func main() { + // Step 1: Set up credentials and the vault config + credentials := common.Credentials{Path: ""} // or ApiKey / Token / CredentialsString + vaultConfig := common.VaultConfig{ + VaultId: "", + ClusterId: "", // from the vault URL: https://{clusterId}.skyvault.skyflowapis.com + Env: common.PROD, // DEV, STAGE, SANDBOX or PROD + Credentials: credentials, + } + + // Step 2: Configure the Skyflow client + skyflowClient, err := client.NewSkyflow(client.WithVaults(vaultConfig), client.WithLogLevel(logger.ERROR)) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + vault, err := skyflowClient.Vault(vaultConfig.VaultId) + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + + // Step 3: Upload the files. Each column takes exactly one of FilePath, Base64 (needs + // FileName) or FileObject. Leave SkyflowId empty to create a new record, or set it to upload + // into an existing record. + file, openErr := os.Open("") + if openErr != nil { + fmt.Println("Could not open file:", openErr) + return + } + defer file.Close() // the caller closes a FileObject + + request := common.UploadFilesRequest{Records: []common.UploadFilesRequestRecord{ + { + TableName: "", + Columns: []common.UploadFilesRequestColumn{ + {Column: "", FilePath: ""}, + {Column: "", Base64: "", FileName: ""}, + {Column: "", FileObject: file}, + }, + }, + // {TableName: "", SkyflowId: "", + // Columns: []common.UploadFilesRequestColumn{{Column: "", FilePath: ""}}}, + }} + response, err := vault.UploadFiles(context.Background(), request, common.UploadFilesOptions{}) + + // Step 4: Handle the response and errors. + // One record per input record, in order. Each has SkyflowId, TableName, Columns, HttpCode, + // Error and RequestId. Each column has Column, FileName, UploadStatus (UPLOADED, FAILED or + // SKIPPED) and Error. + if err != nil { + fmt.Println("Skyflow Specific Error:", err.GetHttpCode(), err.GetMessage(), err.GetDetails()) + return + } + for _, record := range response.Records { + if record.Error != "" { + fmt.Printf("record failed (%d): %s [requestId=%s]\n", record.HttpCode, record.Error, record.RequestId) + } + for _, column := range record.Columns { + fmt.Printf("%s %s: %s %s\n", record.SkyflowId, column.Column, column.UploadStatus, column.Error) + } + } +} From ac53bc9f65f73ff08fce5370b174706490c6b316 Mon Sep 17 00:00:00 2001 From: Himanshu Pal Date: Wed, 21 Oct 2026 09:52:32 +0530 Subject: [PATCH 4/5] SK-3223 update common commit hash Pin common to 6115e81 (shared vault interfaces, HTTP config, unset Env) in v2 and flowvault, and build the flowvault samples against that version instead of the local common folder. Co-Authored-By: Claude Opus 5.5 --- flowvault/go.mod | 5 ++--- flowvault/go.sum | 13 +++---------- samples/flowvault/go.mod | 7 ++----- samples/flowvault/go.sum | 2 ++ v2/go.mod | 2 +- v2/go.sum | 4 ++-- 6 files changed, 12 insertions(+), 21 deletions(-) diff --git a/flowvault/go.mod b/flowvault/go.mod index ebfa281..f4327d1 100644 --- a/flowvault/go.mod +++ b/flowvault/go.mod @@ -4,10 +4,11 @@ go 1.25.0 require ( github.com/golang-jwt/jwt/v4 v4.5.2 // indirect - github.com/skyflowapi/skyflow-go/common v0.0.0-20261014151608-a64b46a2705c + github.com/skyflowapi/skyflow-go/common v0.0.0-20261019104456-6115e81fcef9 ) require ( + github.com/google/uuid v1.6.0 github.com/onsi/ginkgo/v2 v2.33.1 github.com/onsi/gomega v1.44.0 github.com/stretchr/testify v1.8.4 @@ -20,9 +21,7 @@ require ( github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect - github.com/google/uuid v1.6.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/stretchr/objx v0.5.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/mod v0.36.0 // indirect golang.org/x/net v0.56.0 // indirect diff --git a/flowvault/go.sum b/flowvault/go.sum index 4a73ca5..cf05d87 100644 --- a/flowvault/go.sum +++ b/flowvault/go.sum @@ -1,6 +1,5 @@ github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= @@ -41,14 +40,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= -github.com/skyflowapi/skyflow-go/common v0.0.0-20261014151608-a64b46a2705c h1:oWxTNYJ++iBGbcVpJ2oNhyB36WNNVMyBpusEAcSL7pw= -github.com/skyflowapi/skyflow-go/common v0.0.0-20261014151608-a64b46a2705c/go.mod h1:T9roW4Qpv1WFo6tgLjbfLrdOCOz20lVE8ENf2bGb7zI= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/skyflowapi/skyflow-go/common v0.0.0-20261019104456-6115e81fcef9 h1:0SNuvxkS+mr+sy7lDZvwmgjgAIC+HrqEr81lsTTXEgQ= +github.com/skyflowapi/skyflow-go/common v0.0.0-20261019104456-6115e81fcef9/go.mod h1:T9roW4Qpv1WFo6tgLjbfLrdOCOz20lVE8ENf2bGb7zI= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= @@ -75,7 +68,7 @@ golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/samples/flowvault/go.mod b/samples/flowvault/go.mod index 1aeec54..a973bee 100644 --- a/samples/flowvault/go.mod +++ b/samples/flowvault/go.mod @@ -7,11 +7,8 @@ require github.com/skyflowapi/skyflow-go/flowvault v0.0.0 require ( github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/skyflowapi/skyflow-go/common v0.0.0-20261014151608-a64b46a2705c // indirect + github.com/skyflowapi/skyflow-go/common v0.0.0-20261019104456-6115e81fcef9 // indirect ) // FlowVault isn't published yet; build the samples against the code in this repository. -replace ( - github.com/skyflowapi/skyflow-go/common => ../../common - github.com/skyflowapi/skyflow-go/flowvault => ../../flowvault -) +replace github.com/skyflowapi/skyflow-go/flowvault => ../../flowvault diff --git a/samples/flowvault/go.sum b/samples/flowvault/go.sum index 5582f4d..ce8cae7 100644 --- a/samples/flowvault/go.sum +++ b/samples/flowvault/go.sum @@ -20,6 +20,8 @@ github.com/onsi/gomega v1.44.0 h1:eAiGl3Pw5jz5GQdDff0BcxYpAX1JxW8xD7mFUuwNfZQ= github.com/onsi/gomega v1.44.0/go.mod h1:e/C2HwaZ1DhvjzXXuFhcR7hY7Sh9pl7MmoWKEjzwcdA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/skyflowapi/skyflow-go/common v0.0.0-20261019104456-6115e81fcef9 h1:0SNuvxkS+mr+sy7lDZvwmgjgAIC+HrqEr81lsTTXEgQ= +github.com/skyflowapi/skyflow-go/common v0.0.0-20261019104456-6115e81fcef9/go.mod h1:T9roW4Qpv1WFo6tgLjbfLrdOCOz20lVE8ENf2bGb7zI= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= diff --git a/v2/go.mod b/v2/go.mod index dddfeae..ed9499d 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -7,7 +7,7 @@ require ( github.com/google/uuid v1.6.0 github.com/onsi/ginkgo/v2 v2.23.4 github.com/onsi/gomega v1.38.0 - github.com/skyflowapi/skyflow-go/common v0.0.0-20261014151608-a64b46a2705c + github.com/skyflowapi/skyflow-go/common v0.0.0-20261019104456-6115e81fcef9 github.com/stretchr/testify v1.8.4 ) diff --git a/v2/go.sum b/v2/go.sum index 63e5a48..3c0319c 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -24,8 +24,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= -github.com/skyflowapi/skyflow-go/common v0.0.0-20261014151608-a64b46a2705c h1:oWxTNYJ++iBGbcVpJ2oNhyB36WNNVMyBpusEAcSL7pw= -github.com/skyflowapi/skyflow-go/common v0.0.0-20261014151608-a64b46a2705c/go.mod h1:T9roW4Qpv1WFo6tgLjbfLrdOCOz20lVE8ENf2bGb7zI= +github.com/skyflowapi/skyflow-go/common v0.0.0-20261019104456-6115e81fcef9 h1:0SNuvxkS+mr+sy7lDZvwmgjgAIC+HrqEr81lsTTXEgQ= +github.com/skyflowapi/skyflow-go/common v0.0.0-20261019104456-6115e81fcef9/go.mod h1:T9roW4Qpv1WFo6tgLjbfLrdOCOz20lVE8ENf2bGb7zI= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= From 87e934e6a050cd3162c7815bf494749aec90d321 Mon Sep 17 00:00:00 2001 From: Himanshu Pal Date: Thu, 22 Oct 2026 14:58:53 +0530 Subject: [PATCH 5/5] SK-3223-added-exception-tests --- v2/utils/error/skyflow_exception_test.go | 58 ++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 v2/utils/error/skyflow_exception_test.go diff --git a/v2/utils/error/skyflow_exception_test.go b/v2/utils/error/skyflow_exception_test.go new file mode 100644 index 0000000..ddc3268 --- /dev/null +++ b/v2/utils/error/skyflow_exception_test.go @@ -0,0 +1,58 @@ +package errors_test + +import ( + "errors" + "io" + "net/http" + "strings" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + skyflowError "github.com/skyflowapi/skyflow-go/v2/utils/error" +) + +// The error logic lives in skyflow-go/common/errors and is tested there. These specs check +// that v2's wrappers forward to it and keep v2's error codes. + +func TestSkyflowError(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "SkyflowError Suite") +} + +const apiErrorBody = `{"error":{"grpc_code":3,"http_code":400,"http_status":"Bad Request","message":"Invalid request","details":[]}}` + +var _ = Describe("SkyflowError", func() { + It("creates an error with a code and message", func() { + err := skyflowError.NewSkyflowError(skyflowError.INVALID_INPUT_CODE, "bad input") + Expect(err.GetHttpCode()).To(Equal(string(skyflowError.INVALID_INPUT_CODE))) + Expect(err.GetMessage()).To(ContainSubstring("bad input")) + }) + + It("keeps v2's error codes", func() { + Expect(string(skyflowError.SERVER)).To(Equal("500")) + Expect(string(skyflowError.INVALID_INPUT_CODE)).To(Equal("400")) + Expect(string(skyflowError.PARTIAL)).To(Equal("404")) + }) + + It("builds an error from an API response", func() { + response := http.Response{ + StatusCode: http.StatusBadRequest, + Header: http.Header{"Content-Type": {"application/json"}, "X-Request-Id": {"req-1"}}, + Body: io.NopCloser(strings.NewReader(apiErrorBody)), + } + err := skyflowError.SkyflowApiError(response) + Expect(err.GetHttpCode()).To(Equal("400")) + Expect(err.GetMessage()).To(ContainSubstring("Invalid request")) + Expect(err.GetGrpcCode()).To(Equal("3")) + Expect(err.GetRequestId()).To(Equal("req-1")) + }) + + It("builds an error from a generated client error", func() { + header := http.Header{"X-Request-Id": {"req-2"}} + err := skyflowError.SkyflowErrorApi(errors.New("400: "+apiErrorBody), header) + Expect(err.GetHttpCode()).To(Equal("400")) + Expect(err.GetMessage()).To(ContainSubstring("Invalid request")) + Expect(err.GetRequestId()).To(Equal("req-2")) + }) +})