diff --git a/.env.example b/.env.example index a3797df1..819917b9 100644 --- a/.env.example +++ b/.env.example @@ -92,6 +92,13 @@ CCF_EVIDENCE_BATCH_PRUNE_SCHEDULE="0 47 * * * *" # every SSP with automatic filters on; it runs at most hourly. CCF_SSP_SCOPE_AUTO_FILTERS_SCHEDULE="0 23 * * * *" +# River (6-field) cron of SSP template discovery, which creates one SSP per distinct key +# value in the latest evidence and learns their scopes; it runs at most hourly. +CCF_SSP_TEMPLATE_DISCOVERY_SCHEDULE="0 37 * * * *" +# An SSP a template created is reported as stale after this long without evidence for its +# key values (a Go duration). Nothing is deleted. +CCF_SSP_TEMPLATE_STALE_AFTER=168h + # Compliance Map (/api/lineage): built datasets are shared across users and reused for the # TTL; ?refresh=true rebuilds one. 0 disables the cache. CCF_LINEAGE_CACHE_TTL=5m diff --git a/internal/config/config.go b/internal/config/config.go index aab99ff6..f2de8e96 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -53,6 +53,7 @@ type Config struct { EvidenceBatch *EvidenceBatchConfig Lineage *LineageConfig SSPScope *SSPScopeConfig + SSPTemplate *SSPTemplateConfig } func NewConfig(logger *zap.SugaredLogger) *Config { @@ -197,6 +198,11 @@ func NewConfig(logger *zap.SugaredLogger) *Config { logger.Fatalw("Invalid lineage config", "error", err) } + sspTemplateConfig, err := LoadSSPTemplateConfig() + if err != nil { + logger.Fatalw("Invalid SSP template config", "error", err) + } + // Digest configuration digestEnabled := viper.GetBool("digest_enabled") digestSchedule := viper.GetString("digest_schedule") @@ -289,6 +295,7 @@ func NewConfig(logger *zap.SugaredLogger) *Config { EvidenceSubjects: evidenceSubjectConfig, EvidenceBatch: evidenceBatchConfig, SSPScope: LoadSSPScopeConfig(), + SSPTemplate: sspTemplateConfig, Lineage: lineageConfig, DigestEnabled: digestEnabled, DigestSchedule: digestSchedule, diff --git a/internal/config/ssp_template.go b/internal/config/ssp_template.go new file mode 100644 index 00000000..21a80040 --- /dev/null +++ b/internal/config/ssp_template.go @@ -0,0 +1,48 @@ +package config + +import ( + "fmt" + "strings" + "time" + + "github.com/spf13/viper" +) + +const ( + // DefaultSSPTemplateDiscoverySchedule runs SSP template discovery hourly (River + // 6-field cron). + DefaultSSPTemplateDiscoverySchedule = "0 37 * * * *" + // DefaultSSPTemplateStaleAfter is how long an SSP a template created can go without + // evidence for its key values before it is reported as stale. + DefaultSSPTemplateStaleAfter = 7 * 24 * time.Hour +) + +// SSPTemplateConfig configures SSP template discovery. +type SSPTemplateConfig struct { + // DiscoverySchedule is the River cron of the discovery job + // (CCF_SSP_TEMPLATE_DISCOVERY_SCHEDULE). + DiscoverySchedule string + // StaleAfter is how long a created SSP can go without evidence before it is + // reported as stale (CCF_SSP_TEMPLATE_STALE_AFTER, a Go duration). + StaleAfter time.Duration +} + +func LoadSSPTemplateConfig() (*SSPTemplateConfig, error) { + return LoadSSPTemplateConfigFromViper(viper.GetViper()) +} + +func LoadSSPTemplateConfigFromViper(v *viper.Viper) (*SSPTemplateConfig, error) { + cfg := &SSPTemplateConfig{DiscoverySchedule: DefaultSSPTemplateDiscoverySchedule, StaleAfter: DefaultSSPTemplateStaleAfter} + if schedule := strings.TrimSpace(v.GetString("ssp_template_discovery_schedule")); schedule != "" { + cfg.DiscoverySchedule = schedule + } + if v.IsSet("ssp_template_stale_after") { + raw := strings.TrimSpace(v.GetString("ssp_template_stale_after")) + staleAfter, err := time.ParseDuration(raw) + if err != nil || staleAfter <= 0 { + return nil, fmt.Errorf("CCF_SSP_TEMPLATE_STALE_AFTER must be a positive duration such as 168h, got %q", raw) + } + cfg.StaleAfter = staleAfter + } + return cfg, nil +} diff --git a/internal/config/ssp_template_test.go b/internal/config/ssp_template_test.go new file mode 100644 index 00000000..39ab6000 --- /dev/null +++ b/internal/config/ssp_template_test.go @@ -0,0 +1,28 @@ +package config + +import ( + "testing" + "time" + + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +func TestLoadSSPTemplateConfig(t *testing.T) { + v := viper.New() + cfg, err := LoadSSPTemplateConfigFromViper(v) + require.NoError(t, err) + require.Equal(t, &SSPTemplateConfig{DiscoverySchedule: DefaultSSPTemplateDiscoverySchedule, StaleAfter: DefaultSSPTemplateStaleAfter}, cfg) + + v.Set("ssp_template_discovery_schedule", " 0 0 * * * * ") + v.Set("ssp_template_stale_after", "24h") + cfg, err = LoadSSPTemplateConfigFromViper(v) + require.NoError(t, err) + require.Equal(t, &SSPTemplateConfig{DiscoverySchedule: "0 0 * * * *", StaleAfter: 24 * time.Hour}, cfg) + + for _, bad := range []string{"soon", "0", "-1h"} { + v.Set("ssp_template_stale_after", bad) + _, err = LoadSSPTemplateConfigFromViper(v) + require.Error(t, err, bad) + } +} diff --git a/internal/service/relational/ssp_templates.go b/internal/service/relational/ssp_templates.go index 0a0b928f..f3a4aa52 100644 --- a/internal/service/relational/ssp_templates.go +++ b/internal/service/relational/ssp_templates.go @@ -39,6 +39,12 @@ type SSPTemplate struct { func (SSPTemplate) TableName() string { return "ssp_templates" } +// SkippedScopeEntry is a learned scope entry discovery could not store, with the reason. +type SkippedScopeEntry struct { + Labels []ScopeLabel `json:"labels"` + Reason string `json:"reason"` +} + // SSPTemplateInstance records an SSP a template created, keyed by its key values. type SSPTemplateInstance struct { TemplateID uuid.UUID `gorm:"type:uuid;primaryKey"` @@ -52,6 +58,13 @@ type SSPTemplateInstance struct { LastSeenAt time.Time // LastError is the last failure applying the instance (a profile attach, say). LastError *string + // ProfilesPending is set while some of the template's profiles failed to attach to + // the SSP; discovery retries them on its next run. + ProfilesPending bool `gorm:"not null;default:false"` + // SkippedEntries counts the learned entries the last run could not store (a value + // over the length limit, say); SkippedSample keeps a few of them. + SkippedEntries int `gorm:"not null;default:0"` + SkippedSample datatypes.JSONSlice[SkippedScopeEntry] `gorm:"type:jsonb"` Template *SSPTemplate `gorm:"foreignKey:TemplateID;constraint:OnDelete:CASCADE"` } diff --git a/internal/service/relational/ssptemplates/discovery.go b/internal/service/relational/ssptemplates/discovery.go new file mode 100644 index 00000000..00885731 --- /dev/null +++ b/internal/service/relational/ssptemplates/discovery.go @@ -0,0 +1,296 @@ +package ssptemplates + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "errors" + "fmt" + "strings" + "time" + + "github.com/compliance-framework/api/internal/service/relational" + "github.com/compliance-framework/api/internal/service/relational/sspscope" + "github.com/compliance-framework/api/internal/service/relational/sspservice" + "github.com/compliance-framework/api/internal/service/relational/templates" + oscalTypes_1_1_3 "github.com/defenseunicorns/go-oscal/src/types/oscal-1-1-3" + "github.com/google/uuid" + "gorm.io/datatypes" + "gorm.io/gorm" +) + +const ( + // MaxNewSSPsPerRun caps the SSPs one run of a template creates; the next run creates + // the rest. + MaxNewSSPsPerRun = 100 + // maxSkippedSample is how many skipped entries an instance keeps. + maxSkippedSample = 5 +) + +// sspIDNamespace seeds the ids of the SSPs templates create, so two concurrent runs +// cannot create two SSPs for the same key values. +var sspIDNamespace = uuid.MustParse("0bc0400e-f864-41c0-aad4-2a84a628bb2e") + +// SSPID is the id of the SSP the template creates for the key values with keyHash. +func SSPID(templateID uuid.UUID, keyHash string) uuid.UUID { + return uuid.NewSHA1(sspIDNamespace, append(templateID[:], keyHash...)) +} + +// RunResult counts what one discovery run of a template did. +type RunResult struct { + Created int // SSPs created + Updated int // existing SSPs whose instance was refreshed and scope learned + Failed int // groups whose apply failed +} + +// Discovery applies SSP templates: it creates their SSPs and keeps their scopes up to +// date. Each group is applied in its own transaction holding the template's lock, so a +// periodic and a manual run do not interleave. +type Discovery struct { + db *gorm.DB + ssps *sspservice.Service + scope *sspscope.Service +} + +// NewDiscovery returns a Discovery attaching profiles with resolveControls. +func NewDiscovery(db *gorm.DB, resolveControls sspservice.ControlResolver) *Discovery { + return &Discovery{db: db, ssps: sspservice.NewService(resolveControls), scope: sspscope.NewService(db)} +} + +// RunAll runs every enabled template over one load of the latest evidence, going on +// past a failing template. +func (d *Discovery) RunAll(ctx context.Context) error { + db := d.db.WithContext(ctx) + var ids []uuid.UUID + if err := db.Model(&relational.SSPTemplate{}).Where("enabled").Order("created_at").Pluck("id", &ids).Error; err != nil { + return err + } + if len(ids) == 0 { + return nil + } + streams, err := relational.LoadLatestEvidenceStreams(db, nil) + if err != nil { + return err + } + var errs []error + for _, id := range ids { + if _, err := d.run(db, id, streams); err != nil { + errs = append(errs, fmt.Errorf("SSP template %s: %w", id, err)) + } + } + return errors.Join(errs...) +} + +// Run applies the template to the latest evidence. A deleted or disabled template is +// skipped. A group that fails is recorded on its instance (when it has one) and +// reported in the error; the other groups are still applied. +func (d *Discovery) Run(ctx context.Context, templateID uuid.UUID) (RunResult, error) { + db := d.db.WithContext(ctx) + streams, err := relational.LoadLatestEvidenceStreams(db, nil) + if err != nil { + return RunResult{}, err + } + return d.run(db, templateID, streams) +} + +func (d *Discovery) run(db *gorm.DB, templateID uuid.UUID, streams []relational.LatestEvidenceStream) (RunResult, error) { + tmpl, err := get(db, templateID) + if errors.Is(err, ErrNotFound) { + return RunResult{}, nil + } + if err != nil { + return RunResult{}, err + } + if !tmpl.Enabled { + return RunResult{}, nil + } + var instances []relational.SSPTemplateInstance + if err := db.Where("template_id = ?", templateID).Find(&instances).Error; err != nil { + return RunResult{}, err + } + entries, exceptions, err := scopeHashes(db, instances) + if err != nil { + return RunResult{}, err + } + + result := RunResult{} + var errs []error + for _, planned := range BuildPlan(tmpl.SSPTemplate, streams, instances, entries, exceptions).SSPs { + if planned.Exists && planned.SSPID == nil { + continue // the SSP was deleted: never recreate it + } + if !planned.Exists && result.Created >= MaxNewSSPsPerRun { + continue + } + outcome, err := d.apply(db, tmpl.SSPTemplate, planned) + if err != nil { + result.Failed++ + errs = append(errs, fmt.Errorf("key values %v: %w", planned.KeyValues, err)) + if planned.Exists { + errs = append(errs, setLastError(db, templateID, planned.KeyHash, err)) + } + continue + } + switch outcome { + case outcomeCreated: + result.Created++ + case outcomeUpdated: + result.Updated++ + } + } + if err := db.Model(&relational.SSPTemplate{}).Where("id = ?", templateID).UpdateColumn("last_run_at", time.Now()).Error; err != nil { + errs = append(errs, err) + } + return result, errors.Join(errs...) +} + +// outcome is what apply did with a group. +type outcome int + +const ( + outcomeSkipped outcome = iota // the SSP was deleted since the plan was made + outcomeCreated + outcomeUpdated +) + +// apply creates or refreshes one group's SSP and learns its scope, then attaches the +// template's profiles to a new SSP (or retries those that failed). +func (d *Discovery) apply(db *gorm.DB, tmpl relational.SSPTemplate, planned PlannedSSP) (outcome, error) { + var instance relational.SSPTemplateInstance + result := outcomeUpdated + err := db.Transaction(func(tx *gorm.DB) error { + if err := lockTemplate(tx, *tmpl.ID); err != nil { + return err + } + found := tx.Where("template_id = ? AND key_hash = ?", tmpl.ID, planned.KeyHash).Limit(1).Find(&instance) + if found.Error != nil { + return found.Error + } + now := time.Now() + switch { + case found.RowsAffected == 0: + ssp, err := newSSP(tmpl, planned) + if err != nil { + return err + } + stored, err := d.ssps.Create(tx, ssp) + if err != nil { + return err + } + instance = relational.SSPTemplateInstance{ + TemplateID: *tmpl.ID, KeyHash: planned.KeyHash, KeyValues: keyValuesMap(planned.KeyValues), + SSPID: stored.ID, CreatedAt: now, ProfilesPending: len(tmpl.ProfileIDs) > 0, + } + if err := tx.Create(&instance).Error; err != nil { + return err + } + result = outcomeCreated + case instance.SSPID == nil: + result = outcomeSkipped + return nil + } + + skipped := append([]relational.SkippedScopeEntry{}, planned.InvalidEntries...) + if len(planned.Learn) > 0 { + learned, err := d.scope.Learn(tx, *instance.SSPID, *tmpl.ID, planned.Learn) + if err != nil { + return err + } + for _, entry := range learned.Skipped { + skipped = append(skipped, relational.SkippedScopeEntry{Labels: entry.Labels, Reason: entry.Reason}) + } + } + instance.LastSeenAt = now + instance.SkippedEntries = len(skipped) + instance.SkippedSample = skipped[:min(len(skipped), maxSkippedSample)] + updates := map[string]any{"last_seen_at": now, "skipped_entries": instance.SkippedEntries, "skipped_sample": instance.SkippedSample} + if !instance.ProfilesPending { + updates["last_error"] = nil + } + return tx.Model(&instance).Updates(updates).Error + }) + if err != nil || result == outcomeSkipped || !instance.ProfilesPending { + return result, err + } + return result, d.attachProfiles(db, tmpl, instance) +} + +// attachProfiles attaches the template's profiles to the instance's SSP, skipping those +// already bound. A failure does not undo the SSP: it stays in LastError and +// ProfilesPending, and the next run retries. +func (d *Discovery) attachProfiles(db *gorm.DB, tmpl relational.SSPTemplate, instance relational.SSPTemplateInstance) error { + var failures []string + for _, profileID := range tmpl.ProfileIDs { + err := d.ssps.AttachProfile(db, *instance.SSPID, profileID) + if err != nil && !errors.Is(err, sspservice.ErrProfileAlreadyBound) { + failures = append(failures, fmt.Sprintf("attach profile %s: %v", profileID, err)) + } + } + var lastError *string + if len(failures) > 0 { + message := strings.Join(failures, "; ") + lastError = &message + } + return db.Model(&instance).Updates(map[string]any{"profiles_pending": len(failures) > 0, "last_error": lastError}).Error +} + +// newSSP builds the SSP for a group: title, description and props rendered from its key +// values, a prop naming the template, status under-development and nothing else, as the +// UI's "new SSP" form leaves it. +func newSSP(tmpl relational.SSPTemplate, planned PlannedSSP) (oscalTypes_1_1_3.SystemSecurityPlan, error) { + description := planned.Title + if tmpl.DescriptionTemplate != nil { + rendered, err := templates.RenderTemplate(*tmpl.DescriptionTemplate, planned.KeyValues) + if err != nil { + return oscalTypes_1_1_3.SystemSecurityPlan{}, err + } + if strings.TrimSpace(rendered) != "" { + description = rendered + } + } + props := make([]oscalTypes_1_1_3.Property, 0, len(tmpl.Props)+1) + for _, prop := range tmpl.Props { + value, err := templates.RenderTemplate(prop.Value, planned.KeyValues) + if err != nil { + return oscalTypes_1_1_3.SystemSecurityPlan{}, err + } + if value != "" { // OSCAL props need a value + props = append(props, oscalTypes_1_1_3.Property{Name: prop.Name, Value: value}) + } + } + props = append(props, oscalTypes_1_1_3.Property{Name: "ssp-template", Ns: relational.CCFOSCALNamespace, Value: tmpl.ID.String()}) + return oscalTypes_1_1_3.SystemSecurityPlan{ + UUID: SSPID(*tmpl.ID, planned.KeyHash).String(), + Metadata: oscalTypes_1_1_3.Metadata{Title: planned.Title, Version: "1.0", Props: &props}, + SystemCharacteristics: oscalTypes_1_1_3.SystemCharacteristics{ + SystemName: planned.Title, + Description: description, + Status: oscalTypes_1_1_3.Status{State: "under-development"}, + }, + }, nil +} + +func setLastError(db *gorm.DB, templateID uuid.UUID, keyHash string, cause error) error { + message := cause.Error() + return db.Model(&relational.SSPTemplateInstance{}).Where("template_id = ? AND key_hash = ?", templateID, keyHash). + Update("last_error", &message).Error +} + +func keyValuesMap(keyValues map[string]string) datatypes.JSONMap { + out := make(datatypes.JSONMap, len(keyValues)) + for key, value := range keyValues { + out[key] = value + } + return out +} + +// lockTemplate takes the template's transaction-scoped advisory lock (postgres only). +func lockTemplate(tx *gorm.DB, templateID uuid.UUID) error { + if tx.Name() != "postgres" { + return nil + } + sum := sha256.Sum256([]byte("ssp-template:" + templateID.String())) + key1 := int32(binary.BigEndian.Uint32(sum[0:4])) + key2 := int32(binary.BigEndian.Uint32(sum[4:8])) + return tx.Exec("SELECT pg_advisory_xact_lock(?, ?)", key1, key2).Error +} diff --git a/internal/service/relational/ssptemplates/discovery_integration_test.go b/internal/service/relational/ssptemplates/discovery_integration_test.go new file mode 100644 index 00000000..cf3f59d4 --- /dev/null +++ b/internal/service/relational/ssptemplates/discovery_integration_test.go @@ -0,0 +1,210 @@ +//go:build integration + +package ssptemplates_test + +import ( + "context" + "fmt" + "sort" + "strings" + "testing" + "time" + + "github.com/compliance-framework/api/internal/service/relational" + "github.com/compliance-framework/api/internal/service/relational/sspscope" + "github.com/compliance-framework/api/internal/service/relational/ssptemplates" + "github.com/compliance-framework/api/internal/tests" + "github.com/google/uuid" + "github.com/stretchr/testify/suite" +) + +type DiscoveryIntegrationSuite struct { + tests.IntegrationTestSuite + controls []string // what the stub resolver returns for any profile + discovery *ssptemplates.Discovery + templates *ssptemplates.Service +} + +func TestDiscoveryIntegration(t *testing.T) { + suite.Run(t, new(DiscoveryIntegrationSuite)) +} + +func (s *DiscoveryIntegrationSuite) SetupTest() { + s.Require().NoError(s.Migrator.Refresh()) + s.controls = []string{"AC-1", "AC-2"} + s.discovery = ssptemplates.NewDiscovery(s.DB, func(uuid.UUID) ([]string, error) { return s.controls, nil }) + s.templates = ssptemplates.NewService(s.DB) +} + +func (s *DiscoveryIntegrationSuite) evidence(pairs ...string) { + id := uuid.New() + labels := []relational.Labels{} + for i := 0; i+1 < len(pairs); i += 2 { + labels = append(labels, relational.Labels{Name: pairs[i], Value: pairs[i+1]}) + } + s.Require().NoError(s.DB.Create(&relational.Evidence{ + UUIDModel: relational.UUIDModel{ID: &id}, UUID: uuid.New(), Title: "e", + Start: time.Now().Add(-time.Minute), End: time.Now(), Labels: labels, + }).Error) +} + +func (s *DiscoveryIntegrationSuite) template(edit func(*ssptemplates.Input)) uuid.UUID { + description := "Created from _app={{ ._app }}" + in := ssptemplates.Input{ + Name: "Apps", Keys: []string{"_app"}, ScopeLabels: [][]string{{"repo"}}, + TitleTemplate: "SSP for {{ ._app }}", DescriptionTemplate: &description, + Props: []relational.SSPTemplateProp{{Name: "application", Value: "{{ ._app }}"}}, + } + if edit != nil { + edit(&in) + } + tmpl, err := s.templates.Create(in) + s.Require().NoError(err) + return *tmpl.ID +} + +func (s *DiscoveryIntegrationSuite) run(templateID uuid.UUID) ssptemplates.RunResult { + result, err := s.discovery.Run(context.Background(), templateID) + s.Require().NoError(err) + return result +} + +func (s *DiscoveryIntegrationSuite) instance(templateID uuid.UUID) relational.SSPTemplateInstance { + var instance relational.SSPTemplateInstance + s.Require().NoError(s.DB.Where("template_id = ?", templateID).First(&instance).Error) + return instance +} + +func (s *DiscoveryIntegrationSuite) entries(sspID uuid.UUID) []string { + var rows []relational.SSPScopeEntry + s.Require().NoError(s.DB.Where("ssp_id = ?", sspID).Order("match_hash").Find(&rows).Error) + out := []string{} + for _, row := range rows { + s.Equal(relational.ScopeEntryOriginTemplate, row.Origin) + s.Require().NotNil(row.TemplateID) + out = append(out, fmt.Sprint(row.Labels)) + } + sort.Strings(out) + return out +} + +func (s *DiscoveryIntegrationSuite) TestCreatesSSPWithProfile() { + profileID := uuid.New() + s.Require().NoError(s.DB.Create(&relational.Profile{UUIDModel: relational.UUIDModel{ID: &profileID}}).Error) + templateID := s.template(func(in *ssptemplates.Input) { in.ProfileIDs = []uuid.UUID{profileID} }) + s.evidence("_app", "Payments", "repo", "A") + + s.Equal(ssptemplates.RunResult{Created: 1}, s.run(templateID)) + + instance := s.instance(templateID) + s.Require().NotNil(instance.SSPID) + s.Equal(ssptemplates.SSPID(templateID, instance.KeyHash), *instance.SSPID, "deterministic id") + s.False(instance.ProfilesPending) + s.Nil(instance.LastError) + var ssp relational.SystemSecurityPlan + s.Require().NoError(s.DB.Preload("Metadata").Preload("SystemCharacteristics").Preload("Profiles"). + Preload("ControlImplementation.ImplementedRequirements").First(&ssp, "id = ?", instance.SSPID).Error) + oscal := ssp.MarshalOscal() + s.Equal("SSP for Payments", oscal.Metadata.Title) + s.Equal("1.0", oscal.Metadata.Version) + s.Equal("SSP for Payments", oscal.SystemCharacteristics.SystemName) + s.Equal("Created from _app=Payments", oscal.SystemCharacteristics.Description) + s.Equal("under-development", oscal.SystemCharacteristics.Status.State) + s.Require().NotNil(oscal.Metadata.Props) + props := map[string]string{} + for _, prop := range *oscal.Metadata.Props { + props[prop.Name] = prop.Value + } + s.Equal(map[string]string{"application": "Payments", "ssp-template": templateID.String()}, props) + s.Require().Len(ssp.Profiles, 1) + s.Len(ssp.ControlImplementation.ImplementedRequirements, 2) + s.Equal([]string{"[{_app Payments}]", "[{repo A}]"}, s.entries(*instance.SSPID)) + + var tmpl relational.SSPTemplate + s.Require().NoError(s.DB.First(&tmpl, "id = ?", templateID).Error) + s.NotNil(tmpl.LastRunAt) +} + +func (s *DiscoveryIntegrationSuite) TestIdempotentAndLearnsLater() { + templateID := s.template(nil) + s.evidence("_app", "payments", "repo", "a") + s.Equal(ssptemplates.RunResult{Created: 1}, s.run(templateID)) + sspID := *s.instance(templateID).SSPID + firstSeen := s.instance(templateID).LastSeenAt + + s.Equal(ssptemplates.RunResult{Updated: 1}, s.run(templateID)) + var ssps int64 + s.Require().NoError(s.DB.Model(&relational.SystemSecurityPlan{}).Count(&ssps).Error) + s.Equal(int64(1), ssps) + s.Len(s.entries(sspID), 2) + s.True(s.instance(templateID).LastSeenAt.After(firstSeen)) + + // A new value is learned; an excepted one is not. + s.Require().NoError(s.DB.Create(&relational.SSPScopeException{ + SSPID: sspID, Labels: []relational.ScopeLabel{{Key: "repo", Value: "x"}}, + MatchHash: sspscope.MatchHash([]relational.ScopeLabel{{Key: "repo", Value: "x"}}), + }).Error) + s.evidence("_app", "PAYMENTS", "repo", "b") + s.evidence("_app", "payments", "repo", "x") + s.run(templateID) + s.Equal([]string{"[{_app payments}]", "[{repo a}]", "[{repo b}]"}, s.entries(sspID)) + s.Require().NoError(s.DB.Model(&relational.SystemSecurityPlan{}).Count(&ssps).Error) + s.Equal(int64(1), ssps, "PAYMENTS and payments are one group") +} + +func (s *DiscoveryIntegrationSuite) TestNeverRecreatesDeletedSSP() { + templateID := s.template(nil) + s.evidence("_app", "payments") + s.run(templateID) + s.Require().NoError(s.DB.Delete(&relational.SystemSecurityPlan{}, "id = ?", s.instance(templateID).SSPID).Error) + + s.Equal(ssptemplates.RunResult{}, s.run(templateID)) + s.Nil(s.instance(templateID).SSPID) + var ssps int64 + s.Require().NoError(s.DB.Model(&relational.SystemSecurityPlan{}).Count(&ssps).Error) + s.Zero(ssps) +} + +func (s *DiscoveryIntegrationSuite) TestCapsNewSSPsPerRun() { + templateID := s.template(nil) + for i := 0; i <= ssptemplates.MaxNewSSPsPerRun; i++ { + s.evidence("_app", fmt.Sprintf("app-%03d", i)) + } + s.Equal(ssptemplates.RunResult{Created: ssptemplates.MaxNewSSPsPerRun}, s.run(templateID)) + s.Equal(ssptemplates.RunResult{Created: 1, Updated: ssptemplates.MaxNewSSPsPerRun}, s.run(templateID)) +} + +func (s *DiscoveryIntegrationSuite) TestRetriesFailedProfileAttach() { + profileID := uuid.New() + s.Require().NoError(s.DB.Create(&relational.Profile{UUIDModel: relational.UUIDModel{ID: &profileID}}).Error) + templateID := s.template(func(in *ssptemplates.Input) { in.ProfileIDs = []uuid.UUID{profileID} }) + s.evidence("_app", "payments") + s.controls = nil + + s.Equal(ssptemplates.RunResult{Created: 1}, s.run(templateID), "a failed attach does not undo the SSP") + instance := s.instance(templateID) + s.True(instance.ProfilesPending) + s.Require().NotNil(instance.LastError) + s.Contains(*instance.LastError, "no controls were resolved") + + s.controls = []string{"AC-1"} + s.run(templateID) + instance = s.instance(templateID) + s.False(instance.ProfilesPending) + s.Nil(instance.LastError) +} + +func (s *DiscoveryIntegrationSuite) TestSkipsDisabledAndRecordsInvalidEntries() { + disabled := false + off := s.template(func(in *ssptemplates.Input) { in.Enabled = &disabled }) + on := s.template(nil) + s.evidence("_app", "payments", "repo", strings.Repeat("x", 300)) + + s.Require().NoError(s.discovery.RunAll(context.Background())) + var count int64 + s.Require().NoError(s.DB.Model(&relational.SSPTemplateInstance{}).Where("template_id = ?", off).Count(&count).Error) + s.Zero(count, "a disabled template does nothing") + instance := s.instance(on) + s.Equal(1, instance.SkippedEntries) + s.Len(instance.SkippedSample, 1) +} diff --git a/internal/service/relational/ssptemplates/plan.go b/internal/service/relational/ssptemplates/plan.go index 865febb9..08ea90d3 100644 --- a/internal/service/relational/ssptemplates/plan.go +++ b/internal/service/relational/ssptemplates/plan.go @@ -55,10 +55,7 @@ type InvalidGroup struct { } // InvalidEntry is a learned entry a scope cannot hold. -type InvalidEntry struct { - Labels []relational.ScopeLabel `json:"labels"` - Reason string `json:"reason"` -} +type InvalidEntry = relational.SkippedScopeEntry // HashesBySSP holds, per SSP, the MatchHashes of its scope entries or exceptions. type HashesBySSP map[uuid.UUID]map[string]struct{} diff --git a/internal/service/worker/service.go b/internal/service/worker/service.go index 9d3c3054..33696bdf 100644 --- a/internal/service/worker/service.go +++ b/internal/service/worker/service.go @@ -348,12 +348,15 @@ func NewServiceWithDigest( river.AddWorker(workers, river.WorkFunc(evidenceBatchPruneWorker.Work)) sspScopeAutoFiltersWorker := NewSSPScopeAutoFiltersWorker(db, logger) river.AddWorker(workers, river.WorkFunc(sspScopeAutoFiltersWorker.Work)) + sspTemplateDiscoveryWorker := NewSSPTemplateDiscoveryWorker(db, profileResolver, logger) + river.AddWorker(workers, river.WorkFunc(sspTemplateDiscoveryWorker.Work)) // Configure periodic jobs. Evidence batches are always pruned: nothing else bounds // their tables. periodicJobs := periodicJobsFromConfig(digestCfg, logger) periodicJobs = append(periodicJobs, NewEvidenceBatchPrunePeriodicJob(evidenceBatchPruneSchedule, logger)) periodicJobs = append(periodicJobs, NewSSPScopeAutoFiltersPeriodicJob(sspScopeAutoFiltersSchedule(digestCfg), logger)) + periodicJobs = append(periodicJobs, NewSSPTemplateDiscoveryPeriodicJob(sspTemplateDiscoverySchedule(digestCfg), logger)) // Create River client with pgxv5 driver riverConfig := buildRiverConfig(cfg, workers, periodicJobs, aiConfigFromConfig(digestCfg), evidenceBatchConfigFromConfig(digestCfg)) diff --git a/internal/service/worker/ssp_template_discovery.go b/internal/service/worker/ssp_template_discovery.go new file mode 100644 index 00000000..6e21f397 --- /dev/null +++ b/internal/service/worker/ssp_template_discovery.go @@ -0,0 +1,110 @@ +package worker + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/compliance-framework/api/internal/config" + "github.com/compliance-framework/api/internal/service/relational/sspservice" + "github.com/compliance-framework/api/internal/service/relational/ssptemplates" + "github.com/google/uuid" + "github.com/riverqueue/river" + "go.uber.org/zap" + "gorm.io/gorm" +) + +// JobTypeSSPTemplateDiscovery applies SSP templates to the latest evidence. +const JobTypeSSPTemplateDiscovery = "ssp_template_discovery" + +// SSPTemplateDiscoveryArgs runs one template when TemplateID is set (a manual run), or +// every enabled template (the periodic run). +type SSPTemplateDiscoveryArgs struct { + TemplateID *uuid.UUID `json:"template_id,omitempty"` +} + +// Kind implements river.JobArgs. +func (SSPTemplateDiscoveryArgs) Kind() string { return JobTypeSSPTemplateDiscovery } + +// SSPTemplateDiscoveryWorker creates the SSPs of SSP templates and learns their scopes. +type SSPTemplateDiscoveryWorker struct { + db *gorm.DB + resolver ProfileControlResolver + logger *zap.SugaredLogger +} + +// NewSSPTemplateDiscoveryWorker builds the worker; resolver resolves the controls of the +// profiles attached to created SSPs. +func NewSSPTemplateDiscoveryWorker(db *gorm.DB, resolver ProfileControlResolver, logger *zap.SugaredLogger) *SSPTemplateDiscoveryWorker { + if logger == nil { + logger = zap.NewNop().Sugar() + } + return &SSPTemplateDiscoveryWorker{db: db, resolver: resolver, logger: logger} +} + +// Work implements the River worker function. +func (w *SSPTemplateDiscoveryWorker) Work(ctx context.Context, job *river.Job[SSPTemplateDiscoveryArgs]) error { + discovery := ssptemplates.NewDiscovery(w.db, controlIDResolver(ctx, w.resolver)) + if job.Args.TemplateID == nil { + if err := discovery.RunAll(ctx); err != nil { + return fmt.Errorf("ssp template discovery: %w", err) + } + return nil + } + result, err := discovery.Run(ctx, *job.Args.TemplateID) + w.logger.Infow("SSP template discovery run", "templateId", *job.Args.TemplateID, + "created", result.Created, "updated", result.Updated, "failed", result.Failed) + if err != nil { + return fmt.Errorf("ssp template discovery %s: %w", *job.Args.TemplateID, err) + } + return nil +} + +// controlIDResolver adapts the worker's profile resolver to the SSP service's: control +// ids de-duplicated case-insensitively, keeping the first casing seen. +func controlIDResolver(ctx context.Context, resolver ProfileControlResolver) sspservice.ControlResolver { + return func(profileID uuid.UUID) ([]string, error) { + keys, err := resolver.ResolveProfileControlKeys(ctx, profileID) + if err != nil { + return nil, err + } + seen := make(map[string]struct{}, len(keys)) + ids := make([]string, 0, len(keys)) + for _, key := range keys { + folded := strings.ToLower(key.ControlID) + if _, dup := seen[folded]; dup || key.ControlID == "" { + continue + } + seen[folded] = struct{}{} + ids = append(ids, key.ControlID) + } + return ids, nil + } +} + +// NewSSPTemplateDiscoveryPeriodicJob schedules discovery of every enabled template on the +// "scheduler" queue. +func NewSSPTemplateDiscoveryPeriodicJob(schedule string, logger *zap.SugaredLogger) *river.PeriodicJob { + sched := parseCronScheduleWithFallback(schedule, config.DefaultSSPTemplateDiscoverySchedule, "ssp template discovery", logger) + return river.NewPeriodicJob( + sched, + func() (river.JobArgs, *river.InsertOpts) { + return &SSPTemplateDiscoveryArgs{}, &river.InsertOpts{ + Queue: "scheduler", + MaxAttempts: 3, + // Deduplicated per hour: a CCF_SSP_TEMPLATE_DISCOVERY_SCHEDULE more frequent + // than hourly is not honored. + UniqueOpts: river.UniqueOpts{ByArgs: true, ByPeriod: time.Hour}, + } + }, + &river.PeriodicJobOpts{RunOnStart: false}, + ) +} + +func sspTemplateDiscoverySchedule(cfg *config.Config) string { + if cfg == nil || cfg.SSPTemplate == nil { + return config.DefaultSSPTemplateDiscoverySchedule + } + return cfg.SSPTemplate.DiscoverySchedule +} diff --git a/internal/service/worker/ssp_template_discovery_test.go b/internal/service/worker/ssp_template_discovery_test.go new file mode 100644 index 00000000..28b3d30f --- /dev/null +++ b/internal/service/worker/ssp_template_discovery_test.go @@ -0,0 +1,35 @@ +package worker + +import ( + "context" + "testing" + + "github.com/compliance-framework/api/internal/config" + riskrel "github.com/compliance-framework/api/internal/service/relational/risks" + "github.com/google/uuid" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +type staticControlResolver []riskrel.ControlKey + +func (r staticControlResolver) ResolveProfileControlKeys(context.Context, uuid.UUID) ([]riskrel.ControlKey, error) { + return r, nil +} + +func TestSSPTemplateDiscoverySchedule(t *testing.T) { + require.Equal(t, config.DefaultSSPTemplateDiscoverySchedule, sspTemplateDiscoverySchedule(nil)) + require.Equal(t, config.DefaultSSPTemplateDiscoverySchedule, sspTemplateDiscoverySchedule(&config.Config{})) + require.Equal(t, "0 0 * * * *", sspTemplateDiscoverySchedule(&config.Config{SSPTemplate: &config.SSPTemplateConfig{DiscoverySchedule: "0 0 * * * *"}})) + // An invalid schedule falls back to the default instead of failing start-up. + require.NotNil(t, NewSSPTemplateDiscoveryPeriodicJob("not a cron", zap.NewNop().Sugar())) +} + +func TestControlIDResolverDedupesCaseInsensitively(t *testing.T) { + resolve := controlIDResolver(context.Background(), staticControlResolver{ + {CatalogID: "a", ControlID: "AC-1"}, {CatalogID: "b", ControlID: "ac-1"}, {CatalogID: "a", ControlID: ""}, {CatalogID: "a", ControlID: "AC-2"}, + }) + ids, err := resolve(uuid.New()) + require.NoError(t, err) + require.Equal(t, []string{"AC-1", "AC-2"}, ids) +}