feat(ssp-templates): discovery worker and periodic job

.env.example

@@ -92,6 +92,13 @@ CCF_EVIDENCE_BATCH_PRUNE_SCHEDULE="0 47 * * * *"
 # every SSP with automatic filters on; it runs at most hourly.
 CCF_SSP_SCOPE_AUTO_FILTERS_SCHEDULE="0 23 * * * *"
 
+# River (6-field) cron of SSP template discovery, which creates one SSP per distinct key
+# value in the latest evidence and learns their scopes; it runs at most hourly.
+CCF_SSP_TEMPLATE_DISCOVERY_SCHEDULE="0 37 * * * *"
+# An SSP a template created is reported as stale after this long without evidence for its
+# key values (a Go duration). Nothing is deleted.
+CCF_SSP_TEMPLATE_STALE_AFTER=168h
+
 # Compliance Map (/api/lineage): built datasets are shared across users and reused for the
 # TTL; ?refresh=true rebuilds one. 0 disables the cache.
 CCF_LINEAGE_CACHE_TTL=5m

internal/config/config.go

@@ -53,6 +53,7 @@ type Config struct {
 	EvidenceBatch                     *EvidenceBatchConfig
 	Lineage                           *LineageConfig
 	SSPScope                          *SSPScopeConfig
+	SSPTemplate                       *SSPTemplateConfig
 }
 
 func NewConfig(logger *zap.SugaredLogger) *Config {
@@ -197,6 +198,11 @@ func NewConfig(logger *zap.SugaredLogger) *Config {
 		logger.Fatalw("Invalid lineage config", "error", err)
 	}
 
+	sspTemplateConfig, err := LoadSSPTemplateConfig()
+	if err != nil {
+		logger.Fatalw("Invalid SSP template config", "error", err)
+	}
+
 	// Digest configuration
 	digestEnabled := viper.GetBool("digest_enabled")
 	digestSchedule := viper.GetString("digest_schedule")
@@ -289,6 +295,7 @@ func NewConfig(logger *zap.SugaredLogger) *Config {
 		EvidenceSubjects:                  evidenceSubjectConfig,
 		EvidenceBatch:                     evidenceBatchConfig,
 		SSPScope:                          LoadSSPScopeConfig(),
+		SSPTemplate:                       sspTemplateConfig,
 		Lineage:                           lineageConfig,
 		DigestEnabled:                     digestEnabled,
 		DigestSchedule:                    digestSchedule,

internal/config/ssp_template.go

@@ -0,0 +1,48 @@
+package config
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/spf13/viper"
+)
+
+const (
+	// DefaultSSPTemplateDiscoverySchedule runs SSP template discovery hourly (River
+	// 6-field cron).
+	DefaultSSPTemplateDiscoverySchedule = "0 37 * * * *"
+	// DefaultSSPTemplateStaleAfter is how long an SSP a template created can go without
+	// evidence for its key values before it is reported as stale.
+	DefaultSSPTemplateStaleAfter = 7 * 24 * time.Hour
+)
+
+// SSPTemplateConfig configures SSP template discovery.
+type SSPTemplateConfig struct {
+	// DiscoverySchedule is the River cron of the discovery job
+	// (CCF_SSP_TEMPLATE_DISCOVERY_SCHEDULE).
+	DiscoverySchedule string
+	// StaleAfter is how long a created SSP can go without evidence before it is
+	// reported as stale (CCF_SSP_TEMPLATE_STALE_AFTER, a Go duration).
+	StaleAfter time.Duration
+}
+
+func LoadSSPTemplateConfig() (*SSPTemplateConfig, error) {
+	return LoadSSPTemplateConfigFromViper(viper.GetViper())
+}
+
+func LoadSSPTemplateConfigFromViper(v *viper.Viper) (*SSPTemplateConfig, error) {
+	cfg := &SSPTemplateConfig{DiscoverySchedule: DefaultSSPTemplateDiscoverySchedule, StaleAfter: DefaultSSPTemplateStaleAfter}
+	if schedule := strings.TrimSpace(v.GetString("ssp_template_discovery_schedule")); schedule != "" {
+		cfg.DiscoverySchedule = schedule
+	}
+	if v.IsSet("ssp_template_stale_after") {
+		raw := strings.TrimSpace(v.GetString("ssp_template_stale_after"))
+		staleAfter, err := time.ParseDuration(raw)
+		if err != nil || staleAfter <= 0 {
+			return nil, fmt.Errorf("CCF_SSP_TEMPLATE_STALE_AFTER must be a positive duration such as 168h, got %q", raw)
+		}
+		cfg.StaleAfter = staleAfter
+	}
+	return cfg, nil
+}

internal/config/ssp_template_test.go

@@ -0,0 +1,28 @@
+package config
+
+import (
+	"testing"
+	"time"
+
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLoadSSPTemplateConfig(t *testing.T) {
+	v := viper.New()
+	cfg, err := LoadSSPTemplateConfigFromViper(v)
+	require.NoError(t, err)
+	require.Equal(t, &SSPTemplateConfig{DiscoverySchedule: DefaultSSPTemplateDiscoverySchedule, StaleAfter: DefaultSSPTemplateStaleAfter}, cfg)
+
+	v.Set("ssp_template_discovery_schedule", " 0 0 * * * * ")
+	v.Set("ssp_template_stale_after", "24h")
+	cfg, err = LoadSSPTemplateConfigFromViper(v)
+	require.NoError(t, err)
+	require.Equal(t, &SSPTemplateConfig{DiscoverySchedule: "0 0 * * * *", StaleAfter: 24 * time.Hour}, cfg)
+
+	for _, bad := range []string{"soon", "0", "-1h"} {
+		v.Set("ssp_template_stale_after", bad)
+		_, err = LoadSSPTemplateConfigFromViper(v)
+		require.Error(t, err, bad)
+	}
+}

internal/service/relational/ssp_templates.go

@@ -39,6 +39,12 @@ type SSPTemplate struct {
 
 func (SSPTemplate) TableName() string { return "ssp_templates" }
 
+// SkippedScopeEntry is a learned scope entry discovery could not store, with the reason.
+type SkippedScopeEntry struct {
+	Labels []ScopeLabel `json:"labels"`
+	Reason string       `json:"reason"`
+}
+
 // SSPTemplateInstance records an SSP a template created, keyed by its key values.
 type SSPTemplateInstance struct {
 	TemplateID uuid.UUID `gorm:"type:uuid;primaryKey"`
@@ -52,6 +58,13 @@ type SSPTemplateInstance struct {
 	LastSeenAt time.Time
 	// LastError is the last failure applying the instance (a profile attach, say).
 	LastError *string
+	// ProfilesPending is set while some of the template's profiles failed to attach to
+	// the SSP; discovery retries them on its next run.
+	ProfilesPending bool `gorm:"not null;default:false"`
+	// SkippedEntries counts the learned entries the last run could not store (a value
+	// over the length limit, say); SkippedSample keeps a few of them.
+	SkippedEntries int                                    `gorm:"not null;default:0"`
+	SkippedSample  datatypes.JSONSlice[SkippedScopeEntry] `gorm:"type:jsonb"`
 
 	Template *SSPTemplate `gorm:"foreignKey:TemplateID;constraint:OnDelete:CASCADE"`
 }