diff --git a/docs/docs.go b/docs/docs.go index 83965847..31b07d8f 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -2654,6 +2654,63 @@ const docTemplate = `{ ] } }, + "/admin/ssp-templates/preview": { + "post": { + "description": "Show what discovery would do now for a template, saved or not: the SSPs it would create or update, the scope entries it would learn, groups under min-streams, streams lacking a key and groups whose key values cannot be stored. Writes nothing.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "SSP Templates" + ], + "summary": "Preview an SSP template", + "parameters": [ + { + "description": "SSP template, with its id when saved", + "name": "template", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/templates.sspTemplatePreviewRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/templates.sspTemplatePreviewResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/api.Error" + } + } + }, + "security": [ + { + "OAuth2Password": [] + } + ] + } + }, "/admin/ssp-templates/{id}": { "get": { "produces": [ @@ -53261,6 +53318,129 @@ const docTemplate = `{ } } }, + "ssptemplates.GroupStreams": { + "type": "object", + "properties": { + "key-values": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "streams": { + "type": "integer" + } + } + }, + "ssptemplates.InvalidEntry": { + "type": "object", + "properties": { + "labels": { + "type": "array", + "items": { + "$ref": "#/definitions/relational.ScopeLabel" + } + }, + "reason": { + "type": "string" + } + } + }, + "ssptemplates.InvalidGroup": { + "type": "object", + "properties": { + "key-values": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "reason": { + "type": "string" + } + } + }, + "ssptemplates.Plan": { + "type": "object", + "properties": { + "below-threshold": { + "type": "array", + "items": { + "$ref": "#/definitions/ssptemplates.GroupStreams" + } + }, + "invalid": { + "type": "array", + "items": { + "$ref": "#/definitions/ssptemplates.InvalidGroup" + } + }, + "missing-keys": { + "description": "streams that matched the filter but lacked a usable key value", + "type": "integer" + }, + "ssps": { + "type": "array", + "items": { + "$ref": "#/definitions/ssptemplates.PlannedSSP" + } + } + } + }, + "ssptemplates.PlannedSSP": { + "type": "object", + "properties": { + "exists": { + "description": "Exists is true when an instance records this group. Its SSPID is nil when the SSP\nwas deleted: discovery then leaves the group alone.", + "type": "boolean" + }, + "invalid-entries": { + "description": "InvalidEntries are learned entries a scope cannot hold (a value over the length\nlimit, say); discovery skips them.", + "type": "array", + "items": { + "$ref": "#/definitions/ssptemplates.InvalidEntry" + } + }, + "key-hash": { + "type": "string" + }, + "key-values": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "learn": { + "description": "Learn are the scope entries discovery would add: not in the scope yet and not\nexcepted.", + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/relational.ScopeLabel" + } + } + }, + "skipped-by-exception": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/relational.ScopeLabel" + } + } + }, + "ssp-id": { + "type": "string" + }, + "streams": { + "description": "Streams is the number of streams in the group.", + "type": "integer" + }, + "title": { + "type": "string" + } + } + }, "subjects.IdentityLabel": { "type": "object", "properties": { @@ -53930,6 +54110,68 @@ const docTemplate = `{ } } }, + "templates.sspTemplatePreviewRequest": { + "type": "object", + "properties": { + "description-template": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "filter": { + "$ref": "#/definitions/labelfilter.Filter" + }, + "id": { + "description": "ID names the saved template, when there is one, so the preview knows which SSPs\nit already created.", + "type": "string" + }, + "keys": { + "type": "array", + "items": { + "type": "string" + } + }, + "min-streams": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "profile-ids": { + "type": "array", + "items": { + "type": "string" + } + }, + "props": { + "type": "array", + "items": { + "$ref": "#/definitions/relational.SSPTemplateProp" + } + }, + "scope-labels": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "title-template": { + "type": "string" + } + } + }, + "templates.sspTemplatePreviewResponse": { + "type": "object", + "properties": { + "data": { + "$ref": "#/definitions/ssptemplates.Plan" + } + } + }, "templates.sspTemplateRequest": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index 68519494..7e5b1b2b 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2648,6 +2648,63 @@ ] } }, + "/admin/ssp-templates/preview": { + "post": { + "description": "Show what discovery would do now for a template, saved or not: the SSPs it would create or update, the scope entries it would learn, groups under min-streams, streams lacking a key and groups whose key values cannot be stored. Writes nothing.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "SSP Templates" + ], + "summary": "Preview an SSP template", + "parameters": [ + { + "description": "SSP template, with its id when saved", + "name": "template", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/templates.sspTemplatePreviewRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/templates.sspTemplatePreviewResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/api.Error" + } + } + }, + "security": [ + { + "OAuth2Password": [] + } + ] + } + }, "/admin/ssp-templates/{id}": { "get": { "produces": [ @@ -53255,6 +53312,129 @@ } } }, + "ssptemplates.GroupStreams": { + "type": "object", + "properties": { + "key-values": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "streams": { + "type": "integer" + } + } + }, + "ssptemplates.InvalidEntry": { + "type": "object", + "properties": { + "labels": { + "type": "array", + "items": { + "$ref": "#/definitions/relational.ScopeLabel" + } + }, + "reason": { + "type": "string" + } + } + }, + "ssptemplates.InvalidGroup": { + "type": "object", + "properties": { + "key-values": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "reason": { + "type": "string" + } + } + }, + "ssptemplates.Plan": { + "type": "object", + "properties": { + "below-threshold": { + "type": "array", + "items": { + "$ref": "#/definitions/ssptemplates.GroupStreams" + } + }, + "invalid": { + "type": "array", + "items": { + "$ref": "#/definitions/ssptemplates.InvalidGroup" + } + }, + "missing-keys": { + "description": "streams that matched the filter but lacked a usable key value", + "type": "integer" + }, + "ssps": { + "type": "array", + "items": { + "$ref": "#/definitions/ssptemplates.PlannedSSP" + } + } + } + }, + "ssptemplates.PlannedSSP": { + "type": "object", + "properties": { + "exists": { + "description": "Exists is true when an instance records this group. Its SSPID is nil when the SSP\nwas deleted: discovery then leaves the group alone.", + "type": "boolean" + }, + "invalid-entries": { + "description": "InvalidEntries are learned entries a scope cannot hold (a value over the length\nlimit, say); discovery skips them.", + "type": "array", + "items": { + "$ref": "#/definitions/ssptemplates.InvalidEntry" + } + }, + "key-hash": { + "type": "string" + }, + "key-values": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "learn": { + "description": "Learn are the scope entries discovery would add: not in the scope yet and not\nexcepted.", + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/relational.ScopeLabel" + } + } + }, + "skipped-by-exception": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/relational.ScopeLabel" + } + } + }, + "ssp-id": { + "type": "string" + }, + "streams": { + "description": "Streams is the number of streams in the group.", + "type": "integer" + }, + "title": { + "type": "string" + } + } + }, "subjects.IdentityLabel": { "type": "object", "properties": { @@ -53924,6 +54104,68 @@ } } }, + "templates.sspTemplatePreviewRequest": { + "type": "object", + "properties": { + "description-template": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "filter": { + "$ref": "#/definitions/labelfilter.Filter" + }, + "id": { + "description": "ID names the saved template, when there is one, so the preview knows which SSPs\nit already created.", + "type": "string" + }, + "keys": { + "type": "array", + "items": { + "type": "string" + } + }, + "min-streams": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "profile-ids": { + "type": "array", + "items": { + "type": "string" + } + }, + "props": { + "type": "array", + "items": { + "$ref": "#/definitions/relational.SSPTemplateProp" + } + }, + "scope-labels": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "title-template": { + "type": "string" + } + } + }, + "templates.sspTemplatePreviewResponse": { + "type": "object", + "properties": { + "data": { + "$ref": "#/definitions/ssptemplates.Plan" + } + } + }, "templates.sspTemplateRequest": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index d41977de..6ce74b9c 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -11463,6 +11463,94 @@ definitions: totalPages: type: integer type: object + ssptemplates.GroupStreams: + properties: + key-values: + additionalProperties: + type: string + type: object + streams: + type: integer + type: object + ssptemplates.InvalidEntry: + properties: + labels: + items: + $ref: '#/definitions/relational.ScopeLabel' + type: array + reason: + type: string + type: object + ssptemplates.InvalidGroup: + properties: + key-values: + additionalProperties: + type: string + type: object + reason: + type: string + type: object + ssptemplates.Plan: + properties: + below-threshold: + items: + $ref: '#/definitions/ssptemplates.GroupStreams' + type: array + invalid: + items: + $ref: '#/definitions/ssptemplates.InvalidGroup' + type: array + missing-keys: + description: streams that matched the filter but lacked a usable key value + type: integer + ssps: + items: + $ref: '#/definitions/ssptemplates.PlannedSSP' + type: array + type: object + ssptemplates.PlannedSSP: + properties: + exists: + description: |- + Exists is true when an instance records this group. Its SSPID is nil when the SSP + was deleted: discovery then leaves the group alone. + type: boolean + invalid-entries: + description: |- + InvalidEntries are learned entries a scope cannot hold (a value over the length + limit, say); discovery skips them. + items: + $ref: '#/definitions/ssptemplates.InvalidEntry' + type: array + key-hash: + type: string + key-values: + additionalProperties: + type: string + type: object + learn: + description: |- + Learn are the scope entries discovery would add: not in the scope yet and not + excepted. + items: + items: + $ref: '#/definitions/relational.ScopeLabel' + type: array + type: array + skipped-by-exception: + items: + items: + $ref: '#/definitions/relational.ScopeLabel' + type: array + type: array + ssp-id: + type: string + streams: + description: Streams is the number of streams in the group. + type: integer + title: + type: string + type: object subjects.IdentityLabel: properties: key: @@ -11923,6 +12011,49 @@ definitions: data: $ref: '#/definitions/templates.sspTemplateResponse' type: object + templates.sspTemplatePreviewRequest: + properties: + description-template: + type: string + enabled: + type: boolean + filter: + $ref: '#/definitions/labelfilter.Filter' + id: + description: |- + ID names the saved template, when there is one, so the preview knows which SSPs + it already created. + type: string + keys: + items: + type: string + type: array + min-streams: + type: integer + name: + type: string + profile-ids: + items: + type: string + type: array + props: + items: + $ref: '#/definitions/relational.SSPTemplateProp' + type: array + scope-labels: + items: + items: + type: string + type: array + type: array + title-template: + type: string + type: object + templates.sspTemplatePreviewResponse: + properties: + data: + $ref: '#/definitions/ssptemplates.Plan' + type: object templates.sspTemplateRequest: properties: description-template: @@ -15025,6 +15156,45 @@ paths: summary: Update SSP template tags: - SSP Templates + /admin/ssp-templates/preview: + post: + consumes: + - application/json + description: 'Show what discovery would do now for a template, saved or not: + the SSPs it would create or update, the scope entries it would learn, groups + under min-streams, streams lacking a key and groups whose key values cannot + be stored. Writes nothing.' + parameters: + - description: SSP template, with its id when saved + in: body + name: template + required: true + schema: + $ref: '#/definitions/templates.sspTemplatePreviewRequest' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/templates.sspTemplatePreviewResponse' + "400": + description: Bad Request + schema: + $ref: '#/definitions/api.Error' + "404": + description: Not Found + schema: + $ref: '#/definitions/api.Error' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/api.Error' + security: + - OAuth2Password: [] + summary: Preview an SSP template + tags: + - SSP Templates /admin/subject-templates: get: description: List subject templates with optional filters and pagination. diff --git a/internal/api/handler/templates/ssp_template.go b/internal/api/handler/templates/ssp_template.go index 57cc719c..1a6e4e0c 100644 --- a/internal/api/handler/templates/ssp_template.go +++ b/internal/api/handler/templates/ssp_template.go @@ -35,6 +35,7 @@ func NewSSPTemplateHandler(sugar *zap.SugaredLogger, db *gorm.DB) *SSPTemplateHa func (h *SSPTemplateHandler) Register(group *echo.Group) { group.GET("", h.List) group.POST("", h.Create) + group.POST("/preview", h.Preview) group.GET("/:id", h.Get) group.PUT("/:id", h.Update) group.DELETE("/:id", h.Delete) @@ -233,6 +234,43 @@ func (h *SSPTemplateHandler) Delete(ctx echo.Context) error { return ctx.NoContent(http.StatusNoContent) } +type sspTemplatePreviewRequest struct { + // ID names the saved template, when there is one, so the preview knows which SSPs + // it already created. + ID *uuid.UUID `json:"id"` + sspTemplateRequest +} + +type sspTemplatePreviewResponse struct { + Data ssptemplates.Plan `json:"data"` +} + +// Preview godoc +// +// @Summary Preview an SSP template +// @Description Show what discovery would do now for a template, saved or not: the SSPs it would create or update, the scope entries it would learn, groups under min-streams, streams lacking a key and groups whose key values cannot be stored. Writes nothing. +// @Tags SSP Templates +// @Accept json +// @Produce json +// @Param template body sspTemplatePreviewRequest true "SSP template, with its id when saved" +// @Success 200 {object} sspTemplatePreviewResponse +// @Failure 400 {object} api.Error +// @Failure 404 {object} api.Error +// @Failure 500 {object} api.Error +// @Security OAuth2Password +// @Router /admin/ssp-templates/preview [post] +func (h *SSPTemplateHandler) Preview(ctx echo.Context) error { + var req sspTemplatePreviewRequest + if err := ctx.Bind(&req); err != nil { + return ctx.JSON(http.StatusBadRequest, api.NewError(err)) + } + plan, err := h.service.Preview(req.ID, ssptemplates.Input(req.sspTemplateRequest)) + if err != nil { + return h.serviceError(ctx, "failed to preview SSP template", err) + } + return ctx.JSON(http.StatusOK, sspTemplatePreviewResponse{Data: plan}) +} + func (h *SSPTemplateHandler) serviceError(ctx echo.Context, message string, err error) error { switch { case errors.Is(err, ssptemplates.ErrNotFound): diff --git a/internal/api/handler/templates/ssp_template_integration_test.go b/internal/api/handler/templates/ssp_template_integration_test.go index fd91711d..58df4e8a 100644 --- a/internal/api/handler/templates/ssp_template_integration_test.go +++ b/internal/api/handler/templates/ssp_template_integration_test.go @@ -9,6 +9,7 @@ import ( "net/http" "net/http/httptest" "testing" + "time" "github.com/compliance-framework/api/internal/api" handlerpkg "github.com/compliance-framework/api/internal/api/handler" @@ -16,6 +17,7 @@ import ( "github.com/compliance-framework/api/internal/authn" "github.com/compliance-framework/api/internal/authz" "github.com/compliance-framework/api/internal/service/relational" + "github.com/compliance-framework/api/internal/service/relational/sspscope" "github.com/compliance-framework/api/internal/tests" "github.com/google/uuid" "github.com/labstack/echo/v4" @@ -220,3 +222,75 @@ func (s *SSPTemplateApiIntegrationSuite) TestDeleteKeepsSSPs() { s.Zero(instances) s.Equal(int64(1), ssps) } + +func (s *SSPTemplateApiIntegrationSuite) evidence(pairs ...string) { + id, stream := uuid.New(), uuid.New() + labels := []relational.Labels{} + for i := 0; i+1 < len(pairs); i += 2 { + labels = append(labels, relational.Labels{Name: pairs[i], Value: pairs[i+1]}) + } + s.Require().NoError(s.DB.Create(&relational.Evidence{ + UUIDModel: relational.UUIDModel{ID: &id}, UUID: stream, Title: "e", + Start: time.Now().Add(-time.Minute), End: time.Now(), Labels: labels, + }).Error) +} + +func (s *SSPTemplateApiIntegrationSuite) TestPreview() { + s.evidence("_app", "Payments", "repo", "A") + s.evidence("_app", "Payments", "repo", "b") + s.evidence("repo", "c") + body := templateBody("Applications", "_app") + body["scope-labels"] = [][]string{{"repo"}} + saved := s.create(body) + + // A saved template's instance marks its group as existing. + sspID := uuid.New() + s.Require().NoError(s.DB.Create(&relational.SystemSecurityPlan{UUIDModel: relational.UUIDModel{ID: &sspID}}).Error) + _, keyHash, err := sspscope.Normalize([]relational.ScopeLabel{{Key: "_app", Value: "payments"}}) + s.Require().NoError(err) + s.Require().NoError(s.DB.Create(&relational.SSPTemplateInstance{ + TemplateID: uuid.MustParse(saved["id"].(string)), KeyHash: keyHash, + KeyValues: datatypes.JSONMap{"_app": "payments"}, SSPID: &sspID, + }).Error) + s.Require().NoError(s.DB.Create(&relational.SSPScopeException{ + SSPID: sspID, Labels: []relational.ScopeLabel{{Key: "repo", Value: "b"}}, MatchHash: sspscope.MatchHash([]relational.ScopeLabel{{Key: "repo", Value: "b"}}), + }).Error) + + for _, tc := range []struct { + name string + id any + exists bool + learn []any + }{ + {"unsaved", nil, false, []any{ + []any{map[string]any{"key": "_app", "value": "Payments"}}, + []any{map[string]any{"key": "repo", "value": "A"}}, + []any{map[string]any{"key": "repo", "value": "b"}}, + }}, + {"saved", saved["id"], true, []any{ + []any{map[string]any{"key": "_app", "value": "Payments"}}, + []any{map[string]any{"key": "repo", "value": "A"}}, + }}, + } { + body["id"] = tc.id + plan := s.decode(s.do(s.admin, http.MethodPost, "/preview", body), http.StatusOK)["data"].(map[string]any) + s.Equal(float64(1), plan["missing-keys"], tc.name) + ssps := plan["ssps"].([]any) + s.Require().Len(ssps, 1, tc.name) + ssp := ssps[0].(map[string]any) + s.Equal(map[string]any{"_app": "Payments"}, ssp["key-values"], tc.name) + s.Equal("SSP for Payments", ssp["title"], tc.name) + s.Equal(float64(2), ssp["streams"], tc.name) + s.Equal(tc.exists, ssp["exists"], tc.name) + s.Equal(tc.learn, ssp["learn"], tc.name) + } + + var entries int64 + s.Require().NoError(s.DB.Model(&relational.SSPScopeEntry{}).Count(&entries).Error) + s.Zero(entries, "preview writes nothing") + + body["id"] = uuid.NewString() + s.Equal(http.StatusNotFound, s.do(s.admin, http.MethodPost, "/preview", body).Code) + s.Equal(http.StatusBadRequest, s.do(s.admin, http.MethodPost, "/preview", templateBody("No keys")).Code) + s.Equal(http.StatusForbidden, s.do(s.contributor, http.MethodPost, "/preview", templateBody("Apps", "_app")).Code) +} diff --git a/internal/service/relational/evidence.go b/internal/service/relational/evidence.go index 37fd39ea..bedc2fdd 100644 --- a/internal/service/relational/evidence.go +++ b/internal/service/relational/evidence.go @@ -228,6 +228,9 @@ type LatestEvidenceStream struct { Collected time.Time Expires *time.Time Labels map[string][]string + // RawLabels holds the same labels with their values' original casing (trimmed), + // index for index with Labels, for callers that display or store values. + RawLabels map[string][]string } // LoadLatestEvidenceStreams loads the latest evidence per stream (loose index @@ -263,7 +266,7 @@ func LoadLatestEvidenceStreams(db *gorm.DB, componentID *uuid.UUID) ([]LatestEvi if s == nil { s = &LatestEvidenceStream{ ID: r.ID, UUID: r.UUID, Title: r.Title, State: r.State, Reason: r.Reason, - Collected: r.Collected, Expires: r.Expires, Labels: map[string][]string{}, + Collected: r.Collected, Expires: r.Expires, Labels: map[string][]string{}, RawLabels: map[string][]string{}, } byUUID[r.UUID] = s order = append(order, r.UUID) @@ -273,11 +276,12 @@ func LoadLatestEvidenceStreams(db *gorm.DB, componentID *uuid.UUID) ([]LatestEvi if key == "" { continue } - val := "" + raw := "" if r.Value != nil { - val = strings.ToLower(strings.TrimSpace(*r.Value)) + raw = strings.TrimSpace(*r.Value) } - s.Labels[key] = append(s.Labels[key], val) + s.Labels[key] = append(s.Labels[key], strings.ToLower(raw)) + s.RawLabels[key] = append(s.RawLabels[key], raw) } } diff --git a/internal/service/relational/ssptemplates/plan.go b/internal/service/relational/ssptemplates/plan.go new file mode 100644 index 00000000..865febb9 --- /dev/null +++ b/internal/service/relational/ssptemplates/plan.go @@ -0,0 +1,292 @@ +package ssptemplates + +import ( + "errors" + "sort" + "strings" + + "github.com/compliance-framework/api/internal/converters/labelfilter" + "github.com/compliance-framework/api/internal/service/relational" + "github.com/compliance-framework/api/internal/service/relational/sspscope" + "github.com/compliance-framework/api/internal/service/relational/templates" + "github.com/google/uuid" +) + +// Plan is what discovery would do for one template, given the latest evidence. +type Plan struct { + SSPs []PlannedSSP `json:"ssps"` + BelowThreshold []GroupStreams `json:"below-threshold"` + MissingKeys int `json:"missing-keys"` // streams that matched the filter but lacked a usable key value + Invalid []InvalidGroup `json:"invalid"` +} + +// PlannedSSP is one group of streams sharing the template's key values: an SSP to create, +// or the existing instance to update. +type PlannedSSP struct { + KeyValues map[string]string `json:"key-values"` + KeyHash string `json:"key-hash"` + Title string `json:"title"` + // Exists is true when an instance records this group. Its SSPID is nil when the SSP + // was deleted: discovery then leaves the group alone. + Exists bool `json:"exists"` + SSPID *uuid.UUID `json:"ssp-id"` + // Streams is the number of streams in the group. + Streams int `json:"streams"` + // Learn are the scope entries discovery would add: not in the scope yet and not + // excepted. + Learn [][]relational.ScopeLabel `json:"learn"` + SkippedByException [][]relational.ScopeLabel `json:"skipped-by-exception"` + // InvalidEntries are learned entries a scope cannot hold (a value over the length + // limit, say); discovery skips them. + InvalidEntries []InvalidEntry `json:"invalid-entries"` +} + +// GroupStreams is a group under the template's min-streams. +type GroupStreams struct { + KeyValues map[string]string `json:"key-values"` + Streams int `json:"streams"` +} + +// InvalidGroup is a group whose key values cannot be stored as scope labels, or whose +// title does not render or renders empty; no SSP is created for it. +type InvalidGroup struct { + KeyValues map[string]string `json:"key-values"` + Reason string `json:"reason"` +} + +// InvalidEntry is a learned entry a scope cannot hold. +type InvalidEntry struct { + Labels []relational.ScopeLabel `json:"labels"` + Reason string `json:"reason"` +} + +// HashesBySSP holds, per SSP, the MatchHashes of its scope entries or exceptions. +type HashesBySSP map[uuid.UUID]map[string]struct{} + +// group is the streams sharing one combination of key values, compared +// case-insensitively. keyValues keep the casing of the group's first stream (by UUID); +// streams are the members' labels with their original casing, in that order. +type group struct { + keyValues map[string]string + members []relational.LatestEvidenceStream + streams []map[string][]string +} + +// BuildPlan applies tmpl to the latest evidence streams. It is pure: instances are the +// template's instances, and entries and exceptions the scope hashes of their SSPs. +// The filter, the key checks and the grouping use the folded labels (matching is +// case-insensitive); titles, props, key values and learned entries keep the evidence's +// casing, from the first stream (by UUID) that carries them. +func BuildPlan(tmpl relational.SSPTemplate, streams []relational.LatestEvidenceStream, instances []relational.SSPTemplateInstance, entries, exceptions HashesBySSP) Plan { + plan := Plan{SSPs: []PlannedSSP{}, BelowThreshold: []GroupStreams{}, Invalid: []InvalidGroup{}} + var filter *labelfilter.Scope + if tmpl.Filter != nil { + filter = tmpl.Filter.Data().Scope + } + + groups := map[string]*group{} + for _, stream := range streams { + // MatchLabels only fails on an unknown query operator, which Build rejects. + if ok, err := labelfilter.MatchLabels(filter, stream.Labels); err != nil || !ok { + continue + } + keyValues, ok := keyValuesOf(tmpl.Keys, stream.Labels) + if !ok { + plan.MissingKeys++ + continue + } + id := joinPairs(keyValues) + if groups[id] == nil { + groups[id] = &group{} + } + groups[id].members = append(groups[id].members, stream) + } + for _, g := range groups { + sort.Slice(g.members, func(i, j int) bool { return g.members[i].UUID.String() < g.members[j].UUID.String() }) + for _, member := range g.members { + g.streams = append(g.streams, rawLabels(member)) + } + // The keys passed keyValuesOf on the folded labels, and raw labels align with them. + g.keyValues, _ = keyValuesOf(tmpl.Keys, g.streams[0]) + } + + byHash := make(map[string]relational.SSPTemplateInstance, len(instances)) + for _, instance := range instances { + byHash[instance.KeyHash] = instance + } + ids := make([]string, 0, len(groups)) + for id := range groups { + ids = append(ids, id) + } + sort.Strings(ids) + + for _, id := range ids { + g := groups[id] + if len(g.streams) < tmpl.MinStreams { + plan.BelowThreshold = append(plan.BelowThreshold, GroupStreams{KeyValues: g.keyValues, Streams: len(g.streams)}) + continue + } + keyEntry, keyHash, err := sspscope.Normalize(toLabels(g.keyValues)) + if err != nil { + plan.Invalid = append(plan.Invalid, InvalidGroup{KeyValues: g.keyValues, Reason: err.Error()}) + continue + } + title, err := templates.RenderTemplate(tmpl.TitleTemplate, g.keyValues) + if err == nil && strings.TrimSpace(title) == "" { + err = errors.New("title-template renders an empty title") + } + if err != nil { + plan.Invalid = append(plan.Invalid, InvalidGroup{KeyValues: g.keyValues, Reason: err.Error()}) + continue + } + planned := PlannedSSP{ + KeyValues: g.keyValues, KeyHash: keyHash, Title: title, Streams: len(g.streams), + Learn: [][]relational.ScopeLabel{}, SkippedByException: [][]relational.ScopeLabel{}, InvalidEntries: []InvalidEntry{}, + } + var known, excepted map[string]struct{} + if instance, ok := byHash[keyHash]; ok { + planned.Exists, planned.SSPID = true, instance.SSPID + if instance.SSPID != nil { + known, excepted = entries[*instance.SSPID], exceptions[*instance.SSPID] + } + } + if !planned.Exists || planned.SSPID != nil { + candidates := append([][]relational.ScopeLabel{keyEntry}, scopeLabelEntries(tmpl.ScopeLabels.Data(), g.streams, tmpl.MinStreams)...) + classify(&planned, candidates, known, excepted) + } + plan.SSPs = append(plan.SSPs, planned) + } + return plan +} + +// classify sorts candidate entries into planned's Learn, SkippedByException and +// InvalidEntries, dropping those already in the scope and repeats (a scope-label group +// holding a key repeats the key entry). +func classify(planned *PlannedSSP, candidates [][]relational.ScopeLabel, known, excepted map[string]struct{}) { + seen := map[string]struct{}{} + for _, labels := range candidates { + normalized, hash, err := sspscope.Normalize(labels) + if err == nil && contains(seen, hash) { + continue + } + seen[hash] = struct{}{} + switch { + case err != nil: + planned.InvalidEntries = append(planned.InvalidEntries, InvalidEntry{Labels: labels, Reason: err.Error()}) + case contains(excepted, hash): + planned.SkippedByException = append(planned.SkippedByException, normalized) + case !contains(known, hash): + planned.Learn = append(planned.Learn, normalized) + } + } +} + +// scopeLabelEntries returns, for each scope-label group, every combination of the +// group's label values seen on at least minStreams streams, in a stable order. +func scopeLabelEntries(scopeLabels [][]string, streams []map[string][]string, minStreams int) [][]relational.ScopeLabel { + var out [][]relational.ScopeLabel + for _, labelGroup := range scopeLabels { + counts := map[string]int{} + combos := map[string][]relational.ScopeLabel{} + for _, labels := range streams { + // A stream counts once per combination, however many times it carries it. + seen := map[string]struct{}{} + for _, combo := range combinations(labelGroup, labels) { + // Combinations compare case-insensitively; the first stream's casing is kept. + id := strings.ToLower(joinLabels(combo)) + if _, dup := seen[id]; dup { + continue + } + seen[id] = struct{}{} + counts[id]++ + if _, ok := combos[id]; !ok { + combos[id] = combo + } + } + } + ids := make([]string, 0, len(combos)) + for id, count := range counts { + if count >= minStreams { + ids = append(ids, id) + } + } + sort.Strings(ids) + for _, id := range ids { + out = append(out, combos[id]) + } + } + return out +} + +// combinations returns every combination of one non-empty value per label of +// labelGroup that labels carries (a label can have several values); none when a label is +// missing. +func combinations(labelGroup []string, labels map[string][]string) [][]relational.ScopeLabel { + combos := [][]relational.ScopeLabel{{}} + for _, key := range labelGroup { + var next [][]relational.ScopeLabel + for _, value := range labels[key] { + if value == "" { + continue + } + for _, combo := range combos { + extended := append(append([]relational.ScopeLabel{}, combo...), relational.ScopeLabel{Key: key, Value: value}) + next = append(next, extended) + } + } + combos = next + } + return combos +} + +// keyValuesOf returns the stream's value for each key; ok is false when a key is +// missing, empty or carries several values. +func keyValuesOf(keys []string, labels map[string][]string) (map[string]string, bool) { + out := make(map[string]string, len(keys)) + for _, key := range keys { + values := labels[key] + if len(values) != 1 || values[0] == "" { + return nil, false + } + out[key] = values[0] + } + return out, true +} + +// rawLabels is the stream's labels with their original value casing, or the folded +// labels when the loader did not keep it. +func rawLabels(stream relational.LatestEvidenceStream) map[string][]string { + if stream.RawLabels != nil { + return stream.RawLabels + } + return stream.Labels +} + +func toLabels(keyValues map[string]string) []relational.ScopeLabel { + labels := make([]relational.ScopeLabel, 0, len(keyValues)) + for key, value := range keyValues { + labels = append(labels, relational.ScopeLabel{Key: key, Value: value}) + } + return labels +} + +// joinPairs is a stable identity for key values (keys sorted). +func joinPairs(keyValues map[string]string) string { + labels := toLabels(keyValues) + sort.Slice(labels, func(i, j int) bool { return labels[i].Key < labels[j].Key }) + return joinLabels(labels) +} + +// joinLabels joins labels as key=value lines: the grouping identity of a combination. +func joinLabels(labels []relational.ScopeLabel) string { + parts := make([]string, len(labels)) + for i, label := range labels { + parts[i] = label.Key + "=" + label.Value + } + return strings.Join(parts, "\n") +} + +func contains(set map[string]struct{}, key string) bool { + _, ok := set[key] + return ok +} diff --git a/internal/service/relational/ssptemplates/plan_test.go b/internal/service/relational/ssptemplates/plan_test.go new file mode 100644 index 00000000..d72b0799 --- /dev/null +++ b/internal/service/relational/ssptemplates/plan_test.go @@ -0,0 +1,188 @@ +package ssptemplates + +import ( + "strings" + "testing" + + "github.com/compliance-framework/api/internal/converters/labelfilter" + "github.com/compliance-framework/api/internal/service/relational" + "github.com/compliance-framework/api/internal/service/relational/sspscope" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// stream builds a stream's labels from key=value pairs, like LoadLatestEvidenceStreams: +// Labels folded, RawLabels as given. A key may repeat. +func stream(pairs ...string) relational.LatestEvidenceStream { + labels, raw := map[string][]string{}, map[string][]string{} + for _, pair := range pairs { + key, value, _ := strings.Cut(pair, "=") + labels[key] = append(labels[key], strings.ToLower(value)) + raw[key] = append(raw[key], value) + } + return relational.LatestEvidenceStream{UUID: uuid.New(), Labels: labels, RawLabels: raw} +} + +func planTemplate(t *testing.T, edit func(*Input)) relational.SSPTemplate { + in := Input{Name: "Apps", Keys: []string{"_app"}, TitleTemplate: "SSP for {{ ._app }}"} + if edit != nil { + edit(&in) + } + tmpl, err := Build(in) + require.NoError(t, err) + return tmpl +} + +func labels(pairs ...string) []relational.ScopeLabel { + out := []relational.ScopeLabel{} + for _, pair := range pairs { + key, value, _ := strings.Cut(pair, "=") + out = append(out, relational.ScopeLabel{Key: key, Value: value}) + } + return out +} + +func hashOf(t *testing.T, pairs ...string) string { + _, hash, err := sspscope.Normalize(labels(pairs...)) + require.NoError(t, err) + return hash +} + +func TestPlanGroupsByKeyValues(t *testing.T) { + tmpl := planTemplate(t, func(in *Input) { in.Keys = []string{"_app", "env"} }) + plan := BuildPlan(tmpl, []relational.LatestEvidenceStream{ + stream("_app=payments", "env=prod"), + stream("_app=payments", "env=prod", "other=x"), + stream("_app=payments", "env=dev"), + stream("_app=payments"), // missing env + stream("_app=payments", "env="), // empty env + stream("_app=payments", "env=prod", "env=dev"), // multi-valued env + }, nil, nil, nil) + + assert.Equal(t, 3, plan.MissingKeys) + require.Len(t, plan.SSPs, 2) + dev, prod := plan.SSPs[0], plan.SSPs[1] + assert.Equal(t, map[string]string{"_app": "payments", "env": "dev"}, dev.KeyValues) + assert.Equal(t, 1, dev.Streams) + assert.Equal(t, map[string]string{"_app": "payments", "env": "prod"}, prod.KeyValues) + assert.Equal(t, 2, prod.Streams) + assert.Equal(t, "SSP for payments", prod.Title) + assert.False(t, prod.Exists) + assert.Equal(t, hashOf(t, "_app=payments", "env=prod"), prod.KeyHash) + assert.Equal(t, [][]relational.ScopeLabel{labels("_app=payments", "env=prod")}, prod.Learn, "the key entry ANDs every key") +} + +func TestPlanFilterAndMinStreams(t *testing.T) { + tmpl := planTemplate(t, func(in *Input) { + two := 2 + in.MinStreams = &two + in.Filter = &labelfilter.Filter{Scope: &labelfilter.Scope{Condition: &labelfilter.Condition{Label: "auto", Operator: "=", Value: "true"}}} + }) + plan := BuildPlan(tmpl, []relational.LatestEvidenceStream{ + stream("_app=payments", "auto=true"), + stream("_app=payments", "auto=true"), + stream("_app=test", "auto=true"), + stream("_app=ignored"), // does not match the filter + stream("auto=true"), // matches, lacks the key + }, nil, nil, nil) + + assert.Equal(t, 1, plan.MissingKeys, "only streams matching the filter count") + require.Len(t, plan.SSPs, 1) + assert.Equal(t, "payments", plan.SSPs[0].KeyValues["_app"]) + assert.Equal(t, []GroupStreams{{KeyValues: map[string]string{"_app": "test"}, Streams: 1}}, plan.BelowThreshold) +} + +func TestPlanScopeLabelGroups(t *testing.T) { + tmpl := planTemplate(t, func(in *Input) { + two := 2 + in.MinStreams = &two + in.ScopeLabels = [][]string{{"repo"}, {"region", "aws_account"}, {"_app"}} + }) + plan := BuildPlan(tmpl, []relational.LatestEvidenceStream{ + stream("_app=p", "repo=a", "repo=b", "aws_account=1", "region=eu"), + stream("_app=p", "repo=a", "aws_account=1", "region=eu", "region=us"), + stream("_app=p", "repo=c", "aws_account=1"), // no region: no combination + }, nil, nil, nil) + + require.Len(t, plan.SSPs, 1) + assert.Equal(t, [][]relational.ScopeLabel{ + labels("_app=p"), + labels("repo=a"), // b and c are on one stream each, under min-streams + labels("aws_account=1", "region=eu"), + }, plan.SSPs[0].Learn) +} + +func TestPlanExistingInstances(t *testing.T) { + tmpl := planTemplate(t, func(in *Input) { in.ScopeLabels = [][]string{{"repo"}} }) + sspID := uuid.New() + streams := []relational.LatestEvidenceStream{ + stream("_app=kept", "repo=a"), + stream("_app=kept", "repo=oops"), + stream("_app=kept", "repo=new"), + stream("_app=gone", "repo=a"), + } + instances := []relational.SSPTemplateInstance{ + {KeyHash: hashOf(t, "_app=kept"), SSPID: &sspID}, + {KeyHash: hashOf(t, "_app=gone")}, // its SSP was deleted + } + entries := HashesBySSP{sspID: {hashOf(t, "_app=kept"): {}, hashOf(t, "repo=a"): {}}} + exceptions := HashesBySSP{sspID: {hashOf(t, "repo=OOPS"): {}}} + plan := BuildPlan(tmpl, streams, instances, entries, exceptions) + + require.Len(t, plan.SSPs, 2) + gone, kept := plan.SSPs[0], plan.SSPs[1] + assert.True(t, gone.Exists) + assert.Nil(t, gone.SSPID) + assert.Empty(t, gone.Learn, "discovery leaves a deleted SSP alone") + + assert.True(t, kept.Exists) + assert.Equal(t, &sspID, kept.SSPID) + assert.Equal(t, [][]relational.ScopeLabel{labels("repo=new")}, kept.Learn) + assert.Equal(t, [][]relational.ScopeLabel{labels("repo=oops")}, kept.SkippedByException) +} + +func TestPlanInvalidValues(t *testing.T) { + tmpl := planTemplate(t, func(in *Input) { in.ScopeLabels = [][]string{{"repo"}} }) + long := strings.Repeat("x", 300) + plan := BuildPlan(tmpl, []relational.LatestEvidenceStream{ + stream("_app=" + long), + stream("_app=ok", "repo="+long), + }, nil, nil, nil) + + require.Len(t, plan.Invalid, 1) + assert.Equal(t, long, plan.Invalid[0].KeyValues["_app"]) + assert.Contains(t, plan.Invalid[0].Reason, "at most 256 characters") + require.Len(t, plan.SSPs, 1) + assert.Equal(t, [][]relational.ScopeLabel{labels("_app=ok")}, plan.SSPs[0].Learn) + require.Len(t, plan.SSPs[0].InvalidEntries, 1) + assert.Equal(t, labels("repo="+long), plan.SSPs[0].InvalidEntries[0].Labels) +} + +func TestPlanEmptyTitle(t *testing.T) { + tmpl := planTemplate(t, func(in *Input) { in.TitleTemplate = "{{ .APP }}" }) // keys are lower-cased + plan := BuildPlan(tmpl, []relational.LatestEvidenceStream{stream("_app=payments")}, nil, nil, nil) + assert.Empty(t, plan.SSPs) + require.Len(t, plan.Invalid, 1) + assert.Equal(t, "title-template renders an empty title", plan.Invalid[0].Reason) +} + +func TestPlanKeepsValueCasing(t *testing.T) { + tmpl := planTemplate(t, func(in *Input) { + two := 2 + in.MinStreams = &two + in.ScopeLabels = [][]string{{"repo"}} + }) + first, second := stream("_app=Payments-API", "repo=MyOrg/Repo"), stream("_app=payments-api", "repo=myorg/repo") + first.UUID, second.UUID = uuid.MustParse("00000000-0000-0000-0000-000000000001"), uuid.MustParse("00000000-0000-0000-0000-000000000002") + plan := BuildPlan(tmpl, []relational.LatestEvidenceStream{second, first}, nil, nil, nil) + + require.Len(t, plan.SSPs, 1, "grouping is case-insensitive") + planned := plan.SSPs[0] + assert.Equal(t, 2, planned.Streams) + assert.Equal(t, map[string]string{"_app": "Payments-API"}, planned.KeyValues, "the first stream's casing") + assert.Equal(t, "SSP for Payments-API", planned.Title) + assert.Equal(t, hashOf(t, "_app=payments-api"), planned.KeyHash) + assert.Equal(t, [][]relational.ScopeLabel{labels("_app=Payments-API"), labels("repo=MyOrg/Repo")}, planned.Learn, + "combinations count case-insensitively and keep the first stream's casing") +} diff --git a/internal/service/relational/ssptemplates/service.go b/internal/service/relational/ssptemplates/service.go index ffd153f4..c1ec1ff4 100644 --- a/internal/service/relational/ssptemplates/service.go +++ b/internal/service/relational/ssptemplates/service.go @@ -136,3 +136,61 @@ func checkProfiles(db *gorm.DB, ids []uuid.UUID) error { } return nil } + +// Preview plans the template in against the latest evidence, writing nothing. id, when +// set, names the saved template whose instances say which SSPs already exist. +func (s *Service) Preview(id *uuid.UUID, in Input) (Plan, error) { + tmpl, err := Build(in) + if err != nil { + return Plan{}, err + } + var instances []relational.SSPTemplateInstance + if id != nil { + if _, err := get(s.db, *id); err != nil { + return Plan{}, err + } + if err := s.db.Where("template_id = ?", *id).Find(&instances).Error; err != nil { + return Plan{}, err + } + } + streams, err := relational.LoadLatestEvidenceStreams(s.db, nil) + if err != nil { + return Plan{}, err + } + entries, exceptions, err := scopeHashes(s.db, instances) + if err != nil { + return Plan{}, err + } + return BuildPlan(tmpl, streams, instances, entries, exceptions), nil +} + +// scopeHashes loads the MatchHashes of the scope entries and exceptions of the +// instances' SSPs. +func scopeHashes(db *gorm.DB, instances []relational.SSPTemplateInstance) (entries, exceptions HashesBySSP, err error) { + sspIDs := make([]uuid.UUID, 0, len(instances)) + for _, instance := range instances { + if instance.SSPID != nil { + sspIDs = append(sspIDs, *instance.SSPID) + } + } + entries, exceptions = HashesBySSP{}, HashesBySSP{} + if len(sspIDs) == 0 { + return entries, exceptions, nil + } + for model, into := range map[any]HashesBySSP{&relational.SSPScopeEntry{}: entries, &relational.SSPScopeException{}: exceptions} { + var rows []struct { + SSPID uuid.UUID + MatchHash string + } + if err := db.Model(model).Select("ssp_id, match_hash").Where("ssp_id IN ?", sspIDs).Scan(&rows).Error; err != nil { + return nil, nil, err + } + for _, row := range rows { + if into[row.SSPID] == nil { + into[row.SSPID] = map[string]struct{}{} + } + into[row.SSPID][row.MatchHash] = struct{}{} + } + } + return entries, exceptions, nil +}