wip 5

Author: dmcilvaney

change-detection-prd.md

@@ -0,0 +1,56 @@
+# Component Identity & Change Detection
+
+The `component identity` and `component diff-identity` subcommands compute deterministic fingerprints of component build inputs. For example, CI can compute fingerprints for the base and head commits of a PR, then diff them to determine exactly which components have changed and need to be rebuilt/tested.
+
+```bash
+# Typical CI workflow
+git checkout $BASE_REF && azldev component identity -a -O json > base.json
+git checkout $HEAD_REF && azldev component identity -a -O json > head.json
+azldev component diff-identity base.json head.json -O json -c
+# → {"changed": ["curl"], "added": ["wget"], "removed": [], "unchanged": []}
+```
+
+## Fingerprint Inputs
+
+A component's fingerprint is a SHA256 combining:
+
+1. **Config hash** — `hashstructure.Hash()` of the resolved `ComponentConfig` (after all merging). Fields tagged `fingerprint:"-"` are excluded.
+2. **Source identity** — content hash for local specs (all files in the spec directory), commit hash for upstream.
+3. **Overlay file hashes** — SHA256 of each file referenced by overlay `Source` fields.
+4. **Distro name + version**
+5. **Affects commit count** — number of `Affects: <component>` commits in the project repo.
+
+Global change propagation works automatically: the fingerprint operates on the fully-merged config, so a change to a distro or group default changes the resolved config of every inheriting component.
+
+## `fingerprint:"-"` Tag System
+
+The `hashstructure` library uses `TagName: "fingerprint"`. Untagged fields are **included by default** (safe default: false positive > false negative).
+
+A guard test (`TestAllFingerprintedFieldsHaveDecision`) reflects over all fingerprinted structs and maintains a bi-directional allowlist of exclusions. It fails if a `fingerprint:"-"` tag is added without registering it, or if a registered exclusion's tag is removed.
+
+### Adding a New Config Field
+
+1. Add the field to the struct in `internal/projectconfig/`.
+2. **If NOT a build input**: add `fingerprint:"-"` to the struct tag and register it in `expectedExclusions` in `internal/projectconfig/fingerprint_test.go`.
+3. **If a build input**: do nothing — included by default.
+4. Run `mage unit`.
+
+### Adding a New Source Type
+
+1. Implement `SourceIdentityProvider` on your provider (see `localidentity.go` for a simple example).
+2. Add a case to `sourceManager.ResolveSourceIdentity()` in `sourcemanager.go`.
+3. Add tests in `identityprovider_test.go`.
+
+## CLI
+
+### `azldev component identity`
+
+Compute fingerprints. Uses standard component filter flags (`-a`, `-p`, `-g`, `-s`). Exposed as an MCP tool.
+
+### `azldev component diff-identity`
+
+Compare two identity JSON files. The `--changed-only` / `-c` flag filters to only changed and added components (the build queue). Applies to both table and JSON output.
+
+## Known Limitations
+
+- It is difficult to determine WHY a diff occurred (e.g., which specific field changed) since the fingerprint is a single opaque hash. The JSON output includes an `inputs` breakdown (`configHash`, `sourceIdentity`, `overlayFileHashes`, etc.) that can help narrow it down by comparing the two identity files manually.

docs/developer/reference/component-identity.md

@@ -22,6 +22,10 @@ const diffIdentityArgCount = 2
 
 // NewDiffIdentityCommand constructs a [cobra.Command] for "component diff-identity".
 func NewDiffIdentityCommand() *cobra.Command {
+	var options struct {
+		ChangedOnly bool
+	}
+
 	cmd := &cobra.Command{
 		Use:   "diff-identity <base.json> <head.json>",
 		Short: "Compare two identity files and report changed components",
@@ -37,11 +41,14 @@ CI uses the 'changed' and 'added' lists to determine the build queue.`,
 		Args: cobra.ExactArgs(diffIdentityArgCount),
 		RunE: azldev.RunFuncWithoutRequiredConfigWithExtraArgs(
 			func(env *azldev.Env, args []string) (interface{}, error) {
-				return DiffIdentities(env, args[0], args[1])
+				return DiffIdentities(env, args[0], args[1], options.ChangedOnly)
 			},
 		),
 	}
 
+	cmd.Flags().BoolVarP(&options.ChangedOnly, "changed-only", "c", false,
+		"Only show changed and added components (the build queue)")
+
 	return cmd
 }
 
@@ -74,7 +81,7 @@ type IdentityDiffReport struct {
 }
 
 // DiffIdentities reads two identity JSON files and computes the diff.
-func DiffIdentities(env *azldev.Env, basePath string, headPath string) (interface{}, error) {
+func DiffIdentities(env *azldev.Env, basePath string, headPath string, changedOnly bool) (interface{}, error) {
 	baseIdentities, err := readIdentityFile(env, basePath)
 	if err != nil {
 		return nil, fmt.Errorf("reading base identity file %#q:\n%w", basePath, err)
@@ -85,7 +92,7 @@ func DiffIdentities(env *azldev.Env, basePath string, headPath string) (interfac
 		return nil, fmt.Errorf("reading head identity file %#q:\n%w", headPath, err)
 	}
 
-	report := ComputeDiff(baseIdentities, headIdentities)
+	report := ComputeDiff(baseIdentities, headIdentities, changedOnly)
 
 	// Return table-friendly results for table/CSV format, or the report for JSON.
 	if env.DefaultReportFormat() == azldev.ReportFormatJSON {
@@ -121,7 +128,8 @@ func readIdentityFile(
 }
 
 // ComputeDiff compares base and head identity maps and produces a diff report.
-func ComputeDiff(base map[string]string, head map[string]string) *IdentityDiffReport {
+// When changedOnly is true, the Removed and Unchanged lists are left empty.
+func ComputeDiff(base map[string]string, head map[string]string, changedOnly bool) *IdentityDiffReport {
 	report := &IdentityDiffReport{}
 
 	// Check base components against head.
@@ -130,11 +138,15 @@ func ComputeDiff(base map[string]string, head map[string]string) *IdentityDiffRe
 
 		switch {
 		case !exists:
-			report.Removed = append(report.Removed, name)
+			if !changedOnly {
+				report.Removed = append(report.Removed, name)
+			}
 		case baseFP != headFP:
 			report.Changed = append(report.Changed, name)
 		default:
-			report.Unchanged = append(report.Unchanged, name)
+			if !changedOnly {
+				report.Unchanged = append(report.Unchanged, name)
+			}
 		}
 	}
 

docs/user/reference/cli/azldev_component_diff-identity.md

@@ -7,7 +7,11 @@ import (
 	"testing"
 
 	"github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/cmds/component"
+	"github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils"
+	"github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms"
+	"github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestComputeDiff(t *testing.T) {
@@ -25,7 +29,7 @@ func TestComputeDiff(t *testing.T) {
 			"openssl": "sha256:ccc",
 		}
 
-		report := component.ComputeDiff(base, head)
+		report := component.ComputeDiff(base, head, false)
 
 		assert.Equal(t, []string{"wget"}, report.Changed)
 		assert.Equal(t, []string{"libfoo"}, report.Added)
@@ -42,7 +46,7 @@ func TestComputeDiff(t *testing.T) {
 			"curl": "sha256:aaa",
 		}
 
-		report := component.ComputeDiff(base, head)
+		report := component.ComputeDiff(base, head, false)
 
 		assert.Empty(t, report.Changed)
 		assert.Empty(t, report.Added)
@@ -57,7 +61,7 @@ func TestComputeDiff(t *testing.T) {
 			"wget": "sha256:bbb",
 		}
 
-		report := component.ComputeDiff(base, head)
+		report := component.ComputeDiff(base, head, false)
 
 		assert.Empty(t, report.Changed)
 		assert.Equal(t, []string{"curl", "wget"}, report.Added)
@@ -71,7 +75,7 @@ func TestComputeDiff(t *testing.T) {
 		}
 		head := map[string]string{}
 
-		report := component.ComputeDiff(base, head)
+		report := component.ComputeDiff(base, head, false)
 
 		assert.Empty(t, report.Changed)
 		assert.Empty(t, report.Added)
@@ -80,7 +84,7 @@ func TestComputeDiff(t *testing.T) {
 	})
 
 	t.Run("both empty", func(t *testing.T) {
-		report := component.ComputeDiff(map[string]string{}, map[string]string{})
+		report := component.ComputeDiff(map[string]string{}, map[string]string{}, false)
 
 		assert.Empty(t, report.Changed)
 		assert.Empty(t, report.Added)
@@ -94,7 +98,7 @@ func TestComputeDiff(t *testing.T) {
 			"openssl": "sha256:bbb",
 		}
 
-		report := component.ComputeDiff(both, both)
+		report := component.ComputeDiff(both, both, false)
 
 		assert.Empty(t, report.Changed)
 		assert.Empty(t, report.Added)
@@ -114,8 +118,85 @@ func TestComputeDiff(t *testing.T) {
 			"openssl": "sha256:ccc",
 		}
 
-		report := component.ComputeDiff(base, head)
+		report := component.ComputeDiff(base, head, false)
 
 		assert.Equal(t, []string{"curl", "zlib"}, report.Changed, "changed list should be sorted")
 	})
+
+	t.Run("changed only", func(t *testing.T) {
+		base := map[string]string{
+			"curl":    "sha256:aaa",
+			"wget":    "sha256:bbb",
+			"openssl": "sha256:ccc",
+			"libold":  "sha256:fff",
+		}
+		head := map[string]string{
+			"curl":    "sha256:aaa",
+			"wget":    "sha256:ddd",
+			"libfoo":  "sha256:eee",
+			"openssl": "sha256:ccc",
+		}
+
+		report := component.ComputeDiff(base, head, true)
+
+		assert.Equal(t, []string{"wget"}, report.Changed)
+		assert.Equal(t, []string{"libfoo"}, report.Added)
+		assert.Empty(t, report.Removed, "removed should be empty with changedOnly")
+		assert.Empty(t, report.Unchanged, "unchanged should be empty with changedOnly")
+	})
+}
+
+func TestDiffIdentities_MissingFile(t *testing.T) {
+	testEnv := testutils.NewTestEnv(t)
+
+	_, err := component.DiffIdentities(testEnv.Env, "/nonexistent/base.json", "/nonexistent/head.json", false)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "base identity file")
+}
+
+func TestDiffIdentities_MalformedJSON(t *testing.T) {
+	testEnv := testutils.NewTestEnv(t)
+
+	require.NoError(t, fileutils.WriteFile(testEnv.TestFS, "/base.json",
+		[]byte("not valid json"), fileperms.PublicFile))
+	require.NoError(t, fileutils.WriteFile(testEnv.TestFS, "/head.json",
+		[]byte(`[{"component":"a","fingerprint":"sha256:aaa"}]`), fileperms.PublicFile))
+
+	_, err := component.DiffIdentities(testEnv.Env, "/base.json", "/head.json", false)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "base identity file")
+}
+
+func TestDiffIdentities_ValidFiles(t *testing.T) {
+	testEnv := testutils.NewTestEnv(t)
+
+	require.NoError(t, fileutils.WriteFile(testEnv.TestFS, "/base.json",
+		[]byte(`[{"component":"curl","fingerprint":"sha256:aaa"}]`), fileperms.PublicFile))
+	require.NoError(t, fileutils.WriteFile(testEnv.TestFS, "/head.json",
+		[]byte(`[{"component":"curl","fingerprint":"sha256:bbb"},{"component":"wget","fingerprint":"sha256:ccc"}]`),
+		fileperms.PublicFile))
+
+	result, err := component.DiffIdentities(testEnv.Env, "/base.json", "/head.json", false)
+	require.NoError(t, err)
+
+	// Default format is table, so we get []IdentityDiffResult.
+	tableResults, ok := result.([]component.IdentityDiffResult)
+	require.True(t, ok, "expected table results for default report format")
+	require.Len(t, tableResults, 2)
+}
+
+func TestDiffIdentities_EmptyArray(t *testing.T) {
+	testEnv := testutils.NewTestEnv(t)
+
+	require.NoError(t, fileutils.WriteFile(testEnv.TestFS, "/base.json",
+		[]byte(`[]`), fileperms.PublicFile))
+	require.NoError(t, fileutils.WriteFile(testEnv.TestFS, "/head.json",
+		[]byte(`[]`), fileperms.PublicFile))
+
+	result, err := component.DiffIdentities(testEnv.Env, "/base.json", "/head.json", false)
+	require.NoError(t, err)
+
+	tableResults, ok := result.([]component.IdentityDiffResult)
+	require.True(t, ok)
+	assert.Empty(t, tableResults)
 }

internal/app/azldev/cmds/component/diffidentity.go

@@ -336,6 +336,38 @@ func TestComputeIdentity_SourceFilesChange(t *testing.T) {
 	assert.NotEqual(t, fp1, fp2, "different source file hash must produce different fingerprints")
 }
 
+func TestComputeIdentity_SourceFileOriginExcluded(t *testing.T) {
+	ctx := newTestFS(t, map[string]string{
+		"/specs/test.spec": "Name: testpkg\nVersion: 1.0",
+	})
+
+	comp1 := baseComponent()
+	comp1.SourceFiles = []projectconfig.SourceFileReference{
+		{
+			Filename: "source.tar.gz",
+			Hash:     "aaa111",
+			HashType: fileutils.HashTypeSHA256,
+			Origin:   projectconfig.Origin{Type: "download", Uri: "https://old-cdn.example.com/source.tar.gz"},
+		},
+	}
+
+	comp2 := baseComponent()
+	comp2.SourceFiles = []projectconfig.SourceFileReference{
+		{
+			Filename: "source.tar.gz",
+			Hash:     "aaa111",
+			HashType: fileutils.HashTypeSHA256,
+			Origin:   projectconfig.Origin{Type: "download", Uri: "https://new-cdn.example.com/source.tar.gz"},
+		},
+	}
+	distro := baseDistroRef()
+
+	fp1 := computeFingerprint(t, ctx, comp1, distro, 0)
+	fp2 := computeFingerprint(t, ctx, comp2, distro, 0)
+
+	assert.Equal(t, fp1, fp2, "changing source file origin URL must NOT change fingerprint")
+}
+
 func TestComputeIdentity_InputsBreakdown(t *testing.T) {
 	ctx := newTestFS(t, map[string]string{
 		"/specs/test.spec":   "Name: testpkg\nVersion: 1.0",

internal/app/azldev/cmds/component/diffidentity_test.go

@@ -61,7 +61,7 @@ type SourceFileReference struct {
 	HashType fileutils.HashType `toml:"hash-type,omitempty" json:"hashType,omitempty"`
 
 	// Origin for this source file. When omitted, the file is resolved via the lookaside cache.
-	Origin Origin `toml:"origin,omitempty" json:"origin,omitempty"`
+	Origin Origin `toml:"origin,omitempty" json:"origin,omitempty" fingerprint:"-"`
 }
 
 // Defines a component group. Component groups are logical groupings of components (see [ComponentConfig]).

internal/fingerprint/fingerprint_test.go

@@ -75,6 +75,11 @@ func TestAllFingerprintedFieldsHaveDecision(t *testing.T) {
 		// upstream commit hash (captured separately via SourceIdentity) is what matters.
 		// Excluding this prevents a snapshot bump from marking all upstream components as changed.
 		"DistroReference.Snapshot": true,
+
+		// SourceFileReference.Origin — download location metadata (URI, type), not a build input.
+		// The file content is already captured by Filename + Hash; changing a CDN URL should not
+		// trigger a rebuild.
+		"SourceFileReference.Origin": true,
 	}
 
 	// Collect all actual exclusions found via reflection.

internal/projectconfig/component.go

@@ -9,6 +9,7 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io/fs"
+	"path/filepath"
 	"sort"
 
 	"github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components"
@@ -84,7 +85,12 @@ func (provider *LocalIdentityProvider) ResolveSourceIdentity(
 			return "", fmt.Errorf("hashing file %#q:\n%w", filePath, hashErr)
 		}
 
-		fmt.Fprintf(combinedHasher, "%s=%s\n", filePath, fileHash)
+		relPath, relErr := filepath.Rel(provider.specDir, filePath)
+		if relErr != nil {
+			return "", fmt.Errorf("computing relative path for %#q:\n%w", filePath, relErr)
+		}
+
+		fmt.Fprintf(combinedHasher, "%s=%s\n", relPath, fileHash)
 	}
 
 	return hex.EncodeToString(combinedHasher.Sum(nil)), nil

internal/projectconfig/fingerprint_test.go

@@ -173,6 +173,11 @@ func (g *GitProviderImpl) LsRemoteHead(
 		return "", errors.New("repository URL cannot be empty")
 	}
 
+	_, err := url.Parse(repoURL)
+	if err != nil {
+		return "", fmt.Errorf("invalid URL %#q:\n%w", repoURL, err)
+	}
+
 	if branch == "" {
 		return "", errors.New("branch cannot be empty")
 	}