Support multi-document pnpm lockfiles

docs/detectors/pnpm.md

@@ -9,6 +9,12 @@ Pnpm detection relies on the presence of lockfiles generated by the pnpm package
 
 The detector supports lockfile versions 5, 6, and 9, with version 9 being the maximum supported version.
 
+Lockfiles can contain one or two YAML documents. When an environment document is
+present, it comes before the project document. Both documents are scanned so that
+config and package-manager dependencies are included alongside project dependencies.
+Empty documents, more than two documents, and inconsistent declared lockfile versions
+are rejected as invalid lockfiles.
+
 ## Detection strategy
 
 Pnpm detection is performed by parsing lockfiles found in the scan directory. The `PnpmComponentDetectorFactory` acts as a version-aware factory that:

src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V6/PnpmHasDependenciesV6.cs

@@ -14,4 +14,10 @@ internal class PnpmHasDependenciesV6 : PnpmYaml
 
     [YamlMember(Alias = "optionalDependencies")]
     public Dictionary<string, PnpmYamlV6Dependency> OptionalDependencies { get; set; }
+
+    [YamlMember(Alias = "packageManagerDependencies")]
+    public Dictionary<string, PnpmYamlV6Dependency> PackageManagerDependencies { get; set; }
+
+    [YamlMember(Alias = "configDependencies")]
+    public Dictionary<string, PnpmYamlV6Dependency> ConfigDependencies { get; set; }
 }

src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V9/PnpmHasDependenciesV9.cs

@@ -14,4 +14,10 @@ internal class PnpmHasDependenciesV9 : PnpmYaml
 
     [YamlMember(Alias = "optionalDependencies")]
     public Dictionary<string, PnpmYamlV9Dependency> OptionalDependencies { get; set; }
+
+    [YamlMember(Alias = "packageManagerDependencies")]
+    public Dictionary<string, PnpmYamlV9Dependency> PackageManagerDependencies { get; set; }
+
+    [YamlMember(Alias = "configDependencies")]
+    public Dictionary<string, PnpmYamlV9Dependency> ConfigDependencies { get; set; }
 }

src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmParsingUtilitiesBase.cs

@@ -5,17 +5,46 @@ namespace Microsoft.ComponentDetection.Detectors.Pnpm;
 using System.Collections.Generic;
 using System.IO;
 using Microsoft.ComponentDetection.Contracts;
+using YamlDotNet.Core;
+using YamlDotNet.Core.Events;
 using YamlDotNet.Serialization;
 
 internal abstract class PnpmParsingUtilitiesBase<T>
 where T : PnpmYaml
 {
-    public T DeserializePnpmYamlFile(string fileContent)
+    public virtual IReadOnlyList<T> DeserializePnpmYamlFileDocuments(string fileContent)
     {
         var deserializer = new DeserializerBuilder()
             .IgnoreUnmatchedProperties()
             .Build();
-        return deserializer.Deserialize<T>(new StringReader(fileContent));
+
+        using var reader = new StringReader(fileContent);
+        var parser = new Parser(reader);
+        parser.Consume<StreamStart>();
+
+        var documents = new List<T>();
+        var documentCount = 0;
+        while (parser.TryConsume<DocumentStart>(out _))
+        {
+            if (++documentCount > 2)
+            {
+                throw new InvalidOperationException("A pnpm lockfile must contain one or two YAML documents.");
+            }
+
+            var doc = deserializer.Deserialize<T>(parser)
+                ?? throw new InvalidOperationException("A pnpm lockfile must not contain empty YAML documents.");
+
+            documents.Add(doc);
+            parser.TryConsume<DocumentEnd>(out _);
+        }
+
+        parser.Consume<StreamEnd>();
+        if (documentCount == 0)
+        {
+            throw new InvalidOperationException("A pnpm lockfile must contain one or two YAML documents.");
+        }
+
+        return documents.AsReadOnly();
     }
 
     public virtual bool IsPnpmPackageDevDependency(Package pnpmPackage)

src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmParsingUtilitiesFactory.cs

@@ -1,8 +1,8 @@
 #nullable disable
 namespace Microsoft.ComponentDetection.Detectors.Pnpm;
 
-using System.IO;
-using YamlDotNet.Serialization;
+using System;
+using System.Linq;
 
 internal static class PnpmParsingUtilitiesFactory
 {
@@ -20,9 +20,17 @@ public static PnpmParsingUtilitiesBase<T> Create<T>()
 
     public static string DeserializePnpmYamlFileVersion(string fileContent)
     {
-        var deserializer = new DeserializerBuilder()
-            .IgnoreUnmatchedProperties()
-            .Build();
-        return deserializer.Deserialize<PnpmYaml>(new StringReader(fileContent))?.LockfileVersion;
+        var documents = Create<PnpmYaml>().DeserializePnpmYamlFileDocuments(fileContent);
+        var distinctVersions = documents
+            .Select(doc => doc.LockfileVersion)
+            .Where(version => !string.IsNullOrWhiteSpace(version))
+            .Distinct()
+            .ToList();
+        if (distinctVersions.Count > 1)
+        {
+            throw new InvalidOperationException($"Inconsistent lockfile versions found: {string.Join(", ", distinctVersions)}");
+        }
+
+        return distinctVersions.FirstOrDefault();
     }
 }

src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm5Detector.cs

@@ -12,36 +12,39 @@ public class Pnpm5Detector : IPnpmDetector
 
     public void RecordDependencyGraphFromFile(string yamlFileContent, ISingleFileComponentRecorder singleFileComponentRecorder)
     {
-        var yaml = this.pnpmParsingUtilities.DeserializePnpmYamlFile(yamlFileContent);
+        var yamls = this.pnpmParsingUtilities.DeserializePnpmYamlFileDocuments(yamlFileContent);
 
-        foreach (var packageKeyValue in yaml?.Packages ?? Enumerable.Empty<KeyValuePair<string, Package>>())
+        foreach (var yaml in yamls)
         {
-            // Ignore file: as these are local packages.
-            if (packageKeyValue.Key.StartsWith(PnpmConstants.PnpmFileDependencyPath))
+            foreach (var packageKeyValue in yaml?.Packages ?? Enumerable.Empty<KeyValuePair<string, Package>>())
             {
-                continue;
-            }
+                // Ignore file: as these are local packages.
+                if (packageKeyValue.Key.StartsWith(PnpmConstants.PnpmFileDependencyPath))
+                {
+                    continue;
+                }
 
-            var parentDetectedComponent = this.pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath(pnpmPackagePath: packageKeyValue.Key);
-            var isDevDependency = packageKeyValue.Value != null && this.pnpmParsingUtilities.IsPnpmPackageDevDependency(packageKeyValue.Value);
-            singleFileComponentRecorder.RegisterUsage(parentDetectedComponent, isDevelopmentDependency: isDevDependency);
-            parentDetectedComponent = singleFileComponentRecorder.GetComponent(parentDetectedComponent.Component.Id);
+                var parentDetectedComponent = this.pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath(pnpmPackagePath: packageKeyValue.Key);
+                var isDevDependency = packageKeyValue.Value != null && this.pnpmParsingUtilities.IsPnpmPackageDevDependency(packageKeyValue.Value);
+                singleFileComponentRecorder.RegisterUsage(parentDetectedComponent, isDevelopmentDependency: isDevDependency);
+                parentDetectedComponent = singleFileComponentRecorder.GetComponent(parentDetectedComponent.Component.Id);
 
-            if (packageKeyValue.Value.Dependencies != null)
-            {
-                foreach (var dependency in packageKeyValue.Value.Dependencies)
+                if (packageKeyValue.Value?.Dependencies != null)
                 {
-                    // Ignore local packages.
-                    if (this.pnpmParsingUtilities.IsLocalDependency(dependency))
+                    foreach (var dependency in packageKeyValue.Value.Dependencies)
                     {
-                        continue;
-                    }
+                        // Ignore local packages.
+                        if (this.pnpmParsingUtilities.IsLocalDependency(dependency))
+                        {
+                            continue;
+                        }
 
-                    var childDetectedComponent = this.pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath(
-                        pnpmPackagePath: this.CreatePnpmPackagePathFromDependency(dependency.Key, dependency.Value));
+                        var childDetectedComponent = this.pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath(
+                            pnpmPackagePath: this.CreatePnpmPackagePathFromDependency(dependency.Key, dependency.Value));
 
-                    // Older code used the root's dev dependency value. We're leaving this null until we do a second pass to look at each components' top level referrers.
-                    singleFileComponentRecorder.RegisterUsage(childDetectedComponent, parentComponentId: parentDetectedComponent.Component.Id, isDevelopmentDependency: null);
+                        // Older code used the root's dev dependency value. We're leaving this null until we do a second pass to look at each components' top level referrers.
+                        singleFileComponentRecorder.RegisterUsage(childDetectedComponent, parentComponentId: parentDetectedComponent.Component.Id, isDevelopmentDependency: null);
+                    }
                 }
             }
         }

src/Microsoft.ComponentDetection.Detectors/pnpm/Pnpm6Detector.cs

@@ -12,33 +12,68 @@ public class Pnpm6Detector : IPnpmDetector
 
     public void RecordDependencyGraphFromFile(string yamlFileContent, ISingleFileComponentRecorder singleFileComponentRecorder)
     {
-        var yaml = this.pnpmParsingUtilities.DeserializePnpmYamlFile(yamlFileContent);
+        var yamls = this.pnpmParsingUtilities.DeserializePnpmYamlFileDocuments(yamlFileContent);
 
         // There may be multiple instance of the same package (even at the same version) in pnpm differentiated by other aspects of the pnpm dependency path.
         // Therefor all DetectedComponents are tracked by the same full string pnpm uses, the pnpm dependency path, which is used as the key in this dictionary.
         // Some documentation about pnpm dependency paths can be found at https://github.com/pnpm/spec/blob/master/dependency-path.md.
         var components = new Dictionary<string, (DetectedComponent, Package)>();
 
         // Create a component for every package referenced in the lock file.
-        // This includes all directly and transitively referenced dependencies.
-        foreach (var (pnpmDependencyPath, package) in yaml.Packages ?? Enumerable.Empty<KeyValuePair<string, Package>>())
+        // This includes all directly and transitively referenced dependencies across all documents.
+        foreach (var yaml in yamls)
         {
-            // Ignore "file:" as these are local packages.
-            // Such local packages should only be referenced at the top level (via ProcessDependencyList) which also skips them or from other local packages (which this skips).
-            // There should be no cases where a non-local package references a local package, so skipping them here should not result in failed lookups below when adding all the graph references.
-            if (pnpmDependencyPath.StartsWith(PnpmConstants.PnpmFileDependencyPath))
+            foreach (var (pnpmDependencyPath, package) in yaml.Packages ?? Enumerable.Empty<KeyValuePair<string, Package>>())
             {
-                continue;
+                // Ignore "file:" as these are local packages.
+                // Such local packages should only be referenced at the top level (via ProcessDependencyList) which also skips them or from other local packages (which this skips).
+                // There should be no cases where a non-local package references a local package, so skipping them here should not result in failed lookups below when adding all the graph references.
+                if (pnpmDependencyPath.StartsWith(PnpmConstants.PnpmFileDependencyPath))
+                {
+                    continue;
+                }
+
+                if (!components.TryGetValue(pnpmDependencyPath, out var existing))
+                {
+                    var parentDetectedComponent = this.pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath(pnpmPackagePath: pnpmDependencyPath);
+                    components.Add(pnpmDependencyPath, (parentDetectedComponent, package));
+
+                    // Register the component.
+                    // It should get registered again with with additional information (what depended on it) later,
+                    // but registering it now ensures nothing is missed due to a limitation in dependency traversal
+                    // like skipping local dependencies which might have transitively depended on this.
+                    singleFileComponentRecorder.RegisterUsage(parentDetectedComponent, isDevelopmentDependency: this.pnpmParsingUtilities.IsPnpmPackageDevDependency(package));
+                }
+                else
+                {
+                    // If the same package path occurs across multiple documents (e.g. environment and project documents),
+                    // merge package metadata and dependencies, letting a production occurrence win for development classification.
+                    var existingIsDev = this.pnpmParsingUtilities.IsPnpmPackageDevDependency(existing.Item2);
+                    var newIsDev = this.pnpmParsingUtilities.IsPnpmPackageDevDependency(package);
+                    var effectiveIsDev = existingIsDev && newIsDev;
+
+                    if (existingIsDev && !effectiveIsDev)
+                    {
+                        existing.Item2.Dev = bool.FalseString;
+                        singleFileComponentRecorder.RegisterUsage(existing.Item1, isDevelopmentDependency: false);
+                    }
+
+                    if (package.Dependencies != null)
+                    {
+                        if (existing.Item2.Dependencies == null)
+                        {
+                            existing.Item2.Dependencies = new Dictionary<string, string>(package.Dependencies);
+                        }
+                        else
+                        {
+                            foreach (var (name, version) in package.Dependencies)
+                            {
+                                existing.Item2.Dependencies.TryAdd(name, version);
+                            }
+                        }
+                    }
+                }
             }
-
-            var parentDetectedComponent = this.pnpmParsingUtilities.CreateDetectedComponentFromPnpmPath(pnpmPackagePath: pnpmDependencyPath);
-            components.Add(pnpmDependencyPath, (parentDetectedComponent, package));
-
-            // Register the component.
-            // It should get registered again with with additional information (what depended on it) later,
-            // but registering it now ensures nothing is missed due to a limitation in dependency traversal
-            // like skipping local dependencies which might have transitively depended on this.
-            singleFileComponentRecorder.RegisterUsage(parentDetectedComponent, isDevelopmentDependency: this.pnpmParsingUtilities.IsPnpmPackageDevDependency(package));
         }
 
         // Now that the `components` dictionary is populated, make a second pass registering all the dependency edges in the graph.
@@ -56,14 +91,16 @@ public void RecordDependencyGraphFromFile(string yamlFileContent, ISingleFileCom
         }
 
         // Lastly, add all direct dependencies of the current file/project setting isExplicitReferencedDependency to true:
-
-        // "dedicated shrinkwrap" (single package) case:
-        this.ProcessDependencySet(singleFileComponentRecorder, components, yaml);
-
-        // "shared shrinkwrap" (workspace / mono-repos) case:
-        foreach (var (_, package) in yaml.Importers ?? Enumerable.Empty<KeyValuePair<string, PnpmHasDependenciesV6>>())
+        foreach (var yaml in yamls)
         {
-            this.ProcessDependencySet(singleFileComponentRecorder, components, package);
+            // "dedicated shrinkwrap" (single package) case:
+            this.ProcessDependencySet(singleFileComponentRecorder, components, yaml);
+
+            // "shared shrinkwrap" (workspace / mono-repos) case:
+            foreach (var (_, package) in yaml.Importers ?? Enumerable.Empty<KeyValuePair<string, PnpmHasDependenciesV6>>())
+            {
+                this.ProcessDependencySet(singleFileComponentRecorder, components, package);
+            }
         }
     }
 
@@ -72,6 +109,8 @@ private void ProcessDependencySet(ISingleFileComponentRecorder singleFileCompone
         this.ProcessDependencyList(singleFileComponentRecorder, components, item.Dependencies);
         this.ProcessDependencyList(singleFileComponentRecorder, components, item.DevDependencies);
         this.ProcessDependencyList(singleFileComponentRecorder, components, item.OptionalDependencies);
+        this.ProcessDependencyList(singleFileComponentRecorder, components, item.PackageManagerDependencies);
+        this.ProcessDependencyList(singleFileComponentRecorder, components, item.ConfigDependencies);
     }
 
     private void ProcessDependencyList(ISingleFileComponentRecorder singleFileComponentRecorder, Dictionary<string, (DetectedComponent C, Package P)> components, Dictionary<string, PnpmYamlV6Dependency> dependencies)