diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2d2b4c41..e4fe00f0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -54,7 +54,7 @@ jobs: - name: setup Python uses: actions/setup-python@v7 with: - python-version: '3.9' + python-version: '3.13' cache: 'pip' - name: get Python location diff --git a/.gitignore b/.gitignore index 008c5a75..97b71c6f 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ target .npmrc src/providers/*.wasm *.egg-info +*.m2 diff --git a/src/providers/python_controller.js b/src/providers/python_controller.js index 080f1c3f..953f63c8 100644 --- a/src/providers/python_controller.js +++ b/src/providers/python_controller.js @@ -447,7 +447,8 @@ function bringAllDependencies(dependencies, dependencyName, cachedEnvironmentDep function getDependencyTreeJsonFromPipDepTree(pipPath,pythonPath) { let dependencyTree try { - invokeCommand(pipPath, ['install', 'pipdeptree']) + // pipdeptree 4.x depends on nab-* packages that pull typing_extensions>=4.16, polluting the user's env and causing version-mismatch errors + invokeCommand(pipPath, ['install', 'pipdeptree<4']) } catch (error) { throw new Error(`Failed installing pipdeptree utility`, {cause: error}) } diff --git a/src/providers/python_poetry.js b/src/providers/python_poetry.js index 956f6501..cdcd0da1 100644 --- a/src/providers/python_poetry.js +++ b/src/providers/python_poetry.js @@ -58,7 +58,6 @@ export default class Python_poetry extends Base_pyproject { * @param {Object} opts * @returns {Promise<{directDeps: string[], graph: Map}>} */ - // eslint-disable-next-line no-unused-vars async _getDependencyData(manifestDir, _workspaceDir, parsed, opts) { let poetryBin = getCustomPath('poetry', opts) let envBypass = environmentVariableIsPopulated('TRUSTIFY_DA_POETRY_SHOW_TREE') @@ -136,11 +135,12 @@ export default class Python_poetry extends Base_pyproject { * → transitiveMarkers['click']['colorama'] = "sys_platform == 'win32'" * @param {string|null} lockDir * @param {object} parsed - parsed pyproject.toml - * @returns {{directMarkers: Map, transitiveMarkers: Map>, hashMap: Map>}} + * @returns {{directMarkers: Map, transitiveMarkers: Map>, packageMarkers: Map, hashMap: Map>}} */ _extractMarkerData(lockDir, parsed) { let directMarkers = new Map() let transitiveMarkers = new Map() + let packageMarkers = new Map() let hashMap = new Map() // Extract markers from PEP 621 dependency strings: "name[extras]>=ver ; marker" @@ -160,6 +160,9 @@ export default class Python_poetry extends Base_pyproject { let packages = lock.package || [] for (let pkg of packages) { let pkgKey = this._canonicalize(pkg.name) + if (pkg.markers) { + packageMarkers.set(pkgKey, pkg.markers) + } let pkgDeps = pkg.dependencies || {} for (let [depName, depSpec] of Object.entries(pkgDeps)) { let markers = typeof depSpec === 'object' && depSpec != null ? depSpec.markers : null @@ -179,7 +182,7 @@ export default class Python_poetry extends Base_pyproject { } } - return { directMarkers, transitiveMarkers, hashMap } + return { directMarkers, transitiveMarkers, packageMarkers, hashMap } } /** @@ -204,7 +207,7 @@ export default class Python_poetry extends Base_pyproject { * * @param {string} treeOutput * @param {Map} versionMap - canonical name -> resolved version - * @param {{directMarkers: Map, transitiveMarkers: Map>, hashMap: Map>}} markerData + * @param {{directMarkers: Map, transitiveMarkers: Map>, packageMarkers: Map, hashMap: Map>}} markerData * @returns {{directDeps: string[], graph: Map}>}} */ _parsePoetryTree(treeOutput, versionMap, markerData) { @@ -214,6 +217,7 @@ export default class Python_poetry extends Base_pyproject { let stack = [] // [{key, depth}] let currentDirectDep = null + let skipDepth = null for (let line of lines) { if (!line.trim()) { continue } @@ -229,8 +233,10 @@ export default class Python_poetry extends Base_pyproject { if (marker && !evaluateMarker(marker)) { currentDirectDep = null stack = [] + skipDepth = null continue } + skipDepth = null directDeps.push(key) if (!graph.has(key)) { @@ -261,17 +267,31 @@ export default class Python_poetry extends Base_pyproject { let prefix = line.substring(0, nameStart) let depth = (prefix.match(/(?:[├└│ ][\s─]{2} ?)/g) || []).length + // skip entire subtree of a marker-excluded dep + if (skipDepth != null) { + if (depth > skipDepth) { continue } + skipDepth = null + } + // pop stack back to find the parent at depth-1 while (stack.length > 0 && stack[stack.length - 1].depth >= depth) { stack.pop() } + // check package-level markers from poetry.lock + let pkgMarker = markerData.packageMarkers.get(depKey) + if (pkgMarker && !evaluateMarker(pkgMarker)) { + skipDepth = depth + continue + } + let parentKey = stack.length > 0 ? stack[stack.length - 1].key : null if (parentKey) { let parentMarkers = markerData.transitiveMarkers.get(parentKey) if (parentMarkers) { let marker = parentMarkers.get(depKey) if (marker && !evaluateMarker(marker)) { + skipDepth = depth continue } } diff --git a/test/providers/tst_manifests/pip/pip_requirements_txt_ignore/expected_component_sbom.json b/test/providers/tst_manifests/pip/pip_requirements_txt_ignore/expected_component_sbom.json index 86218513..d5299b74 100644 --- a/test/providers/tst_manifests/pip/pip_requirements_txt_ignore/expected_component_sbom.json +++ b/test/providers/tst_manifests/pip/pip_requirements_txt_ignore/expected_component_sbom.json @@ -43,13 +43,6 @@ "type": "library", "bom-ref": "pkg:pypi/asgiref@3.4.1" }, - { - "name": "async-timeout", - "version": "5.0.1", - "purl": "pkg:pypi/async-timeout@5.0.1", - "type": "library", - "bom-ref": "pkg:pypi/async-timeout@5.0.1" - }, { "name": "beautifulsoup4", "version": "4.12.2", @@ -108,10 +101,10 @@ }, { "name": "immutables", - "version": "0.20", - "purl": "pkg:pypi/immutables@0.20", + "version": "0.21", + "purl": "pkg:pypi/immutables@0.21", "type": "library", - "bom-ref": "pkg:pypi/immutables@0.20" + "bom-ref": "pkg:pypi/immutables@0.21" }, { "name": "importlib-metadata", @@ -219,7 +212,6 @@ "pkg:pypi/aiohappyeyeballs@2.6.1", "pkg:pypi/anyio@3.6.2", "pkg:pypi/asgiref@3.4.1", - "pkg:pypi/async-timeout@5.0.1", "pkg:pypi/beautifulsoup4@4.12.2", "pkg:pypi/certifi@2023.7.22", "pkg:pypi/chardet@4.0.0", @@ -228,7 +220,7 @@ "pkg:pypi/flask@2.0.3", "pkg:pypi/h11@0.13.0", "pkg:pypi/idna@2.10", - "pkg:pypi/immutables@0.20", + "pkg:pypi/immutables@0.21", "pkg:pypi/importlib-metadata@4.8.3", "pkg:pypi/itsdangerous@2.0.1", "pkg:pypi/jinja2@3.0.3", @@ -257,10 +249,6 @@ "ref": "pkg:pypi/asgiref@3.4.1", "dependsOn": [] }, - { - "ref": "pkg:pypi/async-timeout@5.0.1", - "dependsOn": [] - }, { "ref": "pkg:pypi/beautifulsoup4@4.12.2", "dependsOn": [] @@ -294,7 +282,7 @@ "dependsOn": [] }, { - "ref": "pkg:pypi/immutables@0.20", + "ref": "pkg:pypi/immutables@0.21", "dependsOn": [] }, { diff --git a/test/providers/tst_manifests/pip/pip_requirements_txt_ignore/expected_stack_sbom.json b/test/providers/tst_manifests/pip/pip_requirements_txt_ignore/expected_stack_sbom.json index 7af92542..f7797ae1 100644 --- a/test/providers/tst_manifests/pip/pip_requirements_txt_ignore/expected_stack_sbom.json +++ b/test/providers/tst_manifests/pip/pip_requirements_txt_ignore/expected_stack_sbom.json @@ -57,13 +57,6 @@ "type": "library", "bom-ref": "pkg:pypi/asgiref@3.4.1" }, - { - "name": "async-timeout", - "version": "5.0.1", - "purl": "pkg:pypi/async-timeout@5.0.1", - "type": "library", - "bom-ref": "pkg:pypi/async-timeout@5.0.1" - }, { "name": "beautifulsoup4", "version": "4.12.2", @@ -164,10 +157,10 @@ }, { "name": "immutables", - "version": "0.20", - "purl": "pkg:pypi/immutables@0.20", + "version": "0.21", + "purl": "pkg:pypi/immutables@0.21", "type": "library", - "bom-ref": "pkg:pypi/immutables@0.20" + "bom-ref": "pkg:pypi/immutables@0.21" }, { "name": "importlib-metadata", @@ -219,7 +212,6 @@ "pkg:pypi/aiohappyeyeballs@2.6.1", "pkg:pypi/anyio@3.6.2", "pkg:pypi/asgiref@3.4.1", - "pkg:pypi/async-timeout@5.0.1", "pkg:pypi/beautifulsoup4@4.12.2", "pkg:pypi/certifi@2023.7.22", "pkg:pypi/chardet@4.0.0", @@ -228,7 +220,7 @@ "pkg:pypi/flask@2.0.3", "pkg:pypi/h11@0.13.0", "pkg:pypi/idna@2.10", - "pkg:pypi/immutables@0.20", + "pkg:pypi/immutables@0.21", "pkg:pypi/importlib-metadata@4.8.3", "pkg:pypi/itsdangerous@2.0.1", "pkg:pypi/jinja2@3.0.3", @@ -268,10 +260,6 @@ "ref": "pkg:pypi/asgiref@3.4.1", "dependsOn": [] }, - { - "ref": "pkg:pypi/async-timeout@5.0.1", - "dependsOn": [] - }, { "ref": "pkg:pypi/beautifulsoup4@4.12.2", "dependsOn": [ @@ -341,7 +329,7 @@ "dependsOn": [] }, { - "ref": "pkg:pypi/immutables@0.20", + "ref": "pkg:pypi/immutables@0.21", "dependsOn": [] }, { diff --git a/test/providers/tst_manifests/pip/pip_requirements_txt_ignore/requirements.txt b/test/providers/tst_manifests/pip/pip_requirements_txt_ignore/requirements.txt index 775171e8..f9316449 100644 --- a/test/providers/tst_manifests/pip/pip_requirements_txt_ignore/requirements.txt +++ b/test/providers/tst_manifests/pip/pip_requirements_txt_ignore/requirements.txt @@ -9,7 +9,7 @@ fastapi==0.75.1 Flask==2.0.3 h11==0.13.0 idna==2.10 -immutables==0.20 +immutables==0.21 importlib-metadata==4.8.3 itsdangerous==2.0.1 Jinja2==3.0.3 diff --git a/test/providers/tst_manifests/pip/pip_requirements_txt_no_ignore/expected_component_sbom.json b/test/providers/tst_manifests/pip/pip_requirements_txt_no_ignore/expected_component_sbom.json index 7e2e9ef0..47668662 100644 --- a/test/providers/tst_manifests/pip/pip_requirements_txt_no_ignore/expected_component_sbom.json +++ b/test/providers/tst_manifests/pip/pip_requirements_txt_no_ignore/expected_component_sbom.json @@ -101,10 +101,10 @@ }, { "name": "immutables", - "version": "0.20", - "purl": "pkg:pypi/immutables@0.20", + "version": "0.21", + "purl": "pkg:pypi/immutables@0.21", "type": "library", - "bom-ref": "pkg:pypi/immutables@0.20" + "bom-ref": "pkg:pypi/immutables@0.21" }, { "name": "importlib-metadata", @@ -227,7 +227,7 @@ "pkg:pypi/flask@2.0.3", "pkg:pypi/h11@0.13.0", "pkg:pypi/idna@2.10", - "pkg:pypi/immutables@0.20", + "pkg:pypi/immutables@0.21", "pkg:pypi/importlib-metadata@4.8.3", "pkg:pypi/itsdangerous@2.0.1", "pkg:pypi/jinja2@3.0.3", @@ -290,7 +290,7 @@ "dependsOn": [] }, { - "ref": "pkg:pypi/immutables@0.20", + "ref": "pkg:pypi/immutables@0.21", "dependsOn": [] }, { diff --git a/test/providers/tst_manifests/pip/pip_requirements_txt_no_ignore/expected_stack_sbom.json b/test/providers/tst_manifests/pip/pip_requirements_txt_no_ignore/expected_stack_sbom.json index 3aef4244..30884c9f 100644 --- a/test/providers/tst_manifests/pip/pip_requirements_txt_no_ignore/expected_stack_sbom.json +++ b/test/providers/tst_manifests/pip/pip_requirements_txt_no_ignore/expected_stack_sbom.json @@ -164,10 +164,10 @@ }, { "name": "immutables", - "version": "0.20", - "purl": "pkg:pypi/immutables@0.20", + "version": "0.21", + "purl": "pkg:pypi/immutables@0.21", "type": "library", - "bom-ref": "pkg:pypi/immutables@0.20" + "bom-ref": "pkg:pypi/immutables@0.21" }, { "name": "importlib-metadata", @@ -227,7 +227,7 @@ "pkg:pypi/flask@2.0.3", "pkg:pypi/h11@0.13.0", "pkg:pypi/idna@2.10", - "pkg:pypi/immutables@0.20", + "pkg:pypi/immutables@0.21", "pkg:pypi/importlib-metadata@4.8.3", "pkg:pypi/itsdangerous@2.0.1", "pkg:pypi/jinja2@3.0.3", @@ -345,7 +345,7 @@ "dependsOn": [] }, { - "ref": "pkg:pypi/immutables@0.20", + "ref": "pkg:pypi/immutables@0.21", "dependsOn": [] }, { diff --git a/test/providers/tst_manifests/pip/pip_requirements_txt_no_ignore/requirements.txt b/test/providers/tst_manifests/pip/pip_requirements_txt_no_ignore/requirements.txt index 3f34b529..cb7d2f51 100644 --- a/test/providers/tst_manifests/pip/pip_requirements_txt_no_ignore/requirements.txt +++ b/test/providers/tst_manifests/pip/pip_requirements_txt_no_ignore/requirements.txt @@ -9,7 +9,7 @@ fastapi==0.75.1 Flask==2.0.3 h11==0.13.0 idna==2.10 -immutables==0.20 +immutables==0.21 importlib-metadata==4.8.3 itsdangerous==2.0.1 Jinja2==3.0.3 diff --git a/test/providers/tst_manifests/pip/pip_requirements_virtual_env_txt_no_ignore/expected_component_sbom.json b/test/providers/tst_manifests/pip/pip_requirements_virtual_env_txt_no_ignore/expected_component_sbom.json index 9d5ab36e..d4bb5790 100644 --- a/test/providers/tst_manifests/pip/pip_requirements_virtual_env_txt_no_ignore/expected_component_sbom.json +++ b/test/providers/tst_manifests/pip/pip_requirements_virtual_env_txt_no_ignore/expected_component_sbom.json @@ -85,10 +85,10 @@ }, { "name" : "immutables", - "version" : "0.20", - "purl" : "pkg:pypi/immutables@0.20", + "version" : "0.21", + "purl" : "pkg:pypi/immutables@0.21", "type" : "library", - "bom-ref" : "pkg:pypi/immutables@0.20" + "bom-ref" : "pkg:pypi/immutables@0.21" }, { "name" : "importlib-metadata", @@ -242,7 +242,7 @@ "dependsOn" : [ ] }, { - "ref" : "pkg:pypi/immutables@0.20", + "ref" : "pkg:pypi/immutables@0.21", "dependsOn" : [ ] }, { diff --git a/test/providers/tst_manifests/pip/pip_requirements_virtual_env_txt_no_ignore/expected_stack_sbom.json b/test/providers/tst_manifests/pip/pip_requirements_virtual_env_txt_no_ignore/expected_stack_sbom.json index 3aef4244..30884c9f 100644 --- a/test/providers/tst_manifests/pip/pip_requirements_virtual_env_txt_no_ignore/expected_stack_sbom.json +++ b/test/providers/tst_manifests/pip/pip_requirements_virtual_env_txt_no_ignore/expected_stack_sbom.json @@ -164,10 +164,10 @@ }, { "name": "immutables", - "version": "0.20", - "purl": "pkg:pypi/immutables@0.20", + "version": "0.21", + "purl": "pkg:pypi/immutables@0.21", "type": "library", - "bom-ref": "pkg:pypi/immutables@0.20" + "bom-ref": "pkg:pypi/immutables@0.21" }, { "name": "importlib-metadata", @@ -227,7 +227,7 @@ "pkg:pypi/flask@2.0.3", "pkg:pypi/h11@0.13.0", "pkg:pypi/idna@2.10", - "pkg:pypi/immutables@0.20", + "pkg:pypi/immutables@0.21", "pkg:pypi/importlib-metadata@4.8.3", "pkg:pypi/itsdangerous@2.0.1", "pkg:pypi/jinja2@3.0.3", @@ -345,7 +345,7 @@ "dependsOn": [] }, { - "ref": "pkg:pypi/immutables@0.20", + "ref": "pkg:pypi/immutables@0.21", "dependsOn": [] }, { diff --git a/test/providers/tst_manifests/pip/pip_requirements_virtual_env_txt_no_ignore/requirements.txt b/test/providers/tst_manifests/pip/pip_requirements_virtual_env_txt_no_ignore/requirements.txt index 3f34b529..cb7d2f51 100644 --- a/test/providers/tst_manifests/pip/pip_requirements_virtual_env_txt_no_ignore/requirements.txt +++ b/test/providers/tst_manifests/pip/pip_requirements_virtual_env_txt_no_ignore/requirements.txt @@ -9,7 +9,7 @@ fastapi==0.75.1 Flask==2.0.3 h11==0.13.0 idna==2.10 -immutables==0.20 +immutables==0.21 importlib-metadata==4.8.3 itsdangerous==2.0.1 Jinja2==3.0.3 diff --git a/test/providers/tst_manifests/pip/pip_requirements_virtual_env_with_ignore/expected_component_sbom.json b/test/providers/tst_manifests/pip/pip_requirements_virtual_env_with_ignore/expected_component_sbom.json index a14a89f2..289a285a 100644 --- a/test/providers/tst_manifests/pip/pip_requirements_virtual_env_with_ignore/expected_component_sbom.json +++ b/test/providers/tst_manifests/pip/pip_requirements_virtual_env_with_ignore/expected_component_sbom.json @@ -93,10 +93,10 @@ }, { "name": "immutables", - "version": "0.20", - "purl": "pkg:pypi/immutables@0.20", + "version": "0.21", + "purl": "pkg:pypi/immutables@0.21", "type": "library", - "bom-ref": "pkg:pypi/immutables@0.20" + "bom-ref": "pkg:pypi/immutables@0.21" }, { "name": "importlib-metadata", @@ -211,7 +211,7 @@ "pkg:pypi/flask@2.0.3", "pkg:pypi/h11@0.13.0", "pkg:pypi/idna@2.10", - "pkg:pypi/immutables@0.20", + "pkg:pypi/immutables@0.21", "pkg:pypi/importlib-metadata@4.8.3", "pkg:pypi/itsdangerous@2.0.1", "pkg:pypi/jinja2@3.0.3", @@ -269,7 +269,7 @@ "dependsOn": [] }, { - "ref": "pkg:pypi/immutables@0.20", + "ref": "pkg:pypi/immutables@0.21", "dependsOn": [] }, { diff --git a/test/providers/tst_manifests/pip/pip_requirements_virtual_env_with_ignore/expected_stack_sbom.json b/test/providers/tst_manifests/pip/pip_requirements_virtual_env_with_ignore/expected_stack_sbom.json index f0720987..d4bdeb88 100644 --- a/test/providers/tst_manifests/pip/pip_requirements_virtual_env_with_ignore/expected_stack_sbom.json +++ b/test/providers/tst_manifests/pip/pip_requirements_virtual_env_with_ignore/expected_stack_sbom.json @@ -150,10 +150,10 @@ }, { "name": "immutables", - "version": "0.20", - "purl": "pkg:pypi/immutables@0.20", + "version": "0.21", + "purl": "pkg:pypi/immutables@0.21", "type": "library", - "bom-ref": "pkg:pypi/immutables@0.20" + "bom-ref": "pkg:pypi/immutables@0.21" }, { "name": "importlib-metadata", @@ -212,7 +212,7 @@ "pkg:pypi/flask@2.0.3", "pkg:pypi/h11@0.13.0", "pkg:pypi/idna@2.10", - "pkg:pypi/immutables@0.20", + "pkg:pypi/immutables@0.21", "pkg:pypi/importlib-metadata@4.8.3", "pkg:pypi/itsdangerous@2.0.1", "pkg:pypi/jinja2@3.0.3", @@ -317,7 +317,7 @@ "dependsOn": [] }, { - "ref": "pkg:pypi/immutables@0.20", + "ref": "pkg:pypi/immutables@0.21", "dependsOn": [] }, { diff --git a/test/providers/tst_manifests/pip/pip_requirements_virtual_env_with_ignore/requirements.txt b/test/providers/tst_manifests/pip/pip_requirements_virtual_env_with_ignore/requirements.txt index 73f1d7cd..cffbfc2d 100644 --- a/test/providers/tst_manifests/pip/pip_requirements_virtual_env_with_ignore/requirements.txt +++ b/test/providers/tst_manifests/pip/pip_requirements_virtual_env_with_ignore/requirements.txt @@ -9,7 +9,7 @@ fastapi==0.75.1 Flask==2.0.3 h11==0.13.0 idna==2.10 -immutables==0.20 +immutables==0.21 importlib-metadata==4.8.3 itsdangerous==2.0.1 Jinja2==3.0.3 diff --git a/test/providers/tst_manifests/pyproject/poetry_only_deps/expected_stack_sbom.json b/test/providers/tst_manifests/pyproject/poetry_only_deps/expected_stack_sbom.json index 4564931d..e1bce3f7 100644 --- a/test/providers/tst_manifests/pyproject/poetry_only_deps/expected_stack_sbom.json +++ b/test/providers/tst_manifests/pyproject/poetry_only_deps/expected_stack_sbom.json @@ -74,19 +74,6 @@ } ] }, - { - "name": "importlib-metadata", - "version": "8.5.0", - "purl": "pkg:pypi/importlib-metadata@8.5.0", - "type": "library", - "bom-ref": "pkg:pypi/importlib-metadata@8.5.0", - "hashes": [ - { - "alg": "SHA-256", - "content": "71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7" - } - ] - }, { "name": "itsdangerous", "version": "2.2.0", @@ -126,19 +113,6 @@ } ] }, - { - "name": "zipp", - "version": "3.20.2", - "purl": "pkg:pypi/zipp@3.20.2", - "type": "library", - "bom-ref": "pkg:pypi/zipp@3.20.2", - "hashes": [ - { - "alg": "SHA-256", - "content": "bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29" - } - ] - }, { "name": "markupsafe", "version": "2.1.5", @@ -223,7 +197,6 @@ "dependsOn": [ "pkg:pypi/blinker@1.8.2", "pkg:pypi/click@8.1.8", - "pkg:pypi/importlib-metadata@8.5.0", "pkg:pypi/itsdangerous@2.2.0", "pkg:pypi/jinja2@3.1.6", "pkg:pypi/werkzeug@3.0.6" @@ -242,12 +215,6 @@ "ref": "pkg:pypi/blinker@1.8.2", "dependsOn": [] }, - { - "ref": "pkg:pypi/importlib-metadata@8.5.0", - "dependsOn": [ - "pkg:pypi/zipp@3.20.2" - ] - }, { "ref": "pkg:pypi/itsdangerous@2.2.0", "dependsOn": [] @@ -264,10 +231,6 @@ "pkg:pypi/markupsafe@2.1.5" ] }, - { - "ref": "pkg:pypi/zipp@3.20.2", - "dependsOn": [] - }, { "ref": "pkg:pypi/markupsafe@2.1.5", "dependsOn": []