diff --git a/docs/detectors/uv.md b/docs/detectors/uv.md index e9835f4b7..88c46c3f3 100644 --- a/docs/detectors/uv.md +++ b/docs/detectors/uv.md @@ -1,6 +1,20 @@ # uv Detection + ## Requirements + [uv](https://docs.astral.sh/uv/) detection relies on a [uv.lock](https://docs.astral.sh/uv/concepts/projects/layout/#the-lockfile) file being present. ## Detection strategy + uv detection is performed by parsing a uv.lock found under the scan directory. + +Full dependency graph generation is supported. + +Dev dependencies across all dependency groups (e.g., `dev`, `lint`, `test`) are identified via transitive reachability analysis. A package reachable from both production and dev roots is classified as non-dev. + +Git-sourced packages are registered as `GitComponent` with the repository URL and commit hash extracted from the lockfile. + +## Known limitations + +1. Editable (`source = { editable = "..." }`) and non-root workspace member packages are registered as regular components rather than being filtered out. +2. Lockfile version validation is not performed; only lockfile version 1 has been tested. diff --git a/src/Microsoft.ComponentDetection.Detectors/uv/UvLock.cs b/src/Microsoft.ComponentDetection.Detectors/uv/UvLock.cs index 0f61f51d7..5734734c9 100644 --- a/src/Microsoft.ComponentDetection.Detectors/uv/UvLock.cs +++ b/src/Microsoft.ComponentDetection.Detectors/uv/UvLock.cs @@ -84,6 +84,7 @@ internal static List ParsePackagesFromModel(object? model) { Registry = sourceTable.TryGetValue("registry", out var regObj) && regObj is string reg ? reg : null, Virtual = sourceTable.TryGetValue("virtual", out var virtObj) && virtObj is string virt ? virt : null, + Git = sourceTable.TryGetValue("git", out var gitObj) && gitObj is string git ? git : null, }; uvPackage.Source = source; } @@ -133,9 +134,12 @@ internal static void ParseMetadata(TomlTable? metadataTable, UvPackage uvPackage if (metadataTable.TryGetValue("requires-dev", out var requiresDevObj) && requiresDevObj is TomlTable requiresDevTable) { - if (requiresDevTable.TryGetValue("dev", out var devObj) && devObj is TomlArray devArr) + foreach (var kvp in requiresDevTable) { - uvPackage.MetadataRequiresDev = ParseDependenciesArray(devArr); + if (kvp.Value is TomlArray groupArr) + { + uvPackage.MetadataRequiresDev.AddRange(ParseDependenciesArray(groupArr)); + } } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/uv/UvLockComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/uv/UvLockComponentDetector.cs index 4b1ec4256..3e8484c84 100644 --- a/src/Microsoft.ComponentDetection.Detectors/uv/UvLockComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/uv/UvLockComponentDetector.cs @@ -26,7 +26,7 @@ public UvLockComponentDetector( public override IList SearchPatterns { get; } = ["uv.lock"]; - public override IEnumerable SupportedComponentTypes => [ComponentType.Pip]; + public override IEnumerable SupportedComponentTypes => [ComponentType.Pip, ComponentType.Git]; public override int Version => 1; @@ -37,6 +37,40 @@ internal static bool IsRootPackage(UvPackage pck) return pck.Source?.Virtual != null; } + internal static (Uri RepositoryUrl, string CommitHash) ParseGitUrl(string gitUrl) + { + var uri = new Uri(gitUrl); + var repoUrl = new Uri(uri.GetLeftPart(UriPartial.Path)); + var commitHash = uri.Fragment.TrimStart('#'); + return (repoUrl, commitHash); + } + + internal static HashSet GetTransitivePackages(IEnumerable roots, List packages) + { + var lookup = packages.ToDictionary(p => p.Name, p => p, StringComparer.OrdinalIgnoreCase); + var visited = new HashSet(StringComparer.OrdinalIgnoreCase); + var queue = new Queue(roots); + + while (queue.Count > 0) + { + var name = queue.Dequeue(); + if (!visited.Add(name)) + { + continue; + } + + if (lookup.TryGetValue(name, out var pkg)) + { + foreach (var dep in pkg.Dependencies) + { + queue.Enqueue(dep.Name); + } + } + } + + return visited; + } + protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, CancellationToken cancellationToken = default) { var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; @@ -49,8 +83,8 @@ protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDiction var uvLock = UvLock.Parse(file.Stream); var rootPackage = uvLock.Packages.FirstOrDefault(IsRootPackage); - var explicitPackages = new HashSet(); - var devPackages = new HashSet(); + var explicitPackages = new HashSet(StringComparer.OrdinalIgnoreCase); + var devRootNames = new HashSet(StringComparer.OrdinalIgnoreCase); if (rootPackage != null) { @@ -61,10 +95,17 @@ protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDiction foreach (var devDep in rootPackage.MetadataRequiresDev) { - devPackages.Add(devDep.Name); + devRootNames.Add(devDep.Name); } } + // Compute dev-only packages via transitive reachability analysis. + // A package is dev-only if it is reachable from dev roots but NOT from production roots. + var prodRoots = rootPackage?.Dependencies.Select(d => d.Name) ?? []; + var prodTransitive = GetTransitivePackages(prodRoots, uvLock.Packages); + var devTransitive = GetTransitivePackages(devRootNames, uvLock.Packages); + var devOnlyPackages = new HashSet(devTransitive.Except(prodTransitive), StringComparer.OrdinalIgnoreCase); + foreach (var pkg in uvLock.Packages) { if (IsRootPackage(pkg)) @@ -72,10 +113,21 @@ protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDiction continue; } - var pipComponent = new PipComponent(pkg.Name, pkg.Version); var isExplicit = explicitPackages.Contains(pkg.Name); - var isDev = devPackages.Contains(pkg.Name); - var detectedComponent = new DetectedComponent(pipComponent); + var isDev = devOnlyPackages.Contains(pkg.Name); + + TypedComponent component; + if (pkg.Source?.Git != null) + { + var (repoUrl, commitHash) = ParseGitUrl(pkg.Source.Git); + component = new GitComponent(repoUrl, commitHash); + } + else + { + component = new PipComponent(pkg.Name, pkg.Version); + } + + var detectedComponent = new DetectedComponent(component); singleFileComponentRecorder.RegisterUsage(detectedComponent, isDevelopmentDependency: isDev, isExplicitReferencedDependency: isExplicit); foreach (var dep in pkg.Dependencies) @@ -83,8 +135,17 @@ protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDiction var depPkg = uvLock.Packages.FirstOrDefault(p => p.Name.Equals(dep.Name, StringComparison.OrdinalIgnoreCase)); if (depPkg != null) { - var depComponentWithVersion = new PipComponent(depPkg.Name, depPkg.Version); - singleFileComponentRecorder.RegisterUsage(new DetectedComponent(depComponentWithVersion), parentComponentId: pipComponent.Id); + TypedComponent depComponent; + if (depPkg.Source?.Git != null) + { + var (depRepoUrl, depCommitHash) = ParseGitUrl(depPkg.Source.Git); + depComponent = new GitComponent(depRepoUrl, depCommitHash); + } + else + { + depComponent = new PipComponent(depPkg.Name, depPkg.Version); + } + singleFileComponentRecorder.RegisterUsage(new DetectedComponent(depComponent), parentComponentId: component.Id, isDevelopmentDependency: isDev); } else { diff --git a/src/Microsoft.ComponentDetection.Detectors/uv/UvSource.cs b/src/Microsoft.ComponentDetection.Detectors/uv/UvSource.cs index 80ea4ba15..6e6f8d01e 100644 --- a/src/Microsoft.ComponentDetection.Detectors/uv/UvSource.cs +++ b/src/Microsoft.ComponentDetection.Detectors/uv/UvSource.cs @@ -5,4 +5,6 @@ public class UvSource public string? Registry { get; set; } public string? Virtual { get; set; } + + public string? Git { get; set; } } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/UvLockDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/UvLockDetectorTests.cs index 0cb0879a1..cfe8749f4 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/UvLockDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/UvLockDetectorTests.cs @@ -267,4 +267,234 @@ public async Task TestUvLockDetector_DevelopmentAndNonDevelopmentDependencies() graph.IsDevelopmentDependency(bazId).Should().BeFalse(); graph.IsDevelopmentDependency(devonlyId).Should().BeTrue(); } + + [TestMethod] + public async Task TestUvLockDetector_MultipleDevGroups_AllMarkedDevAsync() + { + var uvLock = @"[[package]] +name = 'myproject' +version = '0.1.0' +source = { virtual = '.' } +dependencies = [ + { name = 'requests' }, +] +[package.metadata] +requires-dist = [ + { name = 'requests', specifier = '>=2.0' }, +] +[package.metadata.requires-dev] +dev = [ + { name = 'pytest', specifier = '>=8.0' }, +] +lint = [ + { name = 'ruff', specifier = '>=0.4' }, +] +test = [ + { name = 'coverage', specifier = '>=7.0' }, +] +[[package]] +name = 'requests' +version = '2.32.0' +[[package]] +name = 'pytest' +version = '8.0.0' +[[package]] +name = 'ruff' +version = '0.4.0' +[[package]] +name = 'coverage' +version = '7.0.0' +"; + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("uv.lock", uvLock) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var detected = componentRecorder.GetDetectedComponents().ToList(); + var graph = componentRecorder.GetDependencyGraphsByLocation().Values.First(); + + detected.Should().HaveCount(4); + + graph.IsDevelopmentDependency(new PipComponent("requests", "2.32.0").Id).Should().BeFalse(); + graph.IsDevelopmentDependency(new PipComponent("pytest", "8.0.0").Id).Should().BeTrue(); + graph.IsDevelopmentDependency(new PipComponent("ruff", "0.4.0").Id).Should().BeTrue(); + graph.IsDevelopmentDependency(new PipComponent("coverage", "7.0.0").Id).Should().BeTrue(); + } + + [TestMethod] + public async Task TestUvLockDetector_TransitiveDevDeps_MarkedDevAsync() + { + var uvLock = @"[[package]] +name = 'myproject' +version = '0.1.0' +source = { virtual = '.' } +dependencies = [ + { name = 'flask' }, +] +[package.metadata] +requires-dist = [ + { name = 'flask', specifier = '>=3.0' }, +] +[package.metadata.requires-dev] +dev = [ + { name = 'pytest', specifier = '>=8.0' }, +] +[[package]] +name = 'flask' +version = '3.0.0' +[[package]] +name = 'pytest' +version = '8.0.0' +dependencies = [ + { name = 'pluggy' }, +] +[[package]] +name = 'pluggy' +version = '1.5.0' +"; + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("uv.lock", uvLock) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var graph = componentRecorder.GetDependencyGraphsByLocation().Values.First(); + + // flask is a production dependency + graph.IsDevelopmentDependency(new PipComponent("flask", "3.0.0").Id).Should().BeFalse(); + + // pytest is a direct dev dependency + graph.IsDevelopmentDependency(new PipComponent("pytest", "8.0.0").Id).Should().BeTrue(); + + // pluggy is a transitive dep of pytest only — should be dev + graph.IsDevelopmentDependency(new PipComponent("pluggy", "1.5.0").Id).Should().BeTrue(); + } + + [TestMethod] + public async Task TestUvLockDetector_SharedTransitiveDep_NotMarkedDevAsync() + { + var uvLock = @"[[package]] +name = 'myproject' +version = '0.1.0' +source = { virtual = '.' } +dependencies = [ + { name = 'flask' }, +] +[package.metadata] +requires-dist = [ + { name = 'flask', specifier = '>=3.0' }, +] +[package.metadata.requires-dev] +dev = [ + { name = 'pytest', specifier = '>=8.0' }, +] +[[package]] +name = 'flask' +version = '3.0.0' +dependencies = [ + { name = 'click' }, +] +[[package]] +name = 'pytest' +version = '8.0.0' +dependencies = [ + { name = 'click' }, + { name = 'pluggy' }, +] +[[package]] +name = 'click' +version = '8.1.0' +[[package]] +name = 'pluggy' +version = '1.5.0' +"; + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("uv.lock", uvLock) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var graph = componentRecorder.GetDependencyGraphsByLocation().Values.First(); + + // click is shared between flask (prod) and pytest (dev) — should NOT be dev + graph.IsDevelopmentDependency(new PipComponent("click", "8.1.0").Id).Should().BeFalse(); + + // pluggy is only reachable from pytest (dev) — should be dev + graph.IsDevelopmentDependency(new PipComponent("pluggy", "1.5.0").Id).Should().BeTrue(); + } + + [TestMethod] + public async Task TestUvLockDetector_GitSource_RegistersGitComponentAsync() + { + var uvLock = @"[[package]] +name = 'myproject' +version = '0.1.0' +source = { virtual = '.' } +dependencies = [ + { name = 'httpx' }, +] +[package.metadata] +requires-dist = [ + { name = 'httpx' }, +] +[[package]] +name = 'httpx' +version = '0.27.0' +source = { git = 'https://github.com/encode/httpx?tag=0.27.0#abc123def456abc123def456abc123def456abcd' } +"; + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("uv.lock", uvLock) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var detected = componentRecorder.GetDetectedComponents().ToList(); + detected.Should().ContainSingle(); + + var component = detected.First().Component; + component.Should().BeOfType(); + var gitComponent = (GitComponent)component; + gitComponent.RepositoryUrl.Should().Be(new System.Uri("https://github.com/encode/httpx")); + gitComponent.CommitHash.Should().Be("abc123def456abc123def456abc123def456abcd"); + } + + [TestMethod] + public async Task TestUvLockDetector_MixedRegistryAndGitSources_CorrectTypesAsync() + { + var uvLock = @"[[package]] +name = 'myproject' +version = '0.1.0' +source = { virtual = '.' } +dependencies = [ + { name = 'requests' }, + { name = 'httpx' }, +] +[package.metadata] +requires-dist = [ + { name = 'requests', specifier = '>=2.0' }, + { name = 'httpx' }, +] +[[package]] +name = 'requests' +version = '2.32.0' +source = { registry = 'https://pypi.org/simple' } +[[package]] +name = 'httpx' +version = '0.27.0' +source = { git = 'https://github.com/encode/httpx?tag=0.27.0#aabbccdd11223344aabbccdd11223344aabbccdd' } +"; + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("uv.lock", uvLock) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var detected = componentRecorder.GetDetectedComponents().ToList(); + detected.Should().HaveCount(2); + + var pipComponents = detected.Where(d => d.Component is PipComponent).ToList(); + var gitComponents = detected.Where(d => d.Component is GitComponent).ToList(); + + pipComponents.Should().ContainSingle(); + gitComponents.Should().ContainSingle(); + + ((PipComponent)pipComponents.First().Component).Name.Should().Be("requests"); + ((GitComponent)gitComponents.First().Component).RepositoryUrl.Should().Be(new System.Uri("https://github.com/encode/httpx")); + } }