diff --git a/docs/sbom-tool-arguments.md b/docs/sbom-tool-arguments.md index 44095822a..e938ec373 100644 --- a/docs/sbom-tool-arguments.md +++ b/docs/sbom-tool-arguments.md @@ -66,6 +66,9 @@ Actions DeleteManifestDirIfPresent (-D) If set to true, we will delete any previous manifest directories that are already present in the ManifestDirPath without asking the user for confirmation. The new manifest directory will then be created at this location and the generated SBOM will be stored there. FetchLicenseInformation (-li) If set to true, we will attempt to fetch license information of packages detected in the SBOM from the ClearlyDefinedApi. + LicenseInformationTimeoutInSeconds (-lto) Specifies the timeout in seconds for fetching the license information. Defaults to 30 seconds. Has no effect if + FetchLicenseInformation (-li) argument is false or not provided. Valid values are from 1 to 86400. Negative values use the default + value and Values exceeding the maximum are truncated to the maximum. EnablePackageMetadataParsing (-pm) If set to true, we will attempt to parse license and supplier info from the packages metadata file (RubyGems, NuGet, Maven, Npm). Verbosity (-V) Display this amount of detail in the logging output. Verbose diff --git a/src/Microsoft.Sbom.Api/Config/Args/GenerationArgs.cs b/src/Microsoft.Sbom.Api/Config/Args/GenerationArgs.cs index 3d87b243a..e711c0d72 100644 --- a/src/Microsoft.Sbom.Api/Config/Args/GenerationArgs.cs +++ b/src/Microsoft.Sbom.Api/Config/Args/GenerationArgs.cs @@ -116,6 +116,16 @@ public class GenerationArgs : GenerationAndValidationCommonArgs [ArgDescription("If set to true, we will attempt to fetch license information of packages detected in the SBOM from the ClearlyDefinedApi.")] public bool? FetchLicenseInformation { get; set; } + /// + /// Specifies the timeout in seconds for fetching the license information. Defaults to 30 seconds. Has no effect if + /// FetchLicenseInformation (li) argument is false or not provided. A negative value corresponds to an infinite timeout. + /// + [ArgShortcut("lto")] + [ArgDescription("Specifies the timeout in seconds for fetching the license information. Defaults to 30 seconds. " + + "Has no effect if the FetchLicenseInformation (li) argument is false or not provided. A negative value corresponds" + + "to an infinite timeout.")] + public int? LicenseInformationTimeoutInSeconds { get; set; } + /// /// If set to true, we will attempt to parse license and supplier info from the packages metadata file. /// diff --git a/src/Microsoft.Sbom.Api/Config/ConfigSanitizer.cs b/src/Microsoft.Sbom.Api/Config/ConfigSanitizer.cs index e457589b0..1ab6ccc19 100644 --- a/src/Microsoft.Sbom.Api/Config/ConfigSanitizer.cs +++ b/src/Microsoft.Sbom.Api/Config/ConfigSanitizer.cs @@ -6,6 +6,7 @@ using System.IO; using System.Linq; using System.Reflection; +using Microsoft.Sbom.Api.Executors; using Microsoft.Sbom.Api.Hashing; using Microsoft.Sbom.Api.Utils; using Microsoft.Sbom.Common; @@ -81,6 +82,30 @@ public IConfiguration SanitizeConfig(IConfiguration configuration) // Set default package supplier if not provided in configuration. configuration.PackageSupplier = GetPackageSupplierFromAssembly(configuration, logger); + // Prevent null value for LicenseInformationTimeoutInSeconds. + // Values of (0, Constants.MaxLicenseFetchTimeoutInSeconds] are allowed. Negative values are replaced with the default, and + // the higher values are truncated to the maximum of Common.Constants.MaxLicenseFetchTimeoutInSeconds + if (configuration.LicenseInformationTimeoutInSeconds is null) + { + configuration.LicenseInformationTimeoutInSeconds = new(Common.Constants.DefaultLicenseFetchTimeoutInSeconds, SettingSource.Default); + } + else if (configuration.LicenseInformationTimeoutInSeconds.Value <= 0) + { + logger.Warning($"Negative and Zero Values not allowed for timeout. Using the default {Common.Constants.DefaultLicenseFetchTimeoutInSeconds} seconds instead."); + configuration.LicenseInformationTimeoutInSeconds.Value = Common.Constants.DefaultLicenseFetchTimeoutInSeconds; + } + else if (configuration.LicenseInformationTimeoutInSeconds.Value > Common.Constants.MaxLicenseFetchTimeoutInSeconds) + { + logger.Warning($"Specified timeout exceeds maximum allowed. Truncating the timeout to {Common.Constants.MaxLicenseFetchTimeoutInSeconds} seconds."); + configuration.LicenseInformationTimeoutInSeconds.Value = Common.Constants.MaxLicenseFetchTimeoutInSeconds; + } + + // Check if arg -lto is specified but -li is not + if (configuration.FetchLicenseInformation?.Value != true && !configuration.LicenseInformationTimeoutInSeconds.IsDefaultSource) + { + logger.Warning("A license fetching timeout is specified (argument -lto), but this has no effect when FetchLicenseInfo is unspecified or false (argument -li)"); + } + // Replace backslashes in directory paths with the OS-sepcific directory separator character. PathUtils.ConvertToOSSpecificPathSeparators(configuration); diff --git a/src/Microsoft.Sbom.Api/Executors/ComponentDetectionBaseWalker.cs b/src/Microsoft.Sbom.Api/Executors/ComponentDetectionBaseWalker.cs index d4240c50c..e5c1b9e5b 100644 --- a/src/Microsoft.Sbom.Api/Executors/ComponentDetectionBaseWalker.cs +++ b/src/Microsoft.Sbom.Api/Executors/ComponentDetectionBaseWalker.cs @@ -137,7 +137,21 @@ async Task Scan(string path) { licenseInformationRetrieved = true; - var apiResponses = await licenseInformationFetcher.FetchLicenseInformationAsync(listOfComponentsForApi); + List apiResponses; + var licenseInformationFetcher2 = licenseInformationFetcher as ILicenseInformationFetcher2; + if (licenseInformationFetcher2 is null && (bool)!configuration.LicenseInformationTimeoutInSeconds?.IsDefaultSource) + { + log.Warning("Timeout value is specified, but ILicenseInformationFetcher2 is not implemented for the licenseInformationFetcher"); + } + + if (licenseInformationFetcher2 is null || configuration.LicenseInformationTimeoutInSeconds is null) + { + apiResponses = await licenseInformationFetcher.FetchLicenseInformationAsync(listOfComponentsForApi); + } + else + { + apiResponses = await licenseInformationFetcher2.FetchLicenseInformationAsync(listOfComponentsForApi, configuration.LicenseInformationTimeoutInSeconds.Value); + } foreach (var response in apiResponses) { diff --git a/src/Microsoft.Sbom.Api/Executors/ILicenseInformationFetcher.cs b/src/Microsoft.Sbom.Api/Executors/ILicenseInformationFetcher.cs index b5e7a719e..186338cb0 100644 --- a/src/Microsoft.Sbom.Api/Executors/ILicenseInformationFetcher.cs +++ b/src/Microsoft.Sbom.Api/Executors/ILicenseInformationFetcher.cs @@ -18,7 +18,7 @@ public interface ILicenseInformationFetcher List ConvertComponentsToListForApi(IEnumerable scannedComponents); /// - /// Calls the ClearlyDefined API to get the license information for the list of components. + /// Calls the ClearlyDefined API to get the license information for the list of components. Uses a default timeout specified in implementation /// /// A list of strings formatted into a list of strings that can be used to call the batch ClearlyDefined API. /// diff --git a/src/Microsoft.Sbom.Api/Executors/ILicenseInformationFetcher2.cs b/src/Microsoft.Sbom.Api/Executors/ILicenseInformationFetcher2.cs new file mode 100644 index 000000000..26d305ba8 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Executors/ILicenseInformationFetcher2.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.ComponentDetection.Contracts.BcdeModels; + +namespace Microsoft.Sbom.Api.Executors; + +public interface ILicenseInformationFetcher2: ILicenseInformationFetcher +{ + /// + /// Calls the ClearlyDefined API to get the license information for the list of components. + /// + /// A list of strings formatted into a list of strings that can be used to call the batch ClearlyDefined API. + /// Timeout in seconds to use when making web requests + /// + Task> FetchLicenseInformationAsync(List listOfComponentsForApi, int timeoutInSeconds); +} diff --git a/src/Microsoft.Sbom.Api/Executors/ILicenseInformationService2.cs b/src/Microsoft.Sbom.Api/Executors/ILicenseInformationService2.cs new file mode 100644 index 000000000..0e6077be9 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Executors/ILicenseInformationService2.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace Microsoft.Sbom.Api.Executors; + +public interface ILicenseInformationService2 : ILicenseInformationService +{ + public Task> FetchLicenseInformationFromAPI(List listOfComponentsForApi, int timeoutInSeconds); +} diff --git a/src/Microsoft.Sbom.Api/Executors/LicenseInformationFetcher.cs b/src/Microsoft.Sbom.Api/Executors/LicenseInformationFetcher.cs index 47dc6f7d1..bf9c38fe1 100644 --- a/src/Microsoft.Sbom.Api/Executors/LicenseInformationFetcher.cs +++ b/src/Microsoft.Sbom.Api/Executors/LicenseInformationFetcher.cs @@ -14,7 +14,7 @@ namespace Microsoft.Sbom.Api.Executors; -public class LicenseInformationFetcher : ILicenseInformationFetcher +public class LicenseInformationFetcher : ILicenseInformationFetcher2 { private readonly ILogger log; private readonly IRecorder recorder; @@ -87,6 +87,20 @@ public async Task> FetchLicenseInformationAsync(List listOf return await licenseInformationService.FetchLicenseInformationFromAPI(listOfComponentsForApi); } + public async Task> FetchLicenseInformationAsync(List listOfComponentsForApi, int timeoutInSeconds) + { + var licenseInformationService2 = licenseInformationService as ILicenseInformationService2; + if (licenseInformationService2 is null) + { + log.Warning("Timeout is specified in License Fetcher, but licenseInformationService does not implement ILicenseInformationService2"); + return await licenseInformationService.FetchLicenseInformationFromAPI(listOfComponentsForApi); + } + else + { + return await licenseInformationService2.FetchLicenseInformationFromAPI(listOfComponentsForApi, timeoutInSeconds); + } + } + // Will attempt to extract license information from a clearlyDefined batch API response. Will always return a dictionary which may be empty depending on the response. public Dictionary ConvertClearlyDefinedApiResponseToList(string httpResponseContent) { diff --git a/src/Microsoft.Sbom.Api/Executors/LicenseInformationService.cs b/src/Microsoft.Sbom.Api/Executors/LicenseInformationService.cs index c2a2a1504..133ab2330 100644 --- a/src/Microsoft.Sbom.Api/Executors/LicenseInformationService.cs +++ b/src/Microsoft.Sbom.Api/Executors/LicenseInformationService.cs @@ -15,12 +15,11 @@ namespace Microsoft.Sbom.Api.Executors; -public class LicenseInformationService : ILicenseInformationService +public class LicenseInformationService : ILicenseInformationService2 { private readonly ILogger log; private readonly IRecorder recorder; private readonly HttpClient httpClient; - private const int ClientTimeoutSeconds = 30; public LicenseInformationService(ILogger log, IRecorder recorder, HttpClient httpClient) { @@ -30,6 +29,11 @@ public LicenseInformationService(ILogger log, IRecorder recorder, HttpClient htt } public async Task> FetchLicenseInformationFromAPI(List listOfComponentsForApi) + { + return await FetchLicenseInformationFromAPI(listOfComponentsForApi, Common.Constants.MaxLicenseFetchTimeoutInSeconds); + } + + public async Task> FetchLicenseInformationFromAPI(List listOfComponentsForApi, int timeoutInSeconds) { var batchSize = 500; var responses = new List(); @@ -38,7 +42,10 @@ public async Task> FetchLicenseInformationFromAPI(List list var uri = new Uri("https://api.clearlydefined.io/definitions?expand=-files"); httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); - httpClient.Timeout = TimeSpan.FromSeconds(ClientTimeoutSeconds); + if (timeoutInSeconds > 0) + { + httpClient.Timeout = TimeSpan.FromSeconds(timeoutInSeconds); + } // The else cases should be sanitized in Config Sanitizer, and even if not, it'll just use httpClient's default timeout for (var i = 0; i < listOfComponentsForApi.Count; i += batchSize) { diff --git a/src/Microsoft.Sbom.Common/Config/Configuration.cs b/src/Microsoft.Sbom.Common/Config/Configuration.cs index b7cfd6216..aa5789bef 100644 --- a/src/Microsoft.Sbom.Common/Config/Configuration.cs +++ b/src/Microsoft.Sbom.Common/Config/Configuration.cs @@ -47,6 +47,7 @@ public class Configuration : IConfiguration private static readonly AsyncLocal> generationTimestamp = new(); private static readonly AsyncLocal> followSymlinks = new(); private static readonly AsyncLocal> fetchLicenseInformation = new(); + private static readonly AsyncLocal> licenseInformationTimeout = new(); private static readonly AsyncLocal> enablePackageMetadataParsing = new(); private static readonly AsyncLocal> deleteManifestDirIfPresent = new(); private static readonly AsyncLocal> failIfNoPackages = new(); @@ -309,6 +310,14 @@ public ConfigurationSetting FetchLicenseInformation set => fetchLicenseInformation.Value = value; } + /// + [DefaultValue(Constants.DefaultLicenseFetchTimeoutInSeconds)] + public ConfigurationSetting LicenseInformationTimeoutInSeconds + { + get => licenseInformationTimeout.Value; + set => licenseInformationTimeout.Value = value; + } + /// [DefaultValue(false)] public ConfigurationSetting EnablePackageMetadataParsing diff --git a/src/Microsoft.Sbom.Common/Config/IConfiguration.cs b/src/Microsoft.Sbom.Common/Config/IConfiguration.cs index b2ecdc356..d461ce8db 100644 --- a/src/Microsoft.Sbom.Common/Config/IConfiguration.cs +++ b/src/Microsoft.Sbom.Common/Config/IConfiguration.cs @@ -194,6 +194,13 @@ public interface IConfiguration /// ConfigurationSetting FetchLicenseInformation { get; set; } + /// + /// Specifies the timeout in seconds for fetching the license information. Defaults to . + /// Has no effect if FetchLicenseInformation (li) argument is false or not provided. Negative values are set to the default and values exceeding the + /// maximum are truncated to + /// + ConfigurationSetting LicenseInformationTimeoutInSeconds { get; set; } + /// /// If set to true, we will attempt to locate and parse package metadata files for additional information to include in the SBOM such as .nuspec/.pom files in the local package cache. /// diff --git a/src/Microsoft.Sbom.Common/Config/InputConfiguration.cs b/src/Microsoft.Sbom.Common/Config/InputConfiguration.cs index f01022296..151d4ca02 100644 --- a/src/Microsoft.Sbom.Common/Config/InputConfiguration.cs +++ b/src/Microsoft.Sbom.Common/Config/InputConfiguration.cs @@ -141,6 +141,10 @@ public class InputConfiguration : IConfiguration [DefaultValue(false)] public ConfigurationSetting FetchLicenseInformation { get; set; } + /// + [DefaultValue(Constants.DefaultLicenseFetchTimeoutInSeconds)] + public ConfigurationSetting LicenseInformationTimeoutInSeconds { get; set; } + [DefaultValue(false)] public ConfigurationSetting EnablePackageMetadataParsing { get; set; } diff --git a/src/Microsoft.Sbom.Common/Constants.cs b/src/Microsoft.Sbom.Common/Constants.cs index f4fa50769..bb5182193 100644 --- a/src/Microsoft.Sbom.Common/Constants.cs +++ b/src/Microsoft.Sbom.Common/Constants.cs @@ -13,5 +13,8 @@ public static class Constants public const int DefaultParallelism = 8; public const int MaxParallelism = 48; + public const int DefaultLicenseFetchTimeoutInSeconds = 30; + public const int MaxLicenseFetchTimeoutInSeconds = 86400; + public const LogEventLevel DefaultLogLevel = LogEventLevel.Warning; } diff --git a/src/Microsoft.Sbom.Targets/README.md b/src/Microsoft.Sbom.Targets/README.md index f89b8ce69..11382a2ed 100644 --- a/src/Microsoft.Sbom.Targets/README.md +++ b/src/Microsoft.Sbom.Targets/README.md @@ -36,6 +36,7 @@ The custom MSBuild task accepts most of the arguments available for the [SBOM CL | `` | N/A | No | | `` | N/A | No | | `` | `false` | No | +| ``| `30` | No | | `` | `false` | No | | `` | `Information` | No | | `` | `SPDX:2.2` | No | diff --git a/test/Microsoft.Sbom.Api.Tests/Config/ConfigSanitizerTests.cs b/test/Microsoft.Sbom.Api.Tests/Config/ConfigSanitizerTests.cs index 0b6ba5cb7..7d56f8e97 100644 --- a/test/Microsoft.Sbom.Api.Tests/Config/ConfigSanitizerTests.cs +++ b/test/Microsoft.Sbom.Api.Tests/Config/ConfigSanitizerTests.cs @@ -9,6 +9,7 @@ using System.Security.Cryptography; using Microsoft.Sbom.Api.Config; using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Api.Executors; using Microsoft.Sbom.Api.Hashing; using Microsoft.Sbom.Api.Utils; using Microsoft.Sbom.Common; @@ -351,4 +352,48 @@ public void ConfigSantizer_Validate_ReplacesBackslashes_Linux(ManifestToolAction Assert.IsTrue(config.TelemetryFilePath.Value.StartsWith($"/{nameof(config.TelemetryFilePath)}/", StringComparison.Ordinal)); } } + + [TestMethod] + [DataRow(1, DisplayName = "Minimum value of 1")] + [DataRow(Common.Constants.MaxLicenseFetchTimeoutInSeconds, DisplayName = "Maximum Value of 86400")] + public void LicenseInformationTimeoutInSeconds_SanitizeMakesNoChanges(int value) + { + var config = GetConfigurationBaseObject(); + config.LicenseInformationTimeoutInSeconds = new(value, SettingSource.CommandLine); + + configSanitizer.SanitizeConfig(config); + + Assert.AreEqual(value, config.LicenseInformationTimeoutInSeconds.Value, "The value of LicenseInformationTimeoutInSeconds should remain the same through the sanitization process"); + } + + [TestMethod] + [DataRow(int.MinValue, Common.Constants.DefaultLicenseFetchTimeoutInSeconds, DisplayName = "Negative Value is changed to Default")] + [DataRow(0, Common.Constants.DefaultLicenseFetchTimeoutInSeconds, DisplayName = "Zero is changed to Default")] + [DataRow(Common.Constants.MaxLicenseFetchTimeoutInSeconds + 1, Common.Constants.MaxLicenseFetchTimeoutInSeconds, DisplayName = "Max Value + 1 is truncated")] + [DataRow(int.MaxValue, Common.Constants.MaxLicenseFetchTimeoutInSeconds, DisplayName = "int.MaxValue is truncated")] + public void LicenseInformationTimeoutInSeconds_SanitizeExceedsLimits(int value, int expected) + { + var config = GetConfigurationBaseObject(); + config.LicenseInformationTimeoutInSeconds = new(value, SettingSource.CommandLine); + + configSanitizer.SanitizeConfig(config); + + Assert.AreEqual(expected, config.LicenseInformationTimeoutInSeconds.Value, "The value of LicenseInformationTimeoutInSeconds should be sanitized to a valid value"); + } + + [TestMethod] + public void LicenseInformationTimeoutInSeconds_SanitizeNull() + { + var config = GetConfigurationBaseObject(); + config.LicenseInformationTimeoutInSeconds = null; + + configSanitizer.SanitizeConfig(config); + + Assert.AreEqual( + Common.Constants.DefaultLicenseFetchTimeoutInSeconds, + config.LicenseInformationTimeoutInSeconds.Value, + $"The value of LicenseInformationTimeoutInSeconds should be set to {Common.Constants.DefaultLicenseFetchTimeoutInSeconds}s when null"); + + Assert.AreEqual(SettingSource.Default, config.LicenseInformationTimeoutInSeconds.Source, "The source of LicenseInformationTimeoutInSeconds should be set to Default when null"); + } }