From ed2270d5f7944f129345b04241fdddcbb5c96ae7 Mon Sep 17 00:00:00 2001 From: Luka Rukhadze Date: Wed, 25 Jan 2023 19:45:24 +0400 Subject: [PATCH 1/5] feat: add ability to work with more than 20000 workItems Changes: 'AzureDevopsClient.cs' --- .../Client/AzureDevopsClient.cs | 89 ++++++++++++++----- 1 file changed, 67 insertions(+), 22 deletions(-) diff --git a/GherkinSyncTool.Synchronizers.AzureDevOps/Client/AzureDevopsClient.cs b/GherkinSyncTool.Synchronizers.AzureDevOps/Client/AzureDevopsClient.cs index f3cfdef..33dc5df 100644 --- a/GherkinSyncTool.Synchronizers.AzureDevOps/Client/AzureDevopsClient.cs +++ b/GherkinSyncTool.Synchronizers.AzureDevOps/Client/AzureDevopsClient.cs @@ -51,38 +51,28 @@ public WitBatchRequest BuildUpdateTestCaseBatchRequest(int id, JsonPatchDocument public IEnumerable GetAllTestCasesIds() { - var workItemTrackingHttpClient = GetWorkItemTrackingHttpClient(); - - // wiql - Work Item Query Language - var wiql = new Wiql - { - Query = $@"Select [{WorkItemFields.Id}] + var query = $@"Select [{WorkItemFields.Id}] From WorkItems - Where [System.WorkItemType] = '{WorkItemTypes.TestCase}'" - }; - - var workItemIds = workItemTrackingHttpClient.QueryByWiqlAsync(wiql, _azureDevopsSettings.Project).Result; + Where [System.WorkItemType] = '{WorkItemTypes.TestCase}'"; + + var workItems = GetListOfWorkItemsWithQuery(query); - return workItemIds.WorkItems.Select(reference => reference.Id); + return workItems.Select(reference => reference.Id); } public IEnumerable GetSyncedTestCasesIds() { - var workItemTrackingHttpClient = GetWorkItemTrackingHttpClient(); - // wiql - Work Item Query Language - var wiql = new Wiql - { - Query = $@"Select [{WorkItemFields.Id}] + var query = $@"Select [{WorkItemFields.Id}] From WorkItems Where [System.WorkItemType] = '{WorkItemTypes.TestCase}' AND [{WorkItemFields.State}] <> '{TestCaseState.Closed}' - AND [{WorkItemFields.Tags}] Contains '{Tags.GherkinSyncToolIdTagPrefix + _azureDevopsSettings.GherkinSyncToolId}'" - }; - - var workItemIds = workItemTrackingHttpClient.QueryByWiqlAsync(wiql, _azureDevopsSettings.Project).Result; + AND [{WorkItemFields.Tags}] + Contains '{Tags.GherkinSyncToolIdTagPrefix + _azureDevopsSettings.GherkinSyncToolId}'"; + + var workItems = GetListOfWorkItemsWithQuery(query); - return workItemIds.WorkItems.Select(reference => reference.Id); + return workItems.Select(reference => reference.Id); } public List ExecuteWorkItemBatch(List request) @@ -205,5 +195,60 @@ private WorkItemTrackingHttpClient GetWorkItemTrackingHttpClient() return workItemTrackingHttpClient; } + + private IEnumerable GetListOfWorkItemsWithQuery(string query) + { + var results = new List(); + var workItemTrackingHttpClient = GetWorkItemTrackingHttpClient(); + var counter = 10000; + var moreResults = true; + + while (moreResults) + { + // wiql - Work Item Query Language + var wiql = new Wiql + { + Query = $@"{query} + AND [{WorkItemFields.Id}] >= {counter - 10000} + AND [{WorkItemFields.Id}] < {counter}" + }; + + var currentResults = workItemTrackingHttpClient.QueryByWiqlAsync(wiql, _azureDevopsSettings.Project) + .Result.WorkItems.ToList(); + + if (currentResults.Count == 0) + { + try + { + results.AddRange(workItemTrackingHttpClient.QueryByWiqlAsync(new Wiql + { + Query = $@"{query} AND {WorkItemFields.Id} >= {counter}" + }).Result.WorkItems.ToList()); + + moreResults = false; + } + catch(Exception e) + { + if (e.ToString().Contains("VS402337")) + { + // If this exception persists, it means that there are more, than 20000 workItems left, + // so increment and continue + } + else + { + throw; + } + } + } + else + { + results.AddRange(currentResults); + } + + counter += 10000; + } + + return results; + } } -} \ No newline at end of file +} From f0f015b4c541346552043166969c03428a34c3a3 Mon Sep 17 00:00:00 2001 From: Luka Rukhadze Date: Wed, 25 Jan 2023 19:46:04 +0400 Subject: [PATCH 2/5] chore: change assembly version and changelog.md --- CHANGELOG.md | 4 ++-- GherkinSyncTool/GherkinSyncTool.csproj | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c69cf9b..2d67e0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -[3.20.2] - 2022-12-08 +[3.20.3] - 2023-01-25 ### Added -- TestRail: Ability to work with multiple test suite projects. \ No newline at end of file +- Azure DevOps: Ability to work with more, than 20000 work items \ No newline at end of file diff --git a/GherkinSyncTool/GherkinSyncTool.csproj b/GherkinSyncTool/GherkinSyncTool.csproj index 07bedbe..ad4dcd6 100644 --- a/GherkinSyncTool/GherkinSyncTool.csproj +++ b/GherkinSyncTool/GherkinSyncTool.csproj @@ -1,7 +1,7 @@ - 3.20.2 + 3.20.3 Exe netcoreapp3.1;net5.0;net6.0 true From 4a08e393c4248e2a248689e6762dff1daf84ed5c Mon Sep 17 00:00:00 2001 From: Luka Rukhadze Date: Thu, 26 Jan 2023 14:31:36 +0400 Subject: [PATCH 3/5] refactor: change 'GetListOfWorkItemsWithQuery()' method Changes .Count to .Any(); Changes return type to 'IList<>' --- .../Client/AzureDevopsClient.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/GherkinSyncTool.Synchronizers.AzureDevOps/Client/AzureDevopsClient.cs b/GherkinSyncTool.Synchronizers.AzureDevOps/Client/AzureDevopsClient.cs index 33dc5df..ad5a0a5 100644 --- a/GherkinSyncTool.Synchronizers.AzureDevOps/Client/AzureDevopsClient.cs +++ b/GherkinSyncTool.Synchronizers.AzureDevOps/Client/AzureDevopsClient.cs @@ -196,7 +196,7 @@ private WorkItemTrackingHttpClient GetWorkItemTrackingHttpClient() return workItemTrackingHttpClient; } - private IEnumerable GetListOfWorkItemsWithQuery(string query) + private IList GetListOfWorkItemsWithQuery(string query) { var results = new List(); var workItemTrackingHttpClient = GetWorkItemTrackingHttpClient(); @@ -216,13 +216,13 @@ AND [{WorkItemFields.Id}] < {counter}" var currentResults = workItemTrackingHttpClient.QueryByWiqlAsync(wiql, _azureDevopsSettings.Project) .Result.WorkItems.ToList(); - if (currentResults.Count == 0) + if (!currentResults.Any()) { try { results.AddRange(workItemTrackingHttpClient.QueryByWiqlAsync(new Wiql { - Query = $@"{query} AND {WorkItemFields.Id} >= {counter}" + Query = $@"{query} AND [{WorkItemFields.Id}] >= {counter}" }).Result.WorkItems.ToList()); moreResults = false; From 19958af039c59949980aefe52cdd4559f8468d7d Mon Sep 17 00:00:00 2001 From: Luka Rukhadze Date: Thu, 23 Feb 2023 12:33:07 +0400 Subject: [PATCH 4/5] feat: change the logic by using "top" modifier --- .../Client/AzureDevopsClient.cs | 73 ++++++++----------- 1 file changed, 32 insertions(+), 41 deletions(-) diff --git a/GherkinSyncTool.Synchronizers.AzureDevOps/Client/AzureDevopsClient.cs b/GherkinSyncTool.Synchronizers.AzureDevOps/Client/AzureDevopsClient.cs index ad5a0a5..4251968 100644 --- a/GherkinSyncTool.Synchronizers.AzureDevOps/Client/AzureDevopsClient.cs +++ b/GherkinSyncTool.Synchronizers.AzureDevOps/Client/AzureDevopsClient.cs @@ -53,13 +53,14 @@ public IEnumerable GetAllTestCasesIds() { var query = $@"Select [{WorkItemFields.Id}] From WorkItems - Where [System.WorkItemType] = '{WorkItemTypes.TestCase}'"; - + Where [System.WorkItemType] = '{WorkItemTypes.TestCase}' + AND [System.TeamProject] = '{_azureDevopsSettings.Project}'"; + var workItems = GetListOfWorkItemsWithQuery(query); return workItems.Select(reference => reference.Id); } - + public IEnumerable GetSyncedTestCasesIds() { @@ -67,9 +68,10 @@ public IEnumerable GetSyncedTestCasesIds() From WorkItems Where [System.WorkItemType] = '{WorkItemTypes.TestCase}' AND [{WorkItemFields.State}] <> '{TestCaseState.Closed}' - AND [{WorkItemFields.Tags}] + AND [System.TeamProject] = '{_azureDevopsSettings.Project}' + AND [{WorkItemFields.Tags}] Contains '{Tags.GherkinSyncToolIdTagPrefix + _azureDevopsSettings.GherkinSyncToolId}'"; - + var workItems = GetListOfWorkItemsWithQuery(query); return workItems.Select(reference => reference.Id); @@ -146,8 +148,10 @@ private List SendWorkItemBatch(List request) if (witBatchResponse.Code != 200) { - Log.Error($"Something went wrong with the test case synchronization. Title: {request[i].GetFields()[WorkItemFields.Title]}"); - Log.Error($"Status code: {witBatchResponse.Code}{Environment.NewLine}Body: {witBatchResponse.Body}"); + Log.Error( + $"Something went wrong with the test case synchronization. Title: {request[i].GetFields()[WorkItemFields.Title]}"); + Log.Error( + $"Status code: {witBatchResponse.Code}{Environment.NewLine}Body: {witBatchResponse.Body}"); _context.IsRunSuccessful = false; continue; @@ -200,52 +204,39 @@ private IList GetListOfWorkItemsWithQuery(string query) { var results = new List(); var workItemTrackingHttpClient = GetWorkItemTrackingHttpClient(); - var counter = 10000; var moreResults = true; - + var lastIdInList = 0; + while (moreResults) { - // wiql - Work Item Query Language - var wiql = new Wiql + var wiql = new Wiql() { Query = $@"{query} - AND [{WorkItemFields.Id}] >= {counter - 10000} - AND [{WorkItemFields.Id}] < {counter}" + And [{WorkItemFields.Id}] > {lastIdInList} + ORDER BY [{WorkItemFields.Id}] ASC" }; - - var currentResults = workItemTrackingHttpClient.QueryByWiqlAsync(wiql, _azureDevopsSettings.Project) + + //Max number of workItems is 19999, until it gets fixed from the microsoft side + var currentResults = workItemTrackingHttpClient + .QueryByWiqlAsync(wiql, _azureDevopsSettings.Project, top: 19999) .Result.WorkItems.ToList(); - - if (!currentResults.Any()) + + var currentResultsLength = currentResults.Count; + + if (currentResultsLength < 19999) { - try - { - results.AddRange(workItemTrackingHttpClient.QueryByWiqlAsync(new Wiql - { - Query = $@"{query} AND [{WorkItemFields.Id}] >= {counter}" - }).Result.WorkItems.ToList()); - - moreResults = false; - } - catch(Exception e) - { - if (e.ToString().Contains("VS402337")) - { - // If this exception persists, it means that there are more, than 20000 workItems left, - // so increment and continue - } - else - { - throw; - } - } + moreResults = false; } - else + + var lastItemInList = currentResults.LastOrDefault(); + + if (lastItemInList == null) { - results.AddRange(currentResults); + break; } - counter += 10000; + lastIdInList = lastItemInList.Id; + results.AddRange(currentResults); } return results; From cff238c119d9b63f66360b8856b6a506c8df7ee2 Mon Sep 17 00:00:00 2001 From: Luka Rukhadze Date: Tue, 28 Feb 2023 17:00:46 +0400 Subject: [PATCH 5/5] chore: update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d67e0d..89f48e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -[3.20.3] - 2023-01-25 +[3.20.3] - 2023-02-28 ### Added - Azure DevOps: Ability to work with more, than 20000 work items \ No newline at end of file