From 68e61ef2ef7aead938f77b6c46093a0c9aa5cf54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Wed, 25 Dec 2024 03:37:45 +0100 Subject: [PATCH 01/17] Fixing analyzer violations --- .../Attributes/Behaviors/SetsValueReliablyAttribute.cs | 2 ++ Lombiq.Tests.UI/Attributes/ChromeAttribute.cs | 2 ++ Lombiq.Tests.UI/Attributes/EdgeAttribute.cs | 2 ++ Lombiq.Tests.UI/Attributes/FirefoxAttribute.cs | 2 ++ Lombiq.Tests.UI/Attributes/InternetExplorerAttribute.cs | 2 ++ 5 files changed, 10 insertions(+) diff --git a/Lombiq.Tests.UI/Attributes/Behaviors/SetsValueReliablyAttribute.cs b/Lombiq.Tests.UI/Attributes/Behaviors/SetsValueReliablyAttribute.cs index f75dc95c6..5d7f37ecc 100644 --- a/Lombiq.Tests.UI/Attributes/Behaviors/SetsValueReliablyAttribute.cs +++ b/Lombiq.Tests.UI/Attributes/Behaviors/SetsValueReliablyAttribute.cs @@ -1,9 +1,11 @@ using Atata; using Lombiq.Tests.UI.Extensions; using Lombiq.Tests.UI.Helpers; +using System; namespace Lombiq.Tests.UI.Attributes.Behaviors; +[AttributeUsage(AttributeTargets.Property, AllowMultiple = false)] public sealed class SetsValueReliablyAttribute : ValueSetBehaviorAttribute { public override void Execute(IUIComponent component, string value) // #spell-check-ignore-line diff --git a/Lombiq.Tests.UI/Attributes/ChromeAttribute.cs b/Lombiq.Tests.UI/Attributes/ChromeAttribute.cs index b31f469f2..5e68f8de3 100644 --- a/Lombiq.Tests.UI/Attributes/ChromeAttribute.cs +++ b/Lombiq.Tests.UI/Attributes/ChromeAttribute.cs @@ -1,7 +1,9 @@ using Lombiq.Tests.UI.Services; +using System; namespace Lombiq.Tests.UI.Attributes; +[AttributeUsage(AttributeTargets.Method, AllowMultiple = false, Inherited = true)] public sealed class ChromeAttribute : BrowserAttributeBase { protected override Browser Browser => Browser.Chrome; diff --git a/Lombiq.Tests.UI/Attributes/EdgeAttribute.cs b/Lombiq.Tests.UI/Attributes/EdgeAttribute.cs index b46d4db19..dbf71545b 100644 --- a/Lombiq.Tests.UI/Attributes/EdgeAttribute.cs +++ b/Lombiq.Tests.UI/Attributes/EdgeAttribute.cs @@ -1,7 +1,9 @@ using Lombiq.Tests.UI.Services; +using System; namespace Lombiq.Tests.UI.Attributes; +[AttributeUsage(AttributeTargets.Method, AllowMultiple = false, Inherited = true)] public sealed class EdgeAttribute : BrowserAttributeBase { protected override Browser Browser => Browser.Edge; diff --git a/Lombiq.Tests.UI/Attributes/FirefoxAttribute.cs b/Lombiq.Tests.UI/Attributes/FirefoxAttribute.cs index 86d431890..db767f1c4 100644 --- a/Lombiq.Tests.UI/Attributes/FirefoxAttribute.cs +++ b/Lombiq.Tests.UI/Attributes/FirefoxAttribute.cs @@ -1,7 +1,9 @@ using Lombiq.Tests.UI.Services; +using System; namespace Lombiq.Tests.UI.Attributes; +[AttributeUsage(AttributeTargets.Method, AllowMultiple = false, Inherited = true)] public sealed class FirefoxAttribute : BrowserAttributeBase { protected override Browser Browser => Browser.Firefox; diff --git a/Lombiq.Tests.UI/Attributes/InternetExplorerAttribute.cs b/Lombiq.Tests.UI/Attributes/InternetExplorerAttribute.cs index 28642cb76..f78877e52 100644 --- a/Lombiq.Tests.UI/Attributes/InternetExplorerAttribute.cs +++ b/Lombiq.Tests.UI/Attributes/InternetExplorerAttribute.cs @@ -1,7 +1,9 @@ using Lombiq.Tests.UI.Services; +using System; namespace Lombiq.Tests.UI.Attributes; +[AttributeUsage(AttributeTargets.Method, AllowMultiple = false, Inherited = true)] public sealed class InternetExplorerAttribute : BrowserAttributeBase { protected override Browser Browser => Browser.InternetExplorer; From 5114bfa1e2b1822646bc6828f29266486baeade4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Wed, 25 Dec 2024 04:31:43 +0100 Subject: [PATCH 02/17] Fixing more analyzer violations --- Lombiq.Tests.UI/SecurityScanning/YamlDocumentExtensions.cs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Lombiq.Tests.UI/SecurityScanning/YamlDocumentExtensions.cs b/Lombiq.Tests.UI/SecurityScanning/YamlDocumentExtensions.cs index 8b40687d4..cb628c771 100644 --- a/Lombiq.Tests.UI/SecurityScanning/YamlDocumentExtensions.cs +++ b/Lombiq.Tests.UI/SecurityScanning/YamlDocumentExtensions.cs @@ -346,11 +346,7 @@ public static T GetOrAddNode(this YamlNode yamlNode, string key) /// public static void SetMappingChild(this YamlMappingNode node, YamlNode key, YamlNode value) { - if (node.Children.ContainsKey(key)) - { - node.Children.Remove(key); - } - + node.Children.Remove(key); node.Add(key, value); } From f00452294e8cff6e702b0b394aa1df188bdaa8b4 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 29 Dec 2024 02:26:57 +0000 Subject: [PATCH 03/17] Update All packages --- .../Lombiq.Tests.UI.Shortcuts.csproj | 2 +- Lombiq.Tests.UI/.config/dotnet-tools.json | 2 +- Lombiq.Tests.UI/Lombiq.Tests.UI.csproj | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Lombiq.Tests.UI.Shortcuts/Lombiq.Tests.UI.Shortcuts.csproj b/Lombiq.Tests.UI.Shortcuts/Lombiq.Tests.UI.Shortcuts.csproj index e16bb1149..adaa12926 100644 --- a/Lombiq.Tests.UI.Shortcuts/Lombiq.Tests.UI.Shortcuts.csproj +++ b/Lombiq.Tests.UI.Shortcuts/Lombiq.Tests.UI.Shortcuts.csproj @@ -42,7 +42,7 @@ - + diff --git a/Lombiq.Tests.UI/.config/dotnet-tools.json b/Lombiq.Tests.UI/.config/dotnet-tools.json index ab442af6a..1771f4fe8 100644 --- a/Lombiq.Tests.UI/.config/dotnet-tools.json +++ b/Lombiq.Tests.UI/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "rnwood.smtp4dev": { - "version": "3.1.4", + "version": "3.6.1", "commands": [ "smtp4dev" ] diff --git a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj index 924385452..90fa9251e 100644 --- a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj +++ b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj @@ -65,20 +65,20 @@ - + - + - - + + - + @@ -92,7 +92,7 @@ - + From f8eed91a164e916e1eb5759ada91e29dc1234950 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 29 Dec 2024 02:27:05 +0000 Subject: [PATCH 04/17] Update All packages --- Lombiq.Tests.UI.Shortcuts/Lombiq.Tests.UI.Shortcuts.csproj | 2 +- Lombiq.Tests.UI/Lombiq.Tests.UI.csproj | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lombiq.Tests.UI.Shortcuts/Lombiq.Tests.UI.Shortcuts.csproj b/Lombiq.Tests.UI.Shortcuts/Lombiq.Tests.UI.Shortcuts.csproj index e16bb1149..34cda90b8 100644 --- a/Lombiq.Tests.UI.Shortcuts/Lombiq.Tests.UI.Shortcuts.csproj +++ b/Lombiq.Tests.UI.Shortcuts/Lombiq.Tests.UI.Shortcuts.csproj @@ -42,7 +42,7 @@ - + diff --git a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj index 924385452..51a82f4ba 100644 --- a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj +++ b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj @@ -79,7 +79,7 @@ - + @@ -92,7 +92,7 @@ - + From a1cca1133c698e2aa503ffd67b9f26f8b155d25e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sun, 29 Dec 2024 04:06:23 +0100 Subject: [PATCH 05/17] Removing unneeded package-lock --- Lombiq.Tests.UI/package-lock.json | 39 ------------------------------- 1 file changed, 39 deletions(-) delete mode 100644 Lombiq.Tests.UI/package-lock.json diff --git a/Lombiq.Tests.UI/package-lock.json b/Lombiq.Tests.UI/package-lock.json deleted file mode 100644 index 90f7e30fd..000000000 --- a/Lombiq.Tests.UI/package-lock.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "name": "Lombiq.Tests.UI", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "dependencies": { - "gremlins.js": "2.2.0" - } - }, - "node_modules/chance": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/chance/-/chance-1.1.12.tgz", - "integrity": "sha512-vVBIGQVnwtUG+SYe0ge+3MvF78cvSpuCOEUJr7sVEk2vSBuMW6OXNJjSzdtzrlxNUEaoqH2GBd5Y/+18BEB01Q==", - "license": "MIT" - }, - "node_modules/core-js": { - "version": "3.38.1", - "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.38.1.tgz", - "integrity": "sha512-OP35aUorbU3Zvlx7pjsFdu1rGNnD4pgw/CWoYzRY3t2EzoVT7shKHY1dlAy3f41cGIO7ZDPQimhGFTlEYkG/Hw==", - "hasInstallScript": true, - "license": "MIT", - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/core-js" - } - }, - "node_modules/gremlins.js": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/gremlins.js/-/gremlins.js-2.2.0.tgz", - "integrity": "sha512-zVe4+WuCwTheg1OzBOdtiy7nZj52lWyWjCPMw/78wI1uAJdGlc97hVGYsZg0v75Hm+E91y5D+yG4U0lk7j/6xg==", - "license": "MIT", - "dependencies": { - "chance": "^1.1.4", - "core-js": "^3.6.4" - } - } - } -} From c6a0078ef54f5c26044d8f840ef15f30329e8cbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sun, 29 Dec 2024 05:24:21 +0100 Subject: [PATCH 06/17] Fixing SMTP service startup --- Lombiq.Tests.UI/Services/SmtpService.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lombiq.Tests.UI/Services/SmtpService.cs b/Lombiq.Tests.UI/Services/SmtpService.cs index 68111bea2..abd6994e9 100644 --- a/Lombiq.Tests.UI/Services/SmtpService.cs +++ b/Lombiq.Tests.UI/Services/SmtpService.cs @@ -54,7 +54,7 @@ public async Task StartAsync() // The service depends on the smtp4dev .NET CLI tool (https://github.com/rnwood/smtp4dev) to be installed as a // local tool (on local tools see: https://docs.microsoft.com/en-us/dotnet/core/tools/local-tools-how-to-use). // The local tool manifest was already created with dotnet new tool-manifest and the tool installed with: - // dotnet tool install Rnwood.Smtp4dev --version "3.1.4" + // dotnet tool install Rnwood.Smtp4dev --version "" var dotnetToolsConfigFilePath = Path.Combine(".config", "dotnet-tools.json"); if (!File.Exists(dotnetToolsConfigFilePath)) @@ -104,7 +104,8 @@ public async Task StartAsync() // https://github.com/rnwood/smtp4dev/blob/master/Rnwood.Smtp4dev/Program.cs#L132. await CliProgram.DotNet.GetCommand( "tool", "run", "smtp4dev", "--db", string.Empty, "--smtpport", _smtpPort, "--urls", webUIUri) - .ExecuteDotNetApplicationAsync( + .ExecuteUntilOutputAsync( + "Now listening on:", stdErr => throw new IOException( $"The smtp4dev service didn't start properly on SMTP port {smtpPortString} and web UI port " + From 2ba8251d6ecf829272233ad15292239c157b2b72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sun, 29 Dec 2024 05:24:39 +0100 Subject: [PATCH 07/17] Fixing SMTP inbox row lookup --- Lombiq.Tests.UI/Helpers/ByHelper.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Helpers/ByHelper.cs b/Lombiq.Tests.UI/Helpers/ByHelper.cs index bda240752..a33ec0f22 100644 --- a/Lombiq.Tests.UI/Helpers/ByHelper.cs +++ b/Lombiq.Tests.UI/Helpers/ByHelper.cs @@ -17,7 +17,10 @@ public static class ByHelper /// public static By SmtpInboxRow(string text) => By - .XPath($"//tr[contains(@class,'el-table__row')]//div[contains(@class,'cell')][contains(text(), {JsonSerializer.Serialize(text)})]") + .XPath( + "//tr[contains(@class,'el-table__row')]" + + "//div[contains(@class,'cell')]" + + $"[contains(normalize-space(), {JsonSerializer.Serialize(text)})]") .Within(TimeSpan.FromMinutes(2)); /// From c50496f867f4fe41643bb763cb05904d9afd22f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sun, 29 Dec 2024 05:40:19 +0100 Subject: [PATCH 08/17] Updating HelpfulLibraries reference --- Lombiq.Tests.UI/Lombiq.Tests.UI.csproj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj index 90fa9251e..b73995ac5 100644 --- a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj +++ b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj @@ -112,9 +112,9 @@ - - - + + + From f9f022e01a3c9e08a0c17b1d5ed0672dff1b825a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sun, 29 Dec 2024 21:48:10 +0100 Subject: [PATCH 09/17] Removing temporary Renovate group --- renovate.json5 | 4 ---- 1 file changed, 4 deletions(-) diff --git a/renovate.json5 b/renovate.json5 index 93d2a250a..edf4bbd14 100644 --- a/renovate.json5 +++ b/renovate.json5 @@ -20,9 +20,5 @@ 'Microsoft.Extensions.*', ], }, - { - groupName: 'All packages', - matchUpdateTypes: ['*'], - }, ], } From b1d839c688e29459adda502b242d5b4fdc974098 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sun, 29 Dec 2024 23:29:07 +0100 Subject: [PATCH 10/17] Make GetAndEmptyBrowserLog() throw when under Firefox --- Lombiq.Tests.UI/Extensions/LoggingWebDriverExtensions.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Extensions/LoggingWebDriverExtensions.cs b/Lombiq.Tests.UI/Extensions/LoggingWebDriverExtensions.cs index 3c158becd..3224cb16e 100644 --- a/Lombiq.Tests.UI/Extensions/LoggingWebDriverExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/LoggingWebDriverExtensions.cs @@ -1,4 +1,6 @@ using OpenQA.Selenium; +using OpenQA.Selenium.Firefox; +using System; using System.Collections.Generic; namespace Lombiq.Tests.UI.Extensions; @@ -11,5 +13,9 @@ public static class LoggingWebDriverExtensions /// subsequent entries will be included in it. Supports Chrome only. /// public static IEnumerable GetAndEmptyBrowserLog(this IWebDriver driver) => - driver.Manage().Logs.GetLog(LogType.Browser); + driver is FirefoxDriver + ? throw new NotSupportedException( + "Firefox doesn't support getting the browser logs this way, and it will never support it, see " + + "https://github.com/mozilla/geckodriver/issues/284. You can access") + : driver.Manage().Logs.GetLog(LogType.Browser); } From d521837e80c85241b987e1b6adb417603f0f1aff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Mon, 30 Dec 2024 01:58:12 +0100 Subject: [PATCH 11/17] Changing browser logging to BiDi --- .../Tests/ErrorHandlingTests.cs | 8 +- Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs | 13 ++- .../Extensions/LoggingWebDriverExtensions.cs | 21 ----- .../Extensions/SeleniumLogEntryExtensions.cs | 8 +- .../VerificationUITestContextExtensions.cs | 6 +- .../OrchardCoreUITestExecutorConfiguration.cs | 16 ++-- Lombiq.Tests.UI/Services/UITestContext.cs | 93 ++++++++----------- .../Services/UITestExecutionSession.cs | 4 +- Lombiq.Tests.UI/Services/WebDriverFactory.cs | 1 + 9 files changed, 65 insertions(+), 105 deletions(-) delete mode 100644 Lombiq.Tests.UI/Extensions/LoggingWebDriverExtensions.cs diff --git a/Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs b/Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs index dd0df24c5..bf9f62e22 100644 --- a/Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs @@ -60,7 +60,7 @@ public Task ClientSideErrorOnLoadedPageShouldHaltTest() => catch (PageChangeAssertionException) { // Remove browser logs to have a clean slate. - context.ClearHistoricBrowserLog(); + context.ClearCumulativeBrowserLog(); } }); @@ -89,11 +89,9 @@ public Task BrowserLogsShouldPersist() => WriteConsoleLog(); WriteConsoleLog(); - await context.UpdateHistoricBrowserLogAsync(); - context - .HistoricBrowserLog - .Count(entry => entry.Message.Contains(testLog)) + .CumulativeBrowserLog + .Count(entry => entry.Text.Contains(testLog)) .ShouldBe(6); }); diff --git a/Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs b/Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs index 86bbcffc2..a58b6929a 100644 --- a/Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs @@ -3,14 +3,13 @@ using Lombiq.Tests.UI.MonkeyTesting; using Lombiq.Tests.UI.MonkeyTesting.UrlFilters; using Lombiq.Tests.UI.Services; -using OpenQA.Selenium; +using OpenQA.Selenium.BiDi.Modules.Log; using Shouldly; using System; using System.Linq; using System.Threading.Tasks; using Xunit; using Xunit.Abstractions; -using LogLevel = OpenQA.Selenium.LogLevel; namespace Lombiq.Tests.UI.Samples.Tests; @@ -96,13 +95,13 @@ public Task TestAdminBackgroundTasksAsMonkeyRecursivelyShouldWorkWithAdminUser() configuration => configuration.AssertBrowserLog = (logEntries) => logEntries .Where(logEntry => !logEntry - .Message + .Text .Contains("An invalid form control with name='LockTimeout' is not focusable.") && !logEntry - .Message + .Text .Contains("An invalid form control with name='LockExpiration' is not focusable.") && !logEntry.IsNotFoundLogEntry("/favicon.ico") - && logEntry.Level != LogLevel.Info) + && logEntry.Level != Level.Info) .ShouldBeEmpty()); // Monkey testing has its own configuration too. Check out the docs of the options too. @@ -112,11 +111,11 @@ private static MonkeyTestingOptions CreateMonkeyTestingOptions() => PageTestTime = TimeSpan.FromSeconds(5), }; - private static bool IsValidAdminBrowserLogEntry(LogEntry logEntry) => + private static bool IsValidAdminBrowserLogEntry(Entry logEntry) => OrchardCoreUITestExecutorConfiguration.IsValidBrowserLogEntry(logEntry) && // Requests to /api/graphql without further parameters will fail with HTTP 400, but that's OK, since some // parameters are required. - !logEntry.Message.ContainsOrdinalIgnoreCase("/api/graphql - Failed to load resource: the server responded with a status of 400"); + !logEntry.Text.ContainsOrdinalIgnoreCase("/api/graphql - Failed to load resource: the server responded with a status of 400"); } // END OF TRAINING SECTION: Monkey tests. diff --git a/Lombiq.Tests.UI/Extensions/LoggingWebDriverExtensions.cs b/Lombiq.Tests.UI/Extensions/LoggingWebDriverExtensions.cs deleted file mode 100644 index 3224cb16e..000000000 --- a/Lombiq.Tests.UI/Extensions/LoggingWebDriverExtensions.cs +++ /dev/null @@ -1,21 +0,0 @@ -using OpenQA.Selenium; -using OpenQA.Selenium.Firefox; -using System; -using System.Collections.Generic; - -namespace Lombiq.Tests.UI.Extensions; - -public static class LoggingWebDriverExtensions -{ - /// - /// Retrieves the console logs from the browser. This log will contain all the log messages since the start of the - /// session, not just the ones for the current page. NOTE that once you call this the log will be emptied and only - /// subsequent entries will be included in it. Supports Chrome only. - /// - public static IEnumerable GetAndEmptyBrowserLog(this IWebDriver driver) => - driver is FirefoxDriver - ? throw new NotSupportedException( - "Firefox doesn't support getting the browser logs this way, and it will never support it, see " + - "https://github.com/mozilla/geckodriver/issues/284. You can access") - : driver.Manage().Logs.GetLog(LogType.Browser); -} diff --git a/Lombiq.Tests.UI/Extensions/SeleniumLogEntryExtensions.cs b/Lombiq.Tests.UI/Extensions/SeleniumLogEntryExtensions.cs index 69495f1f5..4a3e15b4d 100644 --- a/Lombiq.Tests.UI/Extensions/SeleniumLogEntryExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/SeleniumLogEntryExtensions.cs @@ -1,4 +1,4 @@ -using OpenQA.Selenium; +using OpenQA.Selenium.BiDi.Modules.Log; using System; using System.Collections.Generic; @@ -6,10 +6,10 @@ namespace Lombiq.Tests.UI.Extensions; public static class SeleniumLogEntryExtensions { - public static string ToFormattedString(this IEnumerable logEntries) => + public static string ToFormattedString(this IEnumerable logEntries) => string.Join(Environment.NewLine, logEntries); - public static bool IsNotFoundLogEntry(this LogEntry logEntry, string url) => - logEntry.Message.ContainsOrdinalIgnoreCase( + public static bool IsNotFoundLogEntry(this Entry logEntry, string url) => + logEntry.Text.ContainsOrdinalIgnoreCase( @$"{url} - Failed to load resource: the server responded with a status of 404"); } diff --git a/Lombiq.Tests.UI/Extensions/VerificationUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/VerificationUITestContextExtensions.cs index ab2a509a3..8c0c2fdef 100644 --- a/Lombiq.Tests.UI/Extensions/VerificationUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/VerificationUITestContextExtensions.cs @@ -20,8 +20,6 @@ public static async Task AssertLogsAsync(this UITestContext context) var configuration = context.Configuration; var testOutputHelper = configuration.TestOutputHelper; - if (context.IsBrowserRunning) await context.UpdateHistoricBrowserLogAsync(); - try { await configuration.AssertAppLogsAsync.InvokeFuncAsync(context.Application); @@ -38,12 +36,12 @@ public static async Task AssertLogsAsync(this UITestContext context) { try { - configuration.AssertBrowserLog?.Invoke(context.HistoricBrowserLog); + configuration.AssertBrowserLog?.Invoke(context.CumulativeBrowserLog); } catch (Exception) { testOutputHelper.WriteLine("Browser logs: " + Environment.NewLine); - testOutputHelper.WriteLine(context.HistoricBrowserLog.ToFormattedString()); + testOutputHelper.WriteLine(context.CumulativeBrowserLog.ToFormattedString()); throw; } diff --git a/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs b/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs index 54cff1c0a..8604d70ff 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs @@ -2,7 +2,7 @@ using Lombiq.Tests.UI.Helpers; using Lombiq.Tests.UI.SecurityScanning; using Lombiq.Tests.UI.Services.GitHub; -using OpenQA.Selenium; +using OpenQA.Selenium.BiDi.Modules.Log; using Shouldly; using System; using System.Collections.Generic; @@ -36,20 +36,20 @@ public class OrchardCoreUITestExecutorConfiguration public static readonly Func AssertAppLogsCanContainCacheFolderErrorsAsync = app => app.LogsShouldNotContainAsync(AppLogAssertionHelper.NotMediaCacheEntriesPredicate); - public static readonly Action> AssertBrowserLogIsEmpty = + public static readonly Action> AssertBrowserLogIsEmpty = logEntries => logEntries.ShouldNotContain( logEntry => IsValidBrowserLogEntry(logEntry), logEntries.Where(IsValidBrowserLogEntry).ToFormattedString()); - public static readonly Func IsValidBrowserLogEntry = - logEntry => - logEntry.Level >= LogLevel.Warning && + public static readonly Func IsValidBrowserLogEntry = + entry => + entry.Level >= Level.Warn && // HTML imports are somehow used by Selenium or something but this deprecation notice is always there for // every page. - !logEntry.Message.ContainsOrdinalIgnoreCase("HTML Imports is deprecated") && + !entry.Text.ContainsOrdinalIgnoreCase("HTML Imports is deprecated") && // The 404 is because of how browsers automatically request /favicon.ico even if a favicon is declared to be // under a different URL. - !logEntry.IsNotFoundLogEntry("/favicon.ico"); + !entry.IsNotFoundLogEntry("/favicon.ico"); /// /// Gets the global events available during UI test execution. @@ -108,7 +108,7 @@ public class OrchardCoreUITestExecutorConfiguration : Environment.ProcessorCount; public Func AssertAppLogsAsync { get; set; } = AssertAppLogsCanContainCacheFolderErrorsAsync; - public Action> AssertBrowserLog { get; set; } = AssertBrowserLogIsEmpty; + public Action> AssertBrowserLog { get; set; } = AssertBrowserLogIsEmpty; public ITestOutputHelper TestOutputHelper { get; set; } /// diff --git a/Lombiq.Tests.UI/Services/UITestContext.cs b/Lombiq.Tests.UI/Services/UITestContext.cs index b4cabc373..49131c077 100644 --- a/Lombiq.Tests.UI/Services/UITestContext.cs +++ b/Lombiq.Tests.UI/Services/UITestContext.cs @@ -4,6 +4,8 @@ using Lombiq.Tests.UI.Models; using Lombiq.Tests.UI.SecurityScanning; using OpenQA.Selenium; +using OpenQA.Selenium.BiDi; +using OpenQA.Selenium.BiDi.Modules.Log; using OrchardCore.Environment.Shell; using System; using System.Collections.Generic; @@ -15,7 +17,7 @@ namespace Lombiq.Tests.UI.Services; public class UITestContext { - private readonly List _historicBrowserLog = []; + private readonly List _cumulativeBrowserLog = []; /// /// Gets the globally unique ID of this context. You can use this ID to refer to the current text execution in @@ -103,9 +105,9 @@ public class UITestContext public ZapManager ZapManager { get; } /// - /// Gets a cumulative log of browser console entries. + /// Gets a cumulative log of browser console entries, containing all entries since the start of the test. /// - public IReadOnlyList HistoricBrowserLog => _historicBrowserLog; + public IReadOnlyList CumulativeBrowserLog => _cumulativeBrowserLog; /// /// Gets a dictionary storing some custom contextual data. @@ -149,7 +151,7 @@ public class UITestContext // This is a central context object, we need the data to be passed in the constructor. #pragma warning disable S107 // Methods should not have too many parameters - public UITestContext( + private UITestContext( string id, UITestManifest testManifest, OrchardCoreUITestExecutorConfiguration configuration, @@ -173,64 +175,18 @@ public UITestContext( } /// - /// Updates with current console entries from the browser. + /// Clears accumulated browser log messages from . /// - public Task> UpdateHistoricBrowserLogAsync() - { - var windowHandles = Driver.WindowHandles; - - if (windowHandles.Count > 1) - { - var currentWindowHandle = Driver.CurrentWindowHandle; - - foreach (var windowHandle in windowHandles) - { - // Not using the logging SwitchTo() deliberately as this is not part of what the test does. - Driver.SwitchTo().Window(windowHandle); - _historicBrowserLog.AddRange(Driver.GetAndEmptyBrowserLog()); - } - - try - { - Driver.SwitchTo().Window(currentWindowHandle); - } - catch (NoSuchWindowException) - { - // This can happen in rare instances if the current window/tab was just closed. - Driver.SwitchTo().Window(Driver.WindowHandles[^1]); - } - } - else - { - _historicBrowserLog.AddRange(Driver.GetAndEmptyBrowserLog()); - } - - return Task.FromResult>(_historicBrowserLog); - } - - /// - /// Clears accumulated historic browser log messages from . - /// - public void ClearHistoricBrowserLog() => _historicBrowserLog.Clear(); - - /// - /// Run an assertion on the browser logs of the current tab with the delegate configured in . This doesn't use . - /// - public Task AssertCurrentBrowserLogAsync() - { - Configuration.AssertBrowserLog?.Invoke(Scope.Driver.GetAndEmptyBrowserLog()); - return Task.CompletedTask; - } + public void ClearCumulativeBrowserLog() => _cumulativeBrowserLog.Clear(); /// - /// Clears the application and historic browser logs. + /// Clears the application and cumulative browser logs. /// /// Optional cancellation token for reading the application logs. public async Task ClearLogsAsync(CancellationToken cancellationToken = default) { foreach (var log in await Application.GetLogsAsync(cancellationToken)) await log.RemoveAsync(); - ClearHistoricBrowserLog(); + ClearCumulativeBrowserLog(); } /// @@ -305,6 +261,35 @@ public void SwitchCurrentTenant(string tenantName, Uri baseUri) Scope.BaseUri = baseUri; } + // This is a central context object, we need the data to be passed in the constructor. +#pragma warning disable S107 // Methods should not have too many parameters + public static async Task CreateAsync( + string id, + UITestManifest testManifest, + OrchardCoreUITestExecutorConfiguration configuration, + IWebApplicationInstance application, + AtataScope scope, + Uri testStartUri, + RunningContextContainer runningContextContainer, + ZapManager zapManager) +#pragma warning restore S107 // Methods should not have too many parameters + { + var context = new UITestContext( + id, + testManifest, + configuration, + application, + scope, + testStartUri, + runningContextContainer, + zapManager); + + var biDi = await scope.Driver.AsBiDiAsync(); + await biDi.Log.OnEntryAddedAsync(context._cumulativeBrowserLog.Add); + + return context; + } + /// /// Returns the subdirectory described by inside the current test instance's /// directory. diff --git a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs index 568dd8f55..ef271c67b 100644 --- a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs +++ b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs @@ -650,7 +650,7 @@ Task UITestingBeforeAppStartHandlerAsync(OrchardCoreAppStartContext context, Ins var atataScope = await AtataFactory.StartAtataScopeAsync(contextId, _testOutputHelper, appBaseUri, _configuration); - return new UITestContext( + return await UITestContext.CreateAsync( contextId, _testManifest, _configuration, @@ -814,7 +814,7 @@ private async Task CaptureBrowserUsingDumpsAsync(string debugInformationPath) await File.WriteAllLinesAsync( browserLogPath, - (await _context.UpdateHistoricBrowserLogAsync()).Select(message => message.ToString())); + _context.CumulativeBrowserLog.Select(entry => entry.ToString())); if (_configuration.ReportTeamCityMetadata) { diff --git a/Lombiq.Tests.UI/Services/WebDriverFactory.cs b/Lombiq.Tests.UI/Services/WebDriverFactory.cs index d76022f87..f424bfd8c 100644 --- a/Lombiq.Tests.UI/Services/WebDriverFactory.cs +++ b/Lombiq.Tests.UI/Services/WebDriverFactory.cs @@ -148,6 +148,7 @@ private static TDriverOptions SetCommonOptions(this TDriverOptio driverOptions.AcceptInsecureCertificates = true; driverOptions.UnhandledPromptBehavior = UnhandledPromptBehavior.Ignore; driverOptions.PageLoadStrategy = PageLoadStrategy.Normal; + driverOptions.UseWebSocketUrl = true; return driverOptions; } From de9d6433b43a274a0b219a08c2e84d06a331f248 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Mon, 30 Dec 2024 02:11:45 +0100 Subject: [PATCH 12/17] Not setting up BiDi log event if no browser is configured --- Lombiq.Tests.UI/Services/UITestContext.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Lombiq.Tests.UI/Services/UITestContext.cs b/Lombiq.Tests.UI/Services/UITestContext.cs index 49131c077..c18386c71 100644 --- a/Lombiq.Tests.UI/Services/UITestContext.cs +++ b/Lombiq.Tests.UI/Services/UITestContext.cs @@ -82,7 +82,7 @@ public class UITestContext /// /// Gets a value indicating whether a browser is configured to be used for the test. means /// that no browser will be launched. Note that since the browser is only started on demand, with the first - /// operation requiring it, a browser might not be currently running even if this suggests it may. Check " to check for that. /// public bool IsBrowserConfigured => Configuration.BrowserConfiguration.Browser != Browser.None; @@ -284,8 +284,11 @@ public static async Task CreateAsync( runningContextContainer, zapManager); - var biDi = await scope.Driver.AsBiDiAsync(); - await biDi.Log.OnEntryAddedAsync(context._cumulativeBrowserLog.Add); + if (context.IsBrowserConfigured) + { + var biDi = await scope.Driver.AsBiDiAsync(); + await biDi.Log.OnEntryAddedAsync(context._cumulativeBrowserLog.Add); + } return context; } From 86b4711f297d78400b383c6dc27ea1a6d29b6f80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Mon, 30 Dec 2024 21:09:48 +0100 Subject: [PATCH 13/17] Adding response log to be able to assert on 404s and others, adding browser log filter --- .../Tests/AzureBlobStorageTests.cs | 11 +----- Lombiq.Tests.UI.Samples/Tests/BasicTests.cs | 19 ++++----- .../Tests/ErrorHandlingTests.cs | 24 +++++++++--- Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs | 33 ++++------------ .../Tests/SqlServerTests.cs | 11 +----- .../Extensions/SeleniumLogEntryExtensions.cs | 6 +-- ...eleniumResponseCompletedEventExtensions.cs | 29 ++++++++++++++ .../VerificationUITestContextExtensions.cs | 4 ++ .../OrchardCoreUITestExecutorConfiguration.cs | 35 ++++++++++++----- Lombiq.Tests.UI/Services/UITestContext.cs | 39 +++++++++++++++++-- .../Services/UITestExecutionSession.cs | 14 +++++++ .../UITestExecutorTestDumpConfiguration.cs | 1 + 12 files changed, 150 insertions(+), 76 deletions(-) create mode 100644 Lombiq.Tests.UI/Extensions/SeleniumResponseCompletedEventExtensions.cs diff --git a/Lombiq.Tests.UI.Samples/Tests/AzureBlobStorageTests.cs b/Lombiq.Tests.UI.Samples/Tests/AzureBlobStorageTests.cs index f5818f1d7..f46d065ad 100644 --- a/Lombiq.Tests.UI.Samples/Tests/AzureBlobStorageTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/AzureBlobStorageTests.cs @@ -1,7 +1,5 @@ using Lombiq.Tests.UI.Extensions; using Lombiq.Tests.UI.Samples.Extensions; -using Lombiq.Tests.UI.Services; -using System.Linq; using System.Threading.Tasks; using Xunit; using Xunit.Abstractions; @@ -38,13 +36,8 @@ public Task TogglingFeaturesShouldWorkWithAzureBlobStorage() => { configuration.UseAzureBlobStorage = true; - configuration.AssertBrowserLog = - logEntries => - { - var messagesWithoutToggle = logEntries.Where(logEntry => - !logEntry.IsNotFoundLogEntry(ShortcutsUITestContextExtensions.FeatureToggleTestBenchUrl)); - OrchardCoreUITestExecutorConfiguration.AssertBrowserLogIsEmpty(messagesWithoutToggle); - }; + configuration.ResponseLogFilter = e => + e.IsNonSuccessResponseAndNotExpectedNotFoundResponse(ShortcutsUITestContextExtensions.FeatureToggleTestBenchUrl); }); } diff --git a/Lombiq.Tests.UI.Samples/Tests/BasicTests.cs b/Lombiq.Tests.UI.Samples/Tests/BasicTests.cs index c68601944..e2ffee9e8 100644 --- a/Lombiq.Tests.UI.Samples/Tests/BasicTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/BasicTests.cs @@ -1,9 +1,7 @@ using Lombiq.Tests.UI.Constants; using Lombiq.Tests.UI.Extensions; -using Lombiq.Tests.UI.Services; using OpenQA.Selenium; using Shouldly; -using System.Linq; using System.Threading.Tasks; using Xunit; using Xunit.Abstractions; @@ -74,15 +72,14 @@ public Task TogglingFeaturesShouldWork() => context => context.ExecuteAndAssertTestFeatureToggleAsync(), // You can change the configuration even for each test. configuration => - // By default, apart from some commonly known exceptions, the browser log should be empty. However, - // ExecuteAndAssertTestFeatureToggle() causes a 404 so we need to make sure not to fail on that. - configuration.AssertBrowserLog = - logEntries => - { - var messagesWithoutToggle = logEntries.Where(logEntry => - !logEntry.IsNotFoundLogEntry(ShortcutsUITestContextExtensions.FeatureToggleTestBenchUrl)); - OrchardCoreUITestExecutorConfiguration.AssertBrowserLogIsEmpty(messagesWithoutToggle); - }); + // By default, apart from some commonly known false positives, the response log should be empty. + // However, ExecuteAndAssertTestFeatureToggle() causes a 404 so we need to make sure not to fail on + // that. + // Note that similarly, you can filter out log entries from the browser log with BrowserLogFilter. You + // can also adjust the assertion logic with AssertResponseLog and AssertBrowserLog, but it's best to + // filter out entries in the first place. + configuration.ResponseLogFilter = e => + e.IsNonSuccessResponseAndNotExpectedNotFoundResponse(ShortcutsUITestContextExtensions.FeatureToggleTestBenchUrl)); // Let's see a couple more useful shortcuts in action. [Fact] diff --git a/Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs b/Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs index bf9f62e22..4853b28b2 100644 --- a/Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs @@ -1,7 +1,10 @@ using Lombiq.Tests.UI.Exceptions; using Lombiq.Tests.UI.Extensions; +using Lombiq.Tests.UI.Helpers; using Lombiq.Tests.UI.Pages; using Lombiq.Tests.UI.Samples.Helpers; +using Lombiq.Tests.UI.Services; +using OpenQA.Selenium.BiDi.Modules.Log; using Shouldly; using System; using System.Linq; @@ -59,8 +62,8 @@ public Task ClientSideErrorOnLoadedPageShouldHaltTest() => } catch (PageChangeAssertionException) { - // Remove browser logs to have a clean slate. - context.ClearCumulativeBrowserLog(); + // Remove response logs to have a clean slate. + context.ClearCumulativeResponseLog(); } }); @@ -89,10 +92,21 @@ public Task BrowserLogsShouldPersist() => WriteConsoleLog(); WriteConsoleLog(); - context + // Since the browser log is updated asynchronously, we have to wait for most recent entries to appear. + ReliabilityHelper.DoWithRetriesOrFail(() => + context .CumulativeBrowserLog - .Count(entry => entry.Text.Contains(testLog)) - .ShouldBe(6); + .Count(entry => entry.Text.Contains(testLog)) == 6); + }, + configuration => + { + // By default, anything below warning is not logged to the browser log. So, to allow the info messages + // of the test, we change the filter. + configuration.BrowserLogFilter = logEntry => + OrchardCoreUITestExecutorConfiguration.IsNonSuccessBrowserLogEntry(logEntry) || logEntry.Level == Level.Info; + + // By default, the test will fail if the browser log is not empty. We allow info entries here. + configuration.AssertBrowserLog = logEntries => logEntries.ShouldNotContain(entry => entry.Level > Level.Info); }); [Fact] diff --git a/Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs b/Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs index a58b6929a..4de8984e0 100644 --- a/Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs @@ -1,12 +1,7 @@ -using Atata; using Lombiq.Tests.UI.Extensions; using Lombiq.Tests.UI.MonkeyTesting; using Lombiq.Tests.UI.MonkeyTesting.UrlFilters; -using Lombiq.Tests.UI.Services; -using OpenQA.Selenium.BiDi.Modules.Log; -using Shouldly; using System; -using System.Linq; using System.Threading.Tasks; using Xunit; using Xunit.Abstractions; @@ -68,10 +63,9 @@ public Task TestAdminPagesAsMonkeyRecursivelyShouldWorkWithAdminUser() => return context.TestAdminAsMonkeyRecursivelyAsync(monkeyTestingOptions); }, - configuration => - configuration.AssertBrowserLog = logEntries => logEntries.ShouldNotContain( - logEntry => IsValidAdminBrowserLogEntry(logEntry), - logEntries.Where(IsValidAdminBrowserLogEntry).ToFormattedString())); + // Requests to /api/graphql without further parameters will fail with HTTP 400, but that's OK, since some + // parameters are required. + configuration => configuration.ResponseLogFilter = e => e.IsNonSuccessResponseAndNotExpectedStatusResponse("/api/graphql", 400)); // Let's just test the background tasks management admin area. [Fact] @@ -92,17 +86,10 @@ public Task TestAdminBackgroundTasksAsMonkeyRecursivelyShouldWorkWithAdminUser() await context.SignInDirectlyAndGoToAdminRelativeUrlAsync("/BackgroundTasks"); await context.TestCurrentPageAsMonkeyRecursivelyAsync(monkeyTestingOptions); }, - configuration => configuration.AssertBrowserLog = (logEntries) => logEntries - .Where(logEntry => - !logEntry - .Text - .Contains("An invalid form control with name='LockTimeout' is not focusable.") - && !logEntry - .Text - .Contains("An invalid form control with name='LockExpiration' is not focusable.") - && !logEntry.IsNotFoundLogEntry("/favicon.ico") - && logEntry.Level != Level.Info) - .ShouldBeEmpty()); + configuration => configuration.BrowserLogFilter = logEntry => + logEntry.IsNonSuccessBrowserLogEntry() && + !logEntry.Text.Contains("An invalid form control with name='LockTimeout' is not focusable.") && + !logEntry.Text.Contains("An invalid form control with name='LockExpiration' is not focusable.")); // Monkey testing has its own configuration too. Check out the docs of the options too. private static MonkeyTestingOptions CreateMonkeyTestingOptions() => @@ -110,12 +97,6 @@ private static MonkeyTestingOptions CreateMonkeyTestingOptions() => { PageTestTime = TimeSpan.FromSeconds(5), }; - - private static bool IsValidAdminBrowserLogEntry(Entry logEntry) => - OrchardCoreUITestExecutorConfiguration.IsValidBrowserLogEntry(logEntry) && - // Requests to /api/graphql without further parameters will fail with HTTP 400, but that's OK, since some - // parameters are required. - !logEntry.Text.ContainsOrdinalIgnoreCase("/api/graphql - Failed to load resource: the server responded with a status of 400"); } // END OF TRAINING SECTION: Monkey tests. diff --git a/Lombiq.Tests.UI.Samples/Tests/SqlServerTests.cs b/Lombiq.Tests.UI.Samples/Tests/SqlServerTests.cs index f00da7769..109379013 100644 --- a/Lombiq.Tests.UI.Samples/Tests/SqlServerTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/SqlServerTests.cs @@ -1,7 +1,5 @@ using Lombiq.Tests.UI.Extensions; using Lombiq.Tests.UI.Samples.Extensions; -using Lombiq.Tests.UI.Services; -using System.Linq; using System.Threading.Tasks; using Xunit; using Xunit.Abstractions; @@ -37,13 +35,8 @@ public Task TogglingFeaturesShouldWorkWithSqlServer() => { configuration.UseSqlServer = true; - configuration.AssertBrowserLog = - logEntries => - { - var messagesWithoutToggle = logEntries.Where(logEntry => - !logEntry.IsNotFoundLogEntry(ShortcutsUITestContextExtensions.FeatureToggleTestBenchUrl)); - OrchardCoreUITestExecutorConfiguration.AssertBrowserLogIsEmpty(messagesWithoutToggle); - }; + configuration.ResponseLogFilter = e => + e.IsNonSuccessResponseAndNotExpectedNotFoundResponse(ShortcutsUITestContextExtensions.FeatureToggleTestBenchUrl); }); } diff --git a/Lombiq.Tests.UI/Extensions/SeleniumLogEntryExtensions.cs b/Lombiq.Tests.UI/Extensions/SeleniumLogEntryExtensions.cs index 4a3e15b4d..ecef49664 100644 --- a/Lombiq.Tests.UI/Extensions/SeleniumLogEntryExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/SeleniumLogEntryExtensions.cs @@ -1,3 +1,4 @@ +using Lombiq.Tests.UI.Services; using OpenQA.Selenium.BiDi.Modules.Log; using System; using System.Collections.Generic; @@ -9,7 +10,6 @@ public static class SeleniumLogEntryExtensions public static string ToFormattedString(this IEnumerable logEntries) => string.Join(Environment.NewLine, logEntries); - public static bool IsNotFoundLogEntry(this Entry logEntry, string url) => - logEntry.Text.ContainsOrdinalIgnoreCase( - @$"{url} - Failed to load resource: the server responded with a status of 404"); + public static bool IsNonSuccessBrowserLogEntry(this Entry entry) => + OrchardCoreUITestExecutorConfiguration.IsNonSuccessBrowserLogEntry(entry); } diff --git a/Lombiq.Tests.UI/Extensions/SeleniumResponseCompletedEventExtensions.cs b/Lombiq.Tests.UI/Extensions/SeleniumResponseCompletedEventExtensions.cs new file mode 100644 index 000000000..73a7ab84a --- /dev/null +++ b/Lombiq.Tests.UI/Extensions/SeleniumResponseCompletedEventExtensions.cs @@ -0,0 +1,29 @@ +using Lombiq.Tests.UI.Services; +using OpenQA.Selenium.BiDi.Modules.Network; +using System; +using System.Collections.Generic; + +namespace Lombiq.Tests.UI.Extensions; + +public static class SeleniumResponseCompletedEventExtensions +{ + public static string ToFormattedString(this IEnumerable responses) => + string.Join(Environment.NewLine, responses); + + public static bool IsNonSuccessResponse(this ResponseCompletedEventArgs eventArgs) => + OrchardCoreUITestExecutorConfiguration.IsNonSuccessResponse(eventArgs); + + public static bool IsNonSuccessResponseAndNotExpectedNotFoundResponse(this ResponseCompletedEventArgs eventArgs, string urlContains) => + IsNonSuccessResponse(eventArgs) && + !eventArgs.IsNotFoundResponse(urlContains); + + public static bool IsNonSuccessResponseAndNotExpectedStatusResponse(this ResponseCompletedEventArgs eventArgs, string urlContains, int status) => + IsNonSuccessResponse(eventArgs) && + !(eventArgs.Response.Url.ContainsOrdinalIgnoreCase(urlContains) && eventArgs.Response.Status == status); + + public static bool IsNotFoundResponse(this ResponseCompletedEventArgs eventArgs, string urlContains) => + IsNotFoundResponse(eventArgs.Response, urlContains); + + public static bool IsNotFoundResponse(this ResponseData response, string urlContains) => + response.Status == 404 && response.Url.ContainsOrdinalIgnoreCase(urlContains); +} diff --git a/Lombiq.Tests.UI/Extensions/VerificationUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/VerificationUITestContextExtensions.cs index 8c0c2fdef..3a34b70c1 100644 --- a/Lombiq.Tests.UI/Extensions/VerificationUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/VerificationUITestContextExtensions.cs @@ -36,6 +36,7 @@ public static async Task AssertLogsAsync(this UITestContext context) { try { + configuration.AssertResponseLog?.Invoke(context.CumulativeResponseLog); configuration.AssertBrowserLog?.Invoke(context.CumulativeBrowserLog); } catch (Exception) @@ -43,6 +44,9 @@ public static async Task AssertLogsAsync(this UITestContext context) testOutputHelper.WriteLine("Browser logs: " + Environment.NewLine); testOutputHelper.WriteLine(context.CumulativeBrowserLog.ToFormattedString()); + testOutputHelper.WriteLine("Response logs: " + Environment.NewLine); + testOutputHelper.WriteLine(context.CumulativeResponseLog.ToFormattedString()); + throw; } } diff --git a/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs b/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs index 8604d70ff..2f08cba24 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs @@ -3,11 +3,11 @@ using Lombiq.Tests.UI.SecurityScanning; using Lombiq.Tests.UI.Services.GitHub; using OpenQA.Selenium.BiDi.Modules.Log; +using OpenQA.Selenium.BiDi.Modules.Network; using Shouldly; using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; -using System.Linq; using System.Threading.Tasks; using Xunit.Abstractions; @@ -37,19 +37,22 @@ public class OrchardCoreUITestExecutorConfiguration app => app.LogsShouldNotContainAsync(AppLogAssertionHelper.NotMediaCacheEntriesPredicate); public static readonly Action> AssertBrowserLogIsEmpty = - logEntries => logEntries.ShouldNotContain( - logEntry => IsValidBrowserLogEntry(logEntry), - logEntries.Where(IsValidBrowserLogEntry).ToFormattedString()); + logEntries => logEntries.ShouldBeEmpty(logEntries.ToFormattedString()); - public static readonly Func IsValidBrowserLogEntry = + public static readonly Func IsNonSuccessBrowserLogEntry = entry => entry.Level >= Level.Warn && // HTML imports are somehow used by Selenium or something but this deprecation notice is always there for // every page. - !entry.Text.ContainsOrdinalIgnoreCase("HTML Imports is deprecated") && - // The 404 is because of how browsers automatically request /favicon.ico even if a favicon is declared to be - // under a different URL. - !entry.IsNotFoundLogEntry("/favicon.ico"); + !entry.Text.ContainsOrdinalIgnoreCase("HTML Imports is deprecated"); + + // The 404 is because of how browsers automatically request /favicon.ico even if a favicon is declared to be under a + // different URL. + public static readonly Func IsNonSuccessResponse = e => + e.Response.Status is < 200 or >= 400 && !e.Response.Url.EndsWithOrdinalIgnoreCase("/favicon.ico"); + + public static readonly Action> AssertResponseLogIsEmpty = + responses => responses.ShouldBeEmpty(responses.ToFormattedString()); /// /// Gets the global events available during UI test execution. @@ -108,7 +111,21 @@ public class OrchardCoreUITestExecutorConfiguration : Environment.ProcessorCount; public Func AssertAppLogsAsync { get; set; } = AssertAppLogsCanContainCacheFolderErrorsAsync; + + /// + /// Selects which response data get saved to . + /// + public Func BrowserLogFilter = IsNonSuccessBrowserLogEntry; + public Action> AssertBrowserLog { get; set; } = AssertBrowserLogIsEmpty; + + /// + /// Selects which response data get saved to . + /// + public Func ResponseLogFilter = IsNonSuccessResponse; + + public Action> AssertResponseLog { get; set; } = AssertResponseLogIsEmpty; + public ITestOutputHelper TestOutputHelper { get; set; } /// diff --git a/Lombiq.Tests.UI/Services/UITestContext.cs b/Lombiq.Tests.UI/Services/UITestContext.cs index c18386c71..9d5914771 100644 --- a/Lombiq.Tests.UI/Services/UITestContext.cs +++ b/Lombiq.Tests.UI/Services/UITestContext.cs @@ -1,3 +1,4 @@ +using Atata; using Lombiq.Tests.UI.Constants; using Lombiq.Tests.UI.Exceptions; using Lombiq.Tests.UI.Extensions; @@ -6,8 +7,10 @@ using OpenQA.Selenium; using OpenQA.Selenium.BiDi; using OpenQA.Selenium.BiDi.Modules.Log; +using OpenQA.Selenium.BiDi.Modules.Network; using OrchardCore.Environment.Shell; using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Threading; @@ -17,7 +20,10 @@ namespace Lombiq.Tests.UI.Services; public class UITestContext { - private readonly List _cumulativeBrowserLog = []; + // Multiple browser tabs being open can log at the same time, so we need thread-safe collections. Using a queue to + // preserve the insertion order. + private readonly ConcurrentQueue _cumulativeBrowserLog = []; + private readonly ConcurrentQueue _cumulativeResponseLog = []; /// /// Gets the globally unique ID of this context. You can use this ID to refer to the current text execution in @@ -105,9 +111,19 @@ public class UITestContext public ZapManager ZapManager { get; } /// - /// Gets a cumulative log of browser console entries, containing all entries since the start of the test. + /// Gets a cumulative log of browser console entries, containing all entries since the start of the test. This can + /// be used to assert on the browser log like failing the test on JavaScript exceptions. Note that since the log is + /// updated asynchronously by the browser, entries might appear with some delay. /// - public IReadOnlyList CumulativeBrowserLog => _cumulativeBrowserLog; + public IReadOnlyList CumulativeBrowserLog => _cumulativeBrowserLog.ToReadOnly(); + + /// + /// Gets a cumulative log of browser HTTP responses filtered by , containing all items since the start of the + /// test. This can be used to assert on response details. Note that + /// since the log is updated asynchronously by the browser, entries might appear with some delay. + /// + public IReadOnlyList CumulativeResponseLog => _cumulativeResponseLog.ToReadOnly(); /// /// Gets a dictionary storing some custom contextual data. @@ -179,6 +195,11 @@ private UITestContext( /// public void ClearCumulativeBrowserLog() => _cumulativeBrowserLog.Clear(); + /// + /// Clears accumulated browser log messages from . + /// + public void ClearCumulativeResponseLog() => _cumulativeResponseLog.Clear(); + /// /// Clears the application and cumulative browser logs. /// @@ -187,6 +208,7 @@ public async Task ClearLogsAsync(CancellationToken cancellationToken = default) { foreach (var log in await Application.GetLogsAsync(cancellationToken)) await log.RemoveAsync(); ClearCumulativeBrowserLog(); + ClearCumulativeResponseLog(); } /// @@ -287,7 +309,16 @@ public static async Task CreateAsync( if (context.IsBrowserConfigured) { var biDi = await scope.Driver.AsBiDiAsync(); - await biDi.Log.OnEntryAddedAsync(context._cumulativeBrowserLog.Add); + + await biDi.Log.OnEntryAddedAsync(entry => + { + if (configuration.BrowserLogFilter(entry)) context._cumulativeBrowserLog.Enqueue(entry); + }); + + await biDi.Network.OnResponseCompletedAsync(responseCompleted => + { + if (configuration.ResponseLogFilter(responseCompleted)) context._cumulativeResponseLog.Enqueue(responseCompleted.Response); + }); } return context; diff --git a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs index ef271c67b..199ea03bc 100644 --- a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs +++ b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs @@ -821,6 +821,20 @@ await File.WriteAllLinesAsync( TeamCityMetadataReporter.ReportArtifactLink(_testManifest, "BrowserLog", browserLogPath); } } + + if (_dumpConfiguration.CaptureResponseLog) + { + var responseLogPath = Path.Combine(debugInformationPath, "ResponseLog.log"); + + await File.WriteAllLinesAsync( + responseLogPath, + _context.CumulativeResponseLog.Select(response => response.ToString())); + + if (_configuration.ReportTeamCityMetadata) + { + TeamCityMetadataReporter.ReportArtifactLink(_testManifest, "ResponseLog", responseLogPath); + } + } } private Task TakeScreenshotIfEnabledAsync(UITestContext context) diff --git a/Lombiq.Tests.UI/Services/UITestExecutorTestDumpConfiguration.cs b/Lombiq.Tests.UI/Services/UITestExecutorTestDumpConfiguration.cs index 2724eaed3..198369a46 100644 --- a/Lombiq.Tests.UI/Services/UITestExecutorTestDumpConfiguration.cs +++ b/Lombiq.Tests.UI/Services/UITestExecutorTestDumpConfiguration.cs @@ -17,5 +17,6 @@ public class UITestExecutorTestDumpConfiguration public bool CaptureAppSnapshot { get; set; } = true; public bool CaptureScreenshots { get; set; } = true; public bool CaptureHtmlSource { get; set; } = true; + public bool CaptureResponseLog { get; set; } = true; public bool CaptureBrowserLog { get; set; } = true; } From 3c775347a113d27b440669afcd359f43042c218d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Tue, 31 Dec 2024 00:00:30 +0100 Subject: [PATCH 14/17] Suppressing breaking changes --- Lombiq.Tests.UI/CompatibilitySuppressions.xml | 86 ++++++++++++++++++- 1 file changed, 85 insertions(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/CompatibilitySuppressions.xml b/Lombiq.Tests.UI/CompatibilitySuppressions.xml index 0e0d08d29..5ac55981d 100644 --- a/Lombiq.Tests.UI/CompatibilitySuppressions.xml +++ b/Lombiq.Tests.UI/CompatibilitySuppressions.xml @@ -1,5 +1,5 @@  - + CP0001 @@ -8,6 +8,13 @@ lib/net8.0/Lombiq.Tests.UI.dll true + + CP0001 + T:Lombiq.Tests.UI.Extensions.LoggingWebDriverExtensions + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + CP0002 F:Lombiq.Tests.UI.Services.Browser.InternetExplorer @@ -15,6 +22,76 @@ lib/net8.0/Lombiq.Tests.UI.dll true + + CP0002 + F:Lombiq.Tests.UI.Services.OrchardCoreUITestExecutorConfiguration.AssertBrowserLogIsEmpty + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + F:Lombiq.Tests.UI.Services.OrchardCoreUITestExecutorConfiguration.IsValidBrowserLogEntry + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Extensions.SeleniumLogEntryExtensions.IsNotFoundLogEntry(OpenQA.Selenium.LogEntry,System.String) + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Extensions.SeleniumLogEntryExtensions.ToFormattedString(System.Collections.Generic.IEnumerable{OpenQA.Selenium.LogEntry}) + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Services.OrchardCoreUITestExecutorConfiguration.get_AssertBrowserLog + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Services.UITestContext.#ctor(System.String,Lombiq.Tests.UI.Models.UITestManifest,Lombiq.Tests.UI.Services.OrchardCoreUITestExecutorConfiguration,Lombiq.Tests.UI.Services.IWebApplicationInstance,Lombiq.Tests.UI.Services.AtataScope,System.Uri,Lombiq.Tests.UI.Models.RunningContextContainer,Lombiq.Tests.UI.SecurityScanning.ZapManager) + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Services.UITestContext.AssertCurrentBrowserLogAsync + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Services.UITestContext.ClearHistoricBrowserLog + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Services.UITestContext.get_HistoricBrowserLog + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Services.UITestContext.UpdateHistoricBrowserLogAsync + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + CP0002 M:Lombiq.Tests.UI.Services.WebDriverFactory.CreateInternetExplorerDriverAsync(Lombiq.Tests.UI.Services.BrowserConfiguration,System.TimeSpan) @@ -22,6 +99,13 @@ lib/net8.0/Lombiq.Tests.UI.dll true + + CP0009 + T:Lombiq.Tests.UI.Services.UITestContext + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + CP0011 F:Lombiq.Tests.UI.Services.Browser.None From 18dd2580f10f328741751c8abba3daa3daac6c28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Thu, 23 Jan 2025 21:47:12 +0100 Subject: [PATCH 15/17] Revert "Temporarily disabling SendingTestEmailShouldWork too" This reverts commit 5c2b55c49bf319ba2d822ba4eaaa98b0e6208372. --- Lombiq.Tests.UI.Samples/Tests/EmailTests.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Lombiq.Tests.UI.Samples/Tests/EmailTests.cs b/Lombiq.Tests.UI.Samples/Tests/EmailTests.cs index 4c6c854f6..33afd9c14 100644 --- a/Lombiq.Tests.UI.Samples/Tests/EmailTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/EmailTests.cs @@ -15,10 +15,7 @@ public EmailTests(ITestOutputHelper testOutputHelper) { } - // Will be re-enabled as part of https://github.com/Lombiq/Open-Source-Orchard-Core-Extensions/issues/703. -#pragma warning disable xUnit1004 // Test methods should not be skipped - [Fact(Skip = "Fails with smtp4dev JS exceptions, but works under https://github.com/Lombiq/Open-Source-Orchard-Core-Extensions/issues/703.")] -#pragma warning restore xUnit1004 // Test methods should not be skipped + [Fact] public Task SendingTestEmailShouldWork() => ExecuteTestAfterSetupAsync( async context => From 337acb2254b1d4a7a0818272ff9ec887b9d029e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Tue, 28 Jan 2025 19:27:19 +0100 Subject: [PATCH 16/17] Updating HL reference --- Lombiq.Tests.UI/Lombiq.Tests.UI.csproj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj index 105be7973..8d9127479 100644 --- a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj +++ b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj @@ -111,9 +111,9 @@ - - - + + + From f5f0a0bffbf481206295b5dde2cd3d1fc26436fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Tue, 28 Jan 2025 19:45:03 +0100 Subject: [PATCH 17/17] Updating CompatibilitySuppressions --- Lombiq.Tests.UI/CompatibilitySuppressions.xml | 35 ------------------- 1 file changed, 35 deletions(-) diff --git a/Lombiq.Tests.UI/CompatibilitySuppressions.xml b/Lombiq.Tests.UI/CompatibilitySuppressions.xml index a9e1fba11..3bea5022f 100644 --- a/Lombiq.Tests.UI/CompatibilitySuppressions.xml +++ b/Lombiq.Tests.UI/CompatibilitySuppressions.xml @@ -1,13 +1,6 @@  - - CP0001 - T:Lombiq.Tests.UI.Attributes.InternetExplorerAttribute - lib/net8.0/Lombiq.Tests.UI.dll - lib/net8.0/Lombiq.Tests.UI.dll - true - CP0001 T:Lombiq.Tests.UI.Extensions.LoggingWebDriverExtensions @@ -15,13 +8,6 @@ lib/net8.0/Lombiq.Tests.UI.dll true - - CP0002 - F:Lombiq.Tests.UI.Services.Browser.InternetExplorer - lib/net8.0/Lombiq.Tests.UI.dll - lib/net8.0/Lombiq.Tests.UI.dll - true - CP0002 F:Lombiq.Tests.UI.Services.OrchardCoreUITestExecutorConfiguration.AssertBrowserLogIsEmpty @@ -92,20 +78,6 @@ lib/net8.0/Lombiq.Tests.UI.dll true - - CP0002 - M:Lombiq.Tests.UI.Extensions.HttpClientUITestContextExtensions.CreateAndAuthorizeClientAsync(Lombiq.Tests.UI.Services.UITestContext,System.String,System.String,System.String,System.String,System.String) - lib/net8.0/Lombiq.Tests.UI.dll - lib/net8.0/Lombiq.Tests.UI.dll - true - - - CP0002 - M:Lombiq.Tests.UI.Services.WebDriverFactory.CreateInternetExplorerDriverAsync(Lombiq.Tests.UI.Services.BrowserConfiguration,System.TimeSpan) - lib/net8.0/Lombiq.Tests.UI.dll - lib/net8.0/Lombiq.Tests.UI.dll - true - CP0009 T:Lombiq.Tests.UI.Services.UITestContext @@ -113,11 +85,4 @@ lib/net8.0/Lombiq.Tests.UI.dll true - - CP0011 - F:Lombiq.Tests.UI.Services.Browser.None - lib/net8.0/Lombiq.Tests.UI.dll - lib/net8.0/Lombiq.Tests.UI.dll - true - \ No newline at end of file