Account for incrementally loading pages that have placeholders (#530)

- use a mutation observer to check for page elements that are
placeholders (e.g. elements with the aria-busy="true" acccessibility
attribute).
- use an intersection observer to ensure we only consider placeholder
elements that are visible in the viewport. Many websites would have
placeholders for product lists or search results until the user scrolls
the page section into view.

ts/packages/agents/browser/src/agent/instacart/actionHandler.mts

@@ -140,7 +140,6 @@ export async function handleInstacartAction(
       )) as AllListsInfo;
 
       if (targetList) {
-        // se
         await browser.clickOn(targetList.lists[0].cssSelector);
         await browser.clickOn(targetList.submitButtonCssSelector);
       }
@@ -185,7 +184,7 @@ export async function handleInstacartAction(
     )) as StoreInfo;
 
     await browser.clickOn(targetStore.storeLinkCssSelector);
-    await browser.awaitPageInteraction();
+    await browser.awaitPageInteraction(1000);
     await browser.awaitPageLoad(5000);
   }
 
@@ -197,7 +196,7 @@ export async function handleInstacartAction(
   }
 
   async function searchForRecipe(recipeKeywords: string) {
-    // await goToHomepage();
+    await goToHomepage();
     const selector = (await getComponentFromPage("SearchInput")) as SearchInput;
     const searchSelector = selector.cssSelector;
 
@@ -252,9 +251,10 @@ export async function handleInstacartAction(
 
     if (navigationLink) {
       await browser.clickOn(navigationLink.linkCssSelector);
+      await browser.awaitPageInteraction();
       await browser.awaitPageLoad();
 
-      const request = `List name: ${action.listName}`;
+      const request = `List name: ${action.parameters.listName}`;
       const targetList = (await getComponentFromPage(
         "ListInfo",
         request,
@@ -270,36 +270,59 @@ export async function handleInstacartAction(
 
         if (listDetails && listDetails.products) {
           for (let product of listDetails.products) {
-            if (product.addToCartButton) {
-              await browser.clickOn(product.addToCartButton.cssSelector);
+            if (product.addToCartButtonCssSelector) {
+              await browser.clickOn(product.addToCartButtonCssSelector);
             }
           }
         }
       }
     }
   }
 
+  async function selectStore(storeName: string) {
+    await goToHomepage();
+    const request = `Store name: ${storeName}`;
+    const targetStore = (await getComponentFromPage(
+      "StoreInfo",
+      request,
+    )) as StoreInfo;
+
+    console.log(targetStore);
+
+    if (!targetStore) {
+      return;
+    }
+
+    await browser.clickOn(targetStore.storeLinkCssSelector);
+    await browser.awaitPageInteraction();
+    await browser.awaitPageLoad();
+  }
+
   async function handleBuyItAgain(action: any) {
-    await searchForStore(action.parameters.storeName);
-    await selectStoreSearchResult(action.parameters.storeName);
+    await selectStore(action.parameters.storeName);
 
     const navigationLink = (await getComponentFromPage(
       "BuyItAgainNavigationLink",
     )) as BuyItAgainNavigationLink;
 
+    console.log(navigationLink);
+
     if (navigationLink) {
       await browser.clickOn(navigationLink.linkCssSelector);
+      await browser.awaitPageInteraction();
       await browser.awaitPageLoad();
 
       const headerSection = (await getComponentFromPage(
         "BuyItAgainHeaderSection",
       )) as BuyItAgainHeaderSection;
+      console.log(headerSection);
 
       if (headerSection && headerSection.products) {
         if (action.parameters.allItems) {
           for (let product of headerSection.products) {
-            if (product.addToCartButton) {
-              await browser.clickOn(product.addToCartButton.cssSelector);
+            if (product.addToCartButtonCssSelector) {
+              await browser.clickOn(product.addToCartButtonCssSelector);
+              await browser.awaitPageInteraction();
             }
           }
         } else {
@@ -308,8 +331,9 @@ export async function handleInstacartAction(
             "ProductTile",
             request,
           )) as ProductTile;
-          if (targetProduct && targetProduct.addToCartButton) {
-            await browser.clickOn(targetProduct.addToCartButton.cssSelector);
+          if (targetProduct && targetProduct.addToCartButtonCssSelector) {
+            await browser.clickOn(targetProduct.addToCartButtonCssSelector);
+            await browser.awaitPageInteraction();
           }
         }
       }

ts/packages/agents/browser/src/agent/instacart/schema/pageComponents.mts

@@ -10,10 +10,8 @@ export type ProductTile = {
   // Construct the selector based on the element's Id attribute if the id is present.
   detailsLinkSelector: string;
 
-  addToCartButton?: {
-    // css selector for the add to cart button
-    cssSelector: string;
-  };
+  // css selector for the add to cart button
+  addToCartButtonCssSelector: string;
 };
 
 // This is only present on the Product Details Page

ts/packages/agents/browser/src/extension/contentScript.ts

@@ -3,6 +3,7 @@
 
 import { Readability, isProbablyReaderable } from "@mozilla/readability";
 import { HTMLReducer } from "./htmlReducer";
+import { SkeletonLoadingDetector } from "./loadingDetector";
 import { convert } from "html-to-text";
 import DOMPurify from "dompurify";
 
@@ -518,6 +519,27 @@ function setIdsOnAllElements(frameId: number, useTimestampIds?: boolean) {
     }
 }
 
+async function awaitPageIncrementalUpdates() {
+    return new Promise<string | undefined>((resolve, reject) => {
+        const detector = new SkeletonLoadingDetector({
+            stabilityThresholdMs: 500,
+            // Consider elements visible when they're at least 10% in view
+            intersectionThreshold: 0.1,
+        });
+
+        detector
+            .detect()
+            .then(() => {
+                console.log("Page incremental load completed.");
+                resolve("true");
+            })
+            .catch((error: Error) => {
+                console.error("Failed to detect page load completion:", error);
+                resolve("false");
+            });
+    });
+}
+
 function sendPaleoDbRequest(data: any) {
     document.dispatchEvent(
         new CustomEvent("toPaleoDbAutomation", { detail: data }),
@@ -641,6 +663,12 @@ async function handleScriptAction(
             break;
         }
 
+        case "await_page_incremental_load": {
+            const updated = await awaitPageIncrementalUpdates();
+            sendResponse(updated);
+            break;
+        }
+
         case "run_ui_event": {
             sendUIEventsRequest(message.action);
             sendResponse({});
@@ -688,12 +716,13 @@ async function handleScriptAction(
 }
 
 chrome.runtime?.onMessage.addListener(
-    async (
-        message: any,
-        sender: chrome.runtime.MessageSender,
-        sendResponse,
-    ) => {
-        await handleScriptAction(message, sendResponse);
+    (message: any, sender: chrome.runtime.MessageSender, sendResponse) => {
+        const handleMessage = async () => {
+            await handleScriptAction(message, sendResponse);
+        };
+
+        handleMessage();
+        return true; // Important: indicates we'll send response asynchronously
     },
 );