python[patch]: fix pytest integration example syncing (#1450)

- properly lock access updating shared resource
- correctly pull modified_at when updating example

python/langsmith/pytest_plugin.py

@@ -83,7 +83,7 @@ def pytest_report_teststatus(report, config):
     # The hook normally returns a 3-tuple: (short_letter, verbose_word, color)
     # By returning empty strings, the progress characters won't show.
     if config.getoption("--output") in ("langsmith", "ls"):
-        return "", "", None
+        return "", "", ""
 
 
 class LangSmithPlugin:

python/langsmith/testing/_internal.py

@@ -550,7 +550,7 @@ def get_version(
             if not example_id:
                 return self._version
             else:
-                return self._example_modified_at[example_id]
+                return self._example_modified_at.get(example_id)
 
     def submit_result(
         self,
@@ -620,26 +620,23 @@ def _sync_example(
                 metadata=metadata,
                 created_at=self._experiment.start_time,
             )
-            modified_at = example.modified_at
         else:
             if (
                 (inputs is not None and inputs != example.inputs)
                 or (outputs is not None and outputs != example.outputs)
                 or (metadata is not None and metadata != example.metadata)
                 or str(example.dataset_id) != str(self.id)
             ):
-                response = self.client.update_example(
+                self.client.update_example(
                     example_id=example.id,
                     inputs=inputs,
                     outputs=outputs,
                     metadata=metadata,
                     dataset_id=self.id,
                 )
-                modified_at = datetime.datetime.fromisoformat(response["modified_at"])
-            else:
-                modified_at = example.modified_at
-        if modified_at:
-            self.update_version(modified_at, example_id=example_id)
+                example = self.client.read_example(example_id=example.id)
+        if example.modified_at:
+            self.update_version(example.modified_at, example_id=example_id)
 
     def _submit_feedback(
         self,
@@ -668,8 +665,9 @@ def shutdown(self):
 
     def wait_example_updates(self, example_id: ID_TYPE):
         """Wait for all example updates to complete."""
-        while self._example_futures[example_id]:
-            self._example_futures[example_id].pop().result()
+        with self._lock:
+            while self._example_futures[example_id]:
+                self._example_futures[example_id].pop().result()
 
     def end_run(
         self,
@@ -697,7 +695,7 @@ def _end_run(
         # Ensure example is fully updated
         self.wait_example_updates(example_id)
         # Ensure that run end time is after example modified at.
-        example_modified_at = self.get_version(example_id=example_id)
+        example_modified_at = self.get_version(example_id=example_id) or end_time
         end_time = max(example_modified_at, end_time)
         run_tree.end(outputs=outputs, end_time=end_time)
         run_tree.patch()
@@ -778,11 +776,11 @@ def end_time(self) -> None:
                 self.pytest_nodeid, {"end_time": time.time()}
             )
 
-    def end_run(self, run_tree, outputs: Any) -> Future:
+    def end_run(self, run_tree, outputs: Any) -> None:
         if not (outputs is None or isinstance(outputs, dict)):
             outputs = {"output": outputs}
         end_time = datetime.datetime.now(datetime.timezone.utc)
-        return self.test_suite.end_run(
+        self.test_suite.end_run(
             run_tree,
             self.example_id,
             outputs,

python/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "langsmith"
-version = "0.3.0"
+version = "0.3.1"
 description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform."
 authors = ["LangChain <support@langchain.dev>"]
 license = "MIT"