making testing better

.github/workflows/agentex-tutorials-test.yml

@@ -20,20 +20,17 @@ jobs:
         id: get-tutorials
         run: |
           cd examples/tutorials
-          # Find all tutorials and exclude specific temporal ones
+          # Find all tutorials with a manifest.yaml
           all_tutorials=$(find . -name "manifest.yaml" -exec dirname {} \; | sort | sed 's|^\./||')
 
-          # Filter out the specified temporal tutorials that are being updated
-          filtered_tutorials=$(echo "$all_tutorials" | grep -v -E "(temporal)")
+          # Include all tutorials (temporal tutorials are now included)
+          filtered_tutorials="$all_tutorials"
 
           # Convert to JSON array
           tutorials=$(echo "$filtered_tutorials" | jq -R -s -c 'split("\n") | map(select(length > 0))')
 
           echo "tutorials=$tutorials" >> $GITHUB_OUTPUT
           echo "All tutorials found: $(echo "$all_tutorials" | wc -l)"
-          echo "Filtered tutorials: $(echo "$filtered_tutorials" | wc -l)"
-          echo "Excluded tutorials:"
-          echo "$all_tutorials" | grep -E "(10_temporal/050_|10_temporal/070_|10_temporal/080_)" || echo "  (none matched exclusion pattern)"
           echo "Final tutorial list: $tutorials"
 
   test-tutorial:
@@ -58,8 +55,20 @@ jobs:
       - name: Pull latest AgentEx image
         run: |
           echo "🐳 Pulling latest Scale AgentEx Docker image..."
-          docker pull ghcr.io/scaleapi/scale-agentex/agentex:latest
-          echo "✅ Successfully pulled AgentEx Docker image"
+          max_attempts=3
+          attempt=1
+          while [ $attempt -le $max_attempts ]; do
+            echo "Attempt $attempt of $max_attempts..."
+            if docker pull ghcr.io/scaleapi/scale-agentex/agentex:latest; then
+              echo "✅ Successfully pulled AgentEx Docker image"
+              exit 0
+            fi
+            echo "❌ Pull failed, waiting before retry..."
+            sleep $((attempt * 10))
+            attempt=$((attempt + 1))
+          done
+          echo "❌ Failed to pull image after $max_attempts attempts"
+          exit 1
 
       - name: Checkout scale-agentex repo
         uses: actions/checkout@v4

examples/tutorials/00_sync/020_streaming/tests/test_agent.py

@@ -151,4 +151,3 @@ def test_send_stream_message(self, client: Agentex, agent_name: str, agent_id: s
 
 if __name__ == "__main__":
     pytest.main([__file__, "-v"])
-

examples/tutorials/10_async/00_base/000_hello_acp/tests/test_agent.py

@@ -73,6 +73,8 @@ async def test_send_event_and_poll(self, client: AsyncAgentex, agent_id: str):
         assert task is not None
 
         # Poll for the initial task creation message
+        task_creation_message_found = False
+
         async for message in poll_messages(
             client=client,
             task_id=task.id,
@@ -82,10 +84,15 @@ async def test_send_event_and_poll(self, client: AsyncAgentex, agent_id: str):
             assert isinstance(message, TaskMessage)
             if message.content and message.content.type == "text" and message.content.author == "agent":
                 assert "Hello! I've received your task" in message.content.content
+                task_creation_message_found = True
                 break
 
+        assert task_creation_message_found, "Task creation message not found"
+
         # Send an event and poll for response
         user_message = "Hello, this is a test message!"
+        agent_response_found = False
+
         async for message in send_event_and_poll_yielding(
             client=client,
             agent_id=agent_id,
@@ -97,9 +104,10 @@ async def test_send_event_and_poll(self, client: AsyncAgentex, agent_id: str):
             assert isinstance(message, TaskMessage)
             if message.content and message.content.type == "text" and message.content.author == "agent":
                 assert "Hello! I've received your task" in message.content.content
+                agent_response_found = True
                 break
 
-
+        assert agent_response_found, "Agent response not found"
 class TestStreamingEvents:
     """Test streaming event sending."""
 
@@ -111,7 +119,7 @@ async def test_send_event_and_stream(self, client: AsyncAgentex, agent_id: str):
 
         assert task is not None
         task_creation_found = False
-        # Poll for the initial task creation message
+
         async for message in poll_messages(
             client=client,
             task_id=task.id,
@@ -124,7 +132,7 @@ async def test_send_event_and_stream(self, client: AsyncAgentex, agent_id: str):
                 task_creation_found = True
                 break
 
-        assert task_creation_found, "Task creation message not found in poll"
+        assert task_creation_found, "Task creation message not found"
 
         user_message = "Hello, this is a test message!"
         stream_timeout = 10
@@ -136,9 +144,8 @@ async def test_send_event_and_stream(self, client: AsyncAgentex, agent_id: str):
         user_echo_found = False
         agent_response_found = False
 
-        async def collect_stream_events() -> None:
+        async def stream_messages() -> None:
             nonlocal user_echo_found, agent_response_found
-
             async for event in stream_agent_response(
                 client=client,
                 task_id=task.id,
@@ -161,23 +168,19 @@ async def collect_stream_events() -> None:
                         # Check for user message echo
                         if content.get("content") == user_message:
                             user_echo_found = True
+                elif event_type == "done":
+                    break
 
                 # Exit early if we've found all expected messages
                 if user_echo_found and agent_response_found:
                     break
 
-        # Start streaming task
-        stream_task = asyncio.create_task(collect_stream_events())
+        stream_task = asyncio.create_task(stream_messages())
 
         # Send the event
         event_content = TextContentParam(type="text", author="user", content=user_message)
         await client.agents.send_event(agent_id=agent_id, params={"task_id": task.id, "content": event_content})
-
-        # Wait for the stream to complete (with timeout)
-        try:
-            await asyncio.wait_for(stream_task, timeout=stream_timeout)
-        except asyncio.TimeoutError:
-            pytest.fail(f"Stream timed out after {stream_timeout}s waiting for expected messages")
+        await stream_task
 
         # Verify all expected messages were received (fail if stream ended without finding them)
         assert user_echo_found, "User message echo not found in stream"

examples/tutorials/10_async/00_base/010_multiturn/tests/test_agent.py

@@ -89,6 +89,10 @@ async def test_send_event_and_poll(self, client: AsyncAgentex, agent_id: str):
 
         user_message = "Hello! Here is my test message"
         messages = []
+
+        # Flags to track what we've received
+        user_message_found = False
+        agent_response_found = False
         async for message in send_event_and_poll_yielding(
             client=client,
             agent_id=agent_id,
@@ -98,17 +102,28 @@ async def test_send_event_and_poll(self, client: AsyncAgentex, agent_id: str):
             sleep_interval=1.0,
         ):
             messages.append(message)
-            if len(messages) == 1:
-                assert message.content == TextContent(
-                    author="user",
-                    content=user_message,
-                    type="text",
-                )
-            else:
-                assert message.content is not None
-                assert message.content.author == "agent"
+
+            # Validate messages as they arrive
+            if message.content and hasattr(message.content, "author"):
+                msg_text = getattr(message.content, "content", None)
+                if message.content.author == "user" and msg_text == user_message:
+                    assert message.content == TextContent(
+                        author="user",
+                        content=user_message,
+                        type="text",
+                    )
+                    user_message_found = True
+                elif message.content.author == "agent":
+                    assert user_message_found, "Agent response arrived before user message"
+                    agent_response_found = True
+
+            # Exit early if we've found all expected messages
+            if user_message_found and agent_response_found:
                 break
 
+        assert user_message_found, "User message not found"
+        assert agent_response_found, "Agent response not found"
+
         await asyncio.sleep(1)  # wait for state to be updated
         states = await client.states.list(agent_id=agent_id, task_id=task.id)
         assert len(states) == 1
@@ -152,10 +167,8 @@ async def test_send_event_and_stream(self, client: AsyncAgentex, agent_id: str):
         # Flags to track what we've received
         user_message_found = False
         agent_response_found = False
-
-        async def stream_messages():
+        async def stream_messages() -> None:
             nonlocal user_message_found, agent_response_found
-
             async for event in stream_agent_response(
                 client=client,
                 task_id=task.id,
@@ -175,21 +188,17 @@ async def stream_messages():
                         # Agent response should come after user message
                         assert user_message_found, "Agent response arrived before user message (incorrect order)"
                         agent_response_found = True
+                elif event_type == "done":
+                    break
 
                 # Exit early if we've found both messages
                 if user_message_found and agent_response_found:
                     break
 
         stream_task = asyncio.create_task(stream_messages())
-
         event_content = TextContentParam(type="text", author="user", content=user_message)
         await client.agents.send_event(agent_id=agent_id, params={"task_id": task.id, "content": event_content})
-
-        # Wait for streaming to complete (with timeout)
-        try:
-            await asyncio.wait_for(stream_task, timeout=15)
-        except asyncio.TimeoutError:
-            pytest.fail("Stream timed out after 15s waiting for expected messages")
+        await stream_task
 
         # Validate we received events
         assert len(all_events) > 0, "No events received in streaming response"

examples/tutorials/10_async/00_base/020_streaming/tests/test_agent.py

@@ -28,7 +28,6 @@
 )
 
 from agentex import AsyncAgentex
-from agentex.types import TaskMessage, TextContent
 from agentex.types.agent_rpc_params import ParamsCreateTaskRequest
 from agentex.types.text_content_param import TextContentParam
 
@@ -89,6 +88,10 @@ async def test_send_event_and_poll(self, client: AsyncAgentex, agent_id: str):
 
         user_message = "Hello! Here is my test message"
         messages = []
+
+        # Flags to track what we've received
+        user_message_found = False
+        agent_response_found = False
         async for message in send_event_and_poll_yielding(
             client=client,
             agent_id=agent_id,
@@ -98,23 +101,25 @@ async def test_send_event_and_poll(self, client: AsyncAgentex, agent_id: str):
             sleep_interval=1.0,
             yield_updates=False,
         ):
-
             messages.append(message)
 
-        assert len(messages) > 0
-        # the first message should be the agent re-iterating what the user sent
-        assert isinstance(messages, List)
-        assert len(messages) == 2
-        first_message: TaskMessage = messages[0]
-        assert first_message.content == TextContent(
-            author="user",
-            content=user_message,
-            type="text",
-        )
-
-        second_message: TaskMessage = messages[1]
-        assert second_message.content is not None
-        assert second_message.content.author == "agent"
+            # Validate messages as they come in
+            if message.content and hasattr(message.content, "author"):
+                if message.content.author == "user" and message.content.content == user_message:
+                    user_message_found = True
+                elif message.content.author == "agent":
+                    # Agent response should come after user message
+                    assert user_message_found, "Agent response arrived before user message"
+                    agent_response_found = True
+
+            # Exit early if we've found all expected messages
+            if user_message_found and agent_response_found:
+                break
+
+        # Validate we received expected messages
+        assert len(messages) >= 2, "Expected at least 2 messages (user + agent)"
+        assert user_message_found, "User message not found"
+        assert agent_response_found, "Agent response not found"
 
         # assert the state has been updated
         await asyncio.sleep(1)  # wait for state to be updated
@@ -158,41 +163,43 @@ async def test_send_event_and_stream(self, client: AsyncAgentex, agent_id: str):
         # Collect events from stream
         all_events = []
 
+        # Flags to track what we've received
+        user_message_found = False
+        full_agent_message_found = False
+        delta_messages_found = False
         async def stream_messages() -> None:
+            nonlocal user_message_found, full_agent_message_found, delta_messages_found
             async for event in stream_agent_response(
                 client=client,
                 task_id=task.id,
                 timeout=15,
             ):
                 all_events.append(event)
 
-        stream_task = asyncio.create_task(stream_messages())
+                # Check events as they arrive
+                event_type = event.get("type")
+                if event_type == "full":
+                    content = event.get("content", {})
+                    if content.get("content") == user_message and content.get("author") == "user":
+                        user_message_found = True
+                    elif content.get("author") == "agent":
+                        full_agent_message_found = True
+                elif event_type == "delta":
+                    delta_messages_found = True
+                elif event_type == "done":
+                    break
+
+                # Exit early if we've found all expected messages
+                if user_message_found and full_agent_message_found and delta_messages_found:
+                    break
 
+        stream_task = asyncio.create_task(stream_messages())
         event_content = TextContentParam(type="text", author="user", content=user_message)
         await client.agents.send_event(agent_id=agent_id, params={"task_id": task.id, "content": event_content})
-
-        # Wait for streaming to complete
         await stream_task
 
         # Validate we received events
         assert len(all_events) > 0, "No events received in streaming response"
-
-        # Check for user message, full agent response, and delta messages
-        user_message_found = False
-        full_agent_message_found = False
-        delta_messages_found = False
-
-        for event in all_events:
-            event_type = event.get("type")
-            if event_type == "full":
-                content = event.get("content", {})
-                if content.get("content") == user_message and content.get("author") == "user":
-                    user_message_found = True
-                elif content.get("author") == "agent":
-                    full_agent_message_found = True
-            elif event_type == "delta":
-                delta_messages_found = True
-
         assert user_message_found, "User message not found in stream"
         assert full_agent_message_found, "Full agent message not found in stream"
         assert delta_messages_found, "Delta messages not found in stream (streaming response expected)"