chore(tests): Add unordered mode to cloudflare test runner (#18596)

So far if multiple expected envelopes were chained on the cloudflare
test runner, the runner expected them to arrive in the order pre-defined
by the caller, else the test would fail. This can sometimes be too
strict, resulting in flakes. This PR adds an unordered mode to the
runner that lifts the order restriction on incoming envelopes.

This mode is also used in the hono cloudflare test now.

Closes #18589

Author: nicohrubec

dev-packages/cloudflare-integration-tests/runner.ts

@@ -57,6 +57,9 @@ type StartResult = {
 export function createRunner(...paths: string[]) {
   const testPath = join(...paths);
 
+  // controls whether envelopes are expected in predefined order or not
+  let unordered = false;
+
   if (!existsSync(testPath)) {
     throw new Error(`Test scenario not found: ${testPath}`);
   }
@@ -76,6 +79,10 @@ export function createRunner(...paths: string[]) {
       }
       return this;
     },
+    unordered: function () {
+      unordered = true;
+      return this;
+    },
     ignore: function (...types: EnvelopeItemType[]) {
       types.forEach(t => ignored.add(t));
       return this;
@@ -102,6 +109,14 @@ export function createRunner(...paths: string[]) {
         }
       }
 
+      function assertEnvelopeMatches(expected: Expected, envelope: Envelope): void {
+        if (typeof expected === 'function') {
+          expected(envelope);
+        } else {
+          expect(envelope).toEqual(expected);
+        }
+      }
+
       function newEnvelope(envelope: Envelope): void {
         if (process.env.DEBUG) log('newEnvelope', inspect(envelope, false, null, true));
 
@@ -111,19 +126,36 @@ export function createRunner(...paths: string[]) {
           return;
         }
 
-        const expected = expectedEnvelopes.shift();
-
-        // Catch any error or failed assertions and pass them to done to end the test quickly
         try {
-          if (!expected) {
-            return;
-          }
+          if (unordered) {
+            // find any matching expected envelope
+            const matchIndex = expectedEnvelopes.findIndex(candidate => {
+              try {
+                assertEnvelopeMatches(candidate, envelope);
+                return true;
+              } catch {
+                return false;
+              }
+            });
 
-          if (typeof expected === 'function') {
-            expected(envelope);
+            // no match found
+            if (matchIndex < 0) {
+              return;
+            }
+
+            // remove the matching expected envelope
+            expectedEnvelopes.splice(matchIndex, 1);
           } else {
-            expect(envelope).toEqual(expected);
+            // in ordered mode we just look at the next expected envelope
+            const expected = expectedEnvelopes.shift();
+
+            if (!expected) {
+              return;
+            }
+
+            assertEnvelopeMatches(expected, envelope);
           }
+
           expectCallbackCalled();
         } catch (e) {
           reject(e);

dev-packages/cloudflare-integration-tests/suites/hono/basic/test.ts

@@ -47,6 +47,7 @@ it('Hono app captures errors', async ({ signal }) => {
         }),
       );
     })
+    .unordered()
     .start(signal);
   await runner.makeRequest('get', '/error', { expectError: true });
   await runner.completed();