build: start only the minimum number of Saucelabs browsers required (#50393)

This should save on Saucelabs resources so that if only one saucelabs test is run then only one set of browsers will be started.

PR Close #50393

scripts/test/run-saucelabs-tests.sh

@@ -6,6 +6,7 @@
 set -eu -o pipefail
 
 NUMBER_OF_PARALLEL_BROWSERS="${1:-2}"
+shift
 
 if [[ -z "${SAUCE_USERNAME:-}" ]]; then
   echo "ERROR: SAUCE_USERNAME environment variable must be set; see tools/saucelabs-daemon/README.md for more info."
@@ -50,6 +51,6 @@ trap kill_background_service INT TERM
 sleep 2
 
 # Run all of the saucelabs test targets
-yarn bazel test --config=saucelabs --jobs="$NUMBER_OF_PARALLEL_BROWSERS" ${TESTS}
+yarn bazel test --config=saucelabs --jobs="$NUMBER_OF_PARALLEL_BROWSERS" ${TESTS} "$@"
 
 kill_background_service

tools/saucelabs-daemon/background-service/cli.ts

@@ -43,17 +43,13 @@ if (!parallelExecutions) {
   throw Error(`Please specify a non-zero number of parallel browsers to start.`);
 }
 
-const browserInstances: Browser[] = [];
-for (let i = 0; i < parallelExecutions; i++) {
-  browserInstances.push(...Object.values(customLaunchers) as any);
-}
-
 // Start the daemon and launch the given browser
 const daemon = new SaucelabsDaemon(
     username,
     accessKey,
     process.env.CIRCLE_BUILD_NUM!,
-    browserInstances,
+    Object.values(customLaunchers) as Browser[],
+    parallelExecutions,
     sauceConnect,
     {tunnelIdentifier},
 );

tools/saucelabs-daemon/background-service/saucelabs-daemon.ts

@@ -52,7 +52,7 @@ export class SaucelabsDaemon {
   private _pendingTests = new Map<RemoteBrowser, BrowserTest>();
 
   /** List of active browsers that are managed by the daemon. */
-  private _activeBrowsers = new Set<RemoteBrowser>();
+  private _activeBrowsers: RemoteBrowser[] = [];
 
   /** Map that contains test ids with their claimed browser. */
   private _runningTests = new Map<number, RemoteBrowser>();
@@ -69,11 +69,15 @@ export class SaucelabsDaemon {
   /* Promise  indicating whether we the tunnel is active, or if we are still connecting. */
   private _connection: Promise<void>|undefined = undefined;
 
+  /* Number of parallel executions started */
+  private _parallelExecutions: number = 0;
+
   constructor(
       private _username: string,
       private _accessKey: string,
       private _buildName: string,
       private _browsers: Browser[],
+      private _maxParallelExecutions: number,
       private _sauceConnect: string,
       private _userCapabilities: object = {},
   ) {
@@ -104,7 +108,7 @@ export class SaucelabsDaemon {
       }
     });
     await Promise.all(quitBrowsers);
-    this._activeBrowsers.clear();
+    this._activeBrowsers = [];
     this._runningTests.clear();
     this._pendingTests.clear();
   }
@@ -154,36 +158,26 @@ export class SaucelabsDaemon {
   async startTest(test: BrowserTest): Promise<boolean> {
     await this.connectTunnel();
 
-    const browsers = this._findMatchingBrowsers(test.requestedBrowserId);
-    if (!browsers.length) {
+    if (this._parallelExecutions < this._maxParallelExecutions) {
+      // Start additional browsers on each test start until the max parallel executions are
+      // reached to avoid the race condition of starting a browser and then having another test
+      // start steal it before is claimed by this test.
+      await this.launchBrowserSet();
+    }
+
+    let browser = this._findAvailableBrowser(test.requestedBrowserId);
+    if (!browser) {
+      console.error(`No available browser ${test.requestedBrowserId} for test ${test.testId}!`);
       return false;
     }
 
-    // Find the first available browser and start the test.
-    for (const browser of browsers) {
-      // If the browser is claimed, continue searching.
-      if (browser.state === 'claimed') {
-        continue;
-      }
-
-      // If the browser is launching, check if it can be pre-claimed so that
-      // the test starts once the browser is ready. If it's already claimed,
-      // continue searching.
-      if (browser.state === 'launching') {
-        if (this._pendingTests.has(browser)) {
-          continue;
-        } else {
-          this._pendingTests.set(browser, test);
-          return true;
-        }
-      }
-
+    if (browser.state == 'launching') {
+      this._pendingTests.set(browser, test);
+    } else {
       this._startBrowserTest(browser, test);
-
-      return true;
     }
 
-    return false;
+    return true;
   }
 
   /**
@@ -195,16 +189,18 @@ export class SaucelabsDaemon {
   private async _connect() {
     await openSauceConnectTunnel(
         (this._userCapabilities as any).tunnelIdentifier, this._sauceConnect);
-    await this._launchBrowsers();
   }
 
   /**
    * @internal
-   * Launches all browsers. If there are pending tests waiting for a particular browser to launch
-   * before they can start, those tests are started once the browser is launched.
+   * Launches a set of browsers and increments the count of parallel browser started. If there are
+   * pending tests waiting for a particular browser to launch before they can start, those tests are
+   * started once the browser is launched.
    **/
-  private async _launchBrowsers() {
-    console.debug('Launching browsers...');
+  private async launchBrowserSet() {
+    this._parallelExecutions++;
+    console.debug(
+        `Launching browsers set ${this._parallelExecutions} of ${this._maxParallelExecutions}...`);
 
     // Once the tunnel is established we can launch browsers
     await Promise.all(
@@ -231,7 +227,7 @@ export class SaucelabsDaemon {
 
           // Keep track of the launched browser. We do this before it even completed the
           // launch as we can then handle scheduled tests when the browser is still launching.
-          this._activeBrowsers.add(launched);
+          this._activeBrowsers.push(launched);
 
           // See the following link for public API of the selenium server.
           // https://wiki.saucelabs.com/display/DOCS/Instant+Selenium+Node.js+Tests
@@ -261,7 +257,9 @@ export class SaucelabsDaemon {
           // If a test has been scheduled before the browser completed launching, run
           // it now given that the browser is ready now.
           if (this._pendingTests.has(launched)) {
-            this._startBrowserTest(launched, this._pendingTests.get(launched)!);
+            const test = this._pendingTests.get(launched)!;
+            this._pendingTests.delete(launched);
+            this._startBrowserTest(launched, test);
           }
         }),
     );
@@ -294,16 +292,32 @@ export class SaucelabsDaemon {
 
   /**
    * @internal
-   * Given a browserId, returns a list of matching browsers from the list of active browsers.
+   * Given a browserId, returns a browser that matches the browserId and is free
+   * or launching with no pending test. If no such browser if found, returns
+   * null.
    **/
-  private _findMatchingBrowsers(browserId: string): RemoteBrowser[] {
-    const browsers: RemoteBrowser[] = [];
-    this._activeBrowsers.forEach(b => {
-      if (b.id === browserId) {
-        browsers.push(b);
+  private _findAvailableBrowser(browserId: string): RemoteBrowser|null {
+    for (const browser of this._activeBrowsers) {
+      // If the browser ID doesn't match, continue searching.
+      if (browser.id !== browserId) {
+        continue;
       }
-    });
-    return browsers;
+
+      // If the browser is claimed, continue searching.
+      if (browser.state === 'claimed') {
+        continue;
+      }
+
+      // If the browser is launching, check if it can be pre-claimed so that
+      // the test starts once the browser is ready. If it's already claimed,
+      // continue searching.
+      if (browser.state === 'launching' && this._pendingTests.has(browser)) {
+        continue;
+      }
+
+      return browser;
+    }
+    return null;
   }
 
   /**