Only retry tests in CI

We introduced this change to help with reducing flakiness in CI [1].

This has a slightly different effect locally, where every failed test
will output three times.

This in turn makes it harder to read, especially when you have multiple
failing tests.

Since the original intent for this behaviour was to be used in CI, I'm
proposing we disable it when the CI env variable isn't set.

I've opted to set it for all jobs involving tests, just for consistency.
I'm happy to limit it to just the places where it's required.

[1]: #2059

Author: elenatanasoiu

.github/workflows/main.yml

@@ -199,6 +199,7 @@ jobs:
         run: echo "cli-versions=$(cat ./extensions/ql-vscode/supported_cli_versions.json | jq -rc)" >> $GITHUB_OUTPUT
     outputs:
       cli-versions: ${{ steps.set-variables.outputs.cli-versions }}
+
   cli-test:
     name: CLI Test
     runs-on: ${{ matrix.os }}

extensions/ql-vscode/test/vscode-tests/jest.activated-extension.setup.ts

@@ -5,9 +5,11 @@ import * as tmp from "tmp";
 import { realpathSync } from "fs-extra";
 import { setStoragePath, storagePath } from "./global.helper";
 
-jest.retryTimes(3, {
-  logErrorsBeforeRetry: true,
-});
+if (process.env.CI) {
+  jest.retryTimes(3, {
+    logErrorsBeforeRetry: true,
+  });
+}
 
 // create an extension storage location
 let removeStorage: tmp.DirResult["removeCallback"] | undefined;

extensions/ql-vscode/test/vscode-tests/jest.setup.ts

@@ -1,9 +1,11 @@
 import { env } from "vscode";
 import { beforeEachAction } from "./test-config";
 
-jest.retryTimes(3, {
-  logErrorsBeforeRetry: true,
-});
+if (process.env.CI) {
+  jest.retryTimes(3, {
+    logErrorsBeforeRetry: true,
+  });
+}
 
 beforeEach(async () => {
   jest.spyOn(env, "openExternal").mockResolvedValue(false);