SCANJLIB-216 Rework loading of env variables

Author: henryju

its/it-simple-scanner/src/main/java/com/sonar/scanner/lib/it/Main.java

@@ -21,30 +21,35 @@
 
 import java.util.HashMap;
 import java.util.Map;
+import org.sonarsource.scanner.lib.EnvironmentConfig;
 import org.sonarsource.scanner.lib.LogOutput;
 import org.sonarsource.scanner.lib.ScannerEngineBootstrapper;
 import org.sonarsource.scanner.lib.StdOutLogOutput;
 
 public class Main {
   public static void main(String[] args) {
+    LogOutput logOutput = new StdOutLogOutput();
     try {
       Map<String, String> props = new HashMap<>();
+
+      props.putAll(EnvironmentConfig.load(logOutput));
+
       for (String k : System.getProperties().stringPropertyNames()) {
         if (k.startsWith("sonar.")) {
           props.put(k, System.getProperty(k));
         }
       }
 
-      runProject(props);
+      runProject(props, logOutput);
     } catch (Exception e) {
       e.printStackTrace();
       System.exit(1);
     }
     System.exit(0);
   }
 
-  private static void runProject(Map<String, String> props) throws Exception {
-    LogOutput logOutput = new StdOutLogOutput();
+  private static void runProject(Map<String, String> props, LogOutput logOutput) throws Exception {
+
     try (var scannerEngine = new ScannerEngineBootstrapper("Simple Scanner", "1.0", logOutput)
       .addBootstrapProperties(props)
       .bootstrap()) {

its/it-tests/src/test/java/com/sonar/scanner/lib/it/PropertiesTest.java

@@ -41,14 +41,25 @@ public class PropertiesTest {
   public void testRuntimeEnvironmentPassedAsUserAgent() throws IOException {
     SimpleScanner scanner = new SimpleScanner();
     Map<String, String> params = new HashMap<>();
-    BuildResult buildResult = scanner.executeSimpleProject(project("js-sample"), ORCHESTRATOR.getServer().getUrl(), params);
-    assertThat(buildResult.getLastStatus()).isEqualTo(0);
+    BuildResult buildResult = scanner.executeSimpleProject(project("js-sample"), ORCHESTRATOR.getServer().getUrl(), params, Map.of());
+    assertThat(buildResult.getLastStatus()).isZero();
+    assertThat(buildResult.getLogs()).contains("2 files indexed");
 
     Path accessLogs = ORCHESTRATOR.getServer().getAppLogs().toPath().resolveSibling("access.log");
     String accessLogsContent = new String(Files.readAllBytes(accessLogs));
     assertThat(accessLogsContent).contains("\"Simple Scanner/1.0\"");
   }
 
+  @Test
+  public void passConfigurationUsingEnvVariables() throws IOException {
+    SimpleScanner scanner = new SimpleScanner();
+    Map<String, String> params = new HashMap<>();
+    BuildResult buildResult = scanner.executeSimpleProject(project("js-sample"), ORCHESTRATOR.getServer().getUrl(), params, Map.of("SONAR_SCANNER_JSON_PARAMS", "{\"sonar.exclusions\": \"**/Hello.js\"}"));
+    assertThat(buildResult.getLastStatus()).isZero();
+
+    assertThat(buildResult.getLogs()).contains("1 file indexed");
+  }
+
   private static Path project(String projectName) {
     return Paths.get("..", "projects", projectName);
   }

its/it-tests/src/test/java/com/sonar/scanner/lib/it/ProxyTest.java

@@ -191,7 +191,7 @@ public void simple_analysis_with_proxy_no_auth() throws Exception {
     params.put("http.proxyHost", "localhost");
     params.put("http.proxyPort", "" + httpProxyPort);
 
-    buildResult = scanner.executeSimpleProject(project("js-sample"), ORCHESTRATOR.getServer().getUrl(), params);
+    buildResult = scanner.executeSimpleProject(project("js-sample"), ORCHESTRATOR.getServer().getUrl(), params, Map.of());
     assertThat(buildResult.getLastStatus()).isZero();
     assertThat(seenByProxy).isNotEmpty();
   }
@@ -207,14 +207,14 @@ public void simple_analysis_with_proxy_auth() throws Exception {
     params.put("http.proxyHost", "localhost");
     params.put("http.proxyPort", "" + httpProxyPort);
 
-    BuildResult buildResult = scanner.executeSimpleProject(project("js-sample"), ORCHESTRATOR.getServer().getUrl(), params);
+    BuildResult buildResult = scanner.executeSimpleProject(project("js-sample"), ORCHESTRATOR.getServer().getUrl(), params, Map.of());
     assertThat(buildResult.getLastStatus()).isEqualTo(1);
     assertThat(buildResult.getLogs()).contains("Status returned by url", "is not valid: [407]");
     assertThat(seenByProxy).isEmpty();
 
     params.put("http.proxyUser", PROXY_USER);
     params.put("http.proxyPassword", PROXY_PASSWORD);
-    buildResult = scanner.executeSimpleProject(project("js-sample"), ORCHESTRATOR.getServer().getUrl(), params);
+    buildResult = scanner.executeSimpleProject(project("js-sample"), ORCHESTRATOR.getServer().getUrl(), params, Map.of());
     assertThat(seenByProxy).isNotEmpty();
     if (ORCHESTRATOR.getServer().version().isGreaterThanOrEquals(6, 1)) {
       assertThat(buildResult.getLastStatus()).isZero();

its/it-tests/src/test/java/com/sonar/scanner/lib/it/SSLTest.java

@@ -106,7 +106,7 @@ private static void startSSLTransparentReverseProxy(boolean requireClientAuth) t
 
     // Handler Structure
     HandlerCollection handlers = new HandlerCollection();
-    handlers.setHandlers(new Handler[] {proxyHandler(), new DefaultHandler()});
+    handlers.setHandlers(new Handler[]{proxyHandler(), new DefaultHandler()});
     server.setHandler(handlers);
 
     ServerConnector http = new ServerConnector(server, new HttpConnectionFactory(httpConfig));
@@ -184,7 +184,7 @@ public void simple_analysis_with_server_and_client_certificate() throws Exceptio
     params.put("javax.net.ssl.keyStore", clientKeystore.toString());
     params.put("javax.net.ssl.keyStorePassword", CLIENT_KEYSTORE_PASSWORD);
 
-    buildResult = scanner.executeSimpleProject(project("js-sample"), "https://localhost:" + httpsPort, params);
+    buildResult = scanner.executeSimpleProject(project("js-sample"), "https://localhost:" + httpsPort, params, Map.of());
     assertThat(buildResult.getLastStatus()).isZero();
   }
 
@@ -208,7 +208,7 @@ public void simple_analysis_with_server_and_without_client_certificate_is_failin
     params.put("javax.net.ssl.trustStorePassword", CLIENT_WITH_CA_KEYSTORE_PASSWORD);
     // Voluntary missing client keystore
 
-    buildResult = scanner.executeSimpleProject(project("js-sample"), "https://localhost:" + httpsPort, params);
+    buildResult = scanner.executeSimpleProject(project("js-sample"), "https://localhost:" + httpsPort, params, Map.of());
     assertThat(buildResult.getLastStatus()).isEqualTo(1);
 
     // different exception is thrown depending on the JDK version. See: https://bugs.java.com/bugdatabase/view_bug.do?bug_id=8172163
@@ -251,7 +251,7 @@ private void simple_analysis_with_server_certificate(String clientTrustStore, St
     params.put("javax.net.ssl.trustStore", clientTruststore.toString());
     params.put("javax.net.ssl.trustStorePassword", keyStorePassword);
 
-    buildResult = scanner.executeSimpleProject(project("js-sample"), "https://localhost:" + httpsPort, params);
+    buildResult = scanner.executeSimpleProject(project("js-sample"), "https://localhost:" + httpsPort, params, Map.of());
     assertThat(buildResult.getLastStatus()).isZero();
   }
 }

its/it-tests/src/test/java/com/sonar/scanner/lib/it/ScannerJavaLibraryTestSuite.java

@@ -25,35 +25,24 @@
 import java.time.Instant;
 import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
-import org.apache.commons.lang.StringUtils;
 import org.junit.ClassRule;
 import org.junit.runner.RunWith;
 import org.junit.runners.Suite;
 import org.junit.runners.Suite.SuiteClasses;
 
-import static org.assertj.core.api.Assertions.fail;
-
 @RunWith(Suite.class)
 @SuiteClasses({ProxyTest.class, SSLTest.class, PropertiesTest.class})
 public class ScannerJavaLibraryTestSuite {
   private static final String SONAR_RUNTIME_VERSION = "sonar.runtimeVersion";
 
   @ClassRule
   public static final OrchestratorRule ORCHESTRATOR = OrchestratorRule.builderEnv()
-    .setSonarVersion(getSystemPropertyOrFail(SONAR_RUNTIME_VERSION))
+    .setSonarVersion(System.getProperty(SONAR_RUNTIME_VERSION, "LATEST_RELEASE"))
     .useDefaultAdminCredentialsForBuilds(true)
     // We need to use a plugin compatible with both SonarQube DEV & SonarQube version defined in .cirrus.yml (currently SQ 7.9)
     .addPlugin(MavenLocation.of("org.sonarsource.javascript", "sonar-javascript-plugin", "7.4.4.15624"))
     .build();
 
-  private static String getSystemPropertyOrFail(String orchestratorPropertiesSource) {
-    String propertyValue = System.getProperty(orchestratorPropertiesSource);
-    if (StringUtils.isEmpty(propertyValue)) {
-      fail(orchestratorPropertiesSource + " system property must be defined");
-    }
-    return propertyValue;
-  }
-
   public static void resetData(OrchestratorRule orchestrator) {
     Instant instant = Instant.now();
 

its/it-tests/src/test/java/com/sonar/scanner/lib/it/tools/SimpleScanner.java

@@ -19,19 +19,17 @@
  */
 package com.sonar.scanner.lib.it.tools;
 
+import com.sonar.orchestrator.build.BuildResult;
 import com.sonar.scanner.lib.it.ScannerJavaLibraryTestSuite;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 
-import com.sonar.orchestrator.build.BuildResult;
-
 public class SimpleScanner {
   private static final Path JAR_PATH = Paths.get("..", "it-simple-scanner", "target", "simple-scanner.jar")
     .toAbsolutePath()
@@ -52,18 +50,18 @@ public SimpleScanner() {
   }
 
   public BuildResult executeSimpleProject(Path baseDir, String host) throws IOException {
-    return executeSimpleProject(baseDir, host, Collections.emptyMap());
+    return executeSimpleProject(baseDir, host, Map.of(), Map.of());
   }
 
-  public BuildResult executeSimpleProject(Path baseDir, String host, Map<String, String> extraProps) throws IOException {
+  public BuildResult executeSimpleProject(Path baseDir, String host, Map<String, String> extraProps, Map<String, String> env) throws IOException {
     List<String> params = new ArrayList<>();
     Map<String, String> props = getSimpleProjectProperties(baseDir, host, extraProps);
 
     props.forEach((k, v) -> params.add("-D" + k + "=" + v));
     params.add("-jar");
     params.add(JAR_PATH.toString());
 
-    int status = exec.execute(params.toArray(new String[params.size()]));
+    int status = exec.execute(params.toArray(new String[params.size()]), env);
     BuildResult result = new BuildResult();
 
     result.addStatus(status);

lib/pom.xml

@@ -40,6 +40,10 @@
       <artifactId>sonar-scanner-java-library-batch-interface</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+    </dependency>
 
     <!-- unit tests -->
     <dependency>
@@ -67,11 +71,6 @@
       <artifactId>mockwebserver</artifactId>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>org.apache.commons</groupId>
-      <artifactId>commons-lang3</artifactId>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <!-- used to compare results -->
       <groupId>commons-codec</groupId>

lib/src/main/java/org/sonarsource/scanner/lib/EnvironmentConfig.java

@@ -0,0 +1,105 @@
+/*
+ * SonarScanner Java Library
+ * Copyright (C) 2011-2024 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonarsource.scanner.lib;
+
+import com.google.gson.Gson;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Stream;
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * Utility class to load configuration from environment variables.
+ */
+public class EnvironmentConfig {
+
+  private static final String SONAR_SCANNER_JSON_PARAMS = "SONAR_SCANNER_JSON_PARAMS";
+  private static final String SONARQUBE_SCANNER_PARAMS = "SONARQUBE_SCANNER_PARAMS";
+  private static final String GENERIC_ENV_PREFIX = "SONAR_SCANNER_";
+  private static final String SONAR_HOST_URL_ENV_VAR = "SONAR_HOST_URL";
+
+  private EnvironmentConfig() {
+    // only static methods
+  }
+
+  public static Map<String, String> load(LogOutput logger) {
+    return load(System.getenv(), logger);
+  }
+
+  static Map<String, String> load(Map<String, String> env, LogOutput logger) {
+    var loadedProps = new HashMap<String, String>();
+    Optional.ofNullable(env.get(SONAR_HOST_URL_ENV_VAR)).ifPresent(url -> loadedProps.put(ScannerProperties.HOST_URL, url));
+    env.forEach((key, value) -> {
+      if (!key.equals(SONAR_SCANNER_JSON_PARAMS) && key.startsWith(GENERIC_ENV_PREFIX)) {
+        processEnvVariable(key, value, loadedProps, logger);
+      }
+    });
+    var jsonParams = env.get(SONAR_SCANNER_JSON_PARAMS);
+    var oldJsonParams = env.get(SONARQUBE_SCANNER_PARAMS);
+    if (jsonParams != null) {
+      if (oldJsonParams != null && !oldJsonParams.equals(jsonParams)) {
+        logger.log("Ignoring environment variable '" + SONARQUBE_SCANNER_PARAMS + "' because '" + SONAR_SCANNER_JSON_PARAMS + "' is set", LogOutput.Level.WARN);
+      }
+      parseJsonPropertiesFromEnv(jsonParams, loadedProps, SONAR_SCANNER_JSON_PARAMS, logger);
+    } else if (oldJsonParams != null) {
+      parseJsonPropertiesFromEnv(oldJsonParams, loadedProps, SONARQUBE_SCANNER_PARAMS, logger);
+    }
+    return loadedProps;
+  }
+
+  private static void parseJsonPropertiesFromEnv(String jsonParams, Map<String, String> inputProperties, String envVariableName, LogOutput logger) {
+    try {
+      var jsonProperties = new Gson().<Map<String, String>>fromJson(jsonParams, Map.class);
+      if (jsonProperties != null) {
+        jsonProperties.forEach((key, value) -> {
+          if (inputProperties.containsKey(key)) {
+            if (!inputProperties.get(key).equals(value)) {
+              logger.log("Ignoring property '" + key + "' from env variable '" + envVariableName + "' because it is already defined", LogOutput.Level.WARN);
+            }
+          } else {
+            inputProperties.put(key, value);
+          }
+        });
+      }
+    } catch (Exception e) {
+      throw new IllegalArgumentException("Failed to parse JSON properties from environment variable '" + envVariableName + "'", e);
+    }
+  }
+
+  private static void processEnvVariable(String key, String value, Map<String, String> inputProperties, LogOutput logger) {
+    var suffix = key.substring(GENERIC_ENV_PREFIX.length());
+    if (suffix.isEmpty()) {
+      return;
+    }
+    var toCamelCase = Stream.of(suffix.split("_"))
+      .map(String::toLowerCase)
+      .reduce((a, b) -> a + StringUtils.capitalize(b)).orElseThrow();
+    var propKey = "sonar.scanner." + toCamelCase;
+    if (inputProperties.containsKey(propKey)) {
+      if (!inputProperties.get(propKey).equals(value)) {
+        logger.log("Ignoring environment variable '" + key + "' because it is already defined in the properties", LogOutput.Level.WARN);
+      }
+    } else {
+      inputProperties.put(propKey, value);
+    }
+  }
+
+}

lib/src/main/java/org/sonarsource/scanner/lib/ScannerEngineBootstrapper.java

@@ -27,42 +27,24 @@
 import org.sonarsource.scanner.lib.internal.ClassloadRules;
 import org.sonarsource.scanner.lib.internal.InternalProperties;
 import org.sonarsource.scanner.lib.internal.IsolatedLauncherFactory;
-import org.sonarsource.scanner.lib.internal.cache.Logger;
 
 /**
  * Entry point to run a Sonar analysis programmatically.
  */
 public class ScannerEngineBootstrapper {
-  private static final String SONAR_HOST_URL_ENV_VAR = "SONAR_HOST_URL";
+
   private static final String SONARCLOUD_HOST = "https://sonarcloud.io";
   private final IsolatedLauncherFactory launcherFactory;
   private final LogOutput logOutput;
   private final Map<String, String> bootstrapProperties = new HashMap<>();
-  private final System2 system;
 
   public ScannerEngineBootstrapper(String app, String version, final LogOutput logOutput) {
-    this(app, version, logOutput, new System2());
-  }
-
-  /**
-   * For testing.
-   */
-  public ScannerEngineBootstrapper(String app, String version, final LogOutput logOutput, System2 system2) {
-    this(new LoggerAdapter(logOutput), logOutput, system2);
+    this.logOutput = logOutput;
+    this.launcherFactory = new IsolatedLauncherFactory(new LoggerAdapter(logOutput));
     this.setBootstrapProperty(InternalProperties.SCANNER_APP, app)
       .setBootstrapProperty(InternalProperties.SCANNER_APP_VERSION, version);
   }
 
-  private ScannerEngineBootstrapper(Logger logger, LogOutput logOutput, System2 system) {
-    this(logOutput, system, new IsolatedLauncherFactory(logger));
-  }
-
-  ScannerEngineBootstrapper(LogOutput logOutput, System2 system, IsolatedLauncherFactory launcherFactory) {
-    this.launcherFactory = launcherFactory;
-    this.logOutput = logOutput;
-    this.system = system;
-  }
-
   /**
    * Declare technical properties needed to bootstrap (sonar.host.url, credentials, proxy, ...).
    */
@@ -94,12 +76,7 @@ public ScannerEngineFacade bootstrap() {
   }
 
   private void initBootstrapDefaultValues() {
-    String sonarHostUrl = system.getEnvironmentVariable(SONAR_HOST_URL_ENV_VAR);
-    if (sonarHostUrl != null) {
-      setBootstrapPropertyIfNotAlreadySet(ScannerProperties.HOST_URL, sonarHostUrl);
-    } else {
-      setBootstrapPropertyIfNotAlreadySet(ScannerProperties.HOST_URL, getSonarCloudUrl());
-    }
+    setBootstrapPropertyIfNotAlreadySet(ScannerProperties.HOST_URL, getSonarCloudUrl());
   }
 
   private String getSonarCloudUrl() {