SCANJLIB-306 Restore authentication support for proxy + SSL

Author: henryju

.github/workflows/build.yml

@@ -83,7 +83,7 @@ jobs:
       - name: Run ITs
         run: |
           cd its
-          mvn -B -e verify -Prun-its -Dsonar.runtimeVersion=${{ matrix.SQ_VERSION }} -DjavaVersion=${{ env.JAVA_VERSION }}
+          mvn -B -e verify -Prun-its -Dsonar.runtimeVersion=${{ matrix.SQ_VERSION }}
 
       - name: Upload server logs
         if: failure()

its/it-tests/src/test/java/com/sonar/scanner/lib/it/ProxyTest.java

@@ -26,15 +26,19 @@
 import com.sonar.scanner.lib.it.tools.SimpleScanner;
 import java.io.IOException;
 import java.net.InetAddress;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.Base64;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.ConcurrentLinkedDeque;
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.http.HttpVersion;
+import org.eclipse.jetty.proxy.ConnectHandler;
 import org.eclipse.jetty.proxy.ProxyServlet;
 import org.eclipse.jetty.security.ConstraintMapping;
 import org.eclipse.jetty.security.ConstraintSecurityHandler;
@@ -46,12 +50,15 @@
 import org.eclipse.jetty.server.HttpConnectionFactory;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.server.SslConnectionFactory;
 import org.eclipse.jetty.server.handler.DefaultHandler;
 import org.eclipse.jetty.server.handler.HandlerCollection;
 import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.servlet.ServletHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
 import org.eclipse.jetty.util.security.Constraint;
 import org.eclipse.jetty.util.security.Credential;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.eclipse.jetty.util.thread.QueuedThreadPool;
 import org.junit.After;
 import org.junit.Before;
@@ -64,45 +71,62 @@ public class ProxyTest {
 
   private static final String PROXY_USER = "scott";
   private static final String PROXY_PASSWORD = "tiger";
+
+  // SSL resources reused from SSLTest
+  private static final String SERVER_KEYSTORE = "/SSLTest/server.p12";
+  private static final String SERVER_KEYSTORE_PASSWORD = "pwdServerP12";
+  private static final String KEYSTORE_CLIENT_WITH_CA = "/SSLTest/client-with-ca-keytool.p12";
+  private static final String KEYSTORE_CLIENT_WITH_CA_PASSWORD = "pwdClientCAP12";
+
   private static Server server;
   private static int httpProxyPort;
+  // HTTPS reverse-proxy target, used for the HTTPS CONNECT tests
+  private static Server httpsTargetServer;
+  private static int httpsTargetPort;
 
   @ClassRule
   public static final OrchestratorRule ORCHESTRATOR = ScannerJavaLibraryTestSuite.ORCHESTRATOR;
 
-  private static ConcurrentLinkedDeque<String> seenByProxy = new ConcurrentLinkedDeque<>();
+  private static final ConcurrentLinkedDeque<String> seenByProxy = new ConcurrentLinkedDeque<>();
+  private static final ConcurrentLinkedDeque<String> seenConnectByProxy = new ConcurrentLinkedDeque<>();
 
   @Before
   public void deleteData() {
     ScannerJavaLibraryTestSuite.resetData(ORCHESTRATOR);
     seenByProxy.clear();
+    seenConnectByProxy.clear();
   }
 
   @After
   public void stopProxy() throws Exception {
     if (server != null && server.isStarted()) {
       server.stop();
     }
+    if (httpsTargetServer != null && httpsTargetServer.isStarted()) {
+      httpsTargetServer.stop();
+    }
   }
 
   private static void startProxy(boolean needProxyAuth) throws Exception {
     httpProxyPort = NetworkUtils.getNextAvailablePort(InetAddress.getLocalHost());
 
-    // Setup Threadpool
     QueuedThreadPool threadPool = new QueuedThreadPool();
     threadPool.setMaxThreads(500);
 
     server = new Server(threadPool);
 
-    // HTTP Configuration
     HttpConfiguration httpConfig = new HttpConfiguration();
     httpConfig.setSecureScheme("https");
     httpConfig.setSendServerVersion(true);
     httpConfig.setSendDateHeader(false);
 
-    // Handler Structure
+    // Wrap the ProxyServlet handler with a ConnectHandler so HTTPS CONNECT
+    // tunnels are also handled (and authenticated) by the same proxy.
+    TrackingConnectHandler connectHandler = new TrackingConnectHandler(needProxyAuth);
+    connectHandler.setHandler(proxyHandler(needProxyAuth));
+
     HandlerCollection handlers = new HandlerCollection();
-    handlers.setHandlers(new Handler[] {proxyHandler(needProxyAuth), new DefaultHandler()});
+    handlers.setHandlers(new Handler[] {connectHandler, new DefaultHandler()});
     server.setHandler(handlers);
 
     ServerConnector http = new ServerConnector(server, new HttpConnectionFactory(httpConfig));
@@ -112,6 +136,55 @@ private static void startProxy(boolean needProxyAuth) throws Exception {
     server.start();
   }
 
+  /**
+   * Starts a simple HTTPS reverse-proxy that forwards all traffic to the Orchestrator SonarQube
+   * instance. Used as the HTTPS target in proxy-CONNECT tests.
+   */
+  private static void startHttpsTargetServer() throws Exception {
+    httpsTargetPort = NetworkUtils.getNextAvailablePort(InetAddress.getLocalHost());
+
+    QueuedThreadPool threadPool = new QueuedThreadPool();
+    threadPool.setMaxThreads(500);
+
+    httpsTargetServer = new Server(threadPool);
+
+    HttpConfiguration httpConfig = new HttpConfiguration();
+    httpConfig.setSecureScheme("https");
+    httpConfig.setSecurePort(httpsTargetPort);
+    httpConfig.setSendServerVersion(true);
+    httpConfig.setSendDateHeader(false);
+
+    Path serverKeyStore = Paths.get(ProxyTest.class.getResource(SERVER_KEYSTORE).toURI()).toAbsolutePath();
+    assertThat(serverKeyStore).exists();
+
+    ServerConnector sslConnector = buildServerConnector(serverKeyStore, httpConfig);
+    httpsTargetServer.addConnector(sslConnector);
+
+    // Transparently forward all requests to the Orchestrator instance
+    ServletContextHandler context = new ServletContextHandler();
+    ServletHandler servletHandler = new ServletHandler();
+    ServletHolder holder = servletHandler.addServletWithMapping(ProxyServlet.Transparent.class, "/*");
+    holder.setInitParameter("proxyTo", ORCHESTRATOR.getServer().getUrl());
+    context.setServletHandler(servletHandler);
+    httpsTargetServer.setHandler(context);
+
+    httpsTargetServer.start();
+  }
+
+  private static ServerConnector buildServerConnector(Path serverKeyStore, HttpConfiguration httpConfig) {
+    SslContextFactory.Server sslContextFactory = new SslContextFactory.Server();
+    sslContextFactory.setKeyStorePath(serverKeyStore.toString());
+    sslContextFactory.setKeyStorePassword(SERVER_KEYSTORE_PASSWORD);
+    sslContextFactory.setKeyManagerPassword(SERVER_KEYSTORE_PASSWORD);
+
+    HttpConfiguration httpsConfig = new HttpConfiguration(httpConfig);
+    ServerConnector sslConnector = new ServerConnector(httpsTargetServer,
+      new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString()),
+      new HttpConnectionFactory(httpsConfig));
+    sslConnector.setPort(httpsTargetPort);
+    return sslConnector;
+  }
+
   private static ServletContextHandler proxyHandler(boolean needProxyAuth) {
     ServletContextHandler contextHandler = new ServletContextHandler();
     if (needProxyAuth) {
@@ -155,6 +228,65 @@ private static ServletHandler newServletHandler() {
     return handler;
   }
 
+  /**
+   * ConnectHandler subclass that:
+   * <ul>
+   *   <li>Optionally requires {@code Proxy-Authorization} on CONNECT requests</li>
+   *   <li>Records the host:port of every successfully-authenticated CONNECT</li>
+   * </ul>
+   * <p>
+   * When authentication is required and credentials are missing, the handler sends a {@code 407}
+   * and <em>immediately closes the TCP connection</em> rather than leaving it open for a
+   * challenge-response retry. This is intentional: it forces the JDK {@code HttpClient} to throw
+   * an {@code IOException} (tunnel failed) rather than returning the {@code 407} as a readable HTTP
+   * response to the application. Without the {@link java.net.Authenticator}-based fix in
+   * {@code HttpClientFactory}, the scanner has no way to recover from that {@code IOException} and
+   * the scan fails — reproducing the real-world regression.
+   */
+  private static class TrackingConnectHandler extends ConnectHandler {
+
+    private final boolean requireAuth;
+
+    TrackingConnectHandler(boolean requireAuth) {
+      this.requireAuth = requireAuth;
+    }
+
+    @Override
+    protected void handleConnect(org.eclipse.jetty.server.Request baseRequest, HttpServletRequest request,
+      HttpServletResponse response, String serverAddress) {
+      if (requireAuth && !hasValidCredentials(request)) {
+        // Send a minimal 407 and then immediately shut down the connection so
+        // the JDK cannot read a clean HTTP response — it will throw IOException.
+        try {
+          response.setStatus(HttpServletResponse.SC_PROXY_AUTHENTICATION_REQUIRED);
+          response.setHeader("Proxy-Authenticate", "Basic realm=\"proxy\"");
+          response.flushBuffer();
+        } catch (IOException ignored) {
+          // best-effort flush before closing
+        }
+        baseRequest.setHandled(true);
+        baseRequest.getHttpChannel().getEndPoint().close();
+        return;
+      }
+      seenConnectByProxy.add(serverAddress);
+      super.handleConnect(baseRequest, request, response, serverAddress);
+    }
+
+    private static boolean hasValidCredentials(HttpServletRequest request) {
+      String credentials = request.getHeader("Proxy-Authorization");
+      if (credentials != null && credentials.startsWith("Basic ")) {
+        String decoded = new String(Base64.getDecoder().decode(credentials.substring(6)), StandardCharsets.ISO_8859_1);
+        int colon = decoded.indexOf(':');
+        if (colon > 0) {
+          String user = decoded.substring(0, colon);
+          String pass = decoded.substring(colon + 1);
+          return PROXY_USER.equals(user) && PROXY_PASSWORD.equals(pass);
+        }
+      }
+      return false;
+    }
+  }
+
   public static class MyProxyServlet extends ProxyServlet {
 
     @Override
@@ -202,6 +334,8 @@ public void simple_analysis_with_proxy_auth() throws Exception {
     SimpleScanner scanner = new SimpleScanner();
 
     Map<String, String> params = new HashMap<>();
+    // By default no request to localhost will use proxy
+    params.put("http.nonProxyHosts", "");
     params.put("sonar.scanner.proxyHost", "localhost");
     params.put("sonar.scanner.proxyPort", "" + httpProxyPort);
 
@@ -218,4 +352,46 @@ public void simple_analysis_with_proxy_auth() throws Exception {
     assertThat(buildResult.getLastStatus()).isZero();
   }
 
+  /**
+   * Reproduces the regression reported for SonarScanner CLI 8.0 (java-library 4.0):
+   * HTTPS proxy authentication was broken — the {@code Proxy-Authorization} header was
+   * not sent on the CONNECT tunnel, so the proxy kept returning 407.
+   * <p>
+   * This test uses a local HTTP forward proxy that enforces authentication on CONNECT
+   * requests, plus a local HTTPS reverse-proxy that forwards to the running SonarQube
+   * instance. This mirrors the real-world topology: scanner → HTTP proxy (CONNECT) →
+   * HTTPS SonarQube.
+   */
+  @Test
+  public void simple_analysis_with_https_proxy_auth() throws Exception {
+    startProxy(true);
+    startHttpsTargetServer();
+    SimpleScanner scanner = new SimpleScanner();
+
+    Path clientTruststore = Paths.get(ProxyTest.class.getResource(KEYSTORE_CLIENT_WITH_CA).toURI()).toAbsolutePath();
+    assertThat(clientTruststore).exists();
+
+    Map<String, String> params = new HashMap<>();
+    // By default no request to localhost will use proxy
+    params.put("http.nonProxyHosts", "");
+    params.put("sonar.scanner.proxyHost", "localhost");
+    params.put("sonar.scanner.proxyPort", "" + httpProxyPort);
+    // Trust the self-signed certificate used by the local HTTPS target
+    params.put("sonar.scanner.truststorePath", clientTruststore.toString());
+    params.put("sonar.scanner.truststorePassword", KEYSTORE_CLIENT_WITH_CA_PASSWORD);
+
+    // Without proxy credentials the CONNECT tunnel should be rejected (407)
+    BuildResult buildResult = scanner.executeSimpleProject(project("js-sample"), "https://localhost:" + httpsTargetPort, params, Map.of());
+    assertThat(buildResult.getLastStatus()).isNotZero();
+    assertThat(buildResult.getLogs()).containsIgnoringCase("Failed to query server version");
+    assertThat(seenConnectByProxy).isEmpty();
+
+    // With proxy credentials the CONNECT tunnel must succeed and the full analysis must pass
+    params.put("sonar.scanner.proxyUser", PROXY_USER);
+    params.put("sonar.scanner.proxyPassword", PROXY_PASSWORD);
+    buildResult = scanner.executeSimpleProject(project("js-sample"), "https://localhost:" + httpsTargetPort, params, Map.of());
+    assertThat(buildResult.getLastStatus()).isZero();
+    assertThat(seenConnectByProxy).isNotEmpty();
+  }
+
 }

lib/src/main/java/org/sonarsource/scanner/lib/internal/http/HttpConfig.java

@@ -144,7 +144,6 @@ private static Duration loadDuration(Map<String, String> bootstrapProperties, St
 
   @Nullable
   private static Proxy loadProxy(Map<String, String> bootstrapProperties) {
-    // OkHttp detects 'http.proxyHost' java property already, so just focus on sonar-specific properties
     String proxyHost = defaultIfBlank(bootstrapProperties.get(SONAR_SCANNER_PROXY_HOST), null);
     if (proxyHost != null) {
       int proxyPort;

lib/src/main/java/org/sonarsource/scanner/lib/internal/http/ScannerHttpClient.java

@@ -146,21 +146,21 @@ private <G> G callUrlWithRedirects(String url, boolean authentication, @Nullable
   }
 
   private <G> G callUrlWithRedirectsAndProxyAuth(String url, boolean authentication, @Nullable String acceptHeader, ResponseHandler<G> responseHandler,
-    int redirectCount, boolean proxyAuthAttempted) {
+    int redirectCount, boolean proxyAuthRetry) {
     if (redirectCount > 10) {
       throw new IllegalStateException("Too many redirects (>10) for URL: " + url);
     }
 
-    var request = prepareRequest(url, acceptHeader, authentication, proxyAuthAttempted);
+    var request = prepareRequest(url, acceptHeader, authentication, proxyAuthRetry);
 
     HttpResponse<InputStream> response = null;
     Instant start = Instant.now();
     try {
       LOG.debug("--> {} {}", request.method(), request.uri());
       response = sharedHttpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
 
-      if (response.statusCode() == 407 && !proxyAuthAttempted && httpConfig.getProxyUser() != null) {
-        LOG.debug("Received 407 Proxy Authentication Required, retrying with Proxy-Authorization header");
+      if (response.statusCode() == 407 && !proxyAuthRetry && httpConfig.getProxyUser() != null) {
+        LOG.debug("Received 407 Proxy Authentication Required, retrying request");
         return callUrlWithRedirectsAndProxyAuth(url, authentication, acceptHeader, responseHandler, redirectCount, true);
       }
 
@@ -172,7 +172,7 @@ private <G> G callUrlWithRedirectsAndProxyAuth(String url, boolean authenticatio
             URI originalUri = URI.create(url);
             redirectUrl = originalUri.getScheme() + "://" + originalUri.getAuthority() + redirectUrl;
           }
-          return callUrlWithRedirectsAndProxyAuth(redirectUrl, authentication, acceptHeader, responseHandler, redirectCount + 1, proxyAuthAttempted);
+          return callUrlWithRedirectsAndProxyAuth(redirectUrl, authentication, acceptHeader, responseHandler, redirectCount + 1, proxyAuthRetry);
         }
       }
 
@@ -216,7 +216,7 @@ private interface ResponseHandler<G> {
     G apply(HttpResponse<InputStream> response) throws IOException;
   }
 
-  private HttpRequest prepareRequest(String url, @Nullable String acceptHeader, boolean authentication, boolean addProxyAuth) {
+  private HttpRequest prepareRequest(String url, @Nullable String acceptHeader, boolean authentication, boolean proxyAuthRetry) {
     var timeout = httpConfig.getResponseTimeout().isZero() ? httpConfig.getSocketTimeout() : httpConfig.getResponseTimeout();
 
     var requestBuilder = HttpRequest.newBuilder()
@@ -239,7 +239,11 @@ private HttpRequest prepareRequest(String url, @Nullable String acceptHeader, bo
       }
     }
 
-    if (addProxyAuth && httpConfig.getProxyUser() != null) {
+    // Preemptively send proxy credentials on every request. The JDK HttpClient forwards
+    // Proxy-Authorization from the application request to CONNECT tunnel requests for HTTPS
+    // targets, so sending it upfront avoids a round-trip 407 challenge and works reliably
+    // across JDK versions.
+    if (httpConfig.getProxyUser() != null) {
       String proxyCredentials = httpConfig.getProxyUser() + ":" + (httpConfig.getProxyPassword() != null ? httpConfig.getProxyPassword() : "");
       String encodedProxyCredentials = Base64.getEncoder().encodeToString(proxyCredentials.getBytes(StandardCharsets.UTF_8));
       requestBuilder.header("Proxy-Authorization", "Basic " + encodedProxyCredentials);

lib/src/test/java/org/sonarsource/scanner/lib/internal/http/ScannerHttpClientTest.java

@@ -316,6 +316,26 @@ void should_honor_scanner_proxy_settings_with_auth() {
         .withHeader("Proxy-Authorization", equalTo("Basic " + Base64.getEncoder().encodeToString("proxyUser:proxyPassword".getBytes(StandardCharsets.UTF_8)))));
     }
 
+    @Test
+    void should_send_proxy_auth_preemptively_without_407_challenge() {
+      sonarqube.stubFor(get("/batch/index.txt").willReturn(aResponse().withBody(HELLO_WORLD)));
+
+      Map<String, String> props = new HashMap<>();
+      props.put(ScannerProperties.SONAR_SCANNER_PROXY_HOST, "localhost");
+      props.put(ScannerProperties.SONAR_SCANNER_PROXY_PORT, String.valueOf(proxyMock.getPort()));
+      props.put(ScannerProperties.SONAR_SCANNER_PROXY_USER, "proxyUser");
+      props.put(ScannerProperties.SONAR_SCANNER_PROXY_PASSWORD, "proxyPassword");
+
+      ScannerHttpClient underTest = create(sonarqube.baseUrl(), props);
+      String response = underTest.callWebApi("/batch/index.txt");
+
+      assertThat(response).isEqualTo(HELLO_WORLD);
+      // Proxy-Authorization must be present on the very first request — no 407 round-trip
+      proxyMock.verify(1, getRequestedFor(urlMatching("/batch/.*"))
+        .withHeader("Proxy-Authorization",
+          equalTo("Basic " + Base64.getEncoder().encodeToString("proxyUser:proxyPassword".getBytes(StandardCharsets.UTF_8)))));
+    }
+
     @Test
     @Tag(PROXY_AUTH_ENABLED)
     @RestoreSystemProperties