Spring inconsistent results with RestClient using jdk http client from httpcomponents

I'm not 100% this is a spring bug (as if I ever am)... I haven't fully tracked down exactly what's going on. sorry, I'm kind of giving up on jdk httpclient due to lack of body logging. However, in testing another bug, my test failed when on only switching to jdk http client.

DEBUG 1809075 - .spri.boot.StartupInfoLogger                            : Running with Spring Boot v3.3.0, Spring v6.1.8                                                                            at org.springframework.boot.StartupInfoLogger.logStarting(StartupInfoLogger.java:51)

AuthorizationServerTest > authn() FAILED
    java.lang.AssertionError: [code] 
    Expecting actual:
      {}
    to contain key:
      "code"

The only difference between passing and failing is this commented code

      return RestClient.builder()
        .requestFactory(
          // new HttpComponentsClientHttpRequestFactory(HttpClients.custom().disableRedirectHandling().build())
          new JdkClientHttpRequestFactory(HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NEVER).build())
        )

// © Copyright 2024 Caleb Cushing
// SPDX-License-Identifier: AGPL-3.0-or-later

package com.xenoterracide.test.authorization.server;

import static org.assertj.core.api.Assertions.assertThat;

import java.net.http.HttpClient;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.SecureRandom;
import java.util.Base64;
import java.util.function.Consumer;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.ObjectFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.context.TestConfiguration;
import org.springframework.boot.test.web.server.LocalServerPort;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Lazy;
import org.springframework.http.HttpStatus;
import org.springframework.http.HttpStatusCode;
import org.springframework.http.MediaType;
import org.springframework.http.client.JdkClientHttpRequestFactory;
import org.springframework.security.oauth2.core.AuthorizationGrantType;
import org.springframework.security.oauth2.core.endpoint.OAuth2AccessTokenResponse;
import org.springframework.security.oauth2.core.endpoint.OAuth2ParameterNames;
import org.springframework.security.oauth2.core.endpoint.PkceParameterNames;
import org.springframework.security.oauth2.core.http.converter.OAuth2AccessTokenResponseHttpMessageConverter;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.web.client.RestClient;
import org.springframework.web.util.UriComponentsBuilder;

@ActiveProfiles({ "test", "test-http" })
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
class AuthorizationServerTest {

  @SuppressWarnings("NullAway")
  @Value("${spring.security.user.name}")
  String user;

  @SuppressWarnings("NullAway")
  @Value("${spring.security.user.password}")
  String pass;

  @SuppressWarnings("NullAway")
  @Value("${spring.security.oauth2.authorizationserver.endpoint.authorization-uri}")
  String authorizationUriPath;

  @SuppressWarnings("NullAway")
  @Value("${spring.security.oauth2.authorizationserver.endpoint.token-uri}")
  String tokenUriPath;

  @Autowired
  ObjectFactory<RestClient> oauthTestClient;

  SecureRandom random = new SecureRandom();
  Base64.Encoder encoder = Base64.getUrlEncoder().withoutPadding();

  static byte[] bytesFrom(int size, Consumer<byte[]> setter) {
    var bytes = new byte[size];
    setter.accept(bytes);
    return bytes;
  }

  private static LinkedMultiValueMap<String, String> getAuthParams(String challenge) {
    var authParams = new LinkedMultiValueMap<String, String>();
    authParams.add(PkceParameterNames.CODE_CHALLENGE, challenge);
    authParams.add(PkceParameterNames.CODE_CHALLENGE_METHOD, "S256");
    authParams.add(OAuth2ParameterNames.CLIENT_ID, AuthorizationServer.CLIENT_ID);
    authParams.add(OAuth2ParameterNames.REDIRECT_URI, AuthorizationServer.REDIRECT_URI);
    authParams.add(OAuth2ParameterNames.RESPONSE_TYPE, "code");
    authParams.add(OAuth2ParameterNames.SCOPE, "openid+profile+email");
    authParams.add(OAuth2ParameterNames.STATE, "sUmww5GH");
    authParams.add("nonce", "FVO5cA3");
    authParams.add("audience", "http://localhost");
    authParams.add("response_mode", "query");
    authParams.add("auth0Client", "eyJuY");
    return authParams;
  }

  @Test
  void authn() throws Exception {
    var rc = this.oauthTestClient.getObject();
    var credentials = new LinkedMultiValueMap<String, String>();
    credentials.add("username", this.user);
    credentials.add("password", this.pass);

    var login = rc
      .post()
      .uri("/login")
      .contentType(MediaType.APPLICATION_FORM_URLENCODED)
      .body(credentials)
      .retrieve()
      .onStatus(HttpStatusCode::is4xxClientError, (req, res) -> {})
      .toEntity(String.class);

    assertThat(login).describedAs("login").extracting(res -> res.getStatusCode()).isEqualTo(HttpStatus.FOUND);

    var code = bytesFrom(32, random::nextBytes);
    var verifier = encoder.encodeToString(code);
    var challenge = encoder.encodeToString(
      MessageDigest.getInstance("SHA-256").digest(verifier.getBytes(StandardCharsets.US_ASCII))
    );

    var authParams = getAuthParams(challenge);

    var authorize = rc
      .get()
      .uri(uriBuilder -> uriBuilder.path(this.authorizationUriPath).queryParams(authParams).build())
      .retrieve()
      .toEntity(String.class);

    assertThat(authorize.getStatusCode()).describedAs("authorize").isEqualTo(HttpStatus.FOUND);

    var qp = UriComponentsBuilder.fromUri(authorize.getHeaders().getLocation()).build().getQueryParams();

    assertThat(qp).describedAs("code").containsKey("code");

    var params = new LinkedMultiValueMap<String, String>();
    params.add(OAuth2ParameterNames.CLIENT_ID, AuthorizationServer.CLIENT_ID);
    params.add(OAuth2ParameterNames.GRANT_TYPE, AuthorizationGrantType.AUTHORIZATION_CODE.getValue());
    params.add(OAuth2ParameterNames.CODE, qp.getFirst(OAuth2ParameterNames.CODE));
    params.add(OAuth2ParameterNames.REDIRECT_URI, AuthorizationServer.REDIRECT_URI);
    params.add(PkceParameterNames.CODE_VERIFIER, verifier);

    var tokenResponse = rc
      .post()
      .uri(this.tokenUriPath)
      .body(params)
      .retrieve()
      .toEntity(OAuth2AccessTokenResponse.class);

    assertThat(tokenResponse.getStatusCode()).isEqualTo(HttpStatus.OK);
    assertThat(tokenResponse.getBody().getAccessToken()).isNotNull();
  }

  @TestConfiguration
  static class TestConfig {

    @Bean
    @Lazy
    RestClient oauthTestClient(@LocalServerPort int port) {
      return RestClient.builder()
        .requestFactory(
          // new HttpComponentsClientHttpRequestFactory(HttpClients.custom().disableRedirectHandling().build())
          new JdkClientHttpRequestFactory(HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NEVER).build())
        )
        .baseUrl("http://localhost:" + port)
        .messageConverters(converters -> {
          converters.addFirst(new OAuth2AccessTokenResponseHttpMessageConverter());
        })
        .build();
    }
  }
}

// © Copyright 2024 Caleb Cushing
// SPDX-License-Identifier: AGPL-3.0-or-later

package com.xenoterracide.test.authorization.server;

import com.xenoterracide.tools.java.annotation.ExcludeFromGeneratedCoverageReport;
import java.util.UUID;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Bean;
import org.springframework.core.annotation.Order;
import org.springframework.http.MediaType;
import org.springframework.security.config.Customizer;
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
import org.springframework.security.oauth2.core.AuthorizationGrantType;
import org.springframework.security.oauth2.core.ClientAuthenticationMethod;
import org.springframework.security.oauth2.core.oidc.OidcScopes;
import org.springframework.security.oauth2.server.authorization.client.InMemoryRegisteredClientRepository;
import org.springframework.security.oauth2.server.authorization.client.RegisteredClient;
import org.springframework.security.oauth2.server.authorization.client.RegisteredClientRepository;
import org.springframework.security.oauth2.server.authorization.config.annotation.web.configuration.OAuth2AuthorizationServerConfiguration;
import org.springframework.security.oauth2.server.authorization.config.annotation.web.configurers.OAuth2AuthorizationServerConfigurer;
import org.springframework.security.oauth2.server.authorization.settings.ClientSettings;
import org.springframework.security.web.SecurityFilterChain;
import org.springframework.security.web.authentication.LoginUrlAuthenticationEntryPoint;
import org.springframework.security.web.util.matcher.MediaTypeRequestMatcher;
import org.springframework.web.cors.CorsConfiguration;
import org.springframework.web.cors.CorsConfigurationSource;
import org.springframework.web.cors.UrlBasedCorsConfigurationSource;

/**
 * Test Authorization Server to mimick Auth0.
 */
@SpringBootApplication(proxyBeanMethods = false)
public class AuthorizationServer {

  /**
   * Client ID for the client.
   */
  public static final String CLIENT_ID = "client";
  /**
   * Redirect URI for the client.
   */
  public static final String REDIRECT_URI = "http://localhost:3000";
  private static final String ALL = "*";

  AuthorizationServer() {}

  /**
   * Main.
   *
   * @param args arguments to the program
   */
  @ExcludeFromGeneratedCoverageReport
  public static void main(String[] args) {
    SpringApplication.run(AuthorizationServer.class, args);
  }

  @Bean
  @Order(1)
  SecurityFilterChain authorizationServerSecurityFilterChain(HttpSecurity http) throws Exception {
    OAuth2AuthorizationServerConfiguration.applyDefaultSecurity(http);
    http.getConfigurer(OAuth2AuthorizationServerConfigurer.class).oidc(Customizer.withDefaults());
    http
      // Redirect to the login page when not authenticated from the
      // authorization endpoint
      .exceptionHandling(
        exceptions ->
          exceptions.defaultAuthenticationEntryPointFor(
            new LoginUrlAuthenticationEntryPoint("/login"),
            new MediaTypeRequestMatcher(MediaType.TEXT_HTML)
          )
      )
      // Accept access tokens for User Info and/or Client Registration
      .oauth2ResourceServer(oauth2 -> oauth2.jwt(Customizer.withDefaults()));

    return http.cors(Customizer.withDefaults()).build();
  }

  @Bean
  @Order(2)
  SecurityFilterChain defaultSecurityFilterChain(HttpSecurity http) throws Exception {
    http
      .authorizeHttpRequests(authorize -> authorize.requestMatchers("/oauth/authorize").permitAll())
      .authorizeHttpRequests(authorize -> authorize.anyRequest().authenticated())
      // Form login handles the redirect to the login page from the
      // authorization server filter chain
      .formLogin(Customizer.withDefaults());

    return http.cors(Customizer.withDefaults()).csrf(csrf -> csrf.disable()).build();
  }

  @Bean
  CorsConfigurationSource corsConfigurationSource() {
    var config = new CorsConfiguration();
    config.addAllowedHeader(ALL);
    config.addAllowedMethod(ALL);
    config.addAllowedOrigin(REDIRECT_URI);
    config.setAllowCredentials(true);

    var source = new UrlBasedCorsConfigurationSource();
    source.registerCorsConfiguration("/**", config);
    return source;
  }

  @Bean
  RegisteredClientRepository registeredClientRepository() {
    var publicClient = RegisteredClient.withId(UUID.randomUUID().toString())
      .clientId(CLIENT_ID)
      .clientAuthenticationMethod(ClientAuthenticationMethod.NONE)
      .authorizationGrantType(AuthorizationGrantType.AUTHORIZATION_CODE)
      .redirectUri(REDIRECT_URI)
      .scope(OidcScopes.OPENID)
      .scope(OidcScopes.PROFILE)
      .scope(OidcScopes.EMAIL)
      .clientSettings(ClientSettings.builder().requireAuthorizationConsent(false).requireProofKey(true).build())
      .build();

    return new InMemoryRegisteredClientRepository(publicClient);
  }
}

server.port = 9000
logging.level.root = info
logging.level.com.xenoterracide = debug
logging.level.org.springframework.security = trace
logging.level.jdk.httpclient = info
spring.application.name = auth-server
spring.main.banner-mode = off
spring.security.user.name = user
spring.security.user.password = pass
# match auth0
spring.security.oauth2.authorizationserver.endpoint.authorization-uri = /oauth/authorize
spring.security.oauth2.authorizationserver.endpoint.token-uri = /oauth/token

Comment From: bclozel

Sorry but there's a lot going on in this report and you're not explaining what the problem is. I believe the main issue here is that you expect all supported HTTP libraries to behave the same. RestTemplate and RestClient are APIs driving the clients, but we cannot guarantee that they will behave exactly the same (especially if you're tweaking libraries options).

We can reopen this issue if you can provide a minimal sample that only leverages a client, a local stub (or even https://httpbin.org) and explain how Spring is inconsistent and that this is not a behavior difference between libraries.

Comment From: xenoterracide

Well the only library option I tweaked was to disable follow redirects. Since that causes more problems than it solves. There are no other changes.

As far as being too much it's just an authorization server. I have no idea what it's doing.

As long as I'm not implementing a bunch of interceptors or somehow incompatible options stuff I absolutely do expect the rest client facade to behave the same. Simply disabling redirects in the same way on both libraries is not an incompatible option.

I have to be honest, I don't actually feel like this is a complicated example. I don't know what's going on though because there's no body logging and I suspect it has something to do with the body. Could have something to do with the headers.

Anyways do without Wilt. I'm abandoning the jdk implementation anyways.