Version: Hoxton.SR5

I have two eureka servers configured in application's bootstrap.yml, but only the first is used when application is trying to bootstrap. When the first eureka server is down, application cannot start.

eureka:
    client:
        serviceUrl:
            defaultZone: http://uat4-eureka:8761/eureka,http://uat5-eureka:8761/eureka

Comment From: richard1230

please provide more information about your issue, it's better to provide steps to reproduce this issue

Comment From: holy12345

Hi @wangzw If first eureka server is down then they will auto register to the second one and if the second is down they will auto register to the third one.(Trigger this is use a backend thread). if you want know more info you can check the source code which is RetryableEurekaHttpClient.java

thanks

Comment From: spencergibb

Can you please provide the stack trace?

Comment From: wangzw

Thanks for your fast response.

I do not think RetryableEurekaHttpClient is used when bootstraping.

2020-07-03 20:53:30.980 ERROR 6 --- [           main] o.s.boot.SpringApplication               : Application run failed

org.springframework.web.client.ResourceAccessException: I/O error on GET request for "http://uat4-eureka:8761/eureka/apps/": Connection refused (Connection refused); nested exception is java.net.ConnectException: Connection refused (Connection refused)
    at org.springframework.web.client.RestTemplate.doExecute(RestTemplate.java:748)
    at org.springframework.web.client.RestTemplate.execute(RestTemplate.java:674)
    at org.springframework.web.client.RestTemplate.exchange(RestTemplate.java:583)
    at org.springframework.cloud.netflix.eureka.http.RestTemplateEurekaHttpClient.getApplicationsInternal(RestTemplateEurekaHttpClient.java:154)
    at org.springframework.cloud.netflix.eureka.http.RestTemplateEurekaHttpClient.getApplications(RestTemplateEurekaHttpClient.java:142)
    at org.springframework.cloud.netflix.eureka.config.EurekaConfigServerBootstrapConfiguration.lambda$eurekaConfigServerInstanceProvider$0(EurekaConfigServerBootstrapConfiguration.java:112)
    at org.springframework.cloud.config.client.ConfigServerInstanceProvider.getConfigServerInstances(ConfigServerInstanceProvider.java:50)
    at org.springframework.cloud.config.client.DiscoveryClientConfigServiceBootstrapConfiguration$HeartbeatListener.refresh(DiscoveryClientConfigServiceBootstrapConfiguration.java:120)
    at org.springframework.cloud.config.client.DiscoveryClientConfigServiceBootstrapConfiguration$HeartbeatListener.startup(DiscoveryClientConfigServiceBootstrapConfiguration.java:106)
    at org.springframework.cloud.config.client.DiscoveryClientConfigServiceBootstrapConfiguration$HeartbeatListener.onApplicationEvent(DiscoveryClientConfigServiceBootstrapConfiguration.java:98)
    at org.springframework.context.event.SimpleApplicationEventMulticaster.doInvokeListener(SimpleApplicationEventMulticaster.java:172)
    at org.springframework.context.event.SimpleApplicationEventMulticaster.invokeListener(SimpleApplicationEventMulticaster.java:165)
    at org.springframework.context.event.SimpleApplicationEventMulticaster.multicastEvent(SimpleApplicationEventMulticaster.java:139)
    at org.springframework.context.support.AbstractApplicationContext.publishEvent(AbstractApplicationContext.java:403)
    at org.springframework.context.support.AbstractApplicationContext.publishEvent(AbstractApplicationContext.java:360)
    at org.springframework.context.support.AbstractApplicationContext.finishRefresh(AbstractApplicationContext.java:897)
    at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:553)
    at org.springframework.boot.SpringApplication.refresh(SpringApplication.java:747)
    at org.springframework.boot.SpringApplication.refreshContext(SpringApplication.java:397)
    at org.springframework.boot.SpringApplication.run(SpringApplication.java:315)
    at org.springframework.boot.builder.SpringApplicationBuilder.run(SpringApplicationBuilder.java:140)
    at org.springframework.cloud.bootstrap.BootstrapApplicationListener.bootstrapServiceContext(BootstrapApplicationListener.java:212)
    at org.springframework.cloud.bootstrap.BootstrapApplicationListener.onApplicationEvent(BootstrapApplicationListener.java:117)
    at org.springframework.cloud.bootstrap.BootstrapApplicationListener.onApplicationEvent(BootstrapApplicationListener.java:74)
    at org.springframework.context.event.SimpleApplicationEventMulticaster.doInvokeListener(SimpleApplicationEventMulticaster.java:172)
    at org.springframework.context.event.SimpleApplicationEventMulticaster.invokeListener(SimpleApplicationEventMulticaster.java:165)
    at org.springframework.context.event.SimpleApplicationEventMulticaster.multicastEvent(SimpleApplicationEventMulticaster.java:139)
    at org.springframework.context.event.SimpleApplicationEventMulticaster.multicastEvent(SimpleApplicationEventMulticaster.java:127)
    at org.springframework.boot.context.event.EventPublishingRunListener.environmentPrepared(EventPublishingRunListener.java:76)
    at org.springframework.boot.SpringApplicationRunListeners.environmentPrepared(SpringApplicationRunListeners.java:53)
    at org.springframework.boot.SpringApplication.prepareEnvironment(SpringApplication.java:345)
    at org.springframework.boot.SpringApplication.run(SpringApplication.java:308)
    at org.springframework.boot.builder.SpringApplicationBuilder.run(SpringApplicationBuilder.java:140)
    at cn.hashdata.cloudmgr.teleport.CloudmgrTeleportApplication.main(CloudmgrTeleportApplication.java:18)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at org.springframework.boot.loader.MainMethodRunner.run(MainMethodRunner.java:48)
    at org.springframework.boot.loader.Launcher.launch(Launcher.java:87)
    at org.springframework.boot.loader.Launcher.launch(Launcher.java:50)
    at org.springframework.boot.loader.thin.ThinJarLauncher.launch(ThinJarLauncher.java:193)
    at org.springframework.boot.loader.thin.ThinJarLauncher.main(ThinJarLauncher.java:140)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at org.springframework.boot.loader.wrapper.ThinJarWrapper.launch(ThinJarWrapper.java:140)
    at org.springframework.boot.loader.wrapper.ThinJarWrapper.main(ThinJarWrapper.java:107)
Caused by: java.net.ConnectException: Connection refused (Connection refused)
    at java.net.PlainSocketImpl.socketConnect(Native Method)
    at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
    at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
    at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
    at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
    at java.net.Socket.connect(Socket.java:589)
    at java.net.Socket.connect(Socket.java:538)
    at sun.net.NetworkClient.doConnect(NetworkClient.java:180)
    at sun.net.www.http.HttpClient.openServer(HttpClient.java:463)
    at sun.net.www.http.HttpClient.openServer(HttpClient.java:558)
    at sun.net.www.http.HttpClient.<init>(HttpClient.java:242)
    at sun.net.www.http.HttpClient.New(HttpClient.java:339)
    at sun.net.www.http.HttpClient.New(HttpClient.java:357)
    at sun.net.www.protocol.http.HttpURLConnection.getNewHttpClient(HttpURLConnection.java:1220)
    at sun.net.www.protocol.http.HttpURLConnection.plainConnect0(HttpURLConnection.java:1156)
    at sun.net.www.protocol.http.HttpURLConnection.plainConnect(HttpURLConnection.java:1050)
    at sun.net.www.protocol.http.HttpURLConnection.connect(HttpURLConnection.java:984)
    at org.springframework.http.client.SimpleBufferingClientHttpRequest.executeInternal(SimpleBufferingClientHttpRequest.java:76)
    at org.springframework.http.client.AbstractBufferingClientHttpRequest.executeInternal(AbstractBufferingClientHttpRequest.java:48)
    at org.springframework.http.client.AbstractClientHttpRequest.execute(AbstractClientHttpRequest.java:53)
    at org.springframework.web.client.RestTemplate.doExecute(RestTemplate.java:739)
    ... 48 common frames omitted

Comment From: wangzw

From the code, the first url is return anyway.

https://github.com/spring-cloud/spring-cloud-netflix/blob/d72dd006ad8bb77d6f1e57fa6f0e52fa2e791b03/spring-cloud-netflix-eureka-client/src/main/java/org/springframework/cloud/netflix/eureka/config/EurekaConfigServerBootstrapConfiguration.java#L93-L97

Comment From: wangzw

RestTemplateEurekaHttpClient is used instead of RetryableEurekaHttpClient

https://github.com/spring-cloud/spring-cloud-netflix/blob/d72dd006ad8bb77d6f1e57fa6f0e52fa2e791b03/spring-cloud-netflix-eureka-client/src/main/java/org/springframework/cloud/netflix/eureka/config/EurekaConfigServerBootstrapConfiguration.java#L83-L91

Comment From: Melancholic

I caught this problem too. SpringBoot 2.3.4.RELEASE, spring-cloud-netflix-eureka-client-2.2.3.RELEASE, eureka-client-1.9.21

Comment From: wangzw

Any update?

Comment From: vladotod

I am using Spring Boot 2.6.11, Spring Cloud 2021.0.8 and have same problem. I have two Eureka servers and if one instance down (defined as first URL) then appliction can't start.

Comment From: vladotod

I tested same scenario with Spring Boot 3.1.4 and Spring Cloud 2022.0.4.

First test case: client app uses two Eureka URL-s; Eureka instance for first URL is not available -> app successfully registered on second Eureka instance

Second test case: client app uses two Eureka URL-s; Eureka instance for first URL is not available; client app uses spring-cloud-starter-config and tries to fetch configuration from Config Server using discovery client -> application failed to fetch configuration from Config Server because first Eureka instance is down

This problem is related to the fetching configuration from Config Server via discovery client.

Comment From: vladotod

I found workaround for this problem with custom BootstrapRegistryInitializer class based on org.springframework.cloud.netflix.eureka.config.EurekaConfigServerBootstrapper. Custom initializer class must be defined in META-INF/spring.factories file.

Custom initializer uses com.netflix.discovery.shared.transport.decorator.RetryableEurekaHttpClient. In this implementation, I am not provide com.netflix.discovery.endpoint.EndpointUtils.ServiceUrlRandomizer for URLs resolving.

My custom initializer class (this could be suggested improvement):

public class CustomEurekaConfigServerBootstrapper implements BootstrapRegistryInitializer  {

    @Override
    public void initialize(BootstrapRegistry registry) {
        if (!ClassUtils.isPresent("org.springframework.cloud.config.client.ConfigServerInstanceProvider", null)) {
            return;
        }

        // It is important that we pass a lambda for the Function or else we will get a
        // ClassNotFoundException when config is not on the classpath
        //registry.registerIfAbsent(ConfigServerInstanceProvider.Function.class, null);

        registry.register(ConfigServerInstanceProvider.Function.class, CustomEurekaFunction::create); 
    }

    private static Boolean getDiscoveryEnabled(Binder binder) {
        return binder.bind(ConfigClientProperties.CONFIG_DISCOVERY_ENABLED, Boolean.class).orElse(false)
                && binder.bind("eureka.client.enabled", Boolean.class).orElse(true)
                && binder.bind("spring.cloud.discovery.enabled", Boolean.class).orElse(true);
    }

    final static class CustomEurekaFunction implements ConfigServerInstanceProvider.Function {

        private final BootstrapContext context;

        static CustomEurekaFunction create(BootstrapContext context) {
            return new CustomEurekaFunction(context);
        }

        private CustomEurekaFunction(BootstrapContext context) {
            this.context = context;
        }

        @Override
        public List<ServiceInstance> apply(String serviceId, Binder binder, BindHandler bindHandler, Log log) {
            if (binder == null || !getDiscoveryEnabled(binder)) {
                return Collections.emptyList();
            }

            EurekaClientConfigBean config = binder.bind(EurekaClientConfigBean.PREFIX, EurekaClientConfigBean.class)
                    .orElseGet(EurekaClientConfigBean::new);

            EurekaHttpClient httpClient = new RetryableEurekaHttpClient(serviceId, config.getTransportConfig(), 
                    new ClusterResolver<EurekaEndpoint>() {

                        @Override
                        public String getRegion() {
                            return config.getRegion();
                        }

                        @Override
                        public List<EurekaEndpoint> getClusterEndpoints() {
                            List<String> urls = EndpointUtils.getDiscoveryServiceUrls(config, EurekaClientConfigBean.DEFAULT_ZONE, null);

                            return urls.stream().map(url -> new DefaultEndpoint(url)).collect(Collectors.toList());
                        }
            }, new RestTemplateTransportClientFactory(
                    context.getOrElse(TlsProperties.class, null),
                    context.getOrElse(EurekaClientHttpRequestFactorySupplier.class,
                            new DefaultEurekaClientHttpRequestFactorySupplier()))
                    , 
                    ServerStatusEvaluators.httpSuccessEvaluator(), 
                    3);

            return new EurekaConfigServerInstanceProvider(httpClient, config).getInstances(serviceId);
        }

        @Override
        public List<ServiceInstance> apply(String serviceId) {
            return apply(serviceId, null, null, null);
        }
    }
}