Here is a deadlock as reported by JConsole.
Typically, it happens since I move a code previously executed in an .afterPropertiesSet() (i.g. InitializingBean), into a asynchronous call. This code does remote calls. The main thread ends doing a remote still during the ApplicationContext bootstrapping.
I also recently activated Zipkin/Sleuth, which may take part/be the root cause of the issue.
It would be difficult to isolate in a small reproducible case, but I guess the stacks might help tracking down this issue.
spring-cloud-XXX:2.2.5
Name: main
State: BLOCKED on java.lang.String@dc4fd0d owned by: XXX
Total blocked: 45 Total waited: 17
Stack trace:
app//org.springframework.cloud.context.scope.GenericScope$BeanLifecycleWrapper.getBean(GenericScope.java:388)
app//org.springframework.cloud.context.scope.GenericScope.get(GenericScope.java:186)
app//org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:360)
app//org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:202)
app//org.springframework.aop.target.SimpleBeanTargetSource.getTarget(SimpleBeanTargetSource.java:35)
app//org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:675)
app//brave.sampler.Sampler$$EnhancerBySpringCGLIB$$d2f0e760.isSampled(<generated>)
app//brave.Tracer.decorateContext(Tracer.java:279)
app//brave.Tracer.newRootContext(Tracer.java:215)
app//brave.Tracer.nextContext(Tracer.java:581)
app//brave.Tracer.nextSpan(Tracer.java:555)
app//brave.http.HttpClientHandler.handleSend(HttpClientHandler.java:120)
app//brave.spring.web.TracingClientHttpRequestInterceptor.intercept(TracingClientHttpRequestInterceptor.java:52)
app//org.springframework.cloud.sleuth.instrument.web.client.LazyTracingClientHttpRequestInterceptor.intercept(TraceWebClientAutoConfiguration.java:312)
app//org.springframework.http.client.InterceptingClientHttpRequest$InterceptingRequestExecution.execute(InterceptingClientHttpRequest.java:93)
app//org.springframework.http.client.InterceptingClientHttpRequest.executeInternal(InterceptingClientHttpRequest.java:77)
app//org.springframework.http.client.AbstractBufferingClientHttpRequest.executeInternal(AbstractBufferingClientHttpRequest.java:48)
app//org.springframework.http.client.AbstractClientHttpRequest.execute(AbstractClientHttpRequest.java:53)
[...custom_code...]
app//org.springframework.web.client.RestTemplate.execute(RestTemplate.java:674)
app//org.springframework.web.client.RestTemplate.exchange(RestTemplate.java:583)
[...custom_code...]
java.base@11.0.8/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195)
java.base@11.0.8/java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:948)
java.base@11.0.8/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484)
java.base@11.0.8/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474)
java.base@11.0.8/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:913)
java.base@11.0.8/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
java.base@11.0.8/java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:578)
[...custom_code...]
app//org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.invokeInitMethods(AbstractAutowireCapableBeanFactory.java:1853)
app//org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1790)
app//org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:594)
app//org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:516)
app//org.springframework.beans.factory.support.AbstractBeanFactory.lambda$doGetBean$0(AbstractBeanFactory.java:324)
app//org.springframework.beans.factory.support.AbstractBeanFactory$$Lambda$258/0x000000080031e840.getObject(Unknown Source)
app//org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:234)
- locked java.util.concurrent.ConcurrentHashMap@1a580f7c
app//org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:322)
app//org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:202)
app//org.springframework.beans.factory.config.DependencyDescriptor.resolveCandidate(DependencyDescriptor.java:276)
app//org.springframework.beans.factory.support.DefaultListableBeanFactory.addCandidateEntry(DefaultListableBeanFactory.java:1525)
app//org.springframework.beans.factory.support.DefaultListableBeanFactory.findAutowireCandidates(DefaultListableBeanFactory.java:1489)
app//org.springframework.beans.factory.support.DefaultListableBeanFactory.resolveMultipleBeans(DefaultListableBeanFactory.java:1378)
app//org.springframework.beans.factory.support.DefaultListableBeanFactory.doResolveDependency(DefaultListableBeanFactory.java:1265)
app//org.springframework.beans.factory.support.DefaultListableBeanFactory.resolveDependency(DefaultListableBeanFactory.java:1227)
app//org.springframework.beans.factory.support.ConstructorResolver.resolveAutowiredArgument(ConstructorResolver.java:884)
app//org.springframework.beans.factory.support.ConstructorResolver.createArgumentArray(ConstructorResolver.java:788)
app//org.springframework.beans.factory.support.ConstructorResolver.autowireConstructor(ConstructorResolver.java:227)
app//org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.autowireConstructor(AbstractAutowireCapableBeanFactory.java:1356)
app//org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBeanInstance(AbstractAutowireCapableBeanFactory.java:1203)
app//org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:556)
app//org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:516)
app//org.springframework.beans.factory.support.AbstractBeanFactory.lambda$doGetBean$0(AbstractBeanFactory.java:324)
app//org.springframework.beans.factory.support.AbstractBeanFactory$$Lambda$258/0x000000080031e840.getObject(Unknown Source)
app//org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:234)
- locked java.util.concurrent.ConcurrentHashMap@1a580f7c
app//org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:322)
app//org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:202)
app//org.springframework.beans.factory.config.DependencyDescriptor.resolveCandidate(DependencyDescriptor.java:276)
app//org.springframework.beans.factory.support.DefaultListableBeanFactory.doResolveDependency(DefaultListableBeanFactory.java:1307)
app//org.springframework.beans.factory.support.DefaultListableBeanFactory.resolveDependency(DefaultListableBeanFactory.java:1227)
app//org.springframework.beans.factory.support.ConstructorResolver.resolveAutowiredArgument(ConstructorResolver.java:884)
app//org.springframework.beans.factory.support.ConstructorResolver.createArgumentArray(ConstructorResolver.java:788)
app//org.springframework.beans.factory.support.ConstructorResolver.autowireConstructor(ConstructorResolver.java:227)
app//org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.autowireConstructor(AbstractAutowireCapableBeanFactory.java:1356)
app//org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBeanInstance(AbstractAutowireCapableBeanFactory.java:1203)
app//org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:556)
app//org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:516)
app//org.springframework.beans.factory.support.AbstractBeanFactory.lambda$doGetBean$0(AbstractBeanFactory.java:324)
app//org.springframework.beans.factory.support.AbstractBeanFactory$$Lambda$258/0x000000080031e840.getObject(Unknown Source)
app//org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:234)
- locked java.util.concurrent.ConcurrentHashMap@1a580f7c
app//org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:322)
app//org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:202)
app//org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:897)
app//org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:879)
app//org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:551)
- locked java.lang.Object@4eb37f30
app//org.springframework.boot.web.servlet.context.ServletWebServerApplicationContext.refresh(ServletWebServerApplicationContext.java:143)
app//org.springframework.boot.SpringApplication.refresh(SpringApplication.java:758)
app//org.springframework.boot.SpringApplication.refresh(SpringApplication.java:750)
app//org.springframework.boot.SpringApplication.refreshContext(SpringApplication.java:397)
app//org.springframework.boot.SpringApplication.run(SpringApplication.java:315)
[...custom_code...]
Name: XXX
State: BLOCKED on java.util.concurrent.ConcurrentHashMap@1a580f7c owned by: main
Total blocked: 2 Total waited: 0
Stack trace:
app//org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:216)
app//org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:322)
app//org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:202)
app//org.springframework.beans.factory.support.ConstructorResolver.instantiateUsingFactoryMethod(ConstructorResolver.java:408)
app//org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.instantiateUsingFactoryMethod(AbstractAutowireCapableBeanFactory.java:1336)
app//org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBeanInstance(AbstractAutowireCapableBeanFactory.java:1176)
app//org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:556)
app//org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:516)
app//org.springframework.beans.factory.support.AbstractBeanFactory.lambda$doGetBean$1(AbstractBeanFactory.java:363)
app//org.springframework.beans.factory.support.AbstractBeanFactory$$Lambda$1281/0x0000000800c66440.getObject(Unknown Source)
app//org.springframework.cloud.context.scope.GenericScope$BeanLifecycleWrapper.getBean(GenericScope.java:389)
- locked java.lang.String@dc4fd0d
app//org.springframework.cloud.context.scope.GenericScope.get(GenericScope.java:186)
app//org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:360)
app//org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:202)
app//org.springframework.aop.target.SimpleBeanTargetSource.getTarget(SimpleBeanTargetSource.java:35)
app//org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:675)
app//brave.sampler.Sampler$$EnhancerBySpringCGLIB$$d2f0e760.isSampled(<generated>)
app//brave.Tracer.decorateContext(Tracer.java:279)
app//brave.Tracer.newRootContext(Tracer.java:215)
app//brave.Tracer.nextContext(Tracer.java:581)
app//brave.Tracer.nextSpan(Tracer.java:555)
app//brave.http.HttpClientHandler.handleSend(HttpClientHandler.java:120)
app//brave.spring.web.TracingClientHttpRequestInterceptor.intercept(TracingClientHttpRequestInterceptor.java:52)
app//org.springframework.cloud.sleuth.instrument.web.client.LazyTracingClientHttpRequestInterceptor.intercept(TraceWebClientAutoConfiguration.java:312)
app//org.springframework.http.client.InterceptingClientHttpRequest$InterceptingRequestExecution.execute(InterceptingClientHttpRequest.java:93)
app//org.springframework.http.client.InterceptingClientHttpRequest.executeInternal(InterceptingClientHttpRequest.java:77)
app//org.springframework.http.client.AbstractBufferingClientHttpRequest.executeInternal(AbstractBufferingClientHttpRequest.java:48)
app//org.springframework.http.client.AbstractClientHttpRequest.execute(AbstractClientHttpRequest.java:53)
app//org.springframework.web.client.RestTemplate.doExecute(RestTemplate.java:739)
app//org.springframework.security.oauth2.client.OAuth2RestTemplate.doExecute(OAuth2RestTemplate.java:136)
app//org.springframework.web.client.RestTemplate.execute(RestTemplate.java:674)
app//org.springframework.web.client.RestTemplate.exchange(RestTemplate.java:583)
[...custom_code...]
app//com.google.common.cache.CacheLoader$SupplierToCacheLoader.load(CacheLoader.java:223)
app//com.google.common.cache.CacheLoader$1.load(CacheLoader.java:188)
app//com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3529)
app//com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2278)
app//com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2155)
- locked com.google.common.cache.LocalCache$StrongWriteEntry@2a921ea6
app//com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2045)
app//com.google.common.cache.LocalCache.get(LocalCache.java:3951)
app//com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:3974)
app//com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4958)
app//com.google.common.cache.LocalCache$LocalLoadingCache.getUnchecked(LocalCache.java:4964)
[...custom_code...]
java.base@11.0.8/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
java.base@11.0.8/java.util.concurrent.FutureTask.run(FutureTask.java:264)
java.base@11.0.8/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
java.base@11.0.8/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base@11.0.8/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base@11.0.8/java.lang.Thread.run(Thread.java:834)
Comment From: blacelle
Closed as I now feel this is a Sleuth specific issue https://github.com/spring-cloud/spring-cloud-sleuth/issues/1750