Bug description
AzureOpenAiChatModel.class method: stream(Prompt prompt) // Note: the first chat completions can be ignored when using Azure OpenAI // service which is a known service bug. .skip(1) commit https://github.com/spring-projects/spring-ai/commit/0ccf32780696af284794ee0f8420857dc1cce6a3#diff-008124b444ef3c62065ae60104221890d1bb876c01ea71501d34eeb040e7f210R197
bug: The first data returned specifies the name of FunctionCalling and other information, which will be lost after being skipped, eventually resulting in a null pointer.
model:gpt4o-0513
- code
@Override
public Flux<ChatResponse> stream(Prompt prompt) {
ChatCompletionsOptions options = toAzureChatCompletionsOptions(prompt);
options.setStream(true);
IterableStream<ChatCompletions> chatCompletionsStream = this.openAIClient
.getChatCompletionsStream(options.getModel(), options);
Flux<ChatCompletions> chatCompletionsFlux = Flux.fromIterable(chatCompletionsStream);
final var isFunctionCall = new AtomicBoolean(false);
final var accessibleChatCompletionsFlux = chatCompletionsFlux
// Note: the first chat completions can be ignored when using Azure OpenAI
// service which is a known service bug.
.skip(1)
.map(chatCompletions -> {
final var toolCalls = chatCompletions.getChoices().get(0).getDelta().getToolCalls();
isFunctionCall.set(toolCalls != null && !toolCalls.isEmpty());
return chatCompletions;
})
.windowUntil(chatCompletions -> {
if (isFunctionCall.get() && chatCompletions.getChoices()
.get(0)
.getFinishReason() == CompletionsFinishReason.TOOL_CALLS) {
isFunctionCall.set(false);
return true;
}
return !isFunctionCall.get();
})
.concatMapIterable(window -> {
final var reduce = window.reduce(MergeUtils.emptyChatCompletions(), MergeUtils::mergeChatCompletions);
return List.of(reduce);
})
.flatMap(mono -> mono);
return accessibleChatCompletionsFlux
.switchMap(accessibleChatCompletions -> handleFunctionCallOrReturnStream(options,
Flux.just(accessibleChatCompletions)))
.flatMapIterable(ChatCompletions::getChoices)
.map(choice -> {
var content = Optional.ofNullable(choice.getMessage()).orElse(choice.getDelta()).getContent();
var generation = new Generation(content).withGenerationMetadata(generateChoiceMetadata(choice));
return new ChatResponse(List.of(generation));
});
}
- exception
2024-07-12 15:57:00 [task-2] ERROR org.springframework.ai.chat.model.MessageAggregator - Aggregation Error
java.lang.NullPointerException: Cannot invoke "Object.hashCode()" because "key" is null
at java.base/java.util.concurrent.ConcurrentHashMap.get(ConcurrentHashMap.java:936) ~[?:?]
at java.base/java.util.concurrent.ConcurrentHashMap.containsKey(ConcurrentHashMap.java:964) ~[?:?]
at org.springframework.ai.azure.openai.AzureOpenAiChatModel.doCreateToolResponseRequest(AzureOpenAiChatModel.java:551) ~[spring-ai-azure-openai-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT]
at org.springframework.ai.azure.openai.AzureOpenAiChatModel.doCreateToolResponseRequest(AzureOpenAiChatModel.java:85) ~[spring-ai-azure-openai-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT]
at org.springframework.ai.model.function.AbstractFunctionCallSupport.lambda$handleFunctionCallOrReturnStream$1(AbstractFunctionCallSupport.java:176) ~[spring-ai-core-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT]
at reactor.core.publisher.FluxFlatMap.trySubscribeScalarMap(FluxFlatMap.java:153) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.FluxSwitchMapNoPrefetch.subscribeOrReturn(FluxSwitchMapNoPrefetch.java:61) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.Flux.subscribe(Flux.java:8825) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.FluxSwitchMapNoPrefetch$SwitchMapMain.subscribeInner(FluxSwitchMapNoPrefetch.java:219) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.FluxSwitchMapNoPrefetch$SwitchMapMain.onNext(FluxSwitchMapNoPrefetch.java:164) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.FluxFlatMap$FlatMapMain.tryEmit(FluxFlatMap.java:547) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.FluxFlatMap$FlatMapInner.onNext(FluxFlatMap.java:988) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.Operators$BaseFluxToMonoOperator.completePossiblyEmpty(Operators.java:2097) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.MonoReduceSeed$ReduceSeedSubscriber.onComplete(MonoReduceSeed.java:163) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.FluxWindowPredicate$WindowFlux.checkTerminated(FluxWindowPredicate.java:768) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.FluxWindowPredicate$WindowFlux.drainRegular(FluxWindowPredicate.java:662) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.FluxWindowPredicate$WindowFlux.drain(FluxWindowPredicate.java:748) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.FluxWindowPredicate$WindowFlux.onComplete(FluxWindowPredicate.java:814) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.FluxWindowPredicate$WindowPredicateMain.onNext(FluxWindowPredicate.java:243) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.FluxMap$MapSubscriber.onNext(FluxMap.java:122) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.FluxSkip$SkipSubscriber.onNext(FluxSkip.java:87) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.FluxIterable$IterableSubscription.slowPath(FluxIterable.java:335) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.FluxIterable$IterableSubscription.request(FluxIterable.java:294) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.FluxSkip$SkipSubscriber.request(FluxSkip.java:121) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.FluxMap$MapSubscriber.request(FluxMap.java:164) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.FluxWindowPredicate$WindowPredicateMain.onSubscribe(FluxWindowPredicate.java:188) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.FluxMap$MapSubscriber.onSubscribe(FluxMap.java:92) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.FluxSkip$SkipSubscriber.onSubscribe(FluxSkip.java:78) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.FluxIterable.subscribe(FluxIterable.java:201) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.FluxIterable.subscribe(FluxIterable.java:83) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.Flux.subscribe(Flux.java:8840) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.Flux.subscribeWith(Flux.java:8961) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.Flux.subscribe(Flux.java:8805) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.Flux.subscribe(Flux.java:8729) ~[reactor-core-3.6.4.jar:3.6.4]
at reactor.core.publisher.Flux.subscribe(Flux.java:8647) ~[reactor-core-3.6.4.jar:3.6.4]
at org.intellitech.engine.themeagent.agent.AbsAgent.stream(AbsAgent.java:205) ~[main/:?]
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103) ~[?:?]
at java.base/java.lang.reflect.Method.invoke(Method.java:580) ~[?:?]
at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:351) ~[spring-aop-6.1.5.jar:6.1.5]
at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:196) ~[spring-aop-6.1.5.jar:6.1.5]
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163) ~[spring-aop-6.1.5.jar:6.1.5]
at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:765) ~[spring-aop-6.1.5.jar:6.1.5]
at org.springframework.aop.interceptor.AsyncExecutionInterceptor.lambda$invoke$0(AsyncExecutionInterceptor.java:113) ~[spring-aop-6.1.5.jar:6.1.5]
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317) ~[?:?]
at java.base/java.lang.VirtualThread.run(VirtualThread.java:309) ~[?:?]
Comment From: jacklvyx
@tzolov found This issue is addressed by the https://github.com/spring-projects/spring-ai/commit/feb036d2f6534505ad0bdac12925f93a52737e73
Comment From: tzolov
@jacklvyx is it working as expected now?
Comment From: tzolov
Closing as resolved by https://github.com/spring-projects/spring-ai/commit/feb036d2f6534505ad0bdac12925f93a52737e73 @jacklvyx if the problem is still present please re-open the issue.