From b385e20e1ab596c952c2be155c0f77b25fc36521 Mon Sep 17 00:00:00 2001 From: "sangho.lee" Date: Fri, 15 Dec 2023 16:14:18 +0900 Subject: [PATCH] Deferred ASR.Recognize Event #2452 * If EPD's timeout is longer than request(ASR.Recognize Event)'s timeout, the ASR is not working. * In this case, send request after start speech. --- .../asr/impl/DefaultClientSpeechRecognizer.kt | 93 ++++++++++++------- 1 file changed, 57 insertions(+), 36 deletions(-) diff --git a/nugu-agent/src/main/java/com/skt/nugu/sdk/agent/asr/impl/DefaultClientSpeechRecognizer.kt b/nugu-agent/src/main/java/com/skt/nugu/sdk/agent/asr/impl/DefaultClientSpeechRecognizer.kt index 228fd3eb7..e6ebcc912 100644 --- a/nugu-agent/src/main/java/com/skt/nugu/sdk/agent/asr/impl/DefaultClientSpeechRecognizer.kt +++ b/nugu-agent/src/main/java/com/skt/nugu/sdk/agent/asr/impl/DefaultClientSpeechRecognizer.kt @@ -64,6 +64,7 @@ class DefaultClientSpeechRecognizer( var senderThread: SpeechRecognizeAttachmentSenderThread?, override val eventMessage: EventMessageRequest, val expectSpeechParam: DefaultASRAgent.ExpectSpeechDirectiveParam?, + val epdParam: EndPointDetectorParam, val resultListener: ASRAgentInterface.OnResultListener? ): SpeechRecognizer.Request { override val attributeKey: String? = expectSpeechParam?.directive?.header?.messageId @@ -73,7 +74,7 @@ class DefaultClientSpeechRecognizer( var cancelCause: ASRAgentInterface.CancelCause? = null val shouldBeHandledResult = HashSet() var receiveResponse = false - var call: Call? = null + var recognizeEventCall: Call? = null val eventMessageHeader = with(eventMessage) { Header(dialogRequestId, messageId, name, namespace, version, referrerDialogRequestId) @@ -169,6 +170,7 @@ class DefaultClientSpeechRecognizer( null, eventMessage, expectSpeechDirectiveParam, + epdParam, resultListener ) @@ -203,7 +205,7 @@ class DefaultClientSpeechRecognizer( if(cancel) { - request.call?.cancel() + request.recognizeEventCall?.cancel() } if (isEpdRunning) { @@ -282,46 +284,61 @@ class DefaultClientSpeechRecognizer( return } + sendRecognizeRequestEventIfNotDeferred(recognizeRequest) + + epdState = AudioEndPointDetector.State.EXPECTING_SPEECH + setState(SpeechRecognizer.State.EXPECTING_SPEECH, recognizeRequest) + } + + private fun sendRecognizeRequestEventIfNotDeferred(recognizeRequest: RecognizeRequest) { + val call = createRecognizeRequestCall(recognizeRequest) + + if(call.callTimeout() > recognizeRequest.epdParam.timeoutInSeconds * 1000L) { + sendRecognizeRequest(recognizeRequest, call) + } + } + + private fun createRecognizeRequestCall(recognizeRequest: RecognizeRequest): Call = messageSender.newCall( recognizeRequest.eventMessage, - hashMapOf("Last-Asr-Event-Time" to Preferences.get("Last-Asr-Event-Time").toString()) - ).apply { - recognizeRequest.call = this - if(!this.enqueue(object: MessageSender.Callback { - override fun onFailure(request: MessageRequest, status: Status) { - Logger.d(TAG, "[onFailure] request: $request, status: $status") - if(recognizeRequest == currentRequest && status.error == Status.StatusError.TIMEOUT && recognizeRequest.senderThread != null) { - // if sender thread is working, we handle a timeout error as unknown error. - // this occur when some attachment sent too late(after timeout). - recognizeRequest.errorTypeForCausingEpdStop = ASRAgentInterface.ErrorType.ERROR_UNKNOWN - endPointDetector.stopDetector() - } else { - recognizeRequest.errorTypeForCausingEpdStop = when (status.error) { - Status.StatusError.OK /** cancel, no error **/ , - Status.StatusError.TIMEOUT /** Nothing to do because handle on [onResponseTimeout] **/ , - Status.StatusError.UNKNOWN -> /** Same as return false of [enqueue] **/ return - Status.StatusError.NETWORK -> ASRAgentInterface.ErrorType.ERROR_NETWORK - Status.StatusError.UNAUTHENTICATED -> ASRAgentInterface.ErrorType.ERROR_UNKNOWN - } - endPointDetector.stopDetector() + hashMapOf("Last-Asr-Event-Time" to Preferences.get("Last-Asr-Event-Time")) + ) + + private fun sendRecognizeRequest(recognizeRequest: RecognizeRequest, call: Call) { + recognizeRequest.recognizeEventCall = call + + if(!call.enqueue(object: MessageSender.Callback { + override fun onFailure(request: MessageRequest, status: Status) { + Logger.d(TAG, "[onFailure] request: $request, status: $status") + if(recognizeRequest == currentRequest && status.error == Status.StatusError.TIMEOUT && recognizeRequest.senderThread != null) { + // if sender thread is working, we handle a timeout error as unknown error. + // this occur when some attachment sent too late(after timeout). + recognizeRequest.errorTypeForCausingEpdStop = ASRAgentInterface.ErrorType.ERROR_UNKNOWN + endPointDetector.stopDetector() + } else { + recognizeRequest.errorTypeForCausingEpdStop = when (status.error) { + Status.StatusError.OK /** cancel, no error **/ , + Status.StatusError.TIMEOUT /** Nothing to do because handle on [onResponseTimeout] **/ , + Status.StatusError.UNKNOWN -> /** Same as return false of [enqueue] **/ return + Status.StatusError.NETWORK -> ASRAgentInterface.ErrorType.ERROR_NETWORK + Status.StatusError.UNAUTHENTICATED -> ASRAgentInterface.ErrorType.ERROR_UNKNOWN } + endPointDetector.stopDetector() } + } - override fun onSuccess(request: MessageRequest) { - Logger.d(TAG, "[onSuccess] request: $request") - } + override fun onSuccess(request: MessageRequest) { + Logger.d(TAG, "[onSuccess] request: $request") + } - override fun onResponseStart(request: MessageRequest) { - Logger.d(TAG, "[onResponseStart] request: $request") - Preferences.set("Last-Asr-Event-Time", dateFormat.format(Calendar.getInstance().time)) - } - })) { - recognizeRequest.errorTypeForCausingEpdStop = ASRAgentInterface.ErrorType.ERROR_NETWORK - endPointDetector.stopDetector() - } + override fun onResponseStart(request: MessageRequest) { + Logger.d(TAG, "[onResponseStart] request: $request") + Preferences.set("Last-Asr-Event-Time", dateFormat.format(Calendar.getInstance().time)) + } + })) { + recognizeRequest.errorTypeForCausingEpdStop = ASRAgentInterface.ErrorType.ERROR_NETWORK + endPointDetector.stopDetector() } - epdState = AudioEndPointDetector.State.EXPECTING_SPEECH - setState(SpeechRecognizer.State.EXPECTING_SPEECH, recognizeRequest) } override fun onSpeechStart(eventPosition: Long?) { @@ -336,6 +353,10 @@ class DefaultClientSpeechRecognizer( return } + if(request.recognizeEventCall == null) { + sendRecognizeRequest(request, createRecognizeRequestCall(request)) + } + startSpeechSenderThread(request, eventPosition) epdState = AudioEndPointDetector.State.SPEECH_START setState(SpeechRecognizer.State.SPEECH_START, request) @@ -639,4 +660,4 @@ class DefaultClientSpeechRecognizer( return Pair(sendPosition, sendWakeupBoundary) } -} \ No newline at end of file +}