Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Deferred ASR.Recognize Event #2452 #2453

Merged
merged 1 commit into from
Dec 15, 2023
Merged

Deferred ASR.Recognize Event #2452 #2453

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 57 additions & 36 deletions nugu-agent/src/main/java/com/skt/nugu/sdk/agent/asr/impl/DefaultClientSpeechRecognizer.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ class DefaultClientSpeechRecognizer(
var senderThread: SpeechRecognizeAttachmentSenderThread?,
override val eventMessage: EventMessageRequest,
val expectSpeechParam: DefaultASRAgent.ExpectSpeechDirectiveParam?,
val epdParam: EndPointDetectorParam,
val resultListener: ASRAgentInterface.OnResultListener?
): SpeechRecognizer.Request {
override val attributeKey: String? = expectSpeechParam?.directive?.header?.messageId
Expand All @@ -73,7 +74,7 @@ class DefaultClientSpeechRecognizer(
var cancelCause: ASRAgentInterface.CancelCause? = null
val shouldBeHandledResult = HashSet<String>()
var receiveResponse = false
var call: Call? = null
var recognizeEventCall: Call? = null

val eventMessageHeader = with(eventMessage) {
Header(dialogRequestId, messageId, name, namespace, version, referrerDialogRequestId)
Expand Down Expand Up @@ -169,6 +170,7 @@ class DefaultClientSpeechRecognizer(
null,
eventMessage,
expectSpeechDirectiveParam,
epdParam,
resultListener
)

Expand Down Expand Up @@ -203,7 +205,7 @@ class DefaultClientSpeechRecognizer(


if(cancel) {
request.call?.cancel()
request.recognizeEventCall?.cancel()
}

if (isEpdRunning) {
Expand Down Expand Up @@ -282,46 +284,61 @@ class DefaultClientSpeechRecognizer(
return
}

sendRecognizeRequestEventIfNotDeferred(recognizeRequest)

epdState = AudioEndPointDetector.State.EXPECTING_SPEECH
setState(SpeechRecognizer.State.EXPECTING_SPEECH, recognizeRequest)
}

private fun sendRecognizeRequestEventIfNotDeferred(recognizeRequest: RecognizeRequest) {
val call = createRecognizeRequestCall(recognizeRequest)

if(call.callTimeout() > recognizeRequest.epdParam.timeoutInSeconds * 1000L) {
sendRecognizeRequest(recognizeRequest, call)
}
}

private fun createRecognizeRequestCall(recognizeRequest: RecognizeRequest): Call =
messageSender.newCall(
recognizeRequest.eventMessage,
hashMapOf("Last-Asr-Event-Time" to Preferences.get("Last-Asr-Event-Time").toString())
).apply {
recognizeRequest.call = this
if(!this.enqueue(object: MessageSender.Callback {
override fun onFailure(request: MessageRequest, status: Status) {
Logger.d(TAG, "[onFailure] request: $request, status: $status")
if(recognizeRequest == currentRequest && status.error == Status.StatusError.TIMEOUT && recognizeRequest.senderThread != null) {
// if sender thread is working, we handle a timeout error as unknown error.
// this occur when some attachment sent too late(after timeout).
recognizeRequest.errorTypeForCausingEpdStop = ASRAgentInterface.ErrorType.ERROR_UNKNOWN
endPointDetector.stopDetector()
} else {
recognizeRequest.errorTypeForCausingEpdStop = when (status.error) {
Status.StatusError.OK /** cancel, no error **/ ,
Status.StatusError.TIMEOUT /** Nothing to do because handle on [onResponseTimeout] **/ ,
Status.StatusError.UNKNOWN -> /** Same as return false of [enqueue] **/ return
Status.StatusError.NETWORK -> ASRAgentInterface.ErrorType.ERROR_NETWORK
Status.StatusError.UNAUTHENTICATED -> ASRAgentInterface.ErrorType.ERROR_UNKNOWN
}
endPointDetector.stopDetector()
hashMapOf("Last-Asr-Event-Time" to Preferences.get("Last-Asr-Event-Time"))
)

private fun sendRecognizeRequest(recognizeRequest: RecognizeRequest, call: Call) {
recognizeRequest.recognizeEventCall = call

if(!call.enqueue(object: MessageSender.Callback {
override fun onFailure(request: MessageRequest, status: Status) {
Logger.d(TAG, "[onFailure] request: $request, status: $status")
if(recognizeRequest == currentRequest && status.error == Status.StatusError.TIMEOUT && recognizeRequest.senderThread != null) {
// if sender thread is working, we handle a timeout error as unknown error.
// this occur when some attachment sent too late(after timeout).
recognizeRequest.errorTypeForCausingEpdStop = ASRAgentInterface.ErrorType.ERROR_UNKNOWN
endPointDetector.stopDetector()
} else {
recognizeRequest.errorTypeForCausingEpdStop = when (status.error) {
Status.StatusError.OK /** cancel, no error **/ ,
Status.StatusError.TIMEOUT /** Nothing to do because handle on [onResponseTimeout] **/ ,
Status.StatusError.UNKNOWN -> /** Same as return false of [enqueue] **/ return
Status.StatusError.NETWORK -> ASRAgentInterface.ErrorType.ERROR_NETWORK
Status.StatusError.UNAUTHENTICATED -> ASRAgentInterface.ErrorType.ERROR_UNKNOWN
}
endPointDetector.stopDetector()
}
}

override fun onSuccess(request: MessageRequest) {
Logger.d(TAG, "[onSuccess] request: $request")
}
override fun onSuccess(request: MessageRequest) {
Logger.d(TAG, "[onSuccess] request: $request")
}

override fun onResponseStart(request: MessageRequest) {
Logger.d(TAG, "[onResponseStart] request: $request")
Preferences.set("Last-Asr-Event-Time", dateFormat.format(Calendar.getInstance().time))
}
})) {
recognizeRequest.errorTypeForCausingEpdStop = ASRAgentInterface.ErrorType.ERROR_NETWORK
endPointDetector.stopDetector()
}
override fun onResponseStart(request: MessageRequest) {
Logger.d(TAG, "[onResponseStart] request: $request")
Preferences.set("Last-Asr-Event-Time", dateFormat.format(Calendar.getInstance().time))
}
})) {
recognizeRequest.errorTypeForCausingEpdStop = ASRAgentInterface.ErrorType.ERROR_NETWORK
endPointDetector.stopDetector()
}
epdState = AudioEndPointDetector.State.EXPECTING_SPEECH
setState(SpeechRecognizer.State.EXPECTING_SPEECH, recognizeRequest)
}

override fun onSpeechStart(eventPosition: Long?) {
Expand All @@ -336,6 +353,10 @@ class DefaultClientSpeechRecognizer(
return
}

if(request.recognizeEventCall == null) {
sendRecognizeRequest(request, createRecognizeRequestCall(request))
}

startSpeechSenderThread(request, eventPosition)
epdState = AudioEndPointDetector.State.SPEECH_START
setState(SpeechRecognizer.State.SPEECH_START, request)
Expand Down Expand Up @@ -639,4 +660,4 @@ class DefaultClientSpeechRecognizer(

return Pair(sendPosition, sendWakeupBoundary)
}
}
}
Loading