From 9765c1dd1df409e23f8c6f2602f8437fbae4fc05 Mon Sep 17 00:00:00 2001 From: jayce1116 Date: Tue, 9 Jan 2024 07:44:18 +0900 Subject: [PATCH 1/2] add service field in ASRExpectSpeech --- .../AutomaticSpeechRecognition/ASRExpectSpeech.swift | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NuguAgents/Sources/CapabilityAgents/AutomaticSpeechRecognition/ASRExpectSpeech.swift b/NuguAgents/Sources/CapabilityAgents/AutomaticSpeechRecognition/ASRExpectSpeech.swift index 434388a7a..c901637b5 100644 --- a/NuguAgents/Sources/CapabilityAgents/AutomaticSpeechRecognition/ASRExpectSpeech.swift +++ b/NuguAgents/Sources/CapabilityAgents/AutomaticSpeechRecognition/ASRExpectSpeech.swift @@ -37,6 +37,7 @@ struct ASRExpectSpeech { let asrContext: [String: AnyHashable]? let epd: EPD? let listenTimeoutFailBeep: Bool? + let service: [String: AnyHashable]? struct EPD: Decodable { let timeoutMilliseconds: Int? @@ -55,6 +56,7 @@ extension ASRExpectSpeech.Payload: Decodable { case asrContext case epd case listenTimeoutFailBeep + case service } public init(from decoder: Decoder) throws { @@ -64,6 +66,7 @@ extension ASRExpectSpeech.Payload: Decodable { asrContext = try? container.decodeIfPresent([String: AnyHashable].self, forKey: .asrContext) epd = try? container.decode(EPD.self, forKey: .epd) listenTimeoutFailBeep = try? container.decode(Bool.self, forKey: .listenTimeoutFailBeep) + service = try? container.decodeIfPresent([String: AnyHashable].self, forKey: .service) } } From 093960fbfdfa33af40d8e1c927897028174cb806 Mon Sep 17 00:00:00 2001 From: jayce1116 Date: Tue, 9 Jan 2024 09:39:34 +0900 Subject: [PATCH 2/2] add ASRAgent delegate --- .../AutomaticSpeechRecognition/ASRAgent.swift | 15 +++++++--- .../ASRAgentDelegate.swift | 29 +++++++++++++++++++ .../ASRAgentProtocol.swift | 1 + nugu-ios.xcodeproj/project.pbxproj | 6 +++- 4 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 NuguAgents/Sources/CapabilityAgents/AutomaticSpeechRecognition/ASRAgentDelegate.swift diff --git a/NuguAgents/Sources/CapabilityAgents/AutomaticSpeechRecognition/ASRAgent.swift b/NuguAgents/Sources/CapabilityAgents/AutomaticSpeechRecognition/ASRAgent.swift index 5977a2d7d..3850a6de9 100644 --- a/NuguAgents/Sources/CapabilityAgents/AutomaticSpeechRecognition/ASRAgent.swift +++ b/NuguAgents/Sources/CapabilityAgents/AutomaticSpeechRecognition/ASRAgent.swift @@ -29,10 +29,11 @@ import RxSwift public final class ASRAgent: ASRAgentProtocol { // CapabilityAgentable - // TODO: ASR interface version 1.1 -> ASR.Recognize(wakeup/power) public var capabilityAgentProperty: CapabilityAgentProperty = CapabilityAgentProperty(category: .automaticSpeechRecognition, version: "1.8") private let playSyncProperty = PlaySyncProperty(layerType: .asr, contextType: .sound) + public weak var delegate: ASRAgentDelegate? + // Private private let focusManager: FocusManageable private let contextManager: ContextManageable @@ -93,7 +94,7 @@ public final class ASRAgent: ASRAgentProtocol { log.error("ASR request: \(String(describing: asrRequest)), result: \(String(describing: asrResult))") return } - log.info("\(asrResult)") + log.info("asrResult: \(asrResult)") // `ASRState` -> Event -> `expectSpeechDirective` -> `ASRAgentDelegate` switch asrResult { @@ -453,7 +454,7 @@ private extension ASRAgent { defer { completion(.finished) } self?.asrDispatchQueue.sync { [weak self] in - guard let self = self else { return } + guard let self = self, let delegate = self.delegate else { return } // ex> TTS 도중 stopRecognition 호출. guard let expectSpeech = self.expectSpeech, expectSpeech.messageId == directive.header.messageId else { log.info("Message id does not match") @@ -464,12 +465,18 @@ private extension ASRAgent { log.warning("ExpectSpeech only allowed in IDLE or BUSY state.") return } + let service = expectSpeech.payload.service + guard service == nil || delegate.asrAgentWillStartExpectSpeech(service: service) else { + log.warning("ExpectSpeech service field is not nil. service: \(String(describing: service))") + self.asrResult = nil + return + } self.asrState = .expectingSpeech startRecognition( initiator: .expectSpeech, eventIdentifier: EventIdentifier(), - service: asrRequest?.service, + service: service, requestType: options.requestType, completion: nil ) diff --git a/NuguAgents/Sources/CapabilityAgents/AutomaticSpeechRecognition/ASRAgentDelegate.swift b/NuguAgents/Sources/CapabilityAgents/AutomaticSpeechRecognition/ASRAgentDelegate.swift new file mode 100644 index 000000000..663d46ddb --- /dev/null +++ b/NuguAgents/Sources/CapabilityAgents/AutomaticSpeechRecognition/ASRAgentDelegate.swift @@ -0,0 +1,29 @@ +// +// ASRAgentDelegate.swift +// NuguAgents +// +// Created by Jaycesub on 17/04/2019. +// Copyright (c) 2024 SK Telecom Co., Ltd. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import Foundation + +public protocol ASRAgentDelegate: AnyObject { + /// ASRAgent start recognition after receiving expectSpeech directive. + /// Returns true if start recognition without being affected by the condition. + /// - Parameter service: The service object included in expectSpeech payload + /// - Returns: True if the ASRAgent start recognition after receiving expectSpeech directive. + func asrAgentWillStartExpectSpeech(service: [String: AnyHashable]?) -> Bool +} diff --git a/NuguAgents/Sources/CapabilityAgents/AutomaticSpeechRecognition/ASRAgentProtocol.swift b/NuguAgents/Sources/CapabilityAgents/AutomaticSpeechRecognition/ASRAgentProtocol.swift index 09044852d..9b5055568 100644 --- a/NuguAgents/Sources/CapabilityAgents/AutomaticSpeechRecognition/ASRAgentProtocol.swift +++ b/NuguAgents/Sources/CapabilityAgents/AutomaticSpeechRecognition/ASRAgentProtocol.swift @@ -26,6 +26,7 @@ import NuguUtils /// ASR (AutomaticSpeechRecognition) is responsible for capturing the audio and delivering it to the server and receiving the result of speech recognition. public protocol ASRAgentProtocol: CapabilityAgentable, TypedNotifyable { + var delegate: ASRAgentDelegate? { get set } var options: ASROptions { get set } var asrState: ASRState { get } diff --git a/nugu-ios.xcodeproj/project.pbxproj b/nugu-ios.xcodeproj/project.pbxproj index 5cea510b4..09e5543ec 100644 --- a/nugu-ios.xcodeproj/project.pbxproj +++ b/nugu-ios.xcodeproj/project.pbxproj @@ -3,7 +3,7 @@ archiveVersion = 1; classes = { }; - objectVersion = 52; + objectVersion = 54; objects = { /* Begin PBXBuildFile section */ @@ -529,6 +529,7 @@ E649868A28337649001AD733 /* ControlCenterManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = E649868928337649001AD733 /* ControlCenterManager.swift */; }; E6BE047C2A0BA1A400AE29E3 /* Image+Event.swift in Sources */ = {isa = PBXBuildFile; fileRef = E6BE047B2A0BA1A400AE29E3 /* Image+Event.swift */; }; E6BE047E2A0BA1BD00AE29E3 /* ImageAgent.swift in Sources */ = {isa = PBXBuildFile; fileRef = E6BE047D2A0BA1BD00AE29E3 /* ImageAgent.swift */; }; + E6F3DCBD2B4CB9D800298A20 /* ASRAgentDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = E6F3DCBC2B4CB9D800298A20 /* ASRAgentDelegate.swift */; }; E6FBA02B2A1379C500AF8B05 /* MessengerAgentProtocol.swift in Sources */ = {isa = PBXBuildFile; fileRef = E6FBA02A2A1379C400AF8B05 /* MessengerAgentProtocol.swift */; }; E6FBA02D2A137A9700AF8B05 /* ImageAgentProtocol.swift in Sources */ = {isa = PBXBuildFile; fileRef = E6FBA02C2A137A9700AF8B05 /* ImageAgentProtocol.swift */; }; F6A6B0842AAEFDD900D41DEC /* AudioPlayer2Template.swift in Sources */ = {isa = PBXBuildFile; fileRef = F6A6B0832AAEFDD900D41DEC /* AudioPlayer2Template.swift */; }; @@ -1331,6 +1332,7 @@ E649868928337649001AD733 /* ControlCenterManager.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ControlCenterManager.swift; sourceTree = ""; }; E6BE047B2A0BA1A400AE29E3 /* Image+Event.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Image+Event.swift"; sourceTree = ""; }; E6BE047D2A0BA1BD00AE29E3 /* ImageAgent.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImageAgent.swift; sourceTree = ""; }; + E6F3DCBC2B4CB9D800298A20 /* ASRAgentDelegate.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ASRAgentDelegate.swift; sourceTree = ""; }; E6FBA02A2A1379C400AF8B05 /* MessengerAgentProtocol.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MessengerAgentProtocol.swift; sourceTree = ""; }; E6FBA02C2A137A9700AF8B05 /* ImageAgentProtocol.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImageAgentProtocol.swift; sourceTree = ""; }; F6A6B0832AAEFDD900D41DEC /* AudioPlayer2Template.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioPlayer2Template.swift; sourceTree = ""; }; @@ -2080,6 +2082,7 @@ 737388F024A46C7F0018DDD2 /* ASROptions.swift */, 737388F124A46C7F0018DDD2 /* ASRResult.swift */, 737388F224A46C7F0018DDD2 /* ASRState.swift */, + E6F3DCBC2B4CB9D800298A20 /* ASRAgentDelegate.swift */, ); path = AutomaticSpeechRecognition; sourceTree = ""; @@ -4512,6 +4515,7 @@ 73152FCC23E0415B00F843C3 /* DisplayRenderingInfo.swift in Sources */, 736508652462F7FA00EF4549 /* SktOpusParser.swift in Sources */, 7E284D9C24DD2B4A00BF9640 /* InteractionControlManager.swift in Sources */, + E6F3DCBD2B4CB9D800298A20 /* ASRAgentDelegate.swift in Sources */, 1FD5D19124C82F3B007BA384 /* MediaPlayerAgent+Event.swift in Sources */, 737388C924A46C1D0018DDD2 /* ChipsAgentProtocol.swift in Sources */, 1FD5D17E24A99993007BA384 /* PhoneCallType.swift in Sources */,