From d1435dee2a898381c4ef15142f96b165a8f395fe Mon Sep 17 00:00:00 2001 From: jayce1116 Date: Thu, 6 Jul 2023 16:53:36 +0900 Subject: [PATCH 01/39] =?UTF-8?q?=ED=98=84=EC=9E=AC=20=EC=9E=AC=EC=83=9D?= =?UTF-8?q?=EB=90=98=EA=B3=A0=20=EC=9E=88=EB=8A=94=20=ED=94=8C=EB=A0=88?= =?UTF-8?q?=EC=9D=B4=EC=96=B4=EC=9D=98=20=EB=B3=BC=EB=A5=A8=EC=9D=84=20?= =?UTF-8?q?=EC=A1=B0=EC=A0=88=ED=95=98=EB=8A=94=20=ED=95=A8=EC=88=98=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80=20(#1069)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Description - 현재 재생되고 있는 플레이어의 볼륨을 조절하는 함수 추가 --- .../Sources/CapabilityAgents/TextToSpeech/TTSAgent.swift | 6 ++++++ .../CapabilityAgents/TextToSpeech/TTSAgentProtocol.swift | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/NuguAgents/Sources/CapabilityAgents/TextToSpeech/TTSAgent.swift b/NuguAgents/Sources/CapabilityAgents/TextToSpeech/TTSAgent.swift index f68d4cfbc..7256ffbe6 100644 --- a/NuguAgents/Sources/CapabilityAgents/TextToSpeech/TTSAgent.swift +++ b/NuguAgents/Sources/CapabilityAgents/TextToSpeech/TTSAgent.swift @@ -232,6 +232,12 @@ public extension TTSAgent { self?.stop(player: player, cancelAssociation: cancelAssociation) } } + + func updateLatestPlayerVolume(_ volume: Float) { + ttsDispatchQueue.sync { + latestPlayer?.volume = volume + } + } } // MARK: - FocusChannelDelegate diff --git a/NuguAgents/Sources/CapabilityAgents/TextToSpeech/TTSAgentProtocol.swift b/NuguAgents/Sources/CapabilityAgents/TextToSpeech/TTSAgentProtocol.swift index 8d6271c46..b2503b997 100644 --- a/NuguAgents/Sources/CapabilityAgents/TextToSpeech/TTSAgentProtocol.swift +++ b/NuguAgents/Sources/CapabilityAgents/TextToSpeech/TTSAgentProtocol.swift @@ -63,6 +63,13 @@ public protocol TTSAgentProtocol: CapabilityAgentable, TypedNotifyable { /// Stops playback /// - Parameter cancelAssociation: true: cancel all associated directives, false : only stop tts func stopTTS(cancelAssociation: Bool) + + /** + 현재 재생 중인 플레이어의 볼륨을 설정한다. + + - Parameter volume: 플레이어의 볼륨(0.0 ~ 1.0) + */ + func updateLatestPlayerVolume(_ volume: Float) } // MARK: - Default From ffefe24e45861ad19640dcf81ca20741ff9375b0 Mon Sep 17 00:00:00 2001 From: jayce1116 Date: Tue, 25 Jul 2023 08:12:02 +0900 Subject: [PATCH 02/39] =?UTF-8?q?=EB=A3=A8=ED=8B=B4=20=EB=82=B4=EC=97=90?= =?UTF-8?q?=EC=84=9CTextInput=EC=9A=94=EC=B2=AD=20=EC=8B=9C,=20source?= =?UTF-8?q?=EB=A5=BC=20DYNAMIC=5FROUTINE=5FACTION=EC=9C=BC=EB=A1=9C=20?= =?UTF-8?q?=EC=84=A4=EC=A0=95=20(#1070)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Description - 루틴 내에서TextInput요청 시, source를 DYNAMIC_ROUTINE_ACTION으로 설정 --- .../Routine/RoutineExecuter.swift | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/NuguAgents/Sources/CapabilityAgents/Routine/RoutineExecuter.swift b/NuguAgents/Sources/CapabilityAgents/Routine/RoutineExecuter.swift index dd56565d9..2bedf4775 100644 --- a/NuguAgents/Sources/CapabilityAgents/Routine/RoutineExecuter.swift +++ b/NuguAgents/Sources/CapabilityAgents/Routine/RoutineExecuter.swift @@ -364,11 +364,23 @@ private extension RoutineExecuter { doNextAction() return } + let dynamicSource: TextInputSource = .dynamic("DYNAMIC_ROUTINE_ACTION") if let playServiceId = action.playServiceId { - handlingEvent = textAgent.requestTextInput(text: text, token: action.token, requestType: .specific(playServiceId: playServiceId), completion: completion) - + handlingEvent = textAgent.requestTextInput( + text: text, + token: action.token, + source: dynamicSource, + requestType: .specific(playServiceId: playServiceId), + completion: completion + ) } else { - handlingEvent = textAgent.requestTextInput(text: text, token: action.token, requestType: .normal, completion: completion) + handlingEvent = textAgent.requestTextInput( + text: text, + token: action.token, + source: dynamicSource + requestType: .normal, + completion: completion + ) } case .data: if let eventIdentifier = delegate?.routineExecuterShouldRequestAction(action: action, referrerDialogRequestId: routine.dialogRequestId, completion: completion) { From 8fc63c7cfd85b83ab7f8f2649badc4105ce58f6c Mon Sep 17 00:00:00 2001 From: jayce1116 Date: Tue, 25 Jul 2023 08:38:24 +0900 Subject: [PATCH 03/39] fix build error (#1071) ### Description - fix build error --- .../Sources/CapabilityAgents/Routine/RoutineExecuter.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NuguAgents/Sources/CapabilityAgents/Routine/RoutineExecuter.swift b/NuguAgents/Sources/CapabilityAgents/Routine/RoutineExecuter.swift index 2bedf4775..57abecce8 100644 --- a/NuguAgents/Sources/CapabilityAgents/Routine/RoutineExecuter.swift +++ b/NuguAgents/Sources/CapabilityAgents/Routine/RoutineExecuter.swift @@ -377,7 +377,7 @@ private extension RoutineExecuter { handlingEvent = textAgent.requestTextInput( text: text, token: action.token, - source: dynamicSource + source: dynamicSource, requestType: .normal, completion: completion ) From 64e1c42b279117ee69d1173bf69ae76f1a54d842 Mon Sep 17 00:00:00 2001 From: childc Date: Tue, 1 Aug 2023 11:23:28 +0900 Subject: [PATCH 04/39] =?UTF-8?q?Audio=20session=20deactivation=EC=9D=84?= =?UTF-8?q?=202=EC=B4=88=20=EC=A7=80=EC=97=B0=20=ED=9B=84=20=EC=88=98?= =?UTF-8?q?=ED=96=89=20(#1072)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Description - 2초 안에 AudioSession을 재요청하는 확률이 적지 않음. --- .../AutomaticSpeechRecognition/ASRAgent.swift | 2 +- NuguClientKit/Sources/Client/NuguClient.swift | 11 ++++++++- .../Sources/Client/NuguClientConst.swift | 24 +++++++++++++++++++ nugu-ios.xcodeproj/project.pbxproj | 4 ++++ 4 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 NuguClientKit/Sources/Client/NuguClientConst.swift diff --git a/NuguAgents/Sources/CapabilityAgents/AutomaticSpeechRecognition/ASRAgent.swift b/NuguAgents/Sources/CapabilityAgents/AutomaticSpeechRecognition/ASRAgent.swift index be54911ef..e06dd4742 100644 --- a/NuguAgents/Sources/CapabilityAgents/AutomaticSpeechRecognition/ASRAgent.swift +++ b/NuguAgents/Sources/CapabilityAgents/AutomaticSpeechRecognition/ASRAgent.swift @@ -90,7 +90,7 @@ public final class ASRAgent: ASRAgentProtocol { guard let asrRequest = asrRequest, let asrResult = asrResult else { asrState = .idle expectSpeech = nil - log.error("ASRRequest not exist") + log.error("ASR request: \(String(describing: asrRequest)), result: \(String(describing: asrResult))") return } log.info("\(asrResult)") diff --git a/NuguClientKit/Sources/Client/NuguClient.swift b/NuguClientKit/Sources/Client/NuguClient.swift index 19ef7fc11..e8951c14f 100644 --- a/NuguClientKit/Sources/Client/NuguClient.swift +++ b/NuguClientKit/Sources/Client/NuguClient.swift @@ -278,6 +278,7 @@ public class NuguClient { // Private private var pausedByInterruption = false private let backgroundFocusHolder: BackgroundFocusHolder + private var audioDeactivateWorkItem: DispatchWorkItem? init( contextManager: ContextManageable, @@ -541,6 +542,10 @@ extension NuguClient: FocusDelegate { return delegate?.nuguClientShouldUpdateAudioSessionForFocusAquire() == true } + if let audioDeactivateWorkItem = audioDeactivateWorkItem { + audioDeactivateWorkItem.cancel() + } + return audioSessionManager.updateAudioSession(requestingFocus: true) == true } @@ -550,8 +555,12 @@ extension NuguClient: FocusDelegate { return } - audioSessionManager.notifyAudioSessionDeactivation() + let audioDeactivateWorkItem = DispatchWorkItem { + audioSessionManager.notifyAudioSessionDeactivation() + } + DispatchQueue.global().asyncAfter(deadline: .now() + NuguClientConst.audioSessionDeactivationDelay, execute: audioDeactivateWorkItem) + self.audioDeactivateWorkItem = audioDeactivateWorkItem } } diff --git a/NuguClientKit/Sources/Client/NuguClientConst.swift b/NuguClientKit/Sources/Client/NuguClientConst.swift new file mode 100644 index 000000000..e10efcecb --- /dev/null +++ b/NuguClientKit/Sources/Client/NuguClientConst.swift @@ -0,0 +1,24 @@ +// +// NuguClientConst.swift +// +// +// Created by 김대철 on 2023/07/25. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import Foundation + +enum NuguClientConst { + static let audioSessionDeactivationDelay: TimeInterval = 2 +} diff --git a/nugu-ios.xcodeproj/project.pbxproj b/nugu-ios.xcodeproj/project.pbxproj index ddd9183dc..6219f6f25 100644 --- a/nugu-ios.xcodeproj/project.pbxproj +++ b/nugu-ios.xcodeproj/project.pbxproj @@ -391,6 +391,7 @@ 73E8F35326C3C31500B99140 /* SpeechRecognizerAggregatorState.swift in Sources */ = {isa = PBXBuildFile; fileRef = 73E8F35226C3C31500B99140 /* SpeechRecognizerAggregatorState.swift */; }; 73F0B71925CABC800025FC6A /* NuguClient+Builder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 73F0B71825CABC800025FC6A /* NuguClient+Builder.swift */; }; 73F0B72F25CAC4930025FC6A /* AudioSessionManagerDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 73F0B72E25CAC4930025FC6A /* AudioSessionManagerDelegate.swift */; }; + 73F0DF882A78A13000533B1B /* NuguClientConst.swift in Sources */ = {isa = PBXBuildFile; fileRef = 73F0DF872A78A13000533B1B /* NuguClientConst.swift */; }; 75021E4E259431B50014537C /* DisplayWebViewPresenter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 75021E4D259431B50014537C /* DisplayWebViewPresenter.swift */; }; 75021E64259436930014537C /* DisplayWebViewPresenterDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 75021E63259436930014537C /* DisplayWebViewPresenterDelegate.swift */; }; 75021EAC25945E670014537C /* AudioDisplayViewPresenter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 75021EAB25945E670014537C /* AudioDisplayViewPresenter.swift */; }; @@ -1192,6 +1193,7 @@ 73E8F35226C3C31500B99140 /* SpeechRecognizerAggregatorState.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SpeechRecognizerAggregatorState.swift; sourceTree = ""; }; 73F0B71825CABC800025FC6A /* NuguClient+Builder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "NuguClient+Builder.swift"; sourceTree = ""; }; 73F0B72E25CAC4930025FC6A /* AudioSessionManagerDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioSessionManagerDelegate.swift; sourceTree = ""; }; + 73F0DF872A78A13000533B1B /* NuguClientConst.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = NuguClientConst.swift; sourceTree = ""; }; 7500490F24766B1500B73DD8 /* AudioDisplayView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioDisplayView.swift; sourceTree = ""; }; 75021E4D259431B50014537C /* DisplayWebViewPresenter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DisplayWebViewPresenter.swift; sourceTree = ""; }; 75021E63259436930014537C /* DisplayWebViewPresenterDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DisplayWebViewPresenterDelegate.swift; sourceTree = ""; }; @@ -1656,6 +1658,7 @@ 1F2E803523966B1900569C59 /* Client */ = { isa = PBXGroup; children = ( + 73F0DF872A78A13000533B1B /* NuguClientConst.swift */, 1FFFF3D92375735E00C9A177 /* NuguClient.swift */, 7386DA9823CF279C002BF24C /* NuguClientDelegate.swift */, 73752BC625B8AFA6005C27DA /* NuguClientNotification.swift */, @@ -4186,6 +4189,7 @@ 7E341F3D25AFEEFD00734A45 /* SpeechRecognizerAggregator.swift in Sources */, 0600C3582570E4CF0071AEF3 /* ConfigurationStore.swift in Sources */, 066C9D2E25553EF0000C7800 /* MicInputProvider.swift in Sources */, + 73F0DF882A78A13000533B1B /* NuguClientConst.swift in Sources */, 7330CD84237A77D800FCD6E9 /* KeywordDetector.swift in Sources */, 75E623D625E89EC800EAAE3C /* ASRBeepPlayerResourcesURL.swift in Sources */, ); From ecd29c41129a0376fbf8d84663800c3619340db6 Mon Sep 17 00:00:00 2001 From: jayce1116 Date: Tue, 1 Aug 2023 14:15:20 +0900 Subject: [PATCH 05/39] =?UTF-8?q?requestTextInput=20Method=20=EC=88=98?= =?UTF-8?q?=EC=A0=95=20(#1073)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Description - `RoutineExecuter`내부에서 사용하는 `requestTextInput` 의 경우 playServiceId가 존재하면 `attributes` 내부에 playServiceId 만 전송하고 있었음. - playServiceId와 관계없이 `attributes` 전체 값들을 올려주도록 수정 --- .../Routine/RoutineExecuter.swift | 24 ++++++------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/NuguAgents/Sources/CapabilityAgents/Routine/RoutineExecuter.swift b/NuguAgents/Sources/CapabilityAgents/Routine/RoutineExecuter.swift index 57abecce8..9b4514b36 100644 --- a/NuguAgents/Sources/CapabilityAgents/Routine/RoutineExecuter.swift +++ b/NuguAgents/Sources/CapabilityAgents/Routine/RoutineExecuter.swift @@ -365,23 +365,13 @@ private extension RoutineExecuter { return } let dynamicSource: TextInputSource = .dynamic("DYNAMIC_ROUTINE_ACTION") - if let playServiceId = action.playServiceId { - handlingEvent = textAgent.requestTextInput( - text: text, - token: action.token, - source: dynamicSource, - requestType: .specific(playServiceId: playServiceId), - completion: completion - ) - } else { - handlingEvent = textAgent.requestTextInput( - text: text, - token: action.token, - source: dynamicSource, - requestType: .normal, - completion: completion - ) - } + handlingEvent = textAgent.requestTextInput( + text: text, + token: action.token, + playServiceId: action.playServiceId, + source: dynamicSource, + completion: completion + ) case .data: if let eventIdentifier = delegate?.routineExecuterShouldRequestAction(action: action, referrerDialogRequestId: routine.dialogRequestId, completion: completion) { handlingEvent = eventIdentifier.dialogRequestId From fd6180068181d64538c10d4c12a710423c1590df Mon Sep 17 00:00:00 2001 From: jayce1116 Date: Tue, 1 Aug 2023 14:15:37 +0900 Subject: [PATCH 06/39] =?UTF-8?q?=EB=A3=A8=ED=8B=B4=20=EC=95=A1=EC=85=98?= =?UTF-8?q?=EC=9D=84=20=EC=8B=A4=ED=96=89=ED=95=98=EB=8A=94=20=EC=A4=91=20?= =?UTF-8?q?=EC=98=A4=EB=A5=98=EA=B0=80=20=EB=B0=9C=EC=83=9D=ED=95=98?= =?UTF-8?q?=EB=A9=B4=20=EB=8B=A4=EC=9D=8C=20=EC=95=A1=EC=85=98=EC=9D=84=20?= =?UTF-8?q?=EC=9D=B4=EB=8F=99=ED=95=98=EC=A7=80=20=EC=95=8A=EA=B3=A0=20pau?= =?UTF-8?q?se=20=EC=83=81=ED=83=9C=EB=A1=9C=20=EC=9D=B4=EB=8F=99=ED=95=98?= =?UTF-8?q?=EA=B2=8C=20=EB=B3=80=EA=B2=BD=20(#1074)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Description - 루틴 액션을 실행하는 중 오류가 발생하면 다음 액션을 이동하지 않고 pause 상태로 이동하게 변경 --- .../Sources/CapabilityAgents/Routine/RoutineExecuter.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NuguAgents/Sources/CapabilityAgents/Routine/RoutineExecuter.swift b/NuguAgents/Sources/CapabilityAgents/Routine/RoutineExecuter.swift index 9b4514b36..80f58e6c4 100644 --- a/NuguAgents/Sources/CapabilityAgents/Routine/RoutineExecuter.swift +++ b/NuguAgents/Sources/CapabilityAgents/Routine/RoutineExecuter.swift @@ -344,7 +344,7 @@ private extension RoutineExecuter { let completion: ((StreamDataState) -> Void) = { [weak self] result in log.debug(result) if case .error = result { - self?.doNextAction() + self?.pause() } } From fb0aeb0fc9e33fa8476f5dea9729a6e5cdc2c506 Mon Sep 17 00:00:00 2001 From: jayce1116 Date: Thu, 24 Aug 2023 13:34:06 +0900 Subject: [PATCH 07/39] =?UTF-8?q?ActionTriggerTimout=EB=A5=BC=20actionWork?= =?UTF-8?q?Item=EC=9C=BC=EB=A1=9C=20=EC=9D=BC=EA=B4=84=20=EA=B4=80?= =?UTF-8?q?=EB=A6=AC=ED=95=98=EB=8F=84=EB=A1=9D=20=EB=B3=80=EA=B2=BD=20(#1?= =?UTF-8?q?075)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ###Description - `ActionTriggerTimout`를 actionWorkItem으로 일괄 관리하도록 변경 --- .../Sources/CapabilityAgents/Routine/RoutineExecuter.swift | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/NuguAgents/Sources/CapabilityAgents/Routine/RoutineExecuter.swift b/NuguAgents/Sources/CapabilityAgents/Routine/RoutineExecuter.swift index 80f58e6c4..77d7ed79c 100644 --- a/NuguAgents/Sources/CapabilityAgents/Routine/RoutineExecuter.swift +++ b/NuguAgents/Sources/CapabilityAgents/Routine/RoutineExecuter.swift @@ -351,9 +351,12 @@ private extension RoutineExecuter { if let actionTimeout = action.actionTimeoutInMilliseconds, .zero < actionTimeout { state = .suspended - DispatchQueue.global().asyncAfter(deadline: .now() + NuguTimeInterval(milliseconds: actionTimeout).seconds) { [weak self] in + + let workItem = DispatchWorkItem { [weak self] in self?.delegate?.routineExecuterShouldSendActionTriggerTimout(token: action.token) } + actionWorkItem = workItem + routineDispatchQueue.asyncAfter(deadline: .now() + NuguTimeInterval(milliseconds: actionTimeout).seconds, execute: workItem) } log.debug(action.type) From 4999450a057cc21c39b39bd58675cde5e8415fa3 Mon Sep 17 00:00:00 2001 From: jayce1116 Date: Fri, 25 Aug 2023 16:13:01 +0900 Subject: [PATCH 08/39] =?UTF-8?q?Routine=EC=97=90=EC=84=9C=20=ED=98=84?= =?UTF-8?q?=EC=9E=AC=20=EC=8B=A4=ED=96=89=20=EC=A4=91=EC=9D=B8=20actiondml?= =?UTF-8?q?=20directive=EA=B0=80=20=EB=AA=A8=EB=91=90=20=EC=A2=85=EB=A3=8C?= =?UTF-8?q?=EB=90=98=EC=97=88=EC=9D=84=20=EB=95=8C=20playing=20=EC=83=81?= =?UTF-8?q?=ED=83=9C=EA=B0=80=20=EC=95=84=EB=8B=88=EB=9D=BC=EB=A9=B4=20?= =?UTF-8?q?=EB=8B=A4=EC=9D=8C=20=EC=95=A1=EC=85=98=EC=9D=84=20=EC=8B=A4?= =?UTF-8?q?=ED=96=89=ED=95=98=EC=A7=80=20=EC=95=8A=EB=8F=84=EB=A1=9D=20?= =?UTF-8?q?=EB=B3=80=EA=B2=BD=20(#1077)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Description - Routine에서 현재 실행 중인 actiondml directive가 모두 종료되었을 때 playing 상태가 아니라면 다음 액션을 실행하지 않도록 변경 --- .../Sources/CapabilityAgents/Routine/RoutineExecuter.swift | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/NuguAgents/Sources/CapabilityAgents/Routine/RoutineExecuter.swift b/NuguAgents/Sources/CapabilityAgents/Routine/RoutineExecuter.swift index 77d7ed79c..2efeb7d11 100644 --- a/NuguAgents/Sources/CapabilityAgents/Routine/RoutineExecuter.swift +++ b/NuguAgents/Sources/CapabilityAgents/Routine/RoutineExecuter.swift @@ -385,7 +385,7 @@ private extension RoutineExecuter { } func doNextAction() { - guard let action = currentAction else { return } + guard let action = currentAction, state == .playing else { return } actionWorkItem?.cancel() if shouldDelayAction, let delay = currentAction?.muteDelay { log.debug("Delaying action using mute delay, delay: \(delay.dispatchTimeInterval)") @@ -395,7 +395,6 @@ private extension RoutineExecuter { doActionAfter(delay: delay) } else { delegate?.routineExecuterDidFinishProcessingAction(action) - guard state == .playing else { return } guard hasNextAction else { doFinish() return From 18776ebef9a00af44fef1b4dd971089cb6e06dbf Mon Sep 17 00:00:00 2001 From: jayce1116 Date: Fri, 25 Aug 2023 17:41:45 +0900 Subject: [PATCH 09/39] =?UTF-8?q?SystemAgent=20=EC=9D=98=20Nodirective=20D?= =?UTF-8?q?irective=EB=A5=BC=20notification=EC=97=90=20=EC=B6=94=EA=B0=80?= =?UTF-8?q?=20(#1076)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Description - SystemAgent 의 Nodirective Directive를 notification에 추가 --- .../CapabilityAgents/System/SystemAgent.swift | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/NuguAgents/Sources/CapabilityAgents/System/SystemAgent.swift b/NuguAgents/Sources/CapabilityAgents/System/SystemAgent.swift index 517181e0a..96709c931 100644 --- a/NuguAgents/Sources/CapabilityAgents/System/SystemAgent.swift +++ b/NuguAgents/Sources/CapabilityAgents/System/SystemAgent.swift @@ -41,7 +41,7 @@ public final class SystemAgent: SystemAgentProtocol { DirectiveHandleInfo(namespace: capabilityAgentProperty.name, name: "UpdateState", blockingPolicy: BlockingPolicy(medium: .none, isBlocking: false), directiveHandler: handleUpdateState), DirectiveHandleInfo(namespace: capabilityAgentProperty.name, name: "Exception", blockingPolicy: BlockingPolicy(medium: .none, isBlocking: false), directiveHandler: handleException), DirectiveHandleInfo(namespace: capabilityAgentProperty.name, name: "Revoke", blockingPolicy: BlockingPolicy(medium: .none, isBlocking: false), directiveHandler: handleRevoke), - DirectiveHandleInfo(namespace: capabilityAgentProperty.name, name: "NoDirectives", blockingPolicy: BlockingPolicy(medium: .none, isBlocking: false), directiveHandler: { { $1(.finished) } }), + DirectiveHandleInfo(namespace: capabilityAgentProperty.name, name: "NoDirectives", blockingPolicy: BlockingPolicy(medium: .none, isBlocking: false), directiveHandler: handleNoDirectives), DirectiveHandleInfo(namespace: capabilityAgentProperty.name, name: "Noop", blockingPolicy: BlockingPolicy(medium: .none, isBlocking: false), directiveHandler: { { $1(.finished) } }), DirectiveHandleInfo(namespace: capabilityAgentProperty.name, name: "ResetConnection", blockingPolicy: BlockingPolicy(medium: .none, isBlocking: false), directiveHandler: handleResetConnection) ] @@ -138,6 +138,16 @@ private extension SystemAgent { } } + func handleNoDirectives() -> HandleDirective { + return { [weak self] directive, completion in + defer { completion(.finished) } + + self?.systemDispatchQueue.async { [weak self] in + self?.post(NuguAgentNotification.System.NoDirective(header: directive.header)) + } + } + } + func handleResetConnection() -> HandleDirective { return { [weak self] _, completion in defer { completion(.finished) } @@ -173,6 +183,7 @@ private extension SystemAgent { extension Notification.Name { static let systemAgentDidReceiveExceptionFail = Notification.Name("com.sktelecom.romaine.notification.name.system_agent_did_receive_exception_fail") static let systemAgentDidReceiveRevokeDevice = Notification.Name("com.sktelecom.romaine.notification.name.system_agent_did_revoke_device") + static let systemAgentDidReceiveNoDirective = Notification.Name("com.sktelecom.romaine.notification.name.system_agent_no_directive") } public extension NuguAgentNotification { @@ -202,5 +213,16 @@ public extension NuguAgentNotification { return RevokeDevice(reason: reason, header: header) } } + + public struct NoDirective: TypedNotification { + static public var name: Notification.Name = .systemAgentDidReceiveNoDirective + public let header: Downstream.Header + + public static func make(from: [String: Any]) -> NoDirective? { + guard let header = from["header"] as? Downstream.Header else { return nil } + + return NoDirective(header: header) + } + } } } From 8193b642ab603113c78c97ebcb3a159f7cc03ea3 Mon Sep 17 00:00:00 2001 From: ParkJongSang Date: Mon, 11 Sep 2023 16:50:58 +0900 Subject: [PATCH 10/39] =?UTF-8?q?AudioPlayer2Template=20=EC=83=9D=EC=84=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Display/Models/AudioPlayer2Template.swift | 35 +++++++++++++++++++ nugu-ios.xcodeproj/project.pbxproj | 4 +++ 2 files changed, 39 insertions(+) create mode 100644 NuguAgents/Sources/CapabilityAgents/AudioPlayer/Display/Models/AudioPlayer2Template.swift diff --git a/NuguAgents/Sources/CapabilityAgents/AudioPlayer/Display/Models/AudioPlayer2Template.swift b/NuguAgents/Sources/CapabilityAgents/AudioPlayer/Display/Models/AudioPlayer2Template.swift new file mode 100644 index 000000000..1b65f5b6b --- /dev/null +++ b/NuguAgents/Sources/CapabilityAgents/AudioPlayer/Display/Models/AudioPlayer2Template.swift @@ -0,0 +1,35 @@ +// +// AudioPlayer2Template.swift +// NuguAgents +// +// Created by 박종상님/iOS클라이언트개발팀 on 2023/09/11. +// Copyright © 2023 SK Telecom Co., Ltd. All rights reserved. +// + +import Foundation +/// <#Description#> +public struct AudioPlayer2Template: Decodable { + public let template: Template + + public struct Template: Decodable { + public let type: String + public let title: Title + public let content: Content + public let grammarGuide: [String]? + + public struct Title: Decodable { + public let iconUrl: String? + public let text: String + } + + public struct Content: Decodable { + public let title: String + public let subtitle: String + public let subtitle1: String? + public let imageUrl: String? + public let durationSec: String? + public let backgroundColor: String? + public let lyrics: AudioPlayerLyricsTemplate? + } + } +} diff --git a/nugu-ios.xcodeproj/project.pbxproj b/nugu-ios.xcodeproj/project.pbxproj index 6219f6f25..33a19383b 100644 --- a/nugu-ios.xcodeproj/project.pbxproj +++ b/nugu-ios.xcodeproj/project.pbxproj @@ -531,6 +531,7 @@ E6BE047E2A0BA1BD00AE29E3 /* ImageAgent.swift in Sources */ = {isa = PBXBuildFile; fileRef = E6BE047D2A0BA1BD00AE29E3 /* ImageAgent.swift */; }; E6FBA02B2A1379C500AF8B05 /* MessengerAgentProtocol.swift in Sources */ = {isa = PBXBuildFile; fileRef = E6FBA02A2A1379C400AF8B05 /* MessengerAgentProtocol.swift */; }; E6FBA02D2A137A9700AF8B05 /* ImageAgentProtocol.swift in Sources */ = {isa = PBXBuildFile; fileRef = E6FBA02C2A137A9700AF8B05 /* ImageAgentProtocol.swift */; }; + F6A6B0842AAEFDD900D41DEC /* AudioPlayer2Template.swift in Sources */ = {isa = PBXBuildFile; fileRef = F6A6B0832AAEFDD900D41DEC /* AudioPlayer2Template.swift */; }; F6D49D0D296BAD3500510498 /* AudioPlayerPlaylist.swift in Sources */ = {isa = PBXBuildFile; fileRef = F6D49D0C296BAD3500510498 /* AudioPlayerPlaylist.swift */; }; F6D49D0F296BAE9600510498 /* AudioPlayerAgent+PlaylistEvent.swift in Sources */ = {isa = PBXBuildFile; fileRef = F6D49D0E296BAE9600510498 /* AudioPlayerAgent+PlaylistEvent.swift */; }; F778C001261B096A00B69B32 /* EnumTypedNotification.swift in Sources */ = {isa = PBXBuildFile; fileRef = F778C000261B096A00B69B32 /* EnumTypedNotification.swift */; }; @@ -1332,6 +1333,7 @@ E6BE047D2A0BA1BD00AE29E3 /* ImageAgent.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImageAgent.swift; sourceTree = ""; }; E6FBA02A2A1379C400AF8B05 /* MessengerAgentProtocol.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MessengerAgentProtocol.swift; sourceTree = ""; }; E6FBA02C2A137A9700AF8B05 /* ImageAgentProtocol.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImageAgentProtocol.swift; sourceTree = ""; }; + F6A6B0832AAEFDD900D41DEC /* AudioPlayer2Template.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioPlayer2Template.swift; sourceTree = ""; }; F6D49D0C296BAD3500510498 /* AudioPlayerPlaylist.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioPlayerPlaylist.swift; sourceTree = ""; }; F6D49D0E296BAE9600510498 /* AudioPlayerAgent+PlaylistEvent.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AudioPlayerAgent+PlaylistEvent.swift"; sourceTree = ""; }; F778C000261B096A00B69B32 /* EnumTypedNotification.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = EnumTypedNotification.swift; sourceTree = ""; }; @@ -3113,6 +3115,7 @@ E649867A2833681A001AD733 /* AudioPlayerLyricsTemplate.swift */, E649867B2833681A001AD733 /* AudioPlayer1Template.swift */, F6D49D0C296BAD3500510498 /* AudioPlayerPlaylist.swift */, + F6A6B0832AAEFDD900D41DEC /* AudioPlayer2Template.swift */, ); path = Models; sourceTree = ""; @@ -4480,6 +4483,7 @@ 7E8050AA2518A69300994786 /* ASRInitiator.swift in Sources */, 7ED37D7926044103009A0A24 /* PermissionAgentDirectivePayload.swift in Sources */, 06CF28EB2547DE5700CEC3EC /* ContextManager+Rx.swift in Sources */, + F6A6B0842AAEFDD900D41DEC /* AudioPlayer2Template.swift in Sources */, 7E5F4896243C3C8A00118054 /* SoundAgent.swift in Sources */, 1FCAE15725FF0C59001C8061 /* AlertsAgentDirectivePayload.swift in Sources */, 737388FE24A46C930018DDD2 /* SystemAgentProtocol.swift in Sources */, From 467d6456de33e706b15d957566259debfe97ac89 Mon Sep 17 00:00:00 2001 From: Sokiwar Date: Thu, 14 Sep 2023 14:10:53 +0900 Subject: [PATCH 11/39] =?UTF-8?q?Template2=20=ED=8C=8C=EC=8B=B1=20?= =?UTF-8?q?=EB=A1=9C=EC=A7=81=20=EC=B6=94=EA=B0=80=20(#1081)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description - Template2 모델이 추가됨으로 인해 Template2 파싱로직 추가 - 앱 잠금화면, NotificationCenter 진입시 보이는 컨트롤창에 노출될 데이터들을 위해 필요 --- NuguClientKit/Sources/Audio/ControlCenterManager.swift | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/NuguClientKit/Sources/Audio/ControlCenterManager.swift b/NuguClientKit/Sources/Audio/ControlCenterManager.swift index c9cd20939..f98f85351 100644 --- a/NuguClientKit/Sources/Audio/ControlCenterManager.swift +++ b/NuguClientKit/Sources/Audio/ControlCenterManager.swift @@ -197,6 +197,12 @@ private extension ControlCenterManager { return nil } return (payload.template.content.title, payload.template.content.subtitle1, payload.template.content.subtitle2, payload.template.content.imageUrl) + case "AudioPlayer.Template2": + guard let payload = try? JSONDecoder().decode(AudioPlayer2Template.self, from: payloadAsData) else { + remove() + return nil + } + return (payload.template.content.title, payload.template.content.subtitle, payload.template.content.subtitle1, payload.template.content.imageUrl) default: remove() return nil From c903b557d6e40deb47324e631ff3e6eb781ddeca Mon Sep 17 00:00:00 2001 From: childc Date: Thu, 14 Sep 2023 16:16:01 +0900 Subject: [PATCH 12/39] =?UTF-8?q?[APOLLOQA-11023]=20=ED=82=A4=EC=9B=8C?= =?UTF-8?q?=EB=93=9C/=EC=B9=B4=ED=85=8C=EA=B3=A0=EB=A6=AC=20=EB=89=B4?= =?UTF-8?q?=EC=8A=A4=20=EC=9A=94=EC=95=BD=20=EB=93=A3=EA=B8=B0=20=EC=9D=BC?= =?UTF-8?q?=EC=8B=9C=EC=A0=95=EC=A7=80=20=EC=83=81=ED=83=9C=EC=97=90?= =?UTF-8?q?=EC=84=9C=20=EC=9D=B4=EC=96=B4=EC=84=9C=20=EB=B0=9C=ED=99=94?= =?UTF-8?q?=EC=8B=9C=20=ED=94=8C=EB=A0=88=EC=9D=B4=EC=96=B4=20=EC=A2=85?= =?UTF-8?q?=EB=A3=8C=EB=90=98=EB=8A=94=20=ED=98=84=EC=83=81=20(#1082)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary TTS를 통한 미디어 재생인 경우 예외처리 --- .../Sources/CapabilityAgents/AudioPlayer/AudioPlayer.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/NuguAgents/Sources/CapabilityAgents/AudioPlayer/AudioPlayer.swift b/NuguAgents/Sources/CapabilityAgents/AudioPlayer/AudioPlayer.swift index c887ecf36..12d713aca 100644 --- a/NuguAgents/Sources/CapabilityAgents/AudioPlayer/AudioPlayer.swift +++ b/NuguAgents/Sources/CapabilityAgents/AudioPlayer/AudioPlayer.swift @@ -151,6 +151,7 @@ final class AudioPlayer { } func replacePlayer(_ player: AudioPlayer) { + guard payload.sourceType != .attachment || player.payload.sourceType != .attachment else { return } guard let internalPlayer = player.internalPlayer else { return } self.internalPlayer = internalPlayer From 462b79064a4f43855b225a252444700086ef84f0 Mon Sep 17 00:00:00 2001 From: SeonhoBan Date: Tue, 19 Sep 2023 11:42:12 +0900 Subject: [PATCH 13/39] =?UTF-8?q?phonecall=20context=20=EA=B7=9C=EA=B2=A9?= =?UTF-8?q?=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../PhoneCall/PhoneCallAgentContext.swift | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/NuguAgents/Sources/CapabilityAgents/PhoneCall/PhoneCallAgentContext.swift b/NuguAgents/Sources/CapabilityAgents/PhoneCall/PhoneCallAgentContext.swift index 095d3b1f8..698eae202 100644 --- a/NuguAgents/Sources/CapabilityAgents/PhoneCall/PhoneCallAgentContext.swift +++ b/NuguAgents/Sources/CapabilityAgents/PhoneCall/PhoneCallAgentContext.swift @@ -32,6 +32,8 @@ public struct PhoneCallAgentContext { /// <#Description#> public let recipientIntended: PhoneCallRecipientIntended? /// <#Description#> + public let name: String? + /// <#Description#> public let candidates: [PhoneCallPerson]? /// <#Description#> public let searchScene: String? @@ -41,18 +43,21 @@ public struct PhoneCallAgentContext { /// - intent: <#intent description#> /// - callType: <#callType description#> /// - recipientIntended: <#recipientIntended description#> + /// - name: <#name description#> /// - candidates: <#candidates description#> /// - searchScene: <#searchScene description#> public init( intent: PhoneCallIntent?, callType: PhoneCallType?, recipientIntended: PhoneCallRecipientIntended?, + name: String?, candidates: [PhoneCallPerson]?, searchScene: String? ) { self.intent = intent self.callType = callType self.recipientIntended = recipientIntended + self.name = name self.candidates = candidates self.searchScene = searchScene } @@ -96,18 +101,22 @@ public struct PhoneCallAgentContext { public let recipient: PhoneCallAgentContext.Recipient? /// <#Description#> public let numberBlockable: Bool? + /// <#Description#> + public let aiPhoneUserId: String? /// The initializer for `PhoneCallAgentContext`. public init( state: PhoneCallState, template: PhoneCallAgentContext.Template?, recipient: PhoneCallAgentContext.Recipient?, - numberBlockable: Bool? + numberBlockable: Bool?, + aiPhoneUserId: String? ) { self.state = state self.template = template self.recipient = recipient self.numberBlockable = numberBlockable + self.aiPhoneUserId = aiPhoneUserId } } @@ -119,6 +128,7 @@ extension PhoneCallAgentContext: Codable { case template case recipient case numberBlockable + case aiPhoneUserId } public func encode(to encoder: Encoder) throws { @@ -131,6 +141,8 @@ extension PhoneCallAgentContext: Codable { if let numberBlockableValue = numberBlockable { try container.encodeIfPresent(numberBlockableValue ? "TRUE": "FALSE", forKey: .numberBlockable) } + + try container.encodeIfPresent(aiPhoneUserId, forKey: .aiPhoneUserId) } } From f3bb131158520d3be1d774db9fac55901031cf1f Mon Sep 17 00:00:00 2001 From: ParkJongSang Date: Thu, 21 Sep 2023 10:25:57 +0900 Subject: [PATCH 14/39] =?UTF-8?q?ShowPlaylist=20Directive=20=EC=83=9D?= =?UTF-8?q?=EC=84=B1,=20=EC=84=B1=EA=B3=B5/=EC=8B=A4=ED=8C=A8=20=EC=9D=B4?= =?UTF-8?q?=EB=B2=A4=ED=8A=B8=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../AudioPlayer/AudioPlayerAgent.swift | 19 ++++++++++++++++++- .../Display/AudioPlayerDisplayDelegate.swift | 4 ++++ .../Display/AudioPlayerDisplayManager.swift | 9 +++++++++ .../AudioPlayerAgent+PlaylistEvent.swift | 12 ++++++++++++ 4 files changed, 43 insertions(+), 1 deletion(-) diff --git a/NuguAgents/Sources/CapabilityAgents/AudioPlayer/AudioPlayerAgent.swift b/NuguAgents/Sources/CapabilityAgents/AudioPlayer/AudioPlayerAgent.swift index aec7577b8..1bb8cfe94 100644 --- a/NuguAgents/Sources/CapabilityAgents/AudioPlayer/AudioPlayerAgent.swift +++ b/NuguAgents/Sources/CapabilityAgents/AudioPlayer/AudioPlayerAgent.swift @@ -162,7 +162,8 @@ public final class AudioPlayerAgent: AudioPlayerAgentProtocol { DirectiveHandleInfo(namespace: capabilityAgentProperty.name, name: "UpdateMetadata", blockingPolicy: BlockingPolicy(medium: .none, isBlocking: false), directiveHandler: handleUpdateMetadata), DirectiveHandleInfo(namespace: capabilityAgentProperty.name, name: "ShowLyrics", blockingPolicy: BlockingPolicy(medium: .none, isBlocking: false), directiveHandler: handleShowLyrics), DirectiveHandleInfo(namespace: capabilityAgentProperty.name, name: "HideLyrics", blockingPolicy: BlockingPolicy(medium: .none, isBlocking: false), directiveHandler: handleHideLyrics), - DirectiveHandleInfo(namespace: capabilityAgentProperty.name, name: "ControlLyricsPage", blockingPolicy: BlockingPolicy(medium: .none, isBlocking: false), directiveHandler: handleControlLyricsPage) + DirectiveHandleInfo(namespace: capabilityAgentProperty.name, name: "ControlLyricsPage", blockingPolicy: BlockingPolicy(medium: .none, isBlocking: false), directiveHandler: handleControlLyricsPage), + DirectiveHandleInfo(namespace: capabilityAgentProperty.name, name: "ShowPlaylist", blockingPolicy: BlockingPolicy(medium: .none, isBlocking: false), directiveHandler:) ] public init( @@ -224,6 +225,7 @@ public final class AudioPlayerAgent: AudioPlayerAgentProtocol { if let playlist = self.currentPlaylist { payload["playlist"] = playlist.token + payload["playlistVisible"] = self.currentPlaylist != nil } completion(ContextInfo(contextType: .capability, name: self.capabilityAgentProperty.name, payload: payload.compactMapValues { $0 })) @@ -706,6 +708,21 @@ private extension AudioPlayerAgent { } } + func handleShowPlaylist() -> HandleDirective { + return { [weak self] directive, completion in + guard let self = self, + let playServiceId = directive.payloadDictionary?["playServiceId"] as? String + else { + completion(.failed("Invalid payload")) + return + } + + defer { completion(.finished) } + + self.audioPlayerDisplayManager.showPlaylist(playServiceId: playServiceId, completion: completion) + } + } + func handleAttachment() -> HandleAttachment { return { [weak self] attachment in self?.audioPlayerDispatchQueue.async { [weak self] in diff --git a/NuguAgents/Sources/CapabilityAgents/AudioPlayer/Display/AudioPlayerDisplayDelegate.swift b/NuguAgents/Sources/CapabilityAgents/AudioPlayer/Display/AudioPlayerDisplayDelegate.swift index 5488bfcc3..a5930e564 100644 --- a/NuguAgents/Sources/CapabilityAgents/AudioPlayer/Display/AudioPlayerDisplayDelegate.swift +++ b/NuguAgents/Sources/CapabilityAgents/AudioPlayer/Display/AudioPlayerDisplayDelegate.swift @@ -59,4 +59,8 @@ public protocol AudioPlayerDisplayDelegate: AnyObject { /// <#Description#> /// - Parameter completion: <#completion description#> func audioPlayerIsLyricsVisible(completion: @escaping (Bool) -> Void) + + /// Tells the delegate that the displayed template should show playlist + /// - Parameter completion: Whether succeeded or not + func audioPlayerDisplayShouldShowPlaylist(completion: @escaping (Bool) -> Boid) } diff --git a/NuguAgents/Sources/CapabilityAgents/AudioPlayer/Display/AudioPlayerDisplayManager.swift b/NuguAgents/Sources/CapabilityAgents/AudioPlayer/Display/AudioPlayerDisplayManager.swift index c1d69fdd2..3bd373d6b 100644 --- a/NuguAgents/Sources/CapabilityAgents/AudioPlayer/Display/AudioPlayerDisplayManager.swift +++ b/NuguAgents/Sources/CapabilityAgents/AudioPlayer/Display/AudioPlayerDisplayManager.swift @@ -201,6 +201,15 @@ extension AudioPlayerDisplayManager { delegate.audioPlayerDisplayShouldControlLyricsPage(direction: payload.direction, completion: completion) } + func showPlaylist(playServiceId: String, completion: @escaping (Bool) -> Void) { + guard let delegate = delegate, + currentItem?.mediaPayload.playServiceId == payload.playServiceId else { + completion(false) + return + } + delegate.audioPlayerDisplayShouldShowPlaylist(completion: completion) + } + func notifyUserInteraction() { switch audioPlayerState { case .stopped, .finished: diff --git a/NuguAgents/Sources/CapabilityAgents/AudioPlayer/Event/AudioPlayerAgent+PlaylistEvent.swift b/NuguAgents/Sources/CapabilityAgents/AudioPlayer/Event/AudioPlayerAgent+PlaylistEvent.swift index 9171f7aa1..4b92c3fa7 100644 --- a/NuguAgents/Sources/CapabilityAgents/AudioPlayer/Event/AudioPlayerAgent+PlaylistEvent.swift +++ b/NuguAgents/Sources/CapabilityAgents/AudioPlayer/Event/AudioPlayerAgent+PlaylistEvent.swift @@ -30,6 +30,9 @@ extension AudioPlayerAgent { case playlistItemSelected(token: String, postback: [String: AnyHashable]) case playlistFavoriteSelected(token: String, postback: [String: AnyHashable]) case modifyPlaylist(deletedTokens: [String], tokens: [String]) + + case showPlaylistSucceeded + case showPlaylistFailed(error: [String: String]) } } } @@ -51,7 +54,12 @@ extension AudioPlayerAgent.PlaylistEvent: Eventable { case .modifyPlaylist(deletedTokens: let deletedTokens, tokens: let tokens): eventPayload["deletedTokens"] = deletedTokens eventPayload["tokens"] = tokens + case .showPlaylistSucceeded: + break + case .showPlaylistFailed(error: let error): + eventPayload["error"] = error } + return eventPayload } @@ -61,6 +69,10 @@ extension AudioPlayerAgent.PlaylistEvent: Eventable { return "ElementSelected" case .modifyPlaylist: return "ModifyPlaylist" + case .showPlaylistSucceeded: + return "ShowPlaylistSucceeded" + case .showPlaylistFailed: + return "ShowPlaylistFailed" } } } From b6c177faf18913cca03ac04a981aa449b0008de7 Mon Sep 17 00:00:00 2001 From: SeonhoBan <42695497+SeonhoBan@users.noreply.github.com> Date: Mon, 25 Sep 2023 20:26:43 +0900 Subject: [PATCH 15/39] Update PhoneCallAgent version --- .../Sources/CapabilityAgents/PhoneCall/PhoneCallAgent.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NuguAgents/Sources/CapabilityAgents/PhoneCall/PhoneCallAgent.swift b/NuguAgents/Sources/CapabilityAgents/PhoneCall/PhoneCallAgent.swift index 0e5c33994..9eae1550b 100644 --- a/NuguAgents/Sources/CapabilityAgents/PhoneCall/PhoneCallAgent.swift +++ b/NuguAgents/Sources/CapabilityAgents/PhoneCall/PhoneCallAgent.swift @@ -25,7 +25,7 @@ import NuguCore import RxSwift public class PhoneCallAgent: PhoneCallAgentProtocol { - public var capabilityAgentProperty: CapabilityAgentProperty = CapabilityAgentProperty(category: .phoneCall, version: "1.3") + public var capabilityAgentProperty: CapabilityAgentProperty = CapabilityAgentProperty(category: .phoneCall, version: "1.4") // PhoneCallAgentProtocol public weak var delegate: PhoneCallAgentDelegate? From 3f675cd5eda029f1280c66b623b5f58191c06748 Mon Sep 17 00:00:00 2001 From: SeonhoBan <42695497+SeonhoBan@users.noreply.github.com> Date: Wed, 27 Sep 2023 20:36:24 +0900 Subject: [PATCH 16/39] Update PhoneCallAgentDirectivePayload.swift --- .../PhoneCall/PhoneCallAgentDirectivePayload.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NuguAgents/Sources/CapabilityAgents/PhoneCall/PhoneCallAgentDirectivePayload.swift b/NuguAgents/Sources/CapabilityAgents/PhoneCall/PhoneCallAgentDirectivePayload.swift index ba699ccab..d3630ce26 100644 --- a/NuguAgents/Sources/CapabilityAgents/PhoneCall/PhoneCallAgentDirectivePayload.swift +++ b/NuguAgents/Sources/CapabilityAgents/PhoneCall/PhoneCallAgentDirectivePayload.swift @@ -35,7 +35,7 @@ public enum PhoneCallAgentDirectivePayload { /// The candidate searched for play service. /// /// If nil, there are no search results. - public let candidates: [PhoneCallPerson]? + public var candidates: [PhoneCallPerson]? /// The scene of search target and display tempate public let searchScene: String? /// <#Description#> From 0e43b080eb650132323b9d5cbe0d6ee0016e5876 Mon Sep 17 00:00:00 2001 From: ParkJongSang Date: Fri, 6 Oct 2023 16:19:48 +0900 Subject: [PATCH 17/39] Fix Build Error --- .../CapabilityAgents/AudioPlayer/AudioPlayerAgent.swift | 4 ++-- .../AudioPlayer/Display/AudioPlayerDisplayDelegate.swift | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/NuguAgents/Sources/CapabilityAgents/AudioPlayer/AudioPlayerAgent.swift b/NuguAgents/Sources/CapabilityAgents/AudioPlayer/AudioPlayerAgent.swift index 1bb8cfe94..18ff248b2 100644 --- a/NuguAgents/Sources/CapabilityAgents/AudioPlayer/AudioPlayerAgent.swift +++ b/NuguAgents/Sources/CapabilityAgents/AudioPlayer/AudioPlayerAgent.swift @@ -27,7 +27,7 @@ import RxSwift public final class AudioPlayerAgent: AudioPlayerAgentProtocol { // CapabilityAgentable - public var capabilityAgentProperty: CapabilityAgentProperty = CapabilityAgentProperty(category: .audioPlayer, version: "1.7") + public var capabilityAgentProperty: CapabilityAgentProperty = CapabilityAgentProperty(category: .audioPlayer, version: "1.8") private let playSyncProperty = PlaySyncProperty(layerType: .media, contextType: .sound) // AudioPlayerAgentProtocol @@ -163,7 +163,7 @@ public final class AudioPlayerAgent: AudioPlayerAgentProtocol { DirectiveHandleInfo(namespace: capabilityAgentProperty.name, name: "ShowLyrics", blockingPolicy: BlockingPolicy(medium: .none, isBlocking: false), directiveHandler: handleShowLyrics), DirectiveHandleInfo(namespace: capabilityAgentProperty.name, name: "HideLyrics", blockingPolicy: BlockingPolicy(medium: .none, isBlocking: false), directiveHandler: handleHideLyrics), DirectiveHandleInfo(namespace: capabilityAgentProperty.name, name: "ControlLyricsPage", blockingPolicy: BlockingPolicy(medium: .none, isBlocking: false), directiveHandler: handleControlLyricsPage), - DirectiveHandleInfo(namespace: capabilityAgentProperty.name, name: "ShowPlaylist", blockingPolicy: BlockingPolicy(medium: .none, isBlocking: false), directiveHandler:) + DirectiveHandleInfo(namespace: capabilityAgentProperty.name, name: "ShowPlaylist", blockingPolicy: BlockingPolicy(medium: .none, isBlocking: false), directiveHandler: handleShowPlaylist) ] public init( diff --git a/NuguAgents/Sources/CapabilityAgents/AudioPlayer/Display/AudioPlayerDisplayDelegate.swift b/NuguAgents/Sources/CapabilityAgents/AudioPlayer/Display/AudioPlayerDisplayDelegate.swift index a5930e564..ceb9359d8 100644 --- a/NuguAgents/Sources/CapabilityAgents/AudioPlayer/Display/AudioPlayerDisplayDelegate.swift +++ b/NuguAgents/Sources/CapabilityAgents/AudioPlayer/Display/AudioPlayerDisplayDelegate.swift @@ -62,5 +62,5 @@ public protocol AudioPlayerDisplayDelegate: AnyObject { /// Tells the delegate that the displayed template should show playlist /// - Parameter completion: Whether succeeded or not - func audioPlayerDisplayShouldShowPlaylist(completion: @escaping (Bool) -> Boid) + func audioPlayerDisplayShouldShowPlaylist(completion: @escaping (Bool) -> Void) } From d71a0778ef8f237398ed91ca294e5967596b9ed4 Mon Sep 17 00:00:00 2001 From: ParkJongSang Date: Fri, 6 Oct 2023 16:38:01 +0900 Subject: [PATCH 18/39] Fix Build Error --- .../AudioPlayer/AudioPlayerAgent.swift | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/NuguAgents/Sources/CapabilityAgents/AudioPlayer/AudioPlayerAgent.swift b/NuguAgents/Sources/CapabilityAgents/AudioPlayer/AudioPlayerAgent.swift index 18ff248b2..fbc433b28 100644 --- a/NuguAgents/Sources/CapabilityAgents/AudioPlayer/AudioPlayerAgent.swift +++ b/NuguAgents/Sources/CapabilityAgents/AudioPlayer/AudioPlayerAgent.swift @@ -719,7 +719,15 @@ private extension AudioPlayerAgent { defer { completion(.finished) } - self.audioPlayerDisplayManager.showPlaylist(playServiceId: playServiceId, completion: completion) + self.audioPlayerDisplayManager.showPlaylist(playServiceId: playServiceId) { [weak self] isSuccess in + guard let self = self else { return } + self.sendCompactContextEvent( + PlaylistEvent( + typeInfo: isSuccess ? .showPlaylistSucceeded : .showPlaylistFailed(error: ["message": "show Playlist Failed"]) , + playServiceId: playServiceId + ) + ) + } } } From 86d717cf0e177a8cc7b938208151cca98a11d9f2 Mon Sep 17 00:00:00 2001 From: ParkJongSang Date: Fri, 6 Oct 2023 16:50:27 +0900 Subject: [PATCH 19/39] Fix Build Error --- .../CapabilityAgents/AudioPlayer/AudioPlayerAgent.swift | 4 ++-- .../AudioPlayer/Display/AudioPlayerDisplayManager.swift | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/NuguAgents/Sources/CapabilityAgents/AudioPlayer/AudioPlayerAgent.swift b/NuguAgents/Sources/CapabilityAgents/AudioPlayer/AudioPlayerAgent.swift index fbc433b28..f3cfbe633 100644 --- a/NuguAgents/Sources/CapabilityAgents/AudioPlayer/AudioPlayerAgent.swift +++ b/NuguAgents/Sources/CapabilityAgents/AudioPlayer/AudioPlayerAgent.swift @@ -723,9 +723,9 @@ private extension AudioPlayerAgent { guard let self = self else { return } self.sendCompactContextEvent( PlaylistEvent( - typeInfo: isSuccess ? .showPlaylistSucceeded : .showPlaylistFailed(error: ["message": "show Playlist Failed"]) , + typeInfo: isSuccess ? .showPlaylistSucceeded : .showPlaylistFailed(error: ["message": "show Playlist Failed"]), playServiceId: playServiceId - ) + ).rx ) } } diff --git a/NuguAgents/Sources/CapabilityAgents/AudioPlayer/Display/AudioPlayerDisplayManager.swift b/NuguAgents/Sources/CapabilityAgents/AudioPlayer/Display/AudioPlayerDisplayManager.swift index 3bd373d6b..a7cecc2b7 100644 --- a/NuguAgents/Sources/CapabilityAgents/AudioPlayer/Display/AudioPlayerDisplayManager.swift +++ b/NuguAgents/Sources/CapabilityAgents/AudioPlayer/Display/AudioPlayerDisplayManager.swift @@ -203,7 +203,7 @@ extension AudioPlayerDisplayManager { func showPlaylist(playServiceId: String, completion: @escaping (Bool) -> Void) { guard let delegate = delegate, - currentItem?.mediaPayload.playServiceId == payload.playServiceId else { + currentItem?.mediaPayload.playServiceId == playServiceId else { completion(false) return } From a50df43e5773e4daf4ff4131f6c652348f7544cc Mon Sep 17 00:00:00 2001 From: ParkJongSang Date: Fri, 6 Oct 2023 17:04:56 +0900 Subject: [PATCH 20/39] Fix Build Error --- .../Sources/Presenter/AudioDisplayViewPresenter.swift | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/NuguClientKit/Sources/Presenter/AudioDisplayViewPresenter.swift b/NuguClientKit/Sources/Presenter/AudioDisplayViewPresenter.swift index cdb9d1bd1..4c5027b28 100644 --- a/NuguClientKit/Sources/Presenter/AudioDisplayViewPresenter.swift +++ b/NuguClientKit/Sources/Presenter/AudioDisplayViewPresenter.swift @@ -152,6 +152,12 @@ extension AudioDisplayViewPresenter: AudioPlayerDisplayDelegate { completion(self?.audioDisplayView?.isLyricsVisible ?? false) } } + + public func audioPlayerDisplayShouldShowPlaylist(completion: @escaping (Bool) -> Void) { + DispatchQueue.main.async { [weak self] in + completion(true) + } + } } // MARK: - Private (AudioDisplayView) From 820488af680c77240c6401e1b1f2f2e3ba96dd23 Mon Sep 17 00:00:00 2001 From: childc Date: Mon, 9 Oct 2023 19:45:19 +0900 Subject: [PATCH 21/39] =?UTF-8?q?=EC=A7=81=EB=A0=AC=ED=99=94=20=EC=8B=9C?= =?UTF-8?q?=EB=8F=84=ED=95=A0=20=EB=95=8C=20=EB=B0=9C=EC=83=9D=ED=95=A0=20?= =?UTF-8?q?=EC=88=98=20=EC=9E=88=EB=8A=94=20OBJC=20Exception=20=EC=B2=98?= =?UTF-8?q?=EB=A6=AC=20(#1085)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description - 직렬화를 지원하지 않는 객체에 대해서 NSException이 발생하는데, 이것을 처리하지 않으면 crash가 발생합니다. (unhandled exception) - 직렬화 가능한 객체만 받도록 할 수는 없고, 실패에 대해 crash를 회피하는 정도의 처리만 가능합니다. --- .../Sources/StreamData/Model/Upstream.swift | 19 +++++++++++++++---- Package.swift | 2 +- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/NuguCore/Sources/StreamData/Model/Upstream.swift b/NuguCore/Sources/StreamData/Model/Upstream.swift index 000bd1022..45351c396 100644 --- a/NuguCore/Sources/StreamData/Model/Upstream.swift +++ b/NuguCore/Sources/StreamData/Model/Upstream.swift @@ -20,6 +20,9 @@ import Foundation +import NuguUtils +import NuguObjcUtils + /// An enum that contains the data structures to be send to the server. public enum Upstream { /// A structure that contains a event and contexts. @@ -151,10 +154,18 @@ extension Upstream.Event { "client": client ] - guard - let data = try? JSONSerialization.data(withJSONObject: contextDict.compactMapValues { $0 }, options: []), - let contextString = String(data: data, encoding: .utf8) else { - return "" + var contextString: String = "" + if let error = UnifiedErrorCatcher.try ({ + do { + let data = try JSONSerialization.data(withJSONObject: contextDict.compactMapValues { $0 }, options: []) + contextString = String(data: data, encoding: .utf8) ?? "" + } catch { + return error + } + + return nil + }) { + log.error("context dictionary includes unserializable object. error: \(error)") } return contextString diff --git a/Package.swift b/Package.swift index de6a54e50..886d84a7f 100644 --- a/Package.swift +++ b/Package.swift @@ -83,7 +83,7 @@ let package = Package( ), .target( name: "NuguCore", - dependencies: ["NuguUtils", "NattyLog"], + dependencies: ["NuguUtils", "NuguObjcUtils", "NattyLog"], path: "NuguCore/", exclude: ["Info.plist", "README.md"] ), From 2f1761360467062c655ee896dec088528868a1de Mon Sep 17 00:00:00 2001 From: childc Date: Tue, 10 Oct 2023 10:15:35 +0900 Subject: [PATCH 22/39] =?UTF-8?q?Client=20side=20load=20balance=20?= =?UTF-8?q?=EC=A3=BC=EC=86=8C=20=EC=A7=80=EC=A0=95=20(#1088)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description Server Initiated Directive를 받을 수 있는 주소는 loadBalncedUrl밖에 없기 때문에 직접 지정해서 request하도록 변경 --- NuguCore/Sources/Network/Api/NuguApiProvider.swift | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/NuguCore/Sources/Network/Api/NuguApiProvider.swift b/NuguCore/Sources/Network/Api/NuguApiProvider.swift index f4887cfe8..783a4137c 100644 --- a/NuguCore/Sources/Network/Api/NuguApiProvider.swift +++ b/NuguCore/Sources/Network/Api/NuguApiProvider.swift @@ -137,7 +137,7 @@ class NuguApiProvider: NSObject { return } - guard let resourceServerUrl = self.resourceServerAddress else { + guard let loadBalancedUrl = self.loadBalancedUrl else { single(.failure(NetworkError.noSuitableResourceServer)) return } @@ -146,8 +146,8 @@ class NuguApiProvider: NSObject { self.serverSideEventProcessor = ServerSideEventProcessor() // connect downstream. - guard let downstreamUrl = URL(string: NuguApi.directives.uri(baseUrl: resourceServerUrl)) else { - log.error("invailid url: \(NuguApi.directives.uri(baseUrl: resourceServerUrl))") + guard let downstreamUrl = URL(string: NuguApi.directives.uri(baseUrl: loadBalancedUrl)) else { + log.error("invailid url: \(NuguApi.directives.uri(baseUrl: loadBalancedUrl))") single(.failure(NetworkError.noSuitableResourceServer)) return } @@ -157,7 +157,7 @@ class NuguApiProvider: NSObject { downstreamRequest.allHTTPHeaderFields = header task = self.session.dataTask(with: downstreamRequest) task?.resume() - log.debug("directive request header:\n\(downstreamRequest.allHTTPHeaderFields?.description ?? "")\n") + log.debug("directive request: \(downstreamRequest)\nheader: \(downstreamRequest.allHTTPHeaderFields?.description ?? "")\n") } single(.success(self.serverSideEventProcessor!.subject)) From f095924849ac13c89dca53153be8bb4dd44e5b22 Mon Sep 17 00:00:00 2001 From: childc Date: Tue, 10 Oct 2023 10:17:40 +0900 Subject: [PATCH 23/39] =?UTF-8?q?`startListening`,=20`startListeningWithTi?= =?UTF-8?q?rgger`,=20`stopListening`=EC=9D=98=20=EB=8F=99=EC=9E=91?= =?UTF-8?q?=EC=9D=B4=20=EC=9B=90=EC=9E=90=EC=84=B1=EC=9D=84=20=EA=B0=80?= =?UTF-8?q?=EC=A7=80=EB=8F=84=EB=A1=9D=20=EC=88=98=EC=A0=95=20(#1087)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description - `startListening`, `startListeningWithTirgger`, `stopListening`의 동작이 원자성을 가지도록 수정 - 연속해서 세 함수를 호출하더라도 모든 작업이 중단/중복되지 않고 수행하도록 수정 --- .../Audio/SpeechRecognizerAggregatable.swift | 9 +- .../Audio/SpeechRecognizerAggregator.swift | 142 +++++++++++------- 2 files changed, 92 insertions(+), 59 deletions(-) diff --git a/NuguClientKit/Sources/Audio/SpeechRecognizerAggregatable.swift b/NuguClientKit/Sources/Audio/SpeechRecognizerAggregatable.swift index f94aa5992..9d752ae29 100644 --- a/NuguClientKit/Sources/Audio/SpeechRecognizerAggregatable.swift +++ b/NuguClientKit/Sources/Audio/SpeechRecognizerAggregatable.swift @@ -32,8 +32,7 @@ public protocol SpeechRecognizerAggregatable: AnyObject { /// - Parameters: /// - initiator: The options for recognition. /// - completion: The completion handler to call when the request is complete. - @discardableResult - func startListening(initiator: ASRInitiator, completion: ((StreamDataState) -> Void)?) -> String + func startListening(initiator: ASRInitiator, completion: ((StreamDataState) -> Void)?) /// Start keyword detector with microphone. func startListeningWithTrigger(completion: ((Result) -> Void)?) @@ -43,7 +42,7 @@ public protocol SpeechRecognizerAggregatable: AnyObject { func startMicInputProvider(requestingFocus: Bool, completion: @escaping (EndedUp) -> Void) - func stopMicInputProvider() + func stopMicInputProvider(completion: (() -> Void)?) } public extension SpeechRecognizerAggregatable { @@ -57,4 +56,8 @@ public extension SpeechRecognizerAggregatable { func startListeningWithTrigger() { startListeningWithTrigger(completion: nil) } + + func stopMicInputProvider() { + stopMicInputProvider(completion: nil) + } } diff --git a/NuguClientKit/Sources/Audio/SpeechRecognizerAggregator.swift b/NuguClientKit/Sources/Audio/SpeechRecognizerAggregator.swift index 67e68bc66..a2c5d31e5 100644 --- a/NuguClientKit/Sources/Audio/SpeechRecognizerAggregator.swift +++ b/NuguClientKit/Sources/Audio/SpeechRecognizerAggregator.swift @@ -40,13 +40,12 @@ public class SpeechRecognizerAggregator: SpeechRecognizerAggregatable { private let keywordDetector: KeywordDetector private let micInputProvider = MicInputProvider() - private var audioSessionInterrupted = false private var micInputProviderDelay: DispatchTime = .now() - + private let micQueue = DispatchQueue(label: "com.sktelecom.romaine.NuguClientKit.mic") @Atomic private var startMicWorkItem: DispatchWorkItem? - - // Audio input source - private let micQueue = DispatchQueue(label: "com.sktelecom.romaine.speech_recognizer") + + private var audioSessionInterrupted = false + private let recognizeQueue = DispatchQueue(label: "com.sktelecom.romaine.NuguClientKit.recognize") public var useKeywordDetector = true @@ -102,57 +101,73 @@ public class SpeechRecognizerAggregator: SpeechRecognizerAggregatable { // MARK: - SpeechRecognizerAggregatable public extension SpeechRecognizerAggregator { - @discardableResult - func startListening(initiator: ASRInitiator, completion: ((StreamDataState) -> Void)? = nil) -> String { - switch state { - case .cancelled, - .idle, - .error: - break - case .wakeupTriggering: - keywordDetector.stop() - default: - asrAgent.stopRecognition() - } - - let dialogRequestId = asrAgent.startRecognition(initiator: initiator) { [weak self] state in - guard case .prepared = state else { - completion?(state) - return + func startListening(initiator: ASRInitiator, completion: ((StreamDataState) -> Void)? = nil) { + recognizeQueue.async { [weak self] in + guard let self else { return } + + switch state { + case .cancelled, + .idle, + .error: + break + case .wakeupTriggering: + keywordDetector.stop() + default: + asrAgent.stopRecognition() } - self?.startMicInputProvider(requestingFocus: true) { [weak self] endedUp in - if case let .failure(error) = endedUp { - log.error("Start MicInputProvider failed: \(error)") - self?.asrAgent.stopRecognition() + let sema = DispatchSemaphore(value: .zero) + asrAgent.startRecognition(initiator: initiator) { [weak self] state in + guard case .prepared = state else { + completion?(state) + sema.signal() + return + } + + self?.startMicInputProvider(requestingFocus: true) { [weak self, weak sema] endedUp in + defer { + sema?.signal() + } - var recognizerError: Error { - guard error is MicInputError else { - return error + if case let .failure(error) = endedUp { + log.error("Start MicInputProvider failed: \(error)") + self?.asrAgent.stopRecognition() + + var recognizerError: Error { + guard error is MicInputError else { + return error + } + + return SpeechRecognizerAggregatorError.cannotOpenMicInputForRecognition } + self?.state = .error(recognizerError) + completion?(.error(recognizerError)) - return SpeechRecognizerAggregatorError.cannotOpenMicInputForRecognition + return } - self?.state = .error(recognizerError) - completion?(.error(recognizerError)) - return + completion?(.prepared) } - - completion?(.prepared) } + sema.wait() } - - return dialogRequestId } func startListeningWithTrigger(completion: ((Result) -> Void)?) { guard useKeywordDetector else { return } - _startMicWorkItem.mutate { - $0?.cancel() - $0 = DispatchWorkItem(block: { [weak self] in + log.debug("startListeningWithTrigger") + + recognizeQueue.async { [weak self] in + guard let self else { return } + + let sema = DispatchSemaphore(value: .zero) + let startMicWorkItem = DispatchWorkItem { [weak self] in log.debug("startMicWorkItem start") self?.startMicInputProvider(requestingFocus: false) { (endedUp) in + defer { + sema.signal() + } + if case let .failure(error) = endedUp { log.debug("startMicWorkItem failed: \(error)") var recognizerError: Error { @@ -166,31 +181,45 @@ public extension SpeechRecognizerAggregator { completion?(.failure(recognizerError)) return - } self?.keywordDetector.start() completion?(.success(())) } - }) - } - guard let startMicWorkItem = self.startMicWorkItem else { return } - - if self.micInputProviderDelay > DispatchTime.now() { - DispatchQueue.global().asyncAfter(deadline: self.micInputProviderDelay, execute: startMicWorkItem) - } else { - DispatchQueue.global().async(execute: startMicWorkItem) + } + + _startMicWorkItem.mutate { + $0?.cancel() + $0 = startMicWorkItem + } + + if micInputProviderDelay > DispatchTime.now() { + DispatchQueue.global().asyncAfter(deadline: self.micInputProviderDelay, execute: startMicWorkItem) + } else { + DispatchQueue.global().async(execute: startMicWorkItem) + } + + sema.wait() } } func stopListening() { - if keywordDetector.state == .active { - keywordDetector.stop() - state = .cancelled + log.debug("stopListening") + recognizeQueue.async { [weak self] in + guard let self else { return } + + let sema = DispatchSemaphore(value: .zero) + if keywordDetector.state == .active { + keywordDetector.stop() + state = .cancelled + } + + asrAgent.stopRecognition() + stopMicInputProvider { + sema.signal() + } + sema.wait() } - - stopMicInputProvider() - asrAgent.stopRecognition() } func startMicInputProvider(requestingFocus: Bool, completion: @escaping (EndedUp) -> Void) { @@ -224,10 +253,11 @@ public extension SpeechRecognizerAggregator { } } - func stopMicInputProvider() { + func stopMicInputProvider(completion: (() -> Void)? = nil) { micQueue.async { [weak self] in self?.startMicWorkItem?.cancel() self?.micInputProvider.stop() + completion?() } } } From 8aa691637d60ed9b6fd8b5b5c6712d0a7507fe42 Mon Sep 17 00:00:00 2001 From: childc Date: Tue, 10 Oct 2023 10:18:21 +0900 Subject: [PATCH 24/39] =?UTF-8?q?Server=20sent=20event=20(directive)=20?= =?UTF-8?q?=EC=8B=9C=EC=9E=91/=EC=A2=85=EB=A3=8C=EC=97=90=20=EB=8C=80?= =?UTF-8?q?=ED=95=9C=20=EC=88=9C=EC=84=9C=EB=A5=BC=20=EB=B3=B4=EC=9E=A5?= =?UTF-8?q?=ED=95=9C=EB=8B=A4=20(#1086)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description - `startReceiveServerInitiatedDirective` / `stopReceiveServerInitiatedDirective`의 호출 순서에 따라 동작을 완결성 있게 수행한다. ## Focus - `Semaphore`를 이용한 순서보장 로직 - `Semaphore`객체를 promote하지 않아 signal이 계속 수행되는 것을 막는다 --- NuguClientKit/Sources/Client/NuguClient.swift | 27 +++++++++++++------ .../Sources/StreamData/StreamDataRouter.swift | 6 +++++ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/NuguClientKit/Sources/Client/NuguClient.swift b/NuguClientKit/Sources/Client/NuguClient.swift index e8951c14f..1a80fcd7d 100644 --- a/NuguClientKit/Sources/Client/NuguClient.swift +++ b/NuguClientKit/Sources/Client/NuguClient.swift @@ -279,6 +279,7 @@ public class NuguClient { private var pausedByInterruption = false private let backgroundFocusHolder: BackgroundFocusHolder private var audioDeactivateWorkItem: DispatchWorkItem? + private let directiveConnectionQueue = DispatchQueue(label: "com.sktelecom.romaine.NuguClientKit.directive_connection") init( contextManager: ContextManageable, @@ -422,14 +423,22 @@ public extension NuguClient { The server can send some directives at certain times. */ func startReceiveServerInitiatedDirective(completion: ((StreamDataState) -> Void)? = nil) { - ConfigurationStore.shared.registryServerUrl { [weak self] result in - switch result { - case .failure(let error): - completion?(.error(error)) - case .success(let url): - NuguServerInfo.registryServerAddress = url - self?.streamDataRouter.startReceiveServerInitiatedDirective(completion: completion) + directiveConnectionQueue.async { [weak self] in + let sema = DispatchSemaphore(value: .zero) + ConfigurationStore.shared.registryServerUrl { [weak self] result in + switch result { + case .failure(let error): + sema.signal() + completion?(.error(error)) + case .success(let url): + NuguServerInfo.registryServerAddress = url + self?.streamDataRouter.startReceiveServerInitiatedDirective(completion: { [weak sema] state in + sema?.signal() + completion?(state) + }) + } } + sema.wait() } } @@ -437,7 +446,9 @@ public extension NuguClient { Stop receiving server-initiated-directive. */ func stopReceiveServerInitiatedDirective() { - streamDataRouter.stopReceiveServerInitiatedDirective() + directiveConnectionQueue.async { [weak self] in + self?.streamDataRouter.stopReceiveServerInitiatedDirective() + } } /// Send event that needs a text-based recognition diff --git a/NuguCore/Sources/StreamData/StreamDataRouter.swift b/NuguCore/Sources/StreamData/StreamDataRouter.swift index 7a2b4d323..2c86a05ea 100644 --- a/NuguCore/Sources/StreamData/StreamDataRouter.swift +++ b/NuguCore/Sources/StreamData/StreamDataRouter.swift @@ -60,6 +60,10 @@ public extension StreamDataRouter { serverInitiatedDirectiveStateDisposable = serverInitiatedDirectiveReceiver.stateObserver .subscribe(onNext: { [weak self] state in self?.notificationQueue.async { [weak self] in + if state == .connected { + completion?(.prepared) + } + self?.post(state) } }) @@ -75,6 +79,7 @@ public extension StreamDataRouter { completion?(.error($0)) }, onDisposed: { log.debug("server initiated directive is stopeed") + completion?(.finished) }) serverInitiatedDirectiveDisposable?.disposed(by: disposeBag) } @@ -256,6 +261,7 @@ extension StreamDataRouter { directiveSequencer.processDirective(directive) completion?(.received(part: directive)) + serverInitiatedDirectiveCompletion?(.received(part: directive)) } } else if let attachment = Downstream.Attachment(headerDictionary: part.header, body: part.body) { log.debug("Attachment: \(attachment.header.dialogRequestId), \(attachment.header.type)") From e8444efeb9b0a8acf003fa4a881ccd602d2e34db Mon Sep 17 00:00:00 2001 From: ParkJongSang Date: Mon, 16 Oct 2023 15:46:41 +0900 Subject: [PATCH 25/39] =?UTF-8?q?AudioSession=20Update=20=EB=AC=B4?= =?UTF-8?q?=EC=8B=9C=ED=95=A0=EC=88=98=20=EC=9E=88=EB=8F=84=EB=A1=9D=20?= =?UTF-8?q?=ED=94=84=EB=A1=9C=ED=8D=BC=ED=8B=B0=20=EC=83=9D=EC=84=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Audio/AudioSessionManageable.swift | 1 + .../Sources/Audio/AudioSessionManager.swift | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/NuguClientKit/Sources/Audio/AudioSessionManageable.swift b/NuguClientKit/Sources/Audio/AudioSessionManageable.swift index 4f5363975..5201da5b2 100644 --- a/NuguClientKit/Sources/Audio/AudioSessionManageable.swift +++ b/NuguClientKit/Sources/Audio/AudioSessionManageable.swift @@ -22,6 +22,7 @@ import Foundation public protocol AudioSessionManageable: AnyObject { var delegate: AudioSessionManagerDelegate? { get set } + var allowsUpdateAudioSessionActivation: Bool { get set } func isCarplayConnected() -> Bool func requestRecordPermission(_ response: @escaping (Bool) -> Void) diff --git a/NuguClientKit/Sources/Audio/AudioSessionManager.swift b/NuguClientKit/Sources/Audio/AudioSessionManager.swift index 244e6dd4a..6bcbeee09 100644 --- a/NuguClientKit/Sources/Audio/AudioSessionManager.swift +++ b/NuguClientKit/Sources/Audio/AudioSessionManager.swift @@ -24,6 +24,7 @@ import NuguAgents final public class AudioSessionManager: AudioSessionManageable { public weak var delegate: AudioSessionManagerDelegate? + public var allowsUpdateAudioSessionActivation: Bool = true private let audioPlayerAgent: AudioPlayerAgentProtocol private let defaultCategoryOptions = AVAudioSession.CategoryOptions(arrayLiteral: [.defaultToSpeaker, .allowBluetoothA2DP]) @@ -46,7 +47,7 @@ final public class AudioSessionManager: AudioSessionManageable { // When no other audio is playing, audio session can not detect car play connectivity status even if car play has been already connected. // To resolve this problem, activating audio session should be done in prior to detecting car play connectivity. if AVAudioSession.sharedInstance().isOtherAudioPlaying == false { - try? AVAudioSession.sharedInstance().setActive(true) + try? activeAudioSessionIfNeeded() } } @@ -92,7 +93,8 @@ public extension AudioSessionManager { mode: .default, options: options ) - try AVAudioSession.sharedInstance().setActive(true) + + try activeAudioSessionIfNeeded() return true } catch { log.debug("updateAudioSessionToPlaybackIfNeeded failed: \(error)") @@ -114,7 +116,7 @@ public extension AudioSessionManager { mode: .default, options: [] ) - try AVAudioSession.sharedInstance().setActive(true) + try activeAudioSessionIfNeeded() return true } catch { log.debug("updateAudioSession when carplay connected has failed: \(error)") @@ -154,7 +156,7 @@ public extension AudioSessionManager { ) log.debug("set audio session: \(AVAudioSession.Category.playAndRecord), options: \(options)") - try AVAudioSession.sharedInstance().setActive(true) + try activeAudioSessionIfNeeded() log.debug("audio session activated") return true @@ -273,6 +275,15 @@ private extension AudioSessionManager { } } +// MARK: - Private (audioSessiontActive) + +private extension AudioSessionManager { + func activeAudioSessionIfNeeded() throws { + guard allowsUpdateAudioSessionActivation else { return } + try AVAudioSession.sharedInstance().setActive(true) + } +} + public extension AudioSessionManager { enum AudioSessionInterruptionType { case began From 1f2fb78ffb973dbd327957cbb78f90207ff65346 Mon Sep 17 00:00:00 2001 From: ParkJongSang Date: Mon, 16 Oct 2023 15:59:48 +0900 Subject: [PATCH 26/39] =?UTF-8?q?=EC=BD=94=EB=93=9C=EB=A6=AC=EB=B7=B0=20?= =?UTF-8?q?=EB=B0=98=EC=98=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- NuguClientKit/Sources/Audio/AudioSessionManager.swift | 3 +-- .../Sources/Audio/AudioSessionManagerDelegate.swift | 7 +++++++ NuguClientKit/Sources/Client/NuguClient.swift | 4 ++++ NuguClientKit/Sources/Client/NuguClientDelegate.swift | 3 +++ 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/NuguClientKit/Sources/Audio/AudioSessionManager.swift b/NuguClientKit/Sources/Audio/AudioSessionManager.swift index 6bcbeee09..0195921ec 100644 --- a/NuguClientKit/Sources/Audio/AudioSessionManager.swift +++ b/NuguClientKit/Sources/Audio/AudioSessionManager.swift @@ -24,7 +24,6 @@ import NuguAgents final public class AudioSessionManager: AudioSessionManageable { public weak var delegate: AudioSessionManagerDelegate? - public var allowsUpdateAudioSessionActivation: Bool = true private let audioPlayerAgent: AudioPlayerAgentProtocol private let defaultCategoryOptions = AVAudioSession.CategoryOptions(arrayLiteral: [.defaultToSpeaker, .allowBluetoothA2DP]) @@ -279,7 +278,7 @@ private extension AudioSessionManager { private extension AudioSessionManager { func activeAudioSessionIfNeeded() throws { - guard allowsUpdateAudioSessionActivation else { return } + guard delegate?.allowsUpdateAudioSessionActivation == true else { return } try AVAudioSession.sharedInstance().setActive(true) } } diff --git a/NuguClientKit/Sources/Audio/AudioSessionManagerDelegate.swift b/NuguClientKit/Sources/Audio/AudioSessionManagerDelegate.swift index 26cac2d9b..1d2867722 100644 --- a/NuguClientKit/Sources/Audio/AudioSessionManagerDelegate.swift +++ b/NuguClientKit/Sources/Audio/AudioSessionManagerDelegate.swift @@ -21,8 +21,15 @@ import Foundation public protocol AudioSessionManagerDelegate: AnyObject { + var allowsUpdateAudioSessionActivation: Bool { get } + func audioSessionInterrupted(type: AudioSessionManager.AudioSessionInterruptionType) func audioSessionRouteChanged(reason: AudioSessionManager.AudioSessionRouteChangeReason) func audioSessionWillDeactivate() func audioSessionDidDeactivate() + func audioSessionAllowActive() +} + +public extension AudioSessionManagerDelegate { + var allowsUpdateAudioSessionActivation: Bool { true } } diff --git a/NuguClientKit/Sources/Client/NuguClient.swift b/NuguClientKit/Sources/Client/NuguClient.swift index 1a80fcd7d..d566abbe7 100644 --- a/NuguClientKit/Sources/Client/NuguClient.swift +++ b/NuguClientKit/Sources/Client/NuguClient.swift @@ -670,6 +670,10 @@ extension NuguClient { // MARK: - AudioSessionManagerDelegate extension NuguClient: AudioSessionManagerDelegate { + public var allowsUpdateAudioSessionActivation: Bool { + delegate?.nuguClientAllowsAcitveAudioSession() ?? true + } + private func setupAudioSessionManager() { audioSessionManager?.delegate = self } diff --git a/NuguClientKit/Sources/Client/NuguClientDelegate.swift b/NuguClientKit/Sources/Client/NuguClientDelegate.swift index 60d09c14d..3a6805ca6 100644 --- a/NuguClientKit/Sources/Client/NuguClientDelegate.swift +++ b/NuguClientKit/Sources/Client/NuguClientDelegate.swift @@ -45,6 +45,9 @@ public protocol NuguClientDelegate: AnyObject { /// Notify that nugu client won't play sound anymore. func nuguClientDidReleaseAudioSession() + /// Check Can ActiveAudioSessiong + func nuguClientAllowsAcitveAudioSession() -> Bool + // speech recognizer aggregator state related /// Notify that nugu client speech-related state has been changed From 367e307d5f336fed746439f7576635233523ad05 Mon Sep 17 00:00:00 2001 From: ParkJongSang Date: Mon, 16 Oct 2023 16:00:33 +0900 Subject: [PATCH 27/39] =?UTF-8?q?=EB=88=84=EB=9D=BD=EB=90=9C=20=EC=BD=94?= =?UTF-8?q?=EB=93=9C=20=EB=B0=98=EC=98=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- NuguClientKit/Sources/Audio/AudioSessionManageable.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/NuguClientKit/Sources/Audio/AudioSessionManageable.swift b/NuguClientKit/Sources/Audio/AudioSessionManageable.swift index 5201da5b2..4f5363975 100644 --- a/NuguClientKit/Sources/Audio/AudioSessionManageable.swift +++ b/NuguClientKit/Sources/Audio/AudioSessionManageable.swift @@ -22,7 +22,6 @@ import Foundation public protocol AudioSessionManageable: AnyObject { var delegate: AudioSessionManagerDelegate? { get set } - var allowsUpdateAudioSessionActivation: Bool { get set } func isCarplayConnected() -> Bool func requestRecordPermission(_ response: @escaping (Bool) -> Void) From 04af6360c057bf1195afec8626ff9d1fcf788a75 Mon Sep 17 00:00:00 2001 From: ParkJongSang Date: Mon, 16 Oct 2023 16:03:51 +0900 Subject: [PATCH 28/39] =?UTF-8?q?=EC=9D=98=EB=AF=B8=EC=97=86=EC=9D=B4=20?= =?UTF-8?q?=EB=93=A4=EC=96=B4=EA=B0=84=20=EC=BD=94=EB=93=9C=20=EC=A0=9C?= =?UTF-8?q?=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- NuguClientKit/Sources/Audio/AudioSessionManagerDelegate.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/NuguClientKit/Sources/Audio/AudioSessionManagerDelegate.swift b/NuguClientKit/Sources/Audio/AudioSessionManagerDelegate.swift index 1d2867722..64f5b0c6d 100644 --- a/NuguClientKit/Sources/Audio/AudioSessionManagerDelegate.swift +++ b/NuguClientKit/Sources/Audio/AudioSessionManagerDelegate.swift @@ -27,7 +27,6 @@ public protocol AudioSessionManagerDelegate: AnyObject { func audioSessionRouteChanged(reason: AudioSessionManager.AudioSessionRouteChangeReason) func audioSessionWillDeactivate() func audioSessionDidDeactivate() - func audioSessionAllowActive() } public extension AudioSessionManagerDelegate { From 0159f8f28490f656598a407c25865ddc6d2f5c5d Mon Sep 17 00:00:00 2001 From: ParkJongSang Date: Mon, 16 Oct 2023 16:06:49 +0900 Subject: [PATCH 29/39] =?UTF-8?q?SampleApp=20=EB=B9=8C=EB=93=9C=EC=98=A4?= =?UTF-8?q?=EB=A5=98=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- NuguClientKit/Sources/Client/NuguClientDelegate.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/NuguClientKit/Sources/Client/NuguClientDelegate.swift b/NuguClientKit/Sources/Client/NuguClientDelegate.swift index 3a6805ca6..c3af43dac 100644 --- a/NuguClientKit/Sources/Client/NuguClientDelegate.swift +++ b/NuguClientKit/Sources/Client/NuguClientDelegate.swift @@ -97,6 +97,7 @@ public extension NuguClientDelegate { // even if it is optional delegate method. // Do proper stuffs when NUGU SDK has released using audio session. func nuguClientDidReleaseAudioSession() {} + func nuguClientAllowsAcitveAudioSession() -> Bool { return true } // speech recognizer aggregator state related func nuguClientDidChangeSpeechState(_ state: SpeechRecognizerAggregatorState) {} From 09a4e024486372fbb64c78d0fe411e47bebf2d17 Mon Sep 17 00:00:00 2001 From: ParkJongSang Date: Mon, 16 Oct 2023 17:18:56 +0900 Subject: [PATCH 30/39] =?UTF-8?q?Category=EB=98=90=ED=95=9C=20delegate?= =?UTF-8?q?=EC=9D=98=20=EC=98=81=ED=96=A5=EC=9D=84=20=EB=B0=9B=EB=8F=84?= =?UTF-8?q?=EB=A1=9D=20=EC=A0=81=EC=9A=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Sources/Audio/AudioSessionManager.swift | 38 ++++++++----------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/NuguClientKit/Sources/Audio/AudioSessionManager.swift b/NuguClientKit/Sources/Audio/AudioSessionManager.swift index 0195921ec..8fd9955d9 100644 --- a/NuguClientKit/Sources/Audio/AudioSessionManager.swift +++ b/NuguClientKit/Sources/Audio/AudioSessionManager.swift @@ -87,13 +87,7 @@ public extension AudioSessionManager { AVAudioSession.sharedInstance().category != .playback || AVAudioSession.sharedInstance().categoryOptions != options else { return true } do { - try AVAudioSession.sharedInstance().setCategory( - .playback, - mode: .default, - options: options - ) - - try activeAudioSessionIfNeeded() + try activeAudioSessionIfNeeded(categoryOptions: options) return true } catch { log.debug("updateAudioSessionToPlaybackIfNeeded failed: \(error)") @@ -110,12 +104,7 @@ public extension AudioSessionManager { return true } do { - try AVAudioSession.sharedInstance().setCategory( - .playAndRecord, - mode: .default, - options: [] - ) - try activeAudioSessionIfNeeded() + try activeAudioSessionIfNeeded(categoryOptions: []) return true } catch { log.debug("updateAudioSession when carplay connected has failed: \(error)") @@ -148,15 +137,7 @@ public extension AudioSessionManager { } do { - try AVAudioSession.sharedInstance().setCategory( - .playAndRecord, - mode: .default, - options: options - ) - log.debug("set audio session: \(AVAudioSession.Category.playAndRecord), options: \(options)") - - try activeAudioSessionIfNeeded() - log.debug("audio session activated") + try activeAudioSessionIfNeeded(categoryOptions: options) return true } catch { @@ -277,9 +258,20 @@ private extension AudioSessionManager { // MARK: - Private (audioSessiontActive) private extension AudioSessionManager { - func activeAudioSessionIfNeeded() throws { + func activeAudioSessionIfNeeded(categoryOptions: AVAudioSession.CategoryOptions) throws { guard delegate?.allowsUpdateAudioSessionActivation == true else { return } + + try AVAudioSession.sharedInstance().setCategory( + .playback, + mode: .default, + options: categoryOptions + ) + + log.debug("set audio session: \(AVAudioSession.Category.playAndRecord), options: \(options)") + try AVAudioSession.sharedInstance().setActive(true) + + log.debug("audio session activated") } } From 66f109788707f4b423e27bddd65a72b2fc8af0be Mon Sep 17 00:00:00 2001 From: ParkJongSang Date: Mon, 16 Oct 2023 17:22:56 +0900 Subject: [PATCH 31/39] =?UTF-8?q?=EB=B9=8C=EB=93=9C=EC=98=A4=EB=A5=98=20?= =?UTF-8?q?=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- NuguClientKit/Sources/Audio/AudioSessionManager.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NuguClientKit/Sources/Audio/AudioSessionManager.swift b/NuguClientKit/Sources/Audio/AudioSessionManager.swift index 8fd9955d9..ee53e3786 100644 --- a/NuguClientKit/Sources/Audio/AudioSessionManager.swift +++ b/NuguClientKit/Sources/Audio/AudioSessionManager.swift @@ -46,7 +46,7 @@ final public class AudioSessionManager: AudioSessionManageable { // When no other audio is playing, audio session can not detect car play connectivity status even if car play has been already connected. // To resolve this problem, activating audio session should be done in prior to detecting car play connectivity. if AVAudioSession.sharedInstance().isOtherAudioPlaying == false { - try? activeAudioSessionIfNeeded() + try? activeAudioSessionIfNeeded(categoryOptions: defaultCategoryOptions) } } From ebbd34655a3fe16752694eaf0b4674b68f7bb142 Mon Sep 17 00:00:00 2001 From: ParkJongSang Date: Mon, 16 Oct 2023 17:23:20 +0900 Subject: [PATCH 32/39] =?UTF-8?q?=EB=B9=8C=EB=93=9C=EC=98=A4=EB=A5=98=20?= =?UTF-8?q?=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- NuguClientKit/Sources/Audio/AudioSessionManager.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NuguClientKit/Sources/Audio/AudioSessionManager.swift b/NuguClientKit/Sources/Audio/AudioSessionManager.swift index ee53e3786..45dc62640 100644 --- a/NuguClientKit/Sources/Audio/AudioSessionManager.swift +++ b/NuguClientKit/Sources/Audio/AudioSessionManager.swift @@ -267,7 +267,7 @@ private extension AudioSessionManager { options: categoryOptions ) - log.debug("set audio session: \(AVAudioSession.Category.playAndRecord), options: \(options)") + log.debug("set audio session: \(AVAudioSession.Category.playAndRecord), options: \(categoryOptions)") try AVAudioSession.sharedInstance().setActive(true) From c0a93b57405d039ec1467a1c982a766df4692d44 Mon Sep 17 00:00:00 2001 From: ParkJongSang Date: Mon, 16 Oct 2023 17:31:19 +0900 Subject: [PATCH 33/39] =?UTF-8?q?=EC=BD=94=EB=93=9C=EB=A6=AC=EB=B7=B0=20?= =?UTF-8?q?=EB=B0=98=EC=98=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Sources/Audio/AudioSessionManager.swift | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/NuguClientKit/Sources/Audio/AudioSessionManager.swift b/NuguClientKit/Sources/Audio/AudioSessionManager.swift index 45dc62640..7088540a5 100644 --- a/NuguClientKit/Sources/Audio/AudioSessionManager.swift +++ b/NuguClientKit/Sources/Audio/AudioSessionManager.swift @@ -46,7 +46,7 @@ final public class AudioSessionManager: AudioSessionManageable { // When no other audio is playing, audio session can not detect car play connectivity status even if car play has been already connected. // To resolve this problem, activating audio session should be done in prior to detecting car play connectivity. if AVAudioSession.sharedInstance().isOtherAudioPlaying == false { - try? activeAudioSessionIfNeeded(categoryOptions: defaultCategoryOptions) + try activeAudioSessionIfNeeded() } } @@ -87,7 +87,8 @@ public extension AudioSessionManager { AVAudioSession.sharedInstance().category != .playback || AVAudioSession.sharedInstance().categoryOptions != options else { return true } do { - try activeAudioSessionIfNeeded(categoryOptions: options) + try setAudioCategoryIfNeeded(.playback, options: options) + try activeAudioSessionIfNeeded() return true } catch { log.debug("updateAudioSessionToPlaybackIfNeeded failed: \(error)") @@ -104,7 +105,8 @@ public extension AudioSessionManager { return true } do { - try activeAudioSessionIfNeeded(categoryOptions: []) + try setAudioCategoryIfNeeded(.playAndRecord, options: []) + try activeAudioSessionIfNeeded() return true } catch { log.debug("updateAudioSession when carplay connected has failed: \(error)") @@ -137,7 +139,8 @@ public extension AudioSessionManager { } do { - try activeAudioSessionIfNeeded(categoryOptions: options) + try setAudioCategoryIfNeeded(.playAndRecord, options: options) + try activeAudioSessionIfNeeded() return true } catch { @@ -258,20 +261,24 @@ private extension AudioSessionManager { // MARK: - Private (audioSessiontActive) private extension AudioSessionManager { - func activeAudioSessionIfNeeded(categoryOptions: AVAudioSession.CategoryOptions) throws { + func activeAudioSessionIfNeeded() throws { + guard delegate?.allowsUpdateAudioSessionActivation == true else { return } + + try AVAudioSession.sharedInstance().setActive(true) + + log.debug("audio session activated") + } + + func setAudioCategoryIfNeeded(_ category: AVAudioSession.Category, options: AVAudioSession.CategoryOptions) throws { guard delegate?.allowsUpdateAudioSessionActivation == true else { return } try AVAudioSession.sharedInstance().setCategory( - .playback, + category, mode: .default, options: categoryOptions ) - log.debug("set audio session: \(AVAudioSession.Category.playAndRecord), options: \(categoryOptions)") - - try AVAudioSession.sharedInstance().setActive(true) - - log.debug("audio session activated") + log.debug("set audio session: \(category), options: \(options)") } } From b15e678771732389aa5444c894855a0263297586 Mon Sep 17 00:00:00 2001 From: ParkJongSang Date: Mon, 16 Oct 2023 17:33:11 +0900 Subject: [PATCH 34/39] =?UTF-8?q?=EC=BD=94=EB=93=9C=EB=A6=AC=EB=B7=B0=20?= =?UTF-8?q?=EB=B0=98=EC=98=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- NuguClientKit/Sources/Audio/AudioSessionManager.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NuguClientKit/Sources/Audio/AudioSessionManager.swift b/NuguClientKit/Sources/Audio/AudioSessionManager.swift index 7088540a5..c4b590428 100644 --- a/NuguClientKit/Sources/Audio/AudioSessionManager.swift +++ b/NuguClientKit/Sources/Audio/AudioSessionManager.swift @@ -275,7 +275,7 @@ private extension AudioSessionManager { try AVAudioSession.sharedInstance().setCategory( category, mode: .default, - options: categoryOptions + options: options ) log.debug("set audio session: \(category), options: \(options)") From 655728c0d7aef83c3a19613a32c7017ce04e381b Mon Sep 17 00:00:00 2001 From: ParkJongSang Date: Mon, 16 Oct 2023 17:35:14 +0900 Subject: [PATCH 35/39] =?UTF-8?q?=EB=B9=8C=EB=93=9C=EC=98=A4=EB=A5=98=20?= =?UTF-8?q?=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- NuguClientKit/Sources/Audio/AudioSessionManager.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NuguClientKit/Sources/Audio/AudioSessionManager.swift b/NuguClientKit/Sources/Audio/AudioSessionManager.swift index c4b590428..4b215be7e 100644 --- a/NuguClientKit/Sources/Audio/AudioSessionManager.swift +++ b/NuguClientKit/Sources/Audio/AudioSessionManager.swift @@ -46,7 +46,7 @@ final public class AudioSessionManager: AudioSessionManageable { // When no other audio is playing, audio session can not detect car play connectivity status even if car play has been already connected. // To resolve this problem, activating audio session should be done in prior to detecting car play connectivity. if AVAudioSession.sharedInstance().isOtherAudioPlaying == false { - try activeAudioSessionIfNeeded() + try? activeAudioSessionIfNeeded() } } From ef43828af35bcd04235ce0fc2730772532ad6e83 Mon Sep 17 00:00:00 2001 From: ParkJongSang Date: Mon, 16 Oct 2023 21:48:49 +0900 Subject: [PATCH 36/39] =?UTF-8?q?Deactive=ED=95=A0=EB=95=8C=EB=8F=84=20all?= =?UTF-8?q?owsUpdateAudioSessionActivation=EB=A5=BC=20=EB=B0=94=EB=9D=BC?= =?UTF-8?q?=EB=B3=B4=EB=8F=84=EB=A1=9D=20=EC=A0=81=EC=9A=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- NuguClientKit/Sources/Audio/AudioSessionManager.swift | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/NuguClientKit/Sources/Audio/AudioSessionManager.swift b/NuguClientKit/Sources/Audio/AudioSessionManager.swift index 4b215be7e..92ef3102c 100644 --- a/NuguClientKit/Sources/Audio/AudioSessionManager.swift +++ b/NuguClientKit/Sources/Audio/AudioSessionManager.swift @@ -161,7 +161,7 @@ public extension AudioSessionManager { delegate?.audioSessionWillDeactivate() // Notify audio session deactivation to 3rd party apps - try AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation) + try deactiveAudioSessionIfNeeded() } catch { log.debug("notifyOthersOnDeactivation failed: \(error)") } @@ -269,6 +269,14 @@ private extension AudioSessionManager { log.debug("audio session activated") } + func deactiveAudioSessionIfNeeded() throws { + guard delegate?.allowsUpdateAudioSessionActivation == true else { return } + + try AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation) + + log.debug("audio session deactivated") + } + func setAudioCategoryIfNeeded(_ category: AVAudioSession.Category, options: AVAudioSession.CategoryOptions) throws { guard delegate?.allowsUpdateAudioSessionActivation == true else { return } From 088878f92dbe0ffd879be26b731f00628b061b39 Mon Sep 17 00:00:00 2001 From: jayce1116 Date: Fri, 20 Oct 2023 09:54:00 +0900 Subject: [PATCH 37/39] =?UTF-8?q?startListening=EC=97=90=20=EC=9E=88?= =?UTF-8?q?=EB=8A=94=20semaphore=EC=A0=9C=EA=B1=B0=20(#1093)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Description - startListening에 있는 semaphore제거 - startListening, stopListening을 반복적으로 호출 시 recognizeQueue가 deadlock이 걸리고 있음. - 추후에 deadlock원인 찾아서 고도화 예정 --- .../Audio/SpeechRecognizerAggregator.swift | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/NuguClientKit/Sources/Audio/SpeechRecognizerAggregator.swift b/NuguClientKit/Sources/Audio/SpeechRecognizerAggregator.swift index a2c5d31e5..90641ed03 100644 --- a/NuguClientKit/Sources/Audio/SpeechRecognizerAggregator.swift +++ b/NuguClientKit/Sources/Audio/SpeechRecognizerAggregator.swift @@ -116,19 +116,13 @@ public extension SpeechRecognizerAggregator { asrAgent.stopRecognition() } - let sema = DispatchSemaphore(value: .zero) asrAgent.startRecognition(initiator: initiator) { [weak self] state in guard case .prepared = state else { completion?(state) - sema.signal() return } - self?.startMicInputProvider(requestingFocus: true) { [weak self, weak sema] endedUp in - defer { - sema?.signal() - } - + self?.startMicInputProvider(requestingFocus: true) { [weak self] endedUp in if case let .failure(error) = endedUp { log.error("Start MicInputProvider failed: \(error)") self?.asrAgent.stopRecognition() @@ -149,7 +143,6 @@ public extension SpeechRecognizerAggregator { completion?(.prepared) } } - sema.wait() } } @@ -160,14 +153,9 @@ public extension SpeechRecognizerAggregator { recognizeQueue.async { [weak self] in guard let self else { return } - let sema = DispatchSemaphore(value: .zero) let startMicWorkItem = DispatchWorkItem { [weak self] in log.debug("startMicWorkItem start") self?.startMicInputProvider(requestingFocus: false) { (endedUp) in - defer { - sema.signal() - } - if case let .failure(error) = endedUp { log.debug("startMicWorkItem failed: \(error)") var recognizerError: Error { @@ -198,8 +186,6 @@ public extension SpeechRecognizerAggregator { } else { DispatchQueue.global().async(execute: startMicWorkItem) } - - sema.wait() } } From 3115c9dd35384863e06d0d5d8f3d9a20ecf2b069 Mon Sep 17 00:00:00 2001 From: jayce1116 Date: Fri, 20 Oct 2023 12:28:29 +0900 Subject: [PATCH 38/39] =?UTF-8?q?SessionManager=EC=97=90=EC=84=9C=20sessio?= =?UTF-8?q?n=EC=9D=84=20set=ED=95=A0=20=EB=95=8C=20activeList=EC=97=90=20?= =?UTF-8?q?=EC=A1=B4=EC=9E=AC=ED=95=98=EB=8A=94=20session=EC=9D=BC=20?= =?UTF-8?q?=EA=B2=BD=EC=9A=B0=20=EB=A1=9C=EC=A7=81=20=EB=B3=80=EA=B2=BD=20?= =?UTF-8?q?(#1094)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Description - SessionManager에서 session을 set할 때 activeList에 존재하는 session일 경우 로직 변경 --- .../CapabilityAgents/Session/SessionManager.swift | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/NuguAgents/Sources/CapabilityAgents/Session/SessionManager.swift b/NuguAgents/Sources/CapabilityAgents/Session/SessionManager.swift index 462b556ff..c31788c9e 100644 --- a/NuguAgents/Sources/CapabilityAgents/Session/SessionManager.swift +++ b/NuguAgents/Sources/CapabilityAgents/Session/SessionManager.swift @@ -60,11 +60,17 @@ final public class SessionManager: SessionManageable { public func set(session: Session) { sessionDispatchQueue.async { [weak self] in guard let self = self else { return } - log.debug(session.dialogRequestId) - self.sessions[session.dialogRequestId] = session - self.addTimer(session: session) + let dialogRequestId = session.dialogRequestId + log.debug("Set session, session: \(session)") + self.sessions[dialogRequestId] = session + + if activeList[dialogRequestId] == nil { + log.debug("Session activeList not exist session, dialogRequestId: \(dialogRequestId)") + self.addTimer(session: session) + } else { + self.addActiveSession(dialogRequestId: session.dialogRequestId) + } - self.addActiveSession(dialogRequestId: session.dialogRequestId) self.post(NuguAgentNotification.Session.Set(session: session)) } } @@ -101,6 +107,7 @@ final public class SessionManager: SessionManageable { private extension SessionManager { func addTimer(session: Session) { + log.debug("Start session remove timer. dialogRequestId: \(session.dialogRequestId)") activeTimers[session.dialogRequestId] = Single.timer(SessionConst.sessionTimeout, scheduler: sessionScheduler) .subscribe(onSuccess: { [weak self] _ in log.debug("Timer fired. \(session.dialogRequestId)") From 330feae3d3097e0e856832da8f1c9f9be0b1238c Mon Sep 17 00:00:00 2001 From: ParkJongSang Date: Wed, 25 Oct 2023 11:57:03 +0900 Subject: [PATCH 39/39] =?UTF-8?q?Playlist=20Token=20Key=20=EA=B0=92=20?= =?UTF-8?q?=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Sources/CapabilityAgents/AudioPlayer/AudioPlayerAgent.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NuguAgents/Sources/CapabilityAgents/AudioPlayer/AudioPlayerAgent.swift b/NuguAgents/Sources/CapabilityAgents/AudioPlayer/AudioPlayerAgent.swift index f3cfbe633..a0df40b6d 100644 --- a/NuguAgents/Sources/CapabilityAgents/AudioPlayer/AudioPlayerAgent.swift +++ b/NuguAgents/Sources/CapabilityAgents/AudioPlayer/AudioPlayerAgent.swift @@ -224,7 +224,7 @@ public final class AudioPlayerAgent: AudioPlayerAgentProtocol { } if let playlist = self.currentPlaylist { - payload["playlist"] = playlist.token + payload["playlistToken"] = playlist.token payload["playlistVisible"] = self.currentPlaylist != nil }