From c39d58d421e8019ef1c5e19febcef75b2a22cf2f Mon Sep 17 00:00:00 2001 From: Steven Toribio Date: Tue, 11 Jul 2023 20:03:57 +0000 Subject: [PATCH 1/5] done --- python/tflite_micro/python_ops_resolver.cc | 1 + signal/micro/kernels/BUILD | 28 ++ signal/micro/kernels/overlap_add.cc | 237 +++++++++++++++++ .../overlap_add_flexbuffers_generated_data.cc | 32 +++ .../overlap_add_flexbuffers_generated_data.h | 25 ++ signal/micro/kernels/overlap_add_test.cc | 246 ++++++++++++++++++ signal/src/BUILD | 18 +- signal/src/overlap_add.cc | 52 ++++ signal/src/overlap_add.h | 46 ++++ tensorflow/lite/micro/kernels/Makefile.inc | 5 + tensorflow/lite/micro/kernels/micro_ops.h | 9 +- .../lite/micro/micro_mutable_op_resolver.h | 15 +- tensorflow/lite/micro/tools/make/Makefile | 4 +- 13 files changed, 705 insertions(+), 13 deletions(-) create mode 100644 signal/micro/kernels/overlap_add.cc create mode 100644 signal/micro/kernels/overlap_add_flexbuffers_generated_data.cc create mode 100644 signal/micro/kernels/overlap_add_flexbuffers_generated_data.h create mode 100644 signal/micro/kernels/overlap_add_test.cc create mode 100644 signal/src/overlap_add.cc create mode 100644 signal/src/overlap_add.h diff --git a/python/tflite_micro/python_ops_resolver.cc b/python/tflite_micro/python_ops_resolver.cc index 8d508e1cc67..ed4320ea241 100644 --- a/python/tflite_micro/python_ops_resolver.cc +++ b/python/tflite_micro/python_ops_resolver.cc @@ -80,6 +80,7 @@ PythonOpsResolver::PythonOpsResolver() { AddMul(); AddNeg(); AddNotEqual(); + AddOverlapAdd(); AddPack(); AddPad(); AddPadV2(); diff --git a/signal/micro/kernels/BUILD b/signal/micro/kernels/BUILD index 040332c6924..aea77f61627 100644 --- a/signal/micro/kernels/BUILD +++ b/signal/micro/kernels/BUILD @@ -9,6 +9,7 @@ cc_library( name = "register_signal_ops", srcs = [ "framer.cc", + "overlap_add.cc", "rfft.cc", "window.cc", ], @@ -21,6 +22,7 @@ cc_library( ], deps = [ "//signal/src:circular_buffer", + "//signal/src:overlap_add", "//signal/src:rfft", "//signal/src:window", "//tensorflow/lite:type_to_tflitetype", @@ -113,3 +115,29 @@ cc_test( "//tensorflow/lite/micro/testing:micro_test", ], ) + +cc_library( + name = "overlap_add_flexbuffers_generated_data", + srcs = [ + "overlap_add_flexbuffers_generated_data.cc", + ], + hdrs = [ + "overlap_add_flexbuffers_generated_data.h", + ], +) + +cc_test( + name = "overlap_add_test", + srcs = [ + "overlap_add_test.cc", + ], + deps = [ + ":overlap_add_flexbuffers_generated_data", + ":register_signal_ops", + "//tensorflow/lite/c:common", + "//tensorflow/lite/micro:op_resolvers", + "//tensorflow/lite/micro:test_helpers", + "//tensorflow/lite/micro/kernels:kernel_runner", + "//tensorflow/lite/micro/testing:micro_test", + ], +) diff --git a/signal/micro/kernels/overlap_add.cc b/signal/micro/kernels/overlap_add.cc new file mode 100644 index 00000000000..bc3cfabf9c6 --- /dev/null +++ b/signal/micro/kernels/overlap_add.cc @@ -0,0 +1,237 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "signal/src/overlap_add.h" + +#include + +#include "tensorflow/lite/kernels/internal/tensor_ctypes.h" +#include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/portable_type_to_tflitetype.h" +#include "tensorflow/lite/micro/flatbuffer_utils.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" + +namespace tflite { +namespace { + +constexpr int kInputTensor = 0; +constexpr int kOutputTensor = 0; + +// Indices into the init flexbuffer's vector. +// The parameter's name is in the comment that follows. +// Elements in the vectors are ordered alphabetically by parameter name. +// 'T' is added implicitly by the TensorFlow framework when the type is resolved +// during graph construction. +// constexpr int kTypeIndex = 0; // 'T' (unused) +constexpr int kFrameStepIndex = 1; // 'frame_step' + +template +struct TFLMSignalOverlapAddParams { + int32_t frame_size; + int32_t frame_step; + int32_t outer_dims; + int32_t n_frames; + TfLiteType type; + T** state_buffers; +}; + +template +void ResetState(TFLMSignalOverlapAddParams* params) { + for (int i = 0; i < params->outer_dims; i++) { + memset(params->state_buffers[i], 0, sizeof(T) * params->frame_size); + } +} + +template +void* Init(TfLiteContext* context, const char* buffer, size_t length) { + const uint8_t* buffer_t = reinterpret_cast(buffer); + + auto* params = static_cast*>( + context->AllocatePersistentBuffer( + context, sizeof(TFLMSignalOverlapAddParams))); + + tflite::FlexbufferWrapper fbw(buffer_t, length); + params->type = typeToTfLiteType(); + params->frame_step = fbw.ElementAsInt32(kFrameStepIndex); + return params; +} + +template +TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + TF_LITE_ENSURE_EQ(context, NumInputs(node), 1); + TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); + + MicroContext* micro_context = GetMicroContext(context); + + TfLiteTensor* input = + micro_context->AllocateTempInputTensor(node, kInputTensor); + TF_LITE_ENSURE(context, input != nullptr); + TfLiteTensor* output = + micro_context->AllocateTempOutputTensor(node, kOutputTensor); + TF_LITE_ENSURE(context, output != nullptr); + + TF_LITE_ENSURE_EQ(context, NumDimensions(input), NumDimensions(output) + 1); + + TF_LITE_ENSURE_TYPES_EQ(context, input->type, TfLiteTypeEnum); + TF_LITE_ENSURE_TYPES_EQ(context, output->type, TfLiteTypeEnum); + + auto* params = reinterpret_cast*>( + node->user_data); + RuntimeShape input_shape = GetTensorShape(input); + RuntimeShape output_shape = GetTensorShape(output); + TF_LITE_ENSURE(context, input_shape.DimensionsCount() >= 2); + TF_LITE_ENSURE_EQ(context, input_shape.DimensionsCount(), + output_shape.DimensionsCount() + 1); + + params->frame_size = input_shape.Dims(input_shape.DimensionsCount() - 1); + params->n_frames = input_shape.Dims(input_shape.DimensionsCount() - 2); + params->outer_dims = + input_shape.FlatSize() / (params->frame_size * params->n_frames); + params->state_buffers = static_cast(context->AllocatePersistentBuffer( + context, params->outer_dims * sizeof(T*))); + for (int i = 0; i < params->outer_dims; i++) { + params->state_buffers[i] = + static_cast(context->AllocatePersistentBuffer( + context, params->frame_size * sizeof(T))); + } + ResetState(params); + + micro_context->DeallocateTempTfLiteTensor(input); + micro_context->DeallocateTempTfLiteTensor(output); + return kTfLiteOk; +} + +template +TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { + auto* params = reinterpret_cast*>( + node->user_data); + const TfLiteEvalTensor* input = + tflite::micro::GetEvalInput(context, node, kInputTensor); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); + + const T* input_data = tflite::micro::GetTensorData(input); + T* output_data = tflite::micro::GetTensorData(output); + for (int i = 0; i < params->outer_dims; i++) { + T* buffer = params->state_buffers[i]; + for (int frame = 0; frame < params->n_frames; frame++) { + int input_index = (i * params->n_frames + frame) * params->frame_size; + int output_index = (i * params->n_frames + frame) * params->frame_step; + tflm_signal::OverlapAdd(&input_data[input_index], buffer, params->frame_size, + &output_data[output_index], params->frame_step); + } + } + return kTfLiteOk; +} + +template +void Reset(TfLiteContext* context, void* buffer) { + ResetState(static_cast*>(buffer)); +} + +void* InitAll(TfLiteContext* context, const char* buffer, size_t length) { + const uint8_t* buffer_t = reinterpret_cast(buffer); + const flexbuffers::Map& m = flexbuffers::GetRoot(buffer_t, length).AsMap(); + auto tensor_type = static_cast(m["T"].AsInt32()); + + switch (tensor_type) { + case TensorType_INT16: { + return Init(context, buffer, length); + } + case TensorType_FLOAT32: { + return Init(context, buffer, length); + } + default: + return nullptr; + } +} + +TfLiteStatus PrepareAll(TfLiteContext* context, TfLiteNode* node) { + auto* params = reinterpret_cast*>( + node->user_data); + + switch (params->type) { + case kTfLiteInt16: { + return Prepare(context, node); + } + case kTfLiteFloat32: { + return Prepare(context, node); + } + default: + return kTfLiteError; + } +} + +TfLiteStatus EvalAll(TfLiteContext* context, TfLiteNode* node) { + auto* params = reinterpret_cast*>( + node->user_data); + + switch (params->type) { + case kTfLiteInt16: { + return Eval(context, node); + } + case kTfLiteFloat32: { + return Eval(context, node); + } + default: + return kTfLiteError; + } +} + +void ResetAll(TfLiteContext* context, void* buffer) { + auto* params = + reinterpret_cast*>(buffer); + + switch (params->type) { + case kTfLiteInt16: { + Reset(context, buffer); + break; + } + case kTfLiteFloat32: { + Reset(context, buffer); + break; + } + default: + break; + } +} + +} // namespace + +namespace tflm_signal{ +TFLMRegistration* Register_OVERLAP_ADD() { + static TFLMRegistration r = tflite::micro::RegisterOp( + InitAll, PrepareAll, EvalAll, nullptr, ResetAll); + return &r; +} + + +TFLMRegistration* Register_OVERLAP_ADD_FLOAT() { + static TFLMRegistration r = + tflite::micro::RegisterOp(Init, Prepare, + Eval, nullptr, Reset); + return &r; +} + + +TFLMRegistration* Register_OVERLAP_ADD_INT16() { + static TFLMRegistration r = + tflite::micro::RegisterOp(Init, Prepare, + Eval, nullptr, Reset); + return &r; +} +} // namespace tflm_signal + +} // namespace tflite diff --git a/signal/micro/kernels/overlap_add_flexbuffers_generated_data.cc b/signal/micro/kernels/overlap_add_flexbuffers_generated_data.cc new file mode 100644 index 00000000000..7f810db1ccb --- /dev/null +++ b/signal/micro/kernels/overlap_add_flexbuffers_generated_data.cc @@ -0,0 +1,32 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This file is generated. See: +// tensorflow/lite/micro/kernels/test_data_generation/README.md + +#include "signal/micro/kernels/overlap_add_flexbuffers_generated_data.h" + +const int g_gen_data_size_overlap_add_float = 26; +const unsigned char g_gen_data_overlap_add_float[] = { + 0x66, 0x72, 0x61, 0x6d, 0x65, 0x5f, 0x73, 0x74, 0x65, + 0x70, 0x00, 0x54, 0x00, 0x02, 0x03, 0x0f, 0x02, 0x01, + 0x02, 0x00, 0x01, 0x04, 0x04, 0x04, 0x24, 0x01, +}; + +const int g_gen_data_size_overlap_add_int16 = 26; +const unsigned char g_gen_data_overlap_add_int16[] = { + 0x66, 0x72, 0x61, 0x6d, 0x65, 0x5f, 0x73, 0x74, 0x65, + 0x70, 0x00, 0x54, 0x00, 0x02, 0x03, 0x0f, 0x02, 0x01, + 0x02, 0x07, 0x01, 0x04, 0x04, 0x04, 0x24, 0x01, +}; diff --git a/signal/micro/kernels/overlap_add_flexbuffers_generated_data.h b/signal/micro/kernels/overlap_add_flexbuffers_generated_data.h new file mode 100644 index 00000000000..adb4fbab19d --- /dev/null +++ b/signal/micro/kernels/overlap_add_flexbuffers_generated_data.h @@ -0,0 +1,25 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef SIGNAL_MICRO_KERNELS_TEST_DATA_GENERATION_GENERATE_OVERLAP_ADD_FLEXBUFFERS_DATA_H_ +#define SIGNAL_MICRO_KERNELS_TEST_DATA_GENERATION_GENERATE_OVERLAP_ADD_FLEXBUFFERS_DATA_H_ + +extern const int g_gen_data_size_overlap_add_float; +extern const unsigned char g_gen_data_overlap_add_float[]; + +extern const int g_gen_data_size_overlap_add_int16; +extern const unsigned char g_gen_data_overlap_add_int16[]; + +#endif // SIGNAL_MICRO_KERNELS_TEST_DATA_GENERATION_GENERATE_OVERLAP_ADD_FLEXBUFFERS_DATA_H_ diff --git a/signal/micro/kernels/overlap_add_test.cc b/signal/micro/kernels/overlap_add_test.cc new file mode 100644 index 00000000000..976cdaf2ea0 --- /dev/null +++ b/signal/micro/kernels/overlap_add_test.cc @@ -0,0 +1,246 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "signal/micro/kernels/overlap_add_flexbuffers_generated_data.h" +#include "tensorflow/lite/micro/kernels/kernel_runner.h" +#include "tensorflow/lite/micro/test_helpers.h" +#include "tensorflow/lite/micro/testing/micro_test.h" + +namespace { + +constexpr int kFrameStepIndex = 1; +constexpr int kInputsSize = 1; +constexpr int kOutputsSize = 1; +constexpr int kTensorsSize = kInputsSize + kOutputsSize; + +template +class OverlapAddKernelRunner { + public: + OverlapAddKernelRunner(int* input_dims_data, T* input_data, + int* output_dims_data, T* output_data) + : inputs_array_{tflite::testing::IntArrayFromInts(inputs_array_data_)}, + outputs_array_{tflite::testing::IntArrayFromInts(outputs_array_data_)} { + tensors_[0] = tflite::testing::CreateTensor( + input_data, tflite::testing::IntArrayFromInts(input_dims_data)); + + tensors_[1] = tflite::testing::CreateTensor( + output_data, tflite::testing::IntArrayFromInts(output_dims_data)); + + registration_ = tflite::tflm_signal::Register_OVERLAP_ADD(); + + // go/tflm-static-cleanups for reasoning new is being used like this + kernel_runner_ = new (kernel_runner_buffer) tflite::micro::KernelRunner( + *registration_, tensors_, kTensorsSize, inputs_array_, outputs_array_, + /*builtin_data=*/nullptr); + } + + tflite::micro::KernelRunner& GetKernelRunner() { return *kernel_runner_; } + + private: + uint8_t kernel_runner_buffer[sizeof(tflite::micro::KernelRunner)]; + int inputs_array_data_[kInputsSize + 1] = {1, 0}; + int outputs_array_data_[kOutputsSize + 1] = {1, 1}; + TfLiteTensor tensors_[kTensorsSize] = {}; + TfLiteIntArray* inputs_array_ = nullptr; + TfLiteIntArray* outputs_array_ = nullptr; + TFLMRegistration* registration_ = nullptr; + tflite::micro::KernelRunner* kernel_runner_ = nullptr; +}; + +// We can use any of the templated types here - int16_t was picked arbitrarily +alignas(alignof(OverlapAddKernelRunner)) uint8_t + overlap_add_kernel_runner_buffer[sizeof(OverlapAddKernelRunner)]; + +template +void TestOverlapAddInvoke(int* input_dims_data, T* input_data, int* output_dims_data, + const T* golden_input, const T* golden_output, int iters, + const unsigned char* flexbuffers_data, + const unsigned int flexbuffers_data_size, T* output_data, + tflite::micro::KernelRunner* runner) { + + tflite::FlexbufferWrapper fbw(flexbuffers_data, flexbuffers_data_size); + int frame_step = fbw.ElementAsInt32(kFrameStepIndex); + int frame_size = input_dims_data[input_dims_data[0]]; + int n_frames = input_dims_data[input_dims_data[0] - 1]; + int outer_dims = 1; + for (int i = 1; i < input_dims_data[0] - 1; i++) { + outer_dims *= input_dims_data[i]; + } + for (int i = 0; i < iters; i++) { + for (int outer_dim = 0; outer_dim < outer_dims; outer_dim++) { + int input_idx = outer_dim * n_frames * frame_size; + int golden_input_idx = i * n_frames * frame_size; + memcpy(&input_data[input_idx], &golden_input[golden_input_idx], + n_frames * frame_size * sizeof(T)); + } + TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner->Invoke()); + for (int outer_dim = 0; outer_dim < outer_dims; outer_dim++) { + int output_idx = outer_dim * n_frames * frame_step; + int golden_output_idx = i * n_frames * frame_step; + TF_LITE_MICRO_EXPECT_EQ( + 0, memcmp(&output_data[output_idx], &golden_output[golden_output_idx], + n_frames * frame_step * sizeof(T))); + } + } +} + +template +void TestOverlapAdd(int* input_dims_data, T* input_data, int* output_dims_data, + const T* golden_input, const T* golden_output, int iters, + const unsigned char* flexbuffers_data, + const unsigned int flexbuffers_data_size, T* output_data) { + OverlapAddKernelRunner* overlap_add_runner = + new (overlap_add_kernel_runner_buffer) OverlapAddKernelRunner( + input_dims_data, input_data, output_dims_data, output_data); + // TfLite uses a char* for the raw bytes whereas flexbuffers use an unsigned + // char*. This small discrepancy results in compiler warnings unless we + // reinterpret_cast right before passing in the flexbuffer bytes to the + // KernelRunner. + TF_LITE_MICRO_EXPECT_EQ(overlap_add_runner->GetKernelRunner().InitAndPrepare( + reinterpret_cast(flexbuffers_data), + flexbuffers_data_size), + kTfLiteOk); + TestOverlapAddInvoke(input_dims_data, input_data, output_dims_data, golden_input, + golden_output, iters, flexbuffers_data, + flexbuffers_data_size, output_data, + &overlap_add_runner->GetKernelRunner()); +} + +template +void TestOverlapAddReset(int* input_dims_data, T* input_data, + int* output_dims_data, const T* golden_input, + const T* golden_output, int iters, + const unsigned char* flexbuffers_data, + const unsigned int flexbuffers_data_size, + T* output_data) { + OverlapAddKernelRunner* overlap_add_runner = + new (overlap_add_kernel_runner_buffer) OverlapAddKernelRunner( + input_dims_data, input_data, output_dims_data, output_data); + // TfLite uses a char* for the raw bytes whereas flexbuffers use an unsigned + // char*. This small discrepancy results in compiler warnings unless we + // reinterpret_cast right before passing in the flexbuffer bytes to the + // KernelRunner. + TF_LITE_MICRO_EXPECT_EQ(overlap_add_runner->GetKernelRunner().InitAndPrepare( + reinterpret_cast(flexbuffers_data), + flexbuffers_data_size), + kTfLiteOk); + TestOverlapAddInvoke(input_dims_data, input_data, output_dims_data, golden_input, + golden_output, iters, flexbuffers_data, flexbuffers_data_size, + output_data, &overlap_add_runner->GetKernelRunner()); + overlap_add_runner->GetKernelRunner().Reset(); + TestOverlapAddInvoke(input_dims_data, input_data, output_dims_data, golden_input, + golden_output, iters, flexbuffers_data, flexbuffers_data_size, + output_data, &overlap_add_runner->GetKernelRunner()); +} + +} // namespace + +TF_LITE_MICRO_TESTS_BEGIN + +TF_LITE_MICRO_TEST(OverlapAddTestInt16) { + const int kInputSize = 3; + const int kOutputSize = 1; + int input_dims_data[] = {2, 1, kInputSize}; + int output_dims_data[] = {1, kOutputSize}; + int16_t input_data[kInputSize]; + int16_t output_data = 0; + const int16_t golden_input[] = {125, -12, -895, 1000, 65, -212, + 63, 71, 52, 1, -17, 32}; + const int16_t golden_output[] = {125, 988, -767, -140}; + + TestOverlapAdd(input_dims_data, input_data, output_dims_data, golden_input, + golden_output, sizeof(golden_output) / sizeof(int16_t), + g_gen_data_overlap_add_int16, + g_gen_data_size_overlap_add_int16, &output_data); +} + +TF_LITE_MICRO_TEST(OverlapAddTestFloat) { + const int kInputSize = 3; + const int kOutputSize = 1; + int input_dims_data[] = {2, 1, kInputSize}; + int output_dims_data[] = {1, kOutputSize}; + float input_data[kInputSize]; + float output_data = 0; + const float golden_input[] = {12.5, -1.2, -89.5, 100.0, 6.5, -21.2, + 6.3, 7.1, 5.2, 0.1, -1.7, 3.2}; + const float golden_output[] = {12.5, 98.8, -76.7, -14.0}; + + TestOverlapAdd(input_dims_data, input_data, output_dims_data, golden_input, + golden_output, sizeof(golden_output) / sizeof(float), + g_gen_data_overlap_add_float, + g_gen_data_size_overlap_add_float, &output_data); +} + +TF_LITE_MICRO_TEST(OverlapAddTestNframes4Int16) { + const int kInputSize = 3; + const int kOutputSize = 1; + const int kNFrames = 4; + int input_dims_data[] = {2, kNFrames, kInputSize}; + int output_dims_data[] = {1, kNFrames * kOutputSize}; + int16_t input_data[kNFrames * kInputSize]; + int16_t output_data[kNFrames * kOutputSize]; + const int16_t golden_input[] = {125, -12, -895, 1000, 65, -212, + 63, 71, 52, 1, -17, 32}; + const int16_t golden_output[] = {125, 988, -767, -140}; + + const int kIters = + sizeof(golden_input) / kInputSize / kNFrames / sizeof(int16_t); + TestOverlapAdd(input_dims_data, input_data, output_dims_data, golden_input, + golden_output, kIters, g_gen_data_overlap_add_int16, + g_gen_data_size_overlap_add_int16, output_data); +} + +TF_LITE_MICRO_TEST(OverlapAddTestNframes4OuterDims4Int16) { + const int kInputSize = 3; + const int kOutputSize = 1; + const int kNFrames = 4; + int input_dims_data[] = {4, 2, 2, kNFrames, kInputSize}; + int output_dims_data[] = {3, 2, 2, kNFrames * kOutputSize}; + int16_t input_data[2 * 2 * kNFrames * kInputSize]; + int16_t output_data[2 * 2 * kNFrames * kOutputSize]; + const int16_t golden_input[] = {125, -12, -895, 1000, 65, -212, + 63, 71, 52, 1, -17, 32}; + const int16_t golden_output[] = {125, 988, -767, -140}; + + const int kIters = + sizeof(golden_input) / kInputSize / kNFrames / sizeof(int16_t); + TestOverlapAdd(input_dims_data, input_data, output_dims_data, golden_input, + golden_output, kIters, g_gen_data_overlap_add_int16, + g_gen_data_size_overlap_add_int16, output_data); +} + +TF_LITE_MICRO_TEST(testReset) { + const int kInputSize = 3; + const int kOutputSize = 1; + const int kNFrames = 4; + int input_dims_data[] = {4, 2, 2, kNFrames, kInputSize}; + int output_dims_data[] = {3, 2, 2, kNFrames * kOutputSize}; + int16_t input_data[2 * 2 * kNFrames * kInputSize]; + int16_t output_data[2 * 2 * kNFrames * kOutputSize]; + const int16_t golden_input[] = {125, -12, -895, 1000, 65, -212, + 63, 71, 52, 1, -17, 32}; + const int16_t golden_output[] = {125, 988, -767, -140}; + + const int kIters = + sizeof(golden_input) / kInputSize / kNFrames / sizeof(int16_t); + TestOverlapAddReset(input_dims_data, input_data, output_dims_data, + golden_input, golden_output, kIters, + g_gen_data_overlap_add_int16, + g_gen_data_size_overlap_add_int16, output_data); +} + +TF_LITE_MICRO_TESTS_END diff --git a/signal/src/BUILD b/signal/src/BUILD index c5e51fb2a40..ecb712bb1c8 100644 --- a/signal/src/BUILD +++ b/signal/src/BUILD @@ -3,12 +3,6 @@ package( licenses = ["notice"], ) -cc_library( - name = "circular_buffer", - srcs = ["circular_buffer.cc"], - hdrs = ["circular_buffer.h"], -) - cc_library( name = "complex", hdrs = ["complex.h"], @@ -33,3 +27,15 @@ cc_library( srcs = ["window.cc"], hdrs = ["window.h"], ) + +cc_library( + name = "circular_buffer", + srcs = ["circular_buffer.cc"], + hdrs = ["circular_buffer.h"], +) + +cc_library( + name = "overlap_add", + srcs = ["overlap_add.cc"], + hdrs = ["overlap_add.h"], +) diff --git a/signal/src/overlap_add.cc b/signal/src/overlap_add.cc new file mode 100644 index 00000000000..a9e2402b4be --- /dev/null +++ b/signal/src/overlap_add.cc @@ -0,0 +1,52 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "signal/src/overlap_add.h" + +#include +#include + +namespace tflm_signal { + + +void OverlapAdd(const int16_t* input, int16_t* buffer, int input_size, + int16_t* output, int output_size) { + for (int i = 0; i < input_size; ++i) { + int32_t overlap_added_sample = input[i] + buffer[i]; + if (overlap_added_sample < INT16_MIN) { + buffer[i] = INT16_MIN; + } else if (overlap_added_sample > INT16_MAX) { + buffer[i] = INT16_MAX; + } else { + buffer[i] = (int16_t)overlap_added_sample; + } + } + memcpy(output, buffer, output_size * sizeof(output[0])); + memmove(buffer, &buffer[output_size], + (input_size - output_size) * sizeof(buffer[0])); + memset(&buffer[input_size - output_size], 0, output_size * sizeof(buffer[0])); +} + +void OverlapAdd(const float* input, float* buffer, int input_size, + float* output, int output_size) { + for (int i = 0; i < input_size; ++i) { + buffer[i] += input[i]; + } + memcpy(output, buffer, output_size * sizeof(output[0])); + memmove(buffer, &buffer[output_size], + (input_size - output_size) * sizeof(buffer[0])); + memset(&buffer[input_size - output_size], 0, output_size * sizeof(buffer[0])); +} + +} // namespace tflm_signal diff --git a/signal/src/overlap_add.h b/signal/src/overlap_add.h new file mode 100644 index 00000000000..b4678444f67 --- /dev/null +++ b/signal/src/overlap_add.h @@ -0,0 +1,46 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef SIGNAL_SRC_OVERLAP_ADD_H_ +#define SIGNAL_SRC_OVERLAP_ADD_H_ + +#include +#include + +namespace tflm_signal{ +// Adds (with saturation) the contents of `input` to the contents of `buffer`, +// both of size `input_size`, then copies the first `output_size` elements of +// `buffer` to `output`, shifts the last `input_size`-`output_size` elements of +// `buffer` to the beginning of `buffer` and fills the trailing `output_size` +// samples in `buffer` with zeros. +// input: {input[0] ... input[input_size-1]} +// buffer: {buffer[0] ... buffer[input_size-1]} +// After invocation: +// output: {saturate(input[0] + buffer[0]), +// ... , +// saturate(input[output_size-1] + buffer[output_size-1])} +// buffer: {saturate(input[output_size] + buffer[output_size]), +// ... +// saturate( input[input_size-output_size-1] +// + buffer[input_size-output_size-1]), +// zeros(output_size)} +void OverlapAdd(const int16_t* input, int16_t* buffer, int input_size, + int16_t* output, int output_size); + +// The same as the int16_t variant above, but without saturation +void OverlapAdd(const float* input, float* buffer, int input_size, + float* output, int output_size); + +} // namespace tflm_signal +#endif // SIGNAL_SRC_OVERLAP_ADD_H_ diff --git a/tensorflow/lite/micro/kernels/Makefile.inc b/tensorflow/lite/micro/kernels/Makefile.inc index f07af9b8aa9..96fb8291870 100644 --- a/tensorflow/lite/micro/kernels/Makefile.inc +++ b/tensorflow/lite/micro/kernels/Makefile.inc @@ -59,6 +59,11 @@ $(eval $(call microlite_test,kernel_signal_framer_test,\ $(TENSORFLOW_ROOT)signal/micro/kernels/framer_flexbuffers_generated_data.cc, \ $(TENSORFLOW_ROOT)signal/micro/kernels/framer_flexbuffers_generated_data.h)) +$(eval $(call microlite_test,kernel_signal_overlap_add_test,\ + $(TENSORFLOW_ROOT)signal/micro/kernels/overlap_add_test.cc \ + $(TENSORFLOW_ROOT)signal/micro/kernels/overlap_add_flexbuffers_generated_data.cc, \ + $(TENSORFLOW_ROOT)signal/micro/kernels/overlap_add_flexbuffers_generated_data.h)) + $(eval $(call microlite_test,kernel_signal_window_test,\ $(TENSORFLOW_ROOT)signal/micro/kernels/window_test.cc \ $(TENSORFLOW_ROOT)signal/micro/kernels/window_flexbuffers_generated_data.cc, \ diff --git a/tensorflow/lite/micro/kernels/micro_ops.h b/tensorflow/lite/micro/kernels/micro_ops.h index 0a2391beebe..1a80fe7d32c 100644 --- a/tensorflow/lite/micro/kernels/micro_ops.h +++ b/tensorflow/lite/micro/kernels/micro_ops.h @@ -100,10 +100,8 @@ TFLMRegistration Register_READ_VARIABLE(); TFLMRegistration Register_REDUCE_MAX(); TFLMRegistration Register_RELU(); TFLMRegistration Register_RELU6(); -TFLMRegistration Register_RESHAPE(); TFLMRegistration Register_RESIZE_BILINEAR(); TFLMRegistration Register_RESIZE_NEAREST_NEIGHBOR(); -TFLMRegistration Register_ROUND(); TFLMRegistration Register_RSQRT(); TFLMRegistration Register_SELECT_V2(); TFLMRegistration Register_SHAPE(); @@ -135,9 +133,16 @@ TFLMRegistration Register_ZEROS_LIKE(); // TODO(b/160234179): Change custom OPs to also return by value. namespace tflm_signal { TFLMRegistration* Register_FRAMER(); +TFLMRegistration* Register_OVERLAP_ADD(); TFLMRegistration* Register_WINDOW(); } // namespace tflm_signal +namespace ops { +namespace micro { +TFLMRegistration Register_RESHAPE(); +TFLMRegistration Register_ROUND(); +} // namespace micro +} // namespace ops } // namespace tflite #endif // TENSORFLOW_LITE_MICRO_KERNELS_MICRO_OPS_H_ diff --git a/tensorflow/lite/micro/micro_mutable_op_resolver.h b/tensorflow/lite/micro/micro_mutable_op_resolver.h index 949ce267893..0ff5b194e80 100644 --- a/tensorflow/lite/micro/micro_mutable_op_resolver.h +++ b/tensorflow/lite/micro/micro_mutable_op_resolver.h @@ -22,6 +22,7 @@ limitations under the License. #include "tensorflow/lite/core/api/flatbuffer_conversions.h" #include "tensorflow/lite/kernels/internal/compatibility.h" #include "tensorflow/lite/kernels/op_macros.h" +#include "tensorflow/lite/schema/schema_generated.h" #include "tensorflow/lite/micro/compatibility.h" #include "tensorflow/lite/micro/kernels/add.h" #include "tensorflow/lite/micro/kernels/conv.h" @@ -34,7 +35,6 @@ limitations under the License. #include "tensorflow/lite/micro/kernels/softmax.h" #include "tensorflow/lite/micro/micro_log.h" #include "tensorflow/lite/micro/micro_op_resolver.h" -#include "tensorflow/lite/schema/schema_generated.h" namespace tflite { TFLMRegistration* Register_DETECTION_POSTPROCESS(); @@ -392,6 +392,12 @@ class MicroMutableOpResolver : public MicroOpResolver { ParseNotEqual); } + TfLiteStatus AddOverlapAdd() { + // TODO(b/286250473): change back name to "OverlapAdd" and remove namespace + return AddCustom("SignalOverlapAdd", + tflite::tflm_signal::Register_OVERLAP_ADD()); + } + TfLiteStatus AddPack() { return AddBuiltin(BuiltinOperator_PACK, Register_PACK(), ParsePack); } @@ -434,8 +440,8 @@ class MicroMutableOpResolver : public MicroOpResolver { } TfLiteStatus AddReshape() { - return AddBuiltin(BuiltinOperator_RESHAPE, Register_RESHAPE(), - ParseReshape); + return AddBuiltin(BuiltinOperator_RESHAPE, + tflite::ops::micro::Register_RESHAPE(), ParseReshape); } TfLiteStatus AddResizeBilinear() { @@ -456,7 +462,8 @@ class MicroMutableOpResolver : public MicroOpResolver { } TfLiteStatus AddRound() { - return AddBuiltin(BuiltinOperator_ROUND, Register_ROUND(), ParseRound); + return AddBuiltin(BuiltinOperator_ROUND, + tflite::ops::micro::Register_ROUND(), ParseRound); } TfLiteStatus AddRsqrt() { diff --git a/tensorflow/lite/micro/tools/make/Makefile b/tensorflow/lite/micro/tools/make/Makefile index 9a0ed1904aa..02df92e0b76 100644 --- a/tensorflow/lite/micro/tools/make/Makefile +++ b/tensorflow/lite/micro/tools/make/Makefile @@ -80,7 +80,6 @@ DOWNLOADS_DIR := $(MAKEFILE_DIR)/downloads INCLUDES := \ -I. \ --I$(DOWNLOADS_DIR) \ -I$(DOWNLOADS_DIR)/gemmlowp \ -I$(DOWNLOADS_DIR)/flatbuffers/include \ -I$(DOWNLOADS_DIR)/kissfft \ @@ -299,6 +298,7 @@ $(TENSORFLOW_ROOT)tensorflow/lite/micro/micro_log_test.cc \ $(TENSORFLOW_ROOT)tensorflow/lite/micro/micro_interpreter_test.cc \ $(TENSORFLOW_ROOT)tensorflow/lite/micro/micro_mutable_op_resolver_test.cc \ $(TENSORFLOW_ROOT)tensorflow/lite/micro/micro_resource_variable_test.cc \ +$(TENSORFLOW_ROOT)tensorflow/lite/micro/micro_string_test.cc \ $(TENSORFLOW_ROOT)tensorflow/lite/micro/micro_time_test.cc \ $(TENSORFLOW_ROOT)tensorflow/lite/micro/micro_utils_test.cc \ $(TENSORFLOW_ROOT)tensorflow/lite/micro/recording_micro_allocator_test.cc \ @@ -314,8 +314,10 @@ $(TENSORFLOW_ROOT)tensorflow/lite/micro/memory_planner/non_persistent_buffer_pla MICROLITE_CC_KERNEL_SRCS := \ $(TENSORFLOW_ROOT)signal/micro/kernels/framer.cc \ $(TENSORFLOW_ROOT)signal/micro/kernels/rfft.cc \ +$(TENSORFLOW_ROOT)signal/micro/kernels/overlap_add.cc \ $(TENSORFLOW_ROOT)signal/micro/kernels/window.cc \ $(TENSORFLOW_ROOT)signal/src/circular_buffer.cc \ +$(TENSORFLOW_ROOT)signal/src/overlap_add.cc \ $(TENSORFLOW_ROOT)signal/src/rfft_float.cc \ $(TENSORFLOW_ROOT)signal/src/rfft_int16.cc \ $(TENSORFLOW_ROOT)signal/src/rfft_int32.cc \ From 499a9fbe2e69c2e513d58a4eb13da20fc68355cb Mon Sep 17 00:00:00 2001 From: Steven Toribio Date: Tue, 11 Jul 2023 20:11:41 +0000 Subject: [PATCH 2/5] mistake fix --- tensorflow/lite/micro/kernels/micro_ops.h | 8 ++------ tensorflow/lite/micro/micro_mutable_op_resolver.h | 7 +++---- tensorflow/lite/micro/tools/make/Makefile | 1 + 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/tensorflow/lite/micro/kernels/micro_ops.h b/tensorflow/lite/micro/kernels/micro_ops.h index 1a80fe7d32c..bfc21611a9d 100644 --- a/tensorflow/lite/micro/kernels/micro_ops.h +++ b/tensorflow/lite/micro/kernels/micro_ops.h @@ -100,8 +100,10 @@ TFLMRegistration Register_READ_VARIABLE(); TFLMRegistration Register_REDUCE_MAX(); TFLMRegistration Register_RELU(); TFLMRegistration Register_RELU6(); +TFLMRegistration Register_RESHAPE(); TFLMRegistration Register_RESIZE_BILINEAR(); TFLMRegistration Register_RESIZE_NEAREST_NEIGHBOR(); +TFLMRegistration Register_ROUND(); TFLMRegistration Register_RSQRT(); TFLMRegistration Register_SELECT_V2(); TFLMRegistration Register_SHAPE(); @@ -137,12 +139,6 @@ TFLMRegistration* Register_OVERLAP_ADD(); TFLMRegistration* Register_WINDOW(); } // namespace tflm_signal -namespace ops { -namespace micro { -TFLMRegistration Register_RESHAPE(); -TFLMRegistration Register_ROUND(); -} // namespace micro -} // namespace ops } // namespace tflite #endif // TENSORFLOW_LITE_MICRO_KERNELS_MICRO_OPS_H_ diff --git a/tensorflow/lite/micro/micro_mutable_op_resolver.h b/tensorflow/lite/micro/micro_mutable_op_resolver.h index 0ff5b194e80..bc13e23979b 100644 --- a/tensorflow/lite/micro/micro_mutable_op_resolver.h +++ b/tensorflow/lite/micro/micro_mutable_op_resolver.h @@ -440,8 +440,8 @@ class MicroMutableOpResolver : public MicroOpResolver { } TfLiteStatus AddReshape() { - return AddBuiltin(BuiltinOperator_RESHAPE, - tflite::ops::micro::Register_RESHAPE(), ParseReshape); + return AddBuiltin(BuiltinOperator_RESHAPE, Register_RESHAPE(), + ParseReshape); } TfLiteStatus AddResizeBilinear() { @@ -462,8 +462,7 @@ class MicroMutableOpResolver : public MicroOpResolver { } TfLiteStatus AddRound() { - return AddBuiltin(BuiltinOperator_ROUND, - tflite::ops::micro::Register_ROUND(), ParseRound); + return AddBuiltin(BuiltinOperator_ROUND, Register_ROUND(), ParseRound); } TfLiteStatus AddRsqrt() { diff --git a/tensorflow/lite/micro/tools/make/Makefile b/tensorflow/lite/micro/tools/make/Makefile index 02df92e0b76..411ad369f7f 100644 --- a/tensorflow/lite/micro/tools/make/Makefile +++ b/tensorflow/lite/micro/tools/make/Makefile @@ -80,6 +80,7 @@ DOWNLOADS_DIR := $(MAKEFILE_DIR)/downloads INCLUDES := \ -I. \ +-I$(DOWNLOADS_DIR) \ -I$(DOWNLOADS_DIR)/gemmlowp \ -I$(DOWNLOADS_DIR)/flatbuffers/include \ -I$(DOWNLOADS_DIR)/kissfft \ From b919609f30b9369d36a89683dcdd02dfbe1d77c6 Mon Sep 17 00:00:00 2001 From: Steven Toribio Date: Tue, 11 Jul 2023 20:17:15 +0000 Subject: [PATCH 3/5] style fix --- signal/micro/kernels/overlap_add.cc | 36 +++++++++---------- signal/micro/kernels/overlap_add_test.cc | 34 +++++++++--------- signal/src/overlap_add.cc | 3 +- signal/src/overlap_add.h | 4 +-- .../lite/micro/micro_mutable_op_resolver.h | 2 +- 5 files changed, 39 insertions(+), 40 deletions(-) diff --git a/signal/micro/kernels/overlap_add.cc b/signal/micro/kernels/overlap_add.cc index bc3cfabf9c6..d3725670ff8 100644 --- a/signal/micro/kernels/overlap_add.cc +++ b/signal/micro/kernels/overlap_add.cc @@ -19,9 +19,9 @@ limitations under the License. #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/kernel_util.h" -#include "tensorflow/lite/portable_type_to_tflitetype.h" #include "tensorflow/lite/micro/flatbuffer_utils.h" #include "tensorflow/lite/micro/kernels/kernel_util.h" +#include "tensorflow/lite/portable_type_to_tflitetype.h" namespace tflite { namespace { @@ -59,8 +59,8 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) { const uint8_t* buffer_t = reinterpret_cast(buffer); auto* params = static_cast*>( - context->AllocatePersistentBuffer( - context, sizeof(TFLMSignalOverlapAddParams))); + context->AllocatePersistentBuffer(context, + sizeof(TFLMSignalOverlapAddParams))); tflite::FlexbufferWrapper fbw(buffer_t, length); params->type = typeToTfLiteType(); @@ -87,8 +87,8 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE_TYPES_EQ(context, input->type, TfLiteTypeEnum); TF_LITE_ENSURE_TYPES_EQ(context, output->type, TfLiteTypeEnum); - auto* params = reinterpret_cast*>( - node->user_data); + auto* params = + reinterpret_cast*>(node->user_data); RuntimeShape input_shape = GetTensorShape(input); RuntimeShape output_shape = GetTensorShape(output); TF_LITE_ENSURE(context, input_shape.DimensionsCount() >= 2); @@ -115,8 +115,8 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { template TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { - auto* params = reinterpret_cast*>( - node->user_data); + auto* params = + reinterpret_cast*>(node->user_data); const TfLiteEvalTensor* input = tflite::micro::GetEvalInput(context, node, kInputTensor); TfLiteEvalTensor* output = @@ -129,8 +129,9 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { for (int frame = 0; frame < params->n_frames; frame++) { int input_index = (i * params->n_frames + frame) * params->frame_size; int output_index = (i * params->n_frames + frame) * params->frame_step; - tflm_signal::OverlapAdd(&input_data[input_index], buffer, params->frame_size, - &output_data[output_index], params->frame_step); + tflm_signal::OverlapAdd(&input_data[input_index], buffer, + params->frame_size, &output_data[output_index], + params->frame_step); } } return kTfLiteOk; @@ -159,8 +160,8 @@ void* InitAll(TfLiteContext* context, const char* buffer, size_t length) { } TfLiteStatus PrepareAll(TfLiteContext* context, TfLiteNode* node) { - auto* params = reinterpret_cast*>( - node->user_data); + auto* params = + reinterpret_cast*>(node->user_data); switch (params->type) { case kTfLiteInt16: { @@ -175,8 +176,8 @@ TfLiteStatus PrepareAll(TfLiteContext* context, TfLiteNode* node) { } TfLiteStatus EvalAll(TfLiteContext* context, TfLiteNode* node) { - auto* params = reinterpret_cast*>( - node->user_data); + auto* params = + reinterpret_cast*>(node->user_data); switch (params->type) { case kTfLiteInt16: { @@ -191,8 +192,7 @@ TfLiteStatus EvalAll(TfLiteContext* context, TfLiteNode* node) { } void ResetAll(TfLiteContext* context, void* buffer) { - auto* params = - reinterpret_cast*>(buffer); + auto* params = reinterpret_cast*>(buffer); switch (params->type) { case kTfLiteInt16: { @@ -210,14 +210,13 @@ void ResetAll(TfLiteContext* context, void* buffer) { } // namespace -namespace tflm_signal{ +namespace tflm_signal { TFLMRegistration* Register_OVERLAP_ADD() { static TFLMRegistration r = tflite::micro::RegisterOp( InitAll, PrepareAll, EvalAll, nullptr, ResetAll); return &r; } - TFLMRegistration* Register_OVERLAP_ADD_FLOAT() { static TFLMRegistration r = tflite::micro::RegisterOp(Init, Prepare, @@ -225,13 +224,12 @@ TFLMRegistration* Register_OVERLAP_ADD_FLOAT() { return &r; } - TFLMRegistration* Register_OVERLAP_ADD_INT16() { static TFLMRegistration r = tflite::micro::RegisterOp(Init, Prepare, Eval, nullptr, Reset); return &r; } -} // namespace tflm_signal +} // namespace tflm_signal } // namespace tflite diff --git a/signal/micro/kernels/overlap_add_test.cc b/signal/micro/kernels/overlap_add_test.cc index 976cdaf2ea0..155dad50946 100644 --- a/signal/micro/kernels/overlap_add_test.cc +++ b/signal/micro/kernels/overlap_add_test.cc @@ -66,12 +66,12 @@ alignas(alignof(OverlapAddKernelRunner)) uint8_t overlap_add_kernel_runner_buffer[sizeof(OverlapAddKernelRunner)]; template -void TestOverlapAddInvoke(int* input_dims_data, T* input_data, int* output_dims_data, - const T* golden_input, const T* golden_output, int iters, - const unsigned char* flexbuffers_data, - const unsigned int flexbuffers_data_size, T* output_data, - tflite::micro::KernelRunner* runner) { - +void TestOverlapAddInvoke(int* input_dims_data, T* input_data, + int* output_dims_data, const T* golden_input, + const T* golden_output, int iters, + const unsigned char* flexbuffers_data, + const unsigned int flexbuffers_data_size, + T* output_data, tflite::micro::KernelRunner* runner) { tflite::FlexbufferWrapper fbw(flexbuffers_data, flexbuffers_data_size); int frame_step = fbw.ElementAsInt32(kFrameStepIndex); int frame_size = input_dims_data[input_dims_data[0]]; @@ -114,10 +114,10 @@ void TestOverlapAdd(int* input_dims_data, T* input_data, int* output_dims_data, reinterpret_cast(flexbuffers_data), flexbuffers_data_size), kTfLiteOk); - TestOverlapAddInvoke(input_dims_data, input_data, output_dims_data, golden_input, - golden_output, iters, flexbuffers_data, - flexbuffers_data_size, output_data, - &overlap_add_runner->GetKernelRunner()); + TestOverlapAddInvoke(input_dims_data, input_data, output_dims_data, + golden_input, golden_output, iters, flexbuffers_data, + flexbuffers_data_size, output_data, + &overlap_add_runner->GetKernelRunner()); } template @@ -138,13 +138,15 @@ void TestOverlapAddReset(int* input_dims_data, T* input_data, reinterpret_cast(flexbuffers_data), flexbuffers_data_size), kTfLiteOk); - TestOverlapAddInvoke(input_dims_data, input_data, output_dims_data, golden_input, - golden_output, iters, flexbuffers_data, flexbuffers_data_size, - output_data, &overlap_add_runner->GetKernelRunner()); + TestOverlapAddInvoke(input_dims_data, input_data, output_dims_data, + golden_input, golden_output, iters, flexbuffers_data, + flexbuffers_data_size, output_data, + &overlap_add_runner->GetKernelRunner()); overlap_add_runner->GetKernelRunner().Reset(); - TestOverlapAddInvoke(input_dims_data, input_data, output_dims_data, golden_input, - golden_output, iters, flexbuffers_data, flexbuffers_data_size, - output_data, &overlap_add_runner->GetKernelRunner()); + TestOverlapAddInvoke(input_dims_data, input_data, output_dims_data, + golden_input, golden_output, iters, flexbuffers_data, + flexbuffers_data_size, output_data, + &overlap_add_runner->GetKernelRunner()); } } // namespace diff --git a/signal/src/overlap_add.cc b/signal/src/overlap_add.cc index a9e2402b4be..fe6e6646ab8 100644 --- a/signal/src/overlap_add.cc +++ b/signal/src/overlap_add.cc @@ -19,7 +19,6 @@ limitations under the License. namespace tflm_signal { - void OverlapAdd(const int16_t* input, int16_t* buffer, int input_size, int16_t* output, int output_size) { for (int i = 0; i < input_size; ++i) { @@ -49,4 +48,4 @@ void OverlapAdd(const float* input, float* buffer, int input_size, memset(&buffer[input_size - output_size], 0, output_size * sizeof(buffer[0])); } -} // namespace tflm_signal +} // namespace tflm_signal diff --git a/signal/src/overlap_add.h b/signal/src/overlap_add.h index b4678444f67..61898094c99 100644 --- a/signal/src/overlap_add.h +++ b/signal/src/overlap_add.h @@ -18,7 +18,7 @@ limitations under the License. #include #include -namespace tflm_signal{ +namespace tflm_signal { // Adds (with saturation) the contents of `input` to the contents of `buffer`, // both of size `input_size`, then copies the first `output_size` elements of // `buffer` to `output`, shifts the last `input_size`-`output_size` elements of @@ -42,5 +42,5 @@ void OverlapAdd(const int16_t* input, int16_t* buffer, int input_size, void OverlapAdd(const float* input, float* buffer, int input_size, float* output, int output_size); -} // namespace tflm_signal +} // namespace tflm_signal #endif // SIGNAL_SRC_OVERLAP_ADD_H_ diff --git a/tensorflow/lite/micro/micro_mutable_op_resolver.h b/tensorflow/lite/micro/micro_mutable_op_resolver.h index bc13e23979b..c472b55d6c5 100644 --- a/tensorflow/lite/micro/micro_mutable_op_resolver.h +++ b/tensorflow/lite/micro/micro_mutable_op_resolver.h @@ -22,7 +22,6 @@ limitations under the License. #include "tensorflow/lite/core/api/flatbuffer_conversions.h" #include "tensorflow/lite/kernels/internal/compatibility.h" #include "tensorflow/lite/kernels/op_macros.h" -#include "tensorflow/lite/schema/schema_generated.h" #include "tensorflow/lite/micro/compatibility.h" #include "tensorflow/lite/micro/kernels/add.h" #include "tensorflow/lite/micro/kernels/conv.h" @@ -35,6 +34,7 @@ limitations under the License. #include "tensorflow/lite/micro/kernels/softmax.h" #include "tensorflow/lite/micro/micro_log.h" #include "tensorflow/lite/micro/micro_op_resolver.h" +#include "tensorflow/lite/schema/schema_generated.h" namespace tflite { TFLMRegistration* Register_DETECTION_POSTPROCESS(); From 8c2b2834722079b03ececc640bb3b8d2aa80f9a8 Mon Sep 17 00:00:00 2001 From: Steven Toribio Date: Tue, 11 Jul 2023 20:28:36 +0000 Subject: [PATCH 4/5] makefile --- tensorflow/lite/micro/tools/make/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/lite/micro/tools/make/Makefile b/tensorflow/lite/micro/tools/make/Makefile index 411ad369f7f..2cb056d8f23 100644 --- a/tensorflow/lite/micro/tools/make/Makefile +++ b/tensorflow/lite/micro/tools/make/Makefile @@ -299,7 +299,6 @@ $(TENSORFLOW_ROOT)tensorflow/lite/micro/micro_log_test.cc \ $(TENSORFLOW_ROOT)tensorflow/lite/micro/micro_interpreter_test.cc \ $(TENSORFLOW_ROOT)tensorflow/lite/micro/micro_mutable_op_resolver_test.cc \ $(TENSORFLOW_ROOT)tensorflow/lite/micro/micro_resource_variable_test.cc \ -$(TENSORFLOW_ROOT)tensorflow/lite/micro/micro_string_test.cc \ $(TENSORFLOW_ROOT)tensorflow/lite/micro/micro_time_test.cc \ $(TENSORFLOW_ROOT)tensorflow/lite/micro/micro_utils_test.cc \ $(TENSORFLOW_ROOT)tensorflow/lite/micro/recording_micro_allocator_test.cc \ From 4947dad075997b957dec95e0ed7b0ff9a499d125 Mon Sep 17 00:00:00 2001 From: Steven Toribio Date: Tue, 11 Jul 2023 23:02:40 +0000 Subject: [PATCH 5/5] fixes --- signal/micro/kernels/overlap_add.cc | 6 +++ signal/micro/kernels/overlap_add_test.cc | 62 +++++++++++++----------- 2 files changed, 39 insertions(+), 29 deletions(-) diff --git a/signal/micro/kernels/overlap_add.cc b/signal/micro/kernels/overlap_add.cc index d3725670ff8..8de74630214 100644 --- a/signal/micro/kernels/overlap_add.cc +++ b/signal/micro/kernels/overlap_add.cc @@ -62,6 +62,10 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) { context->AllocatePersistentBuffer(context, sizeof(TFLMSignalOverlapAddParams))); + if (params == nullptr) { + return nullptr; + } + tflite::FlexbufferWrapper fbw(buffer_t, length); params->type = typeToTfLiteType(); params->frame_step = fbw.ElementAsInt32(kFrameStepIndex); @@ -101,6 +105,8 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { input_shape.FlatSize() / (params->frame_size * params->n_frames); params->state_buffers = static_cast(context->AllocatePersistentBuffer( context, params->outer_dims * sizeof(T*))); + TF_LITE_ENSURE(context, params != nullptr); + for (int i = 0; i < params->outer_dims; i++) { params->state_buffers[i] = static_cast(context->AllocatePersistentBuffer( diff --git a/signal/micro/kernels/overlap_add_test.cc b/signal/micro/kernels/overlap_add_test.cc index 155dad50946..f7b9f522748 100644 --- a/signal/micro/kernels/overlap_add_test.cc +++ b/signal/micro/kernels/overlap_add_test.cc @@ -20,7 +20,7 @@ limitations under the License. #include "tensorflow/lite/micro/test_helpers.h" #include "tensorflow/lite/micro/testing/micro_test.h" -namespace { +namespace tflite { constexpr int kFrameStepIndex = 1; constexpr int kInputsSize = 1; @@ -32,15 +32,15 @@ class OverlapAddKernelRunner { public: OverlapAddKernelRunner(int* input_dims_data, T* input_data, int* output_dims_data, T* output_data) - : inputs_array_{tflite::testing::IntArrayFromInts(inputs_array_data_)}, - outputs_array_{tflite::testing::IntArrayFromInts(outputs_array_data_)} { - tensors_[0] = tflite::testing::CreateTensor( - input_data, tflite::testing::IntArrayFromInts(input_dims_data)); + : inputs_array_{testing::IntArrayFromInts(inputs_array_data_)}, + outputs_array_{testing::IntArrayFromInts(outputs_array_data_)} { + tensors_[0] = testing::CreateTensor( + input_data, testing::IntArrayFromInts(input_dims_data)); tensors_[1] = tflite::testing::CreateTensor( - output_data, tflite::testing::IntArrayFromInts(output_dims_data)); + output_data, testing::IntArrayFromInts(output_dims_data)); - registration_ = tflite::tflm_signal::Register_OVERLAP_ADD(); + registration_ = tflm_signal::Register_OVERLAP_ADD(); // go/tflm-static-cleanups for reasoning new is being used like this kernel_runner_ = new (kernel_runner_buffer) tflite::micro::KernelRunner( @@ -48,17 +48,17 @@ class OverlapAddKernelRunner { /*builtin_data=*/nullptr); } - tflite::micro::KernelRunner& GetKernelRunner() { return *kernel_runner_; } + micro::KernelRunner& GetKernelRunner() { return *kernel_runner_; } private: - uint8_t kernel_runner_buffer[sizeof(tflite::micro::KernelRunner)]; + uint8_t kernel_runner_buffer[sizeof(micro::KernelRunner)]; int inputs_array_data_[kInputsSize + 1] = {1, 0}; int outputs_array_data_[kOutputsSize + 1] = {1, 1}; TfLiteTensor tensors_[kTensorsSize] = {}; TfLiteIntArray* inputs_array_ = nullptr; TfLiteIntArray* outputs_array_ = nullptr; TFLMRegistration* registration_ = nullptr; - tflite::micro::KernelRunner* kernel_runner_ = nullptr; + micro::KernelRunner* kernel_runner_ = nullptr; }; // We can use any of the templated types here - int16_t was picked arbitrarily @@ -149,7 +149,7 @@ void TestOverlapAddReset(int* input_dims_data, T* input_data, &overlap_add_runner->GetKernelRunner()); } -} // namespace +} // namespace tflite TF_LITE_MICRO_TESTS_BEGIN @@ -164,10 +164,11 @@ TF_LITE_MICRO_TEST(OverlapAddTestInt16) { 63, 71, 52, 1, -17, 32}; const int16_t golden_output[] = {125, 988, -767, -140}; - TestOverlapAdd(input_dims_data, input_data, output_dims_data, golden_input, - golden_output, sizeof(golden_output) / sizeof(int16_t), - g_gen_data_overlap_add_int16, - g_gen_data_size_overlap_add_int16, &output_data); + tflite::TestOverlapAdd(input_dims_data, input_data, output_dims_data, + golden_input, golden_output, + sizeof(golden_output) / sizeof(int16_t), + g_gen_data_overlap_add_int16, + g_gen_data_size_overlap_add_int16, &output_data); } TF_LITE_MICRO_TEST(OverlapAddTestFloat) { @@ -181,10 +182,11 @@ TF_LITE_MICRO_TEST(OverlapAddTestFloat) { 6.3, 7.1, 5.2, 0.1, -1.7, 3.2}; const float golden_output[] = {12.5, 98.8, -76.7, -14.0}; - TestOverlapAdd(input_dims_data, input_data, output_dims_data, golden_input, - golden_output, sizeof(golden_output) / sizeof(float), - g_gen_data_overlap_add_float, - g_gen_data_size_overlap_add_float, &output_data); + tflite::TestOverlapAdd(input_dims_data, input_data, output_dims_data, + golden_input, golden_output, + sizeof(golden_output) / sizeof(float), + g_gen_data_overlap_add_float, + g_gen_data_size_overlap_add_float, &output_data); } TF_LITE_MICRO_TEST(OverlapAddTestNframes4Int16) { @@ -201,9 +203,10 @@ TF_LITE_MICRO_TEST(OverlapAddTestNframes4Int16) { const int kIters = sizeof(golden_input) / kInputSize / kNFrames / sizeof(int16_t); - TestOverlapAdd(input_dims_data, input_data, output_dims_data, golden_input, - golden_output, kIters, g_gen_data_overlap_add_int16, - g_gen_data_size_overlap_add_int16, output_data); + tflite::TestOverlapAdd(input_dims_data, input_data, output_dims_data, + golden_input, golden_output, kIters, + g_gen_data_overlap_add_int16, + g_gen_data_size_overlap_add_int16, output_data); } TF_LITE_MICRO_TEST(OverlapAddTestNframes4OuterDims4Int16) { @@ -220,9 +223,10 @@ TF_LITE_MICRO_TEST(OverlapAddTestNframes4OuterDims4Int16) { const int kIters = sizeof(golden_input) / kInputSize / kNFrames / sizeof(int16_t); - TestOverlapAdd(input_dims_data, input_data, output_dims_data, golden_input, - golden_output, kIters, g_gen_data_overlap_add_int16, - g_gen_data_size_overlap_add_int16, output_data); + tflite::TestOverlapAdd(input_dims_data, input_data, output_dims_data, + golden_input, golden_output, kIters, + g_gen_data_overlap_add_int16, + g_gen_data_size_overlap_add_int16, output_data); } TF_LITE_MICRO_TEST(testReset) { @@ -239,10 +243,10 @@ TF_LITE_MICRO_TEST(testReset) { const int kIters = sizeof(golden_input) / kInputSize / kNFrames / sizeof(int16_t); - TestOverlapAddReset(input_dims_data, input_data, output_dims_data, - golden_input, golden_output, kIters, - g_gen_data_overlap_add_int16, - g_gen_data_size_overlap_add_int16, output_data); + tflite::TestOverlapAddReset(input_dims_data, input_data, output_dims_data, + golden_input, golden_output, kIters, + g_gen_data_overlap_add_int16, + g_gen_data_size_overlap_add_int16, output_data); } TF_LITE_MICRO_TESTS_END