Replacing the ancient vm_util with function_io/function_util. (iree-o…

…rg#16351)

The new code is written using iree_io_stream_t instead of directly using
files so that we have a consistent I/O interface. This also will allow
us to use the flag handling code on platforms without stdio or from
tools where we may want to avoid hitting the filesystem.

The old `iree/base/internal/file_io.h` utilities aren't dead yet as
they're currently the only place we support memory mapping files. Future
changes will add a platform-specific file-backed stream type that allows
for mapping so that we can change/enhance all APIs dealing with files to
support mapping and bare metal execution without the need for the giant
IREE_FILE_IO_ENABLE flag.

The existing flag parsing is mostly compatible modulo changing numpy
multi-array expansion to the explicit form of `--input=*foo.npy` to
indicate that all remaining arrays in a file should be inserted. The
parsing now supports consuming multiple inputs from a single binary file
or singular arrays from a numpy file by using the same syntax as
`--output=` has supported with `@` indicating start of file and `+`
indicating continuation.

To support all of this two new stream types were added:
`iree_io_stdio_stream_t` using stdio to read/write files and
`iree_io_vec_stream_t` using a growable vector of blocks (ala iovec)
that allocates new blocks of memory as the stream is written.

compiler/src/iree/compiler/ConstEval/BUILD.bazel

@@ -79,7 +79,6 @@ iree_compiler_cc_library(
         "//runtime/src/iree/hal",
         "//runtime/src/iree/hal/drivers/local_task/registration",
         "//runtime/src/iree/modules/hal",
-        "//runtime/src/iree/tooling:vm_util",
         "//runtime/src/iree/vm",
         "//runtime/src/iree/vm/bytecode:module",
         "@llvm-project//llvm:Support",

compiler/src/iree/compiler/ConstEval/CMakeLists.txt

@@ -71,7 +71,6 @@ iree_cc_library(
     iree::hal
     iree::hal::drivers::local_task::registration
     iree::modules::hal
-    iree::tooling::vm_util
     iree::vm
     iree::vm::bytecode::module
   PUBLIC

runtime/src/iree/base/string_builder.c

@@ -104,6 +104,10 @@ IREE_API_EXPORT iree_status_t iree_string_builder_reserve(
   return iree_ok_status();
 }
 
+IREE_API_EXPORT void iree_string_builder_reset(iree_string_builder_t* builder) {
+  builder->size = 0;
+}
+
 IREE_API_EXPORT iree_status_t iree_string_builder_append_inline(
     iree_string_builder_t* builder, iree_host_size_t count, char** out_head) {
   *out_head = NULL;

runtime/src/iree/base/string_builder.h

@@ -106,6 +106,9 @@ IREE_API_EXPORT IREE_MUST_USE_RESULT char* iree_string_builder_take_storage(
 IREE_API_EXPORT iree_status_t iree_string_builder_reserve(
     iree_string_builder_t* builder, iree_host_size_t minimum_capacity);
 
+// Resets the string builder length to 0 without releasing storage.
+IREE_API_EXPORT void iree_string_builder_reset(iree_string_builder_t* builder);
+
 // Reserves storage for |count| characters (including NUL) and returns a mutable
 // pointer in |out_head| for the caller to write the characters.
 // The pointer is only valid so long as the string builder is initialized and

runtime/src/iree/io/BUILD.bazel

@@ -14,27 +14,42 @@ package(
 
 iree_runtime_cc_library(
     name = "file_handle",
-    srcs = [
-        "file_handle.c",
-    ],
-    hdrs = [
-        "file_handle.h",
-    ],
+    srcs = ["file_handle.c"],
+    hdrs = ["file_handle.h"],
     deps = [
+        ":memory_stream",
         ":stream",
         "//runtime/src/iree/base",
         "//runtime/src/iree/base/internal",
     ],
 )
 
 iree_runtime_cc_library(
-    name = "parameter_index",
-    srcs = [
-        "parameter_index.c",
+    name = "memory_stream",
+    srcs = ["memory_stream.c"],
+    hdrs = ["memory_stream.h"],
+    deps = [
+        ":stream",
+        "//runtime/src/iree/base",
+        "//runtime/src/iree/base/internal",
     ],
-    hdrs = [
-        "parameter_index.h",
+)
+
+iree_runtime_cc_test(
+    name = "memory_stream_test",
+    srcs = ["memory_stream_test.cc"],
+    deps = [
+        ":memory_stream",
+        "//runtime/src/iree/base",
+        "//runtime/src/iree/testing:gtest",
+        "//runtime/src/iree/testing:gtest_main",
     ],
+)
+
+iree_runtime_cc_library(
+    name = "parameter_index",
+    srcs = ["parameter_index.c"],
+    hdrs = ["parameter_index.h"],
     deps = [
         ":file_handle",
         "//runtime/src/iree/base",
@@ -45,12 +60,8 @@ iree_runtime_cc_library(
 
 iree_runtime_cc_library(
     name = "parameter_index_provider",
-    srcs = [
-        "parameter_index_provider.c",
-    ],
-    hdrs = [
-        "parameter_index_provider.h",
-    ],
+    srcs = ["parameter_index_provider.c"],
+    hdrs = ["parameter_index_provider.h"],
     deps = [
         ":parameter_index",
         ":parameter_provider",
@@ -62,12 +73,8 @@ iree_runtime_cc_library(
 
 iree_runtime_cc_library(
     name = "parameter_provider",
-    srcs = [
-        "parameter_provider.c",
-    ],
-    hdrs = [
-        "parameter_provider.h",
-    ],
+    srcs = ["parameter_provider.c"],
+    hdrs = ["parameter_provider.h"],
     deps = [
         "//runtime/src/iree/base",
         "//runtime/src/iree/hal",
@@ -76,12 +83,8 @@ iree_runtime_cc_library(
 
 iree_runtime_cc_library(
     name = "scope_map",
-    srcs = [
-        "scope_map.c",
-    ],
-    hdrs = [
-        "scope_map.h",
-    ],
+    srcs = ["scope_map.c"],
+    hdrs = ["scope_map.h"],
     deps = [
         ":parameter_index",
         "//runtime/src/iree/base",
@@ -90,24 +93,42 @@ iree_runtime_cc_library(
 )
 
 iree_runtime_cc_library(
-    name = "stream",
-    srcs = [
-        "stream.c",
+    name = "stdio_stream",
+    srcs = ["stdio_stream.c"],
+    hdrs = ["stdio_stream.h"],
+    deps = [
+        ":stream",
+        "//runtime/src/iree/base",
+        "//runtime/src/iree/base/internal",
     ],
-    hdrs = [
-        "stream.h",
+)
+
+iree_runtime_cc_library(
+    name = "stream",
+    srcs = ["stream.c"],
+    hdrs = ["stream.h"],
+    deps = [
+        "//runtime/src/iree/base",
+        "//runtime/src/iree/base/internal",
     ],
+)
+
+iree_runtime_cc_library(
+    name = "vec_stream",
+    srcs = ["vec_stream.c"],
+    hdrs = ["vec_stream.h"],
     deps = [
+        ":stream",
         "//runtime/src/iree/base",
         "//runtime/src/iree/base/internal",
     ],
 )
 
 iree_runtime_cc_test(
-    name = "stream_test",
-    srcs = ["stream_test.cc"],
+    name = "vec_stream_test",
+    srcs = ["vec_stream_test.cc"],
     deps = [
-        ":stream",
+        ":vec_stream",
         "//runtime/src/iree/base",
         "//runtime/src/iree/testing:gtest",
         "//runtime/src/iree/testing:gtest_main",

runtime/src/iree/io/CMakeLists.txt

@@ -18,12 +18,39 @@ iree_cc_library(
   SRCS
     "file_handle.c"
   DEPS
+    ::memory_stream
     ::stream
     iree::base
     iree::base::internal
   PUBLIC
 )
 
+iree_cc_library(
+  NAME
+    memory_stream
+  HDRS
+    "memory_stream.h"
+  SRCS
+    "memory_stream.c"
+  DEPS
+    ::stream
+    iree::base
+    iree::base::internal
+  PUBLIC
+)
+
+iree_cc_test(
+  NAME
+    memory_stream_test
+  SRCS
+    "memory_stream_test.cc"
+  DEPS
+    ::memory_stream
+    iree::base
+    iree::testing::gtest
+    iree::testing::gtest_main
+)
+
 iree_cc_library(
   NAME
     parameter_index
@@ -82,6 +109,20 @@ iree_cc_library(
   PUBLIC
 )
 
+iree_cc_library(
+  NAME
+    stdio_stream
+  HDRS
+    "stdio_stream.h"
+  SRCS
+    "stdio_stream.c"
+  DEPS
+    ::stream
+    iree::base
+    iree::base::internal
+  PUBLIC
+)
+
 iree_cc_library(
   NAME
     stream
@@ -95,14 +136,28 @@ iree_cc_library(
   PUBLIC
 )
 
-iree_cc_test(
+iree_cc_library(
   NAME
-    stream_test
+    vec_stream
+  HDRS
+    "vec_stream.h"
   SRCS
-    "stream_test.cc"
+    "vec_stream.c"
   DEPS
     ::stream
     iree::base
+    iree::base::internal
+  PUBLIC
+)
+
+iree_cc_test(
+  NAME
+    vec_stream_test
+  SRCS
+    "vec_stream_test.cc"
+  DEPS
+    ::vec_stream
+    iree::base
     iree::testing::gtest
     iree::testing::gtest_main
 )

runtime/src/iree/io/file_handle.c

@@ -7,6 +7,7 @@
 #include "iree/io/file_handle.h"
 
 #include "iree/base/internal/atomics.h"
+#include "iree/io/memory_stream.h"
 
 //===----------------------------------------------------------------------===//
 // iree_io_file_handle_t

runtime/src/iree/io/file_handle.h

@@ -155,6 +155,9 @@ iree_io_file_handle_flush(iree_io_file_handle_t* handle);
 // iree_io_stream_t utilities
 //===----------------------------------------------------------------------===//
 
+// TODO(benvanik): remove/rework iree_io_stream_open so that it doesn't pull in
+// any implementations by putting callbacks on the file handle constructors.
+
 // Opens a stream from the given |file_handle| at the absolute |file_offset|.
 // The returned stream will retain the file until it is released.
 IREE_API_EXPORT iree_status_t iree_io_stream_open(