Replacing the ancient vm_util with function_io/function_util.

The new code is written using iree_io_stream_t instead of directly
using files so that we have a consistent I/O interface. This also will
allow us to use the flag handling code on platforms without stdio
or from tools where we may want to avoid hitting the filesystem.

The old iree/base/internal/file_io.h utilities aren't dead yet as they're
currently the only place we support memory mapping files. Future changes
will add a platform-specific file-backed stream type that allows for
mapping so that we can change/enhance all APIs dealing with files to
support mapping and bare metal execution without the need for the
giant IREE_FILE_IO_ENABLE flag.

The existing flag parsing is mostly compatible modulo changing numpy
multi-array expansion to the explicit form of `--input=*foo.npy` to
indicate that all remaining arrays in a file should be inserted. The
parsing now supports consuming multiple inputs from a single binary
file or singular arrays from a numpy file by using the same syntax as
`--output=` has supported with `@` indicating start of file and `+`
indicating continuation.

compiler/src/iree/compiler/ConstEval/BUILD.bazel

@@ -79,7 +79,6 @@ iree_compiler_cc_library(
         "//runtime/src/iree/hal",
         "//runtime/src/iree/hal/drivers/local_task/registration",
         "//runtime/src/iree/modules/hal",
-        "//runtime/src/iree/tooling:vm_util",
         "//runtime/src/iree/vm",
         "//runtime/src/iree/vm/bytecode:module",
         "@llvm-project//llvm:Support",

compiler/src/iree/compiler/ConstEval/CMakeLists.txt

@@ -71,7 +71,6 @@ iree_cc_library(
     iree::hal
     iree::hal::drivers::local_task::registration
     iree::modules::hal
-    iree::tooling::vm_util
     iree::vm
     iree::vm::bytecode::module
   PUBLIC

runtime/src/iree/tooling/BUILD.bazel

@@ -43,7 +43,6 @@ iree_runtime_cc_library(
     hdrs = ["comparison.h"],
     deps = [
         ":buffer_view_matchers",
-        ":vm_util",
         "//runtime/src/iree/base",
         "//runtime/src/iree/base/internal:flags",
         "//runtime/src/iree/base/internal:span",
@@ -58,7 +57,7 @@ iree_runtime_cc_test(
     srcs = ["comparison_test.cc"],
     deps = [
         ":comparison",
-        ":vm_util",
+        ":function_io",
         "//runtime/src/iree/base",
         "//runtime/src/iree/base/internal:span",
         "//runtime/src/iree/hal",
@@ -108,6 +107,49 @@ iree_runtime_cc_library(
     ],
 )
 
+iree_runtime_cc_library(
+    name = "function_io",
+    srcs = ["function_io.c"],
+    hdrs = ["function_io.h"],
+    deps = [
+        ":numpy_io",
+        "//runtime/src/iree/base",
+        "//runtime/src/iree/hal",
+        "//runtime/src/iree/io:stdio_stream",
+        "//runtime/src/iree/io:stream",
+        "//runtime/src/iree/io:vec_stream",
+        "//runtime/src/iree/modules/hal",
+        "//runtime/src/iree/vm",
+    ],
+)
+
+iree_runtime_cc_test(
+    name = "function_io_test",
+    srcs = ["function_io_test.cc"],
+    deps = [
+        ":function_io",
+        "//runtime/src/iree/base",
+        "//runtime/src/iree/hal",
+        "//runtime/src/iree/io:vec_stream",
+        "//runtime/src/iree/modules/hal",
+        "//runtime/src/iree/testing:gtest",
+        "//runtime/src/iree/testing:gtest_main",
+        "//runtime/src/iree/vm",
+    ],
+)
+
+iree_runtime_cc_library(
+    name = "function_util",
+    srcs = ["function_util.c"],
+    hdrs = ["function_util.h"],
+    deps = [
+        "//runtime/src/iree/base",
+        "//runtime/src/iree/hal",
+        "//runtime/src/iree/modules/hal",
+        "//runtime/src/iree/vm",
+    ],
+)
+
 iree_runtime_cc_library(
     name = "instrument_util",
     srcs = ["instrument_util.c"],
@@ -129,6 +171,7 @@ iree_runtime_cc_library(
     deps = [
         "//runtime/src/iree/base",
         "//runtime/src/iree/hal",
+        "//runtime/src/iree/io:stream",
     ],
 )
 
@@ -139,7 +182,8 @@ iree_runtime_cc_test(
     deps = [
         ":device_util",
         ":numpy_io",
-        "//runtime/src/iree/base/internal:file_io",
+        "//runtime/src/iree/io:memory_stream",
+        "//runtime/src/iree/io:vec_stream",
         "//runtime/src/iree/testing:gtest",
         "//runtime/src/iree/testing:gtest_main",
         "//runtime/src/iree/tooling/testdata/npy",
@@ -176,45 +220,15 @@ iree_runtime_cc_library(
         ":comparison",
         ":context_util",
         ":device_util",
+        ":function_io",
+        ":function_util",
         ":instrument_util",
-        ":vm_util",
         "//runtime/src/iree/base",
         "//runtime/src/iree/base/internal:flags",
         "//runtime/src/iree/hal",
+        "//runtime/src/iree/io:stdio_stream",
         "//runtime/src/iree/modules/hal:types",
         "//runtime/src/iree/vm",
         "//runtime/src/iree/vm/bytecode:module",
     ],
 )
-
-# TODO(benvanik): fold these into iree/runtime and use that instead.
-iree_runtime_cc_library(
-    name = "vm_util",
-    srcs = ["vm_util.c"],
-    hdrs = ["vm_util.h"],
-    deps = [
-        ":numpy_io",
-        "//runtime/src/iree/base",
-        "//runtime/src/iree/base/internal:file_io",
-        "//runtime/src/iree/hal",
-        "//runtime/src/iree/modules/hal",
-        "//runtime/src/iree/vm",
-        "//runtime/src/iree/vm/bytecode:module",
-    ],
-)
-
-iree_runtime_cc_test(
-    name = "vm_util_test",
-    srcs = ["vm_util_test.cc"],
-    deps = [
-        ":device_util",
-        ":vm_util",
-        "//runtime/src/iree/base",
-        "//runtime/src/iree/base/internal:span",
-        "//runtime/src/iree/hal",
-        "//runtime/src/iree/modules/hal",
-        "//runtime/src/iree/testing:gtest",
-        "//runtime/src/iree/testing:gtest_main",
-        "//runtime/src/iree/vm",
-    ],
-)

runtime/src/iree/tooling/CMakeLists.txt

@@ -48,7 +48,6 @@ iree_cc_library(
     "comparison.cc"
   DEPS
     ::buffer_view_matchers
-    ::vm_util
     iree::base
     iree::base::internal::flags
     iree::base::internal::span
@@ -65,7 +64,7 @@ iree_cc_test(
     "comparison_test.cc"
   DEPS
     ::comparison
-    ::vm_util
+    ::function_io
     iree::base
     iree::base::internal::span
     iree::hal
@@ -120,6 +119,56 @@ iree_cc_library(
   PUBLIC
 )
 
+iree_cc_library(
+  NAME
+    function_io
+  HDRS
+    "function_io.h"
+  SRCS
+    "function_io.c"
+  DEPS
+    ::numpy_io
+    iree::base
+    iree::hal
+    iree::io::stdio_stream
+    iree::io::stream
+    iree::io::vec_stream
+    iree::modules::hal
+    iree::vm
+  PUBLIC
+)
+
+iree_cc_test(
+  NAME
+    function_io_test
+  SRCS
+    "function_io_test.cc"
+  DEPS
+    ::function_io
+    iree::base
+    iree::hal
+    iree::io::vec_stream
+    iree::modules::hal
+    iree::testing::gtest
+    iree::testing::gtest_main
+    iree::vm
+)
+
+iree_cc_library(
+  NAME
+    function_util
+  HDRS
+    "function_util.h"
+  SRCS
+    "function_util.c"
+  DEPS
+    iree::base
+    iree::hal
+    iree::modules::hal
+    iree::vm
+  PUBLIC
+)
+
 iree_cc_library(
   NAME
     instrument_util
@@ -147,6 +196,7 @@ iree_cc_library(
   DEPS
     iree::base
     iree::hal
+    iree::io::stream
   PUBLIC
 )
 
@@ -158,7 +208,8 @@ iree_cc_test(
   DEPS
     ::device_util
     ::numpy_io
-    iree::base::internal::file_io
+    iree::io::memory_stream
+    iree::io::vec_stream
     iree::testing::gtest
     iree::testing::gtest_main
     iree::tooling::testdata::npy
@@ -202,52 +253,19 @@ iree_cc_library(
     ::comparison
     ::context_util
     ::device_util
+    ::function_io
+    ::function_util
     ::instrument_util
-    ::vm_util
     iree::base
     iree::base::internal::flags
     iree::hal
+    iree::io::stdio_stream
     iree::modules::hal::types
     iree::vm
     iree::vm::bytecode::module
   PUBLIC
 )
 
-iree_cc_library(
-  NAME
-    vm_util
-  HDRS
-    "vm_util.h"
-  SRCS
-    "vm_util.c"
-  DEPS
-    ::numpy_io
-    iree::base
-    iree::base::internal::file_io
-    iree::hal
-    iree::modules::hal
-    iree::vm
-    iree::vm::bytecode::module
-  PUBLIC
-)
-
-iree_cc_test(
-  NAME
-    vm_util_test
-  SRCS
-    "vm_util_test.cc"
-  DEPS
-    ::device_util
-    ::vm_util
-    iree::base
-    iree::base::internal::span
-    iree::hal
-    iree::modules::hal
-    iree::testing::gtest
-    iree::testing::gtest_main
-    iree::vm
-)
-
 ### BAZEL_TO_CMAKE_PRESERVES_ALL_CONTENT_BELOW_THIS_LINE ###
 
 # We're co-opting the VMVX module loader option for this as the inline-static

runtime/src/iree/tooling/comparison.cc

@@ -14,7 +14,6 @@
 #include "iree/hal/api.h"
 #include "iree/modules/hal/module.h"
 #include "iree/tooling/buffer_view_matchers.h"
-#include "iree/tooling/vm_util.h"
 #include "iree/vm/api.h"
 
 using namespace iree;