diff --git a/frontends/PyCDE/integration_test/esi_ram.py b/frontends/PyCDE/integration_test/esi_ram.py index adf903858ed7..b7a8cff63c23 100644 --- a/frontends/PyCDE/integration_test/esi_ram.py +++ b/frontends/PyCDE/integration_test/esi_ram.py @@ -2,7 +2,7 @@ # RUN: rm -rf %t # RUN: mkdir %t && cd %t # RUN: %PYTHON% %s %t 2>&1 -# RUN: esi-cosim-runner.py --tmpdir %t --exec %S/test_software/esi_ram.py `ls %t/hw/*.sv | grep -v driver.sv` +# RUN: esi-cosim.py -- %PYTHON% %S/test_software/esi_ram.py cosim env import pycde from pycde import (AppID, Clock, Input, Module, generator) diff --git a/frontends/PyCDE/integration_test/esi_test.py b/frontends/PyCDE/integration_test/esi_test.py index 69262213d67b..6f57c967cfe2 100644 --- a/frontends/PyCDE/integration_test/esi_test.py +++ b/frontends/PyCDE/integration_test/esi_test.py @@ -2,7 +2,7 @@ # RUN: rm -rf %t # RUN: mkdir %t && cd %t # RUN: %PYTHON% %s %t 2>&1 -# RUN: esi-cosim-runner.py --tmpdir %t --exec %S/test_software/esi_test.py `ls %t/hw/*.sv | grep -v driver.sv` +# RUN: esi-cosim.py -- %PYTHON% %S/test_software/esi_test.py cosim env import pycde from pycde import (AppID, Clock, Input, Module, generator) diff --git a/frontends/PyCDE/integration_test/lit.cfg.py b/frontends/PyCDE/integration_test/lit.cfg.py index 903f77060508..12fc346e5ace 100644 --- a/frontends/PyCDE/integration_test/lit.cfg.py +++ b/frontends/PyCDE/integration_test/lit.cfg.py @@ -42,6 +42,11 @@ llvm_config.use_default_substitutions() +llvm_config.with_environment('LIBRARY_PATH', [config.llvm_lib_dir], + append_path=True) +llvm_config.with_environment('LD_LIBRARY_PATH', [config.llvm_lib_dir], + append_path=True) + # Set the timeout, if requested. if config.timeout is not None and config.timeout != "": lit_config.maxIndividualTestTime = int(config.timeout) @@ -151,6 +156,8 @@ llvm_config.with_environment('PYTHONPATH', [f"{config.esi_runtime_path}/python/"], append_path=True) + tools.append("esi-cosim.py") + tool_dirs.append(f"{config.esi_runtime_path}/cosim") # Enable ESI cosim tests if they have been built. if config.esi_cosim_path != "": diff --git a/frontends/PyCDE/src/bsp/cosim.py b/frontends/PyCDE/src/bsp/cosim.py index 7a2840805bf6..94f37b02c799 100644 --- a/frontends/PyCDE/src/bsp/cosim.py +++ b/frontends/PyCDE/src/bsp/cosim.py @@ -20,7 +20,7 @@ def CosimBSP(user_module): """Wrap and return a cosimulation 'board support package' containing 'user_module'""" - class top(Module): + class ESI_Cosim_Top(Module): clk = Clock() rst = Input(types.int(1)) @@ -35,4 +35,4 @@ def build(ports): System.current().add_packaging_step(esi.package) - return top + return ESI_Cosim_Top diff --git a/lib/Dialect/ESI/CMakeLists.txt b/lib/Dialect/ESI/CMakeLists.txt index 9533448fd809..bf8fcfc9fbb2 100644 --- a/lib/Dialect/ESI/CMakeLists.txt +++ b/lib/Dialect/ESI/CMakeLists.txt @@ -105,6 +105,7 @@ set(ESI_RUNTIME_SRCS runtime/cosim/Cosim_Manifest.sv runtime/cosim/Cosim_Endpoint.sv runtime/cosim/CosimDpi.capnp + runtime/cosim/esi-cosim.py runtime/python/esi/__init__.py runtime/python/esi/accelerator.py runtime/python/esi/esiCppAccel.cpp diff --git a/lib/Dialect/ESI/runtime/cosim/driver.cpp b/lib/Dialect/ESI/runtime/cosim/driver.cpp new file mode 100644 index 000000000000..a94bd761b6c7 --- /dev/null +++ b/lib/Dialect/ESI/runtime/cosim/driver.cpp @@ -0,0 +1,110 @@ +//===- driver.cpp - ESI Verilator software driver -------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// A fairly standard, boilerplate Verilator C++ simulation driver. Assumes the +// top level exposes just two signals: 'clk' and 'rst'. +// +//===----------------------------------------------------------------------===// + +#ifndef TOP_MODULE +#define TOP_MODULE ESI_Cosim_Top +#endif // TOP_MODULE + +// Macro black magic to get the header file name and class name from the +// TOP_MODULE macro. Need to disable formatting for this section, as +// clang-format messes it up by inserting spaces. + +// clang-format off +#define STRINGIFY_MACRO(x) STR(x) +#define STR(x) #x +#define EXPAND(x)x +#define CONCAT3(n1, n2, n3) STRINGIFY_MACRO(EXPAND(n1)EXPAND(n2)EXPAND(n3)) +#define TOKENPASTE(x, y) x ## y +#define CLASSNAME(x, y) TOKENPASTE(x, y) + +#include CONCAT3(V,TOP_MODULE,.h) +// clang-format on + +#include "verilated_vcd_c.h" + +#include "signal.h" +#include + +vluint64_t timeStamp; + +// Stop the simulation gracefully on ctrl-c. +volatile bool stopSimulation = false; +void handle_sigint(int) { stopSimulation = true; } + +// Called by $time in Verilog. +double sc_time_stamp() { return timeStamp; } + +int main(int argc, char **argv) { + // Register graceful exit handler. + signal(SIGINT, handle_sigint); + + Verilated::commandArgs(argc, argv); + + // Construct the simulated module's C++ model. + auto &dut = *new CLASSNAME(V, TOP_MODULE)(); + char *waveformFile = getenv("SAVE_WAVE"); + + VerilatedVcdC *tfp = nullptr; + if (waveformFile) { +#ifdef TRACE + tfp = new VerilatedVcdC(); + Verilated::traceEverOn(true); + dut.trace(tfp, 99); // Trace 99 levels of hierarchy + tfp->open(waveformFile); + std::cout << "[driver] Writing trace to " << waveformFile << std::endl; +#else + std::cout + << "[driver] Warning: waveform file specified, but not a debug build" + << std::endl; +#endif + } + + std::cout << "[driver] Starting simulation" << std::endl; + + // TODO: Add max speed (cycles per second) option for small, interactive + // simulations to reduce waveform for debugging. Should this be a command line + // option or configurable over the cosim interface? + + // Reset. + dut.rst = 1; + dut.clk = 0; + + // TODO: Support ESI reset handshake in the future. + // Run for a few cycles with reset held. + for (timeStamp = 0; timeStamp < 8 && !Verilated::gotFinish(); timeStamp++) { + dut.eval(); + dut.clk = !dut.clk; + if (tfp) + tfp->dump(timeStamp); + } + + // Take simulation out of reset. + dut.rst = 0; + + // Run for the specified number of cycles out of reset. + for (; !Verilated::gotFinish() && !stopSimulation; timeStamp++) { + dut.eval(); + dut.clk = !dut.clk; + if (tfp) + tfp->dump(timeStamp); + } + + // Tell the simulator that we're going to exit. This flushes the output(s) and + // frees whatever memory may have been allocated. + dut.final(); + if (tfp) + tfp->close(); + + std::cout << "[driver] Ending simulation at tick #" << timeStamp << std::endl; + return 0; +} diff --git a/lib/Dialect/ESI/runtime/cosim/esi-cosim.py b/lib/Dialect/ESI/runtime/cosim/esi-cosim.py new file mode 100755 index 000000000000..cc6c902c7e6a --- /dev/null +++ b/lib/Dialect/ESI/runtime/cosim/esi-cosim.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 + +# ===- esi-cosim.py - ESI cosimulation launch utility --------*- python -*-===// +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ===----------------------------------------------------------------------===// +# +# Utility script to start a simulation and launch a command to interact with it +# via ESI cosimulation. +# +# ===----------------------------------------------------------------------===// + +import argparse +import os +from pathlib import Path +import re +import signal +import socket +import subprocess +import sys +import textwrap +import time +from typing import List + +CosimCollateralDir = Path(os.path.dirname(os.path.realpath(__file__))) + + +def is_port_open(port) -> bool: + """Check if a TCP port is open locally.""" + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + result = sock.connect_ex(('127.0.0.1', port)) + sock.close() + return True if result == 0 else False + + +class SourceFiles: + + def __init__(self, top: str) -> None: + # User source files. + self.user: List[Path] = [] + # DPI shared objects. + self.dpi_so: List[str] = ["EsiCosimDpiServer"] + # DPI SV files. + self.dpi_sv: List[Path] = [ + CosimCollateralDir / "Cosim_DpiPkg.sv", + CosimCollateralDir / "Cosim_Endpoint.sv", + CosimCollateralDir / "Cosim_Manifest.sv", + CosimCollateralDir / "Cosim_MMIO.sv", + ] + # Name of the top module. + self.top = top + + def add_dir(self, dir: Path): + """Add all the RTL files in a directory to the source list.""" + for file in dir.iterdir(): + if file.is_file() and (file.suffix == ".sv" or file.suffix == ".v"): + self.user.append(file) + + @property + def rtl_sources(self) -> List[Path]: + """Return a list of all the RTL source files.""" + return self.dpi_sv + self.user + + +class Simulator: + + def __init__(self, sources: SourceFiles, run_dir: Path, debug: bool): + self.sources = sources + self.run_dir = run_dir + self.debug = debug + + def compile(self): + """Compile the sources. Returns the exit code of the simulation compiler.""" + assert False, "Must be implemented by subclass" + + def run_command(self) -> List[str]: + """Return the command to run the simulation.""" + assert False, "Must be implemented by subclass" + + def run(self, inner_command: str) -> int: + """Start the simulation then run the command specified. Kill the simulation + when the command exits.""" + + # 'simProc' is accessed in the finally block. Declare it here to avoid + # syntax errors in that block. + simProc = None + try: + # Open log files + self.run_dir.mkdir(parents=True, exist_ok=True) + simStdout = open(self.run_dir / "sim_stdout.log", "w") + simStderr = open(self.run_dir / "sim_stderr.log", "w") + + # Erase the config file if it exists. We don't want to read + # an old config. + portFileName = self.run_dir / "cosim.cfg" + if os.path.exists(portFileName): + os.remove(portFileName) + + # Run the simulation. + simEnv = os.environ.copy() + if self.debug: + simEnv["COSIM_DEBUG_FILE"] = "cosim_debug.log" + simProc = subprocess.Popen(self.run_command(), + stdout=simStdout, + stderr=simStderr, + env=simEnv, + cwd=self.run_dir, + preexec_fn=os.setsid) + simStderr.close() + simStdout.close() + + # Get the port which the simulation RPC selected. + checkCount = 0 + while (not os.path.exists(portFileName)) and \ + simProc.poll() is None: + time.sleep(0.1) + checkCount += 1 + if checkCount > 200: + raise Exception(f"Cosim never wrote cfg file: {portFileName}") + port = -1 + while port < 0: + portFile = open(portFileName, "r") + for line in portFile.readlines(): + m = re.match("port: (\\d+)", line) + if m is not None: + port = int(m.group(1)) + portFile.close() + + # Wait for the simulation to start accepting RPC connections. + checkCount = 0 + while not is_port_open(port): + checkCount += 1 + if checkCount > 200: + raise Exception(f"Cosim RPC port ({port}) never opened") + if simProc.poll() is not None: + raise Exception("Simulation exited early") + time.sleep(0.05) + + # Run the inner command, passing the connection info via environment vars. + testEnv = os.environ.copy() + testEnv["ESI_COSIM_PORT"] = str(port) + testEnv["ESI_COSIM_HOST"] = "localhost" + return subprocess.run(inner_command, cwd=os.getcwd(), + env=testEnv).returncode + finally: + # Make sure to stop the simulation no matter what. + if simProc: + os.killpg(os.getpgid(simProc.pid), signal.SIGINT) + # Allow the simulation time to flush its outputs. + try: + simProc.wait(timeout=1.0) + except subprocess.TimeoutExpired: + # If the simulation doesn't exit of its own free will, kill it. + simProc.kill() + + +class Verilator(Simulator): + """Run and compile funcs for Verilator.""" + + DefaultDriver = CosimCollateralDir / "driver.cpp" + + def __init__(self, sources: SourceFiles, run_dir: Path, debug: bool): + super().__init__(sources, run_dir, debug) + + self.verilator = "verilator" + if "VERILATOR_PATH" in os.environ: + self.verilator = os.environ["VERILATOR_PATH"] + + def compile(self) -> int: + compile_cmd: List[str] = [ + self.verilator, + "--cc", + "--top-module", + self.sources.top, + "-sv", + "--build", + "--exe", + "--assert", + str(Verilator.DefaultDriver), + ] + cflags = [] + if self.debug: + compile_cmd += ["--trace", "--trace-params", "--trace-structs"] + cflags.append("-DTRACE") + if len(cflags) > 0: + compile_cmd += ["-CFLAGS", " ".join(cflags)] + if len(self.sources.dpi_so) > 0: + compile_cmd += [ + "-LDFLAGS", " ".join(["-l" + so for so in self.sources.dpi_so]) + ] + compile_cmd += [str(p) for p in self.sources.rtl_sources] + + cp = subprocess.run(compile_cmd, capture_output=True, text=True) + self.run_dir.mkdir(parents=True, exist_ok=True) + open(self.run_dir / "compile_stdout.log", "w").write(cp.stdout) + open(self.run_dir / "compile_stderr.log", "w").write(cp.stderr) + if cp.returncode != 0: + print("====== Compilation failure:") + print(cp.stderr) + return cp.returncode + + def run_command(self): + exe = Path.cwd() / "obj_dir" / ("V" + self.sources.top) + return [str(exe)] + + +def __main__(args): + argparser = argparse.ArgumentParser( + description="Wrap a 'inner_cmd' in an ESI cosimulation environment.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=textwrap.dedent(""" + Notes: + - For Verilator, libEsiCosimDpiServer.so must be in the dynamic + library runtime search path (LD_LIBRARY_PATH) and link time path + (LIBRARY_PATH). If it is installed to a standard location (e.g. + /usr/lib), this should be handled automatically. + - This script needs to sit in the same directory as the ESI support + SystemVerilog (e.g. Cosim_DpiPkg.sv, Cosim_MMIO.sv, etc.). It can, + however, be soft linked to a different location. + - The simulator executable(s) must be in your PATH. + """)) + + argparser.add_argument( + "--sim", + type=str, + default="verilator", + help="Name of the RTL simulator to use or path to an executable.") + argparser.add_argument("--rundir", + default="run", + help="Directory in which simulation should be run.") + argparser.add_argument( + "--top", + default="ESI_Cosim_Top", + help="Name of the 'top' module to use in the simulation.") + argparser.add_argument("--debug", + action="store_true", + help="Enable debug output.") + argparser.add_argument("--source", + help="Directories containing the source files.", + nargs="+", + default=["hw"]) + + argparser.add_argument("inner_cmd", + nargs=argparse.REMAINDER, + help="Command to run in the simulation environment.") + + if len(args) <= 1: + argparser.print_help() + return + args = argparser.parse_args(args[1:]) + + sources = SourceFiles(args.top) + for src in args.source: + sources.add_dir(Path(src)) + + if args.sim == "verilator": + sim = Verilator(sources, Path(args.rundir), args.debug) + else: + print("Unknown simulator: " + args.sim) + print("Supported simulators: ") + print(" - verilator") + return 1 + + rc = sim.compile() + if rc != 0: + return rc + return sim.run(args.inner_cmd[1:]) + + +if __name__ == '__main__': + sys.exit(__main__(sys.argv)) diff --git a/lib/Dialect/ESI/runtime/cpp/lib/backends/Cosim.cpp b/lib/Dialect/ESI/runtime/cpp/lib/backends/Cosim.cpp index 70d876c96a87..acde367fc2b9 100644 --- a/lib/Dialect/ESI/runtime/cpp/lib/backends/Cosim.cpp +++ b/lib/Dialect/ESI/runtime/cpp/lib/backends/Cosim.cpp @@ -42,7 +42,7 @@ CosimAccelerator::connect(string connectionString) { if ((colon = connectionString.find(':')) != string::npos) { portStr = connectionString.substr(colon + 1); host = connectionString.substr(0, colon); - } else { + } else if (connectionString.ends_with("cosim.cfg")) { ifstream cfg(connectionString); string line, key, value; @@ -58,6 +58,19 @@ CosimAccelerator::connect(string connectionString) { if (portStr.size() == 0) throw runtime_error("port line not found in file"); + } else if (connectionString == "env") { + char *hostEnv = getenv("ESI_COSIM_HOST"); + if (hostEnv) + host = hostEnv; + else + host = "localhost"; + char *portEnv = getenv("ESI_COSIM_PORT"); + if (portEnv) + portStr = portEnv; + else + throw runtime_error("ESI_COSIM_PORT environment variable not set"); + } else { + throw runtime_error("Invalid connection string '" + connectionString + "'"); } uint16_t port = stoul(portStr); return make_unique(host, port);