diff --git a/.gitignore b/.gitignore index 3b89454b..3edfa035 100644 --- a/.gitignore +++ b/.gitignore @@ -6,8 +6,6 @@ dev/ *.class *.jar !gradle/wrapper/gradle-wrapper.jar -!lib/json-20230618.jar -!lib/toml4j-0.7.2.jar *.war *.ear @@ -47,7 +45,6 @@ Thumbs.db *.kotlin_builtins # Ignore development config files -lib/src/test/resources/reai-dev-config.toml bin .antProperties.xml diff --git a/.revengai/features.json b/.revengai/features.json index a02eb508..3447ce37 100644 --- a/.revengai/features.json +++ b/.revengai/features.json @@ -35,7 +35,8 @@ "status": "yes" }, "fs_nns_filter": { - "status": "partial" + "note": "Results per function is fixed: 1 for binary-level matching, 25 for function-level. Not user configurable.", + "status": "absent" }, "fs_similarity_filter": { "status": "yes" diff --git a/CLAUDE.md b/CLAUDE.md index 952c9cbd..aff41eea 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,10 +4,12 @@ Ghidra extension for the RevEng.AI toolkit (Java 21). Build with `GHIDRA_INSTALL ## API access -**All RevEng.AI API calls must go through the generated `ai.reveng:sdk` client** (the `*Api` classes such as `CollectionsApi`, `SearchApi`, `AnalysesCoreApi`). Do not hand-roll HTTP requests. The legacy manual path in `TypedApiImplementation` (`requestBuilderForEndpoint` / `sendRequest` / `sendVersion2Request`) is being migrated away from — do not extend it. +**All RevEng.AI API calls must go through the generated `ai.reveng:sdk` client** (the `*Api` classes such as `CollectionsApi`, `SearchApi`, `AnalysesCoreApi`). Do not hand-roll HTTP requests. Where a response cannot go through a generated model, use the generated `*Call` form and read the body directly, so the path, query and auth still come from the SDK. If a generated SDK model rejects a live response (e.g. strict validation throwing on an undeclared field), fix it by bumping the SDK to a version whose model matches the API — not by falling back to a manual request. `SdkSchemaTest` guards the SDK version floor and the specific API/model surface the plugin depends on; update it when you change which SDK methods are used. ## Dependencies in the built extension -Runtime dependencies are copied into `lib/` and bundled into the extension zip. `lib/` is gitignored. The copy step does not prune old versions, so after bumping a dependency delete the previous jar from `lib/` before rebuilding — otherwise the zip ships two versions and the classloader may load the stale one. +Runtime dependencies are copied into `lib/` by Ghidra's `copyDependencies` task, put on the compile classpath, and bundled into the extension zip. The jars themselves are gitignored. + +`copyDependencies` never removes anything, so `build.gradle` registers a `pruneStaleJars` task that deletes `lib/*.jar` and runs before it. Every build therefore starts from an empty `lib/` and ships exactly the jars that resolved; bumping or dropping a dependency needs no manual cleanup. diff --git a/README.md b/README.md index 8c639a16..0295c84c 100644 --- a/README.md +++ b/README.md @@ -32,9 +32,13 @@ and use it for Binary Code Similarity to help you Reverse Engineer stripped bina ## Key features -* Upload the current binary for analysis -* Automatically rename all functions above a confidence threshold -* Show similar functions and their names for one selected function +* Create a RevEng.AI analysis for the open binary, or attach to one that already exists in the portal +* Match functions against the RevEng.AI dataset and rename them, either one at a time or across the whole binary +* A Similar Functions window that follows the cursor and diffs the selected function against each match +* AI Decompilation, with a natural language explanation of what the function does +* Agent Chat: ask the RevEng.AI agent about the current binary and let it rename and re-type functions +* Sync With Portal: apply the names, function signatures and data types the portal holds for your analysis, and push local renames back up. Local type and signature edits go up on their own as you make them +* Automatic sync of the names and data types recovered by the server-side auto-unstrip pass ## Installation @@ -67,7 +71,9 @@ Once installed, you can enable the plugin via the `Configure` tool. 1. Navigate to Ghidra's Configure tool - `File` -> `Configure` 2. Click `Configure` under the `RevEng.AI` plugin group -3. Select the checkbox next to each of the plugins except the `DevPlugin` (unless you are doing development on the plugin itself) +3. Select the checkbox next to every plugin in the list: `AgentChatPlugin`, `AnalysisManagementPlugin`, `BinarySimilarityPlugin`, `LoggingPlugin` and `ReaiAPIServicePlugin` + +`ReaiAPIServicePlugin` and `LoggingPlugin` provide the API and logging services that the three feature plugins require, so all five need to be enabled. ![Plugins Configuration Window](screenshots/plugins-configuration-window.png) @@ -94,7 +100,7 @@ When you load the plugin for the first time, or by selecting `RevEng.AI -> Confi You are now ready to analyse a binary. Import `src/test/resources/fdupes` into Ghidra and then create a new RevEng analysis, by going to `RevEng.AI -> Analysis -> Create New`. -Usually it's enough to use the default options, but you can also select specific platforms or architectures if you want to. +Usually it's enough to use the default options, but you can also select a specific architecture if you want to. ![Upload Dialog](screenshots/upload-dialog-v2.png) @@ -115,7 +121,7 @@ applying them. We now have uploaded `fdupes` to our dataset, meaning we can now use it for our binary similarity tasks. Let's see how this works on a stripped version of `fdupes`. -Import `src/test/resourcesfdupes.stripped` using the same steps as before. Once this has been completed, you can move on to the next step. +Import `src/test/resources/fdupes.stripped` using the same steps as before. Once this has been completed, you can move on to the next step. With `fdupes.stripped` open in Ghidra, select a function in Ghidra's listing or decompiler view, and `Right-Click -> Match function`. This will open the function matching and renaming window. @@ -123,7 +129,7 @@ This will open the function matching and renaming window. ![Function Matching Action](screenshots/function-matching-action.png) ![Function Matching Window](screenshots/function-matching-window-2.png) -Adjust the filters as necessary and when ready click `Match Functions`. This will return up to 10 functions that match +Adjust the filters as necessary and when ready click `Match Functions`. This will return up to 25 candidate matches for the selected function. You can then decide to rename the function to one of the suggested names by clicking `Rename Selected`. You can always update the filters and click `Match Functions` again to update the returned functions based on updated filters. @@ -160,11 +166,15 @@ The plugin is still undergoing active development currently, and we are looking ### Code Overview -We have tried to decompose the plugin into a series of individual plugins dependent on a **CorePlugin**. +The extension is decomposed into several Ghidra plugins, all in `src/main/java/ai/reveng/toolkit/ghidra/plugins`. -The **CorePlugin** provides services that are shared across all parts of the toolkit, namely configuration and API Services. +Two of them exist only to provide shared services: **ReaiAPIServicePlugin** handles the API credentials and +provides `GhidraRevengService`, the single entry point to the RevEng.AI API, and **LoggingPlugin** provides +`ReaiLoggingService`. The feature plugins — **AnalysisManagementPlugin**, **BinarySimilarityPlugin** and +**AgentChatPlugin** — declare those services in their `servicesRequired` and acquire them from the tool. -You should therefore group related features into a Feature Plugin, and then acquire services from the CorePlugin as required. This gives users the flexiblity to enable / disable features based on their use-case and/or preferences. +You should therefore group related features into a feature plugin, and then acquire services as required. +This gives users the flexibility to enable / disable features based on their use-case and/or preferences. ### Building from source @@ -177,10 +187,10 @@ Gradle can be used to build the plugin from its source code. git clone https://github.com/RevEngAI/plugin-ghidra.git ``` -2. Enter the repository and build with gradle. +2. Enter the repository and build with the Gradle wrapper. ``` cd plugin-ghidra - gradle -PGHIDRA_INSTALL_DIR= + ./gradlew -PGHIDRA_INSTALL_DIR= buildExtension ``` * Replace `` with the path to your local Ghidra installation path. diff --git a/build.gradle b/build.gradle index adb1f639..99707a9a 100644 --- a/build.gradle +++ b/build.gradle @@ -56,6 +56,8 @@ repositories { // dropped into the lib/ directory. // See https://docs.gradle.org/current/userguide/declaring_repositories.html for more info. // Ex: mavenCentral() + // mavenLocal() is listed first so that locally installed artifacts take precedence. + mavenLocal() mavenCentral() } @@ -64,23 +66,46 @@ dependencies { // this extension is built. implementation 'io.github.java-diff-utils:java-diff-utils:4.12' implementation 'org.json:json:20250107' - implementation "com.google.guava:guava:33.2.0-jre" implementation group: 'com.fifesoft', name: 'rsyntaxtextarea', version: '3.5.2' implementation 'org.commonmark:commonmark:0.24.0' implementation 'org.commonmark:commonmark-ext-gfm-tables:0.24.0' - implementation('ai.reveng:sdk:3.123.0') + implementation('ai.reveng:sdk:4.4.0') testImplementation('junit:junit:4.13.1') testRuntimeOnly("org.junit.vintage:junit-vintage-engine:5.8.2") + // Version comes from the junit-bom pulled in by junit-vintage-engine. + testRuntimeOnly('org.junit.platform:junit-platform-launcher') // https://mvnrepository.com/artifact/org.jetbrains/annotations implementation group: 'org.jetbrains', name: 'annotations', version: '26.0.2' } + +// Build against a different ai.reveng:sdk than the pin above, e.g. one installed into the local +// Maven repository: +// +// ./gradlew buildExtension -PrevengSdkVersion= +// +// The pin in the dependencies block stays the literal default so that +// .github/scripts/bump_revengai.py can still find and rewrite it. +if (project.hasProperty('revengSdkVersion')) { + configurations.all { + resolutionStrategy.force "ai.reveng:sdk:${project.revengSdkVersion}" + } +} + +// Ghidra's copyDependencies task copies runtime jars into lib/ but never prunes, and everything +// in lib/ ends up on the compile classpath and in the extension zip, and copyDependencies never +// removes anything. Without this, dropping or bumping a dependency leaves the old jar behind and +// it keeps shipping. copyDependencies puts back exactly the jars that resolved. +tasks.register('pruneStaleJars', Delete) { + delete fileTree(dir: 'lib', include: '*.jar') +} +copyDependencies.dependsOn pruneStaleJars + test { maxParallelForks = 3 useJUnitPlatform() jvmArgs '--add-exports=java.desktop/sun.awt=ALL-UNNAMED' -// systemProperty 'java.awt.headless', 'true' systemProperty "ghidra.test.property.batch.mode", 'true' // Ghidra 12.1+ installs a serial filter factory during application initialization. The JVM @@ -99,8 +124,16 @@ test { systemProperty 'jdk.serialFilterFactory', filterFactoryClass } } -// Exclude additional files from the built extension -// Ex: buildExtension.exclude '.idea/**' +// Ghidra's buildExtension zips the whole project directory, excluding only build output, IDE +// dotfiles and src/. These paths are development-only and nothing reads them at runtime. +buildExtension.exclude 'screenshots/**' +buildExtension.exclude '.github/**' +buildExtension.exclude 'docs/**' +buildExtension.exclude 'scripts/**' +buildExtension.exclude '.revengai/**' +buildExtension.exclude 'CLAUDE.md' +buildExtension.exclude '.worktreeinclude' +buildExtension.exclude '.java-version' tasks.register('updateReadmeScreenshots', Copy) { diff --git a/data/README.txt b/data/README.txt deleted file mode 100644 index 1222f673..00000000 --- a/data/README.txt +++ /dev/null @@ -1,15 +0,0 @@ -The "data" directory is intended to hold data files that will be used by this module and will -not end up in the .jar file, but will be present in the zip or tar file. Typically, data -files are placed here rather than in the resources directory if the user may need to edit them. - -An optional data/languages directory can exist for the purpose of containing various Sleigh language -specification files and importer opinion files. - -The data/buildLanguage.xml is used for building the contents of the data/languages directory. - -The skel language definition has been commented-out within the skel.ldefs file so that the -skeleton language does not show-up within Ghidra. - -See the Sleigh language documentation (docs/languages/index.html) for details Sleigh language -specification syntax. - \ No newline at end of file diff --git a/lib/README.txt b/lib/README.txt index 528dbc6c..254c3e31 100644 --- a/lib/README.txt +++ b/lib/README.txt @@ -1,3 +1,6 @@ -The "lib" directory is intended to hold Jar files which this module is dependent upon. Jar files -may be placed in this directory manually, or automatically by maven via the dependencies block -of this module's build.gradle file. \ No newline at end of file +The "lib" directory holds the Jar files which this module depends upon at runtime. They are put +here by Gradle from the dependencies block of this module's build.gradle, and end up in the built +extension zip. The jars themselves are not checked in. + +Do not place jars here by hand: every build first deletes lib/*.jar (see the pruneStaleJars task in +build.gradle) so that only the dependencies that actually resolved are shipped. diff --git a/os/linux_x86_64/README.txt b/os/linux_x86_64/README.txt deleted file mode 100644 index 7dd33ce2..00000000 --- a/os/linux_x86_64/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -The "os/linux_x86_64" directory is intended to hold Linux native binaries -which this module is dependent upon. This directory may be eliminated for a specific -module if native binaries are not provided for the corresponding platform. diff --git a/os/mac_x86_64/README.txt b/os/mac_x86_64/README.txt deleted file mode 100644 index fbf2469e..00000000 --- a/os/mac_x86_64/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -The "os/mac_x86_64" directory is intended to hold macOS (OS X) native binaries -which this module is dependent upon. This directory may be eliminated for a specific -module if native binaries are not provided for the corresponding platform. diff --git a/os/win_x86_64/README.txt b/os/win_x86_64/README.txt deleted file mode 100644 index e0359950..00000000 --- a/os/win_x86_64/README.txt +++ /dev/null @@ -1,3 +0,0 @@ -The "os/win_x86_64" directory is intended to hold MS Windows native binaries (.exe) -which this module is dependent upon. This directory may be eliminated for a specific -module if native binaries are not provided for the corresponding platform. diff --git a/scripts/emit_features.py b/scripts/emit_features.py index 2a7a2d17..7f95c738 100644 --- a/scripts/emit_features.py +++ b/scripts/emit_features.py @@ -20,7 +20,10 @@ "fs_collection_filter": {"status": "yes"}, "fs_binary_filter": {"status": "yes"}, "fs_debug_filter": {"status": "yes"}, - "fs_nns_filter": {"status": "partial"}, + "fs_nns_filter": { + "status": "absent", + "note": "Results per function is fixed: 1 for binary-level matching, 25 for function-level. Not user configurable.", + }, "fs_similarity_filter": {"status": "yes"}, "upload_function_names": {"status": "yes"}, "data_types_sync": {"status": "yes"}, diff --git a/src/main/help/help/TOC_Source.xml b/src/main/help/help/TOC_Source.xml deleted file mode 100644 index a34f62e8..00000000 --- a/src/main/help/help/TOC_Source.xml +++ /dev/null @@ -1,57 +0,0 @@ - - - - - - - diff --git a/src/main/help/help/topics/reait/help.html b/src/main/help/help/topics/reait/help.html deleted file mode 100644 index 1f9d6a1f..00000000 --- a/src/main/help/help/topics/reait/help.html +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - - - - - - Skeleton Help File for a Module - - - - -

Skeleton Help File for a Module

- -

This is a simple skeleton help topic. For a better description of what should and should not - go in here, see the "sample" Ghidra extension in the Extensions/Ghidra directory, or see your - favorite help topic. In general, language modules do not have their own help topics.

- - diff --git a/src/main/java/ai/reveng/toolkit/ghidra/FunctionExplanation/FunctionExplanationPlugin.java b/src/main/java/ai/reveng/toolkit/ghidra/FunctionExplanation/FunctionExplanationPlugin.java deleted file mode 100644 index 1d060b9b..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/FunctionExplanation/FunctionExplanationPlugin.java +++ /dev/null @@ -1,48 +0,0 @@ -package ai.reveng.toolkit.ghidra.FunctionExplanation; - -/** - * This plugin provides features for generating function comments summarising what the function is doing, and what its role in the wider program might be - */ -//@formatter:off - -import ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage; -import ai.reveng.toolkit.ghidra.FunctionExplanation.actions.AskForFunctionExplanationAction; -import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface;import ai.reveng.toolkit.ghidra.core.services.logging.ReaiLoggingService; -import ghidra.app.plugin.PluginCategoryNames; -import ghidra.app.plugin.ProgramPlugin; -import ghidra.app.services.ProgramManager; -import ghidra.framework.plugintool.PluginInfo; -import ghidra.framework.plugintool.PluginTool; -import ghidra.framework.plugintool.util.PluginStatus; - -@PluginInfo( - status = PluginStatus.HIDDEN, - packageName = ReaiPluginPackage.NAME, - category = PluginCategoryNames.COMMON, - shortDescription = "Provide Function Explanation using AI", - description = "Provides support for annotating functions in the decompiler view with human read comments on what the function does", - servicesRequired = { TypedApiInterface.class, ProgramManager.class, ReaiLoggingService.class } -) -//@formatter:on -public class FunctionExplanationPlugin extends ProgramPlugin { - /** - * Plugin constructor. - * - * @param tool The plugin tool that this plugin is added to. - */ - public FunctionExplanationPlugin(PluginTool tool) { - super(tool); - - setupActions(); - } - - private void setupActions() { - AskForFunctionExplanationAction feAction = new AskForFunctionExplanationAction(tool); - tool.addAction(feAction); - } - - @Override - public void init() { - super.init(); - } -} \ No newline at end of file diff --git a/src/main/java/ai/reveng/toolkit/ghidra/FunctionExplanation/actions/AskForFunctionExplanationAction.java b/src/main/java/ai/reveng/toolkit/ghidra/FunctionExplanation/actions/AskForFunctionExplanationAction.java deleted file mode 100644 index 42e55968..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/FunctionExplanation/actions/AskForFunctionExplanationAction.java +++ /dev/null @@ -1,107 +0,0 @@ -package ai.reveng.toolkit.ghidra.FunctionExplanation.actions; - -import ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage; -import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; -import ai.reveng.toolkit.ghidra.core.services.logging.ReaiLoggingService; -import docking.ActionContext; -import docking.action.DockingAction; -import docking.action.MenuData; -import ghidra.app.decompiler.ClangTokenGroup; -import ghidra.app.decompiler.DecompInterface; -import ghidra.app.decompiler.DecompileOptions; -import ghidra.app.decompiler.DecompileResults; -import ghidra.app.plugin.core.decompile.DecompilerActionContext; -import ghidra.app.services.ProgramManager; -import ghidra.framework.plugintool.PluginTool; -import ghidra.program.model.listing.Function; -import ghidra.program.model.listing.Program; -import ghidra.util.Msg; - -import javax.help.UnsupportedOperationException; - -public class AskForFunctionExplanationAction extends DockingAction { - - private PluginTool tool; - private Function fau; - private GhidraRevengService apiService; - private ReaiLoggingService loggingService; - - public AskForFunctionExplanationAction(PluginTool tool) { - super("CustomDecompilerAction", tool.getName()); - setPopupMenuData(new MenuData(new String[] { "Explain this function" }, ReaiPluginPackage.NAME)); - this.tool = tool; - loggingService = tool.getService(ReaiLoggingService.class); - if (loggingService == null) { - Msg.error(this, "Unable to access logging service"); - } - } - - @Override - public boolean isEnabledForContext(ActionContext context) { - if (!(context instanceof DecompilerActionContext)) { - return false; - } - - return true; - } - - @Override - public void actionPerformed(ActionContext context) { - if (!(context instanceof DecompilerActionContext)) { - return; - } - - DecompilerActionContext decompilerContext = (DecompilerActionContext) context; - DecompInterface decompiler = new DecompInterface(); - ProgramManager programManager = tool.getService(ProgramManager.class); - Program currentProgram = programManager.getCurrentProgram(); - - decompiler.openProgram(currentProgram); - - boolean initialized = decompiler.openProgram(currentProgram); - if (!initialized) { - loggingService.error("Failed to initialize DecompInterface"); - return; - } - - this.fau = currentProgram.getFunctionManager().getFunctionAt(decompilerContext.getAddress()); - - if (this.fau == null) { - loggingService.error("No function at given address"); - return; - } - - DecompileOptions options = decompiler.getOptions(); - if (options == null) { - options = new DecompileOptions(); - decompiler.setOptions(options); - } - - int timeout = options.getDefaultTimeout(); - - DecompileResults results = decompiler.decompileFunction(this.fau, timeout, null); - loggingService.info("Decomp:\n\n" + results.toString()); - - if (!results.decompileCompleted()) { - loggingService.error("Issue decompiling function"); - return; - } - - ClangTokenGroup decompiledFunction = results.getCCodeMarkup(); - - apiService = tool.getService(GhidraRevengService.class); -// Object res = apiService.explain(decompiledFunction.toString()); - throw new UnsupportedOperationException("FunctionExplaination not implemented yet"); -// if (res.getJsonObject().has("error")) { -// loggingService.error("Error with function explaination: " + res.getJsonObject().get("error").toString()); -// Msg.showError(this, null, "", "Error getting function explaination: " + res.getJsonObject().get("error")); -// return; -// } -// -// int transactionID = currentProgram.startTransaction("Set function pre-comment based on RevEng.ai description"); -// String fComment = String.format("RevEng.AI Autogenerated\n\n%s", res.getJsonObject().getString("explanation")); -// fau.setComment(fComment); -// currentProgram.endTransaction(transactionID, true); -// loggingService.info(fComment); - } -} \ No newline at end of file diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/cmds/ComputeTypeInfoTask.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/cmds/ComputeTypeInfoTask.java deleted file mode 100644 index 63cfb80c..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/cmds/ComputeTypeInfoTask.java +++ /dev/null @@ -1,80 +0,0 @@ -package ai.reveng.toolkit.ghidra.binarysimilarity.cmds; - -import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; -import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; -import ai.reveng.toolkit.ghidra.core.services.api.types.*; -import ghidra.util.exception.CancelledException; -import ghidra.util.task.Task; -import ghidra.util.task.TaskMonitor; - -import javax.annotation.Nullable; -import java.time.Duration; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import static java.util.stream.Collectors.groupingBy; - -/** - * Task to compute type information for a given function or list of functions - * - * uses https://api.reveng.ai/v2/docs#tag/Function-Overview/operation/generate_function_datatypes_v2_analyses__analysis_id__functions_data_types_post - * - */ -public class ComputeTypeInfoTask extends Task { - private final List functions; - private final GhidraRevengService service; - private final DataTypeAvailableCallback callback; - - public ComputeTypeInfoTask(GhidraRevengService service, - List functions, - @Nullable DataTypeAvailableCallback callback) { - super("Computing Type Info", true, true, false); - this.service = service; - this.functions = functions; - this.callback = callback; - } - - @Override - public void run(TaskMonitor monitor) throws CancelledException { - monitor.setMessage("Generating Type Info"); - monitor.setProgress(0); - monitor.setMaximum(functions.size()); - - functions.stream() - .map( f -> service.getApi().getFunctionDetails(f)) - .collect(groupingBy(FunctionDetails::analysisId)) - .forEach((analysisID, functions) -> { - service.getApi().generateFunctionDataTypes(analysisID, functions.stream().map(FunctionDetails::functionId).toList()); - }); - - Set missing = new HashSet<>(functions); - - while (!missing.isEmpty()) { - try { - // JDK 17 doesn't support passing a Duration to Thread.sleep - Thread.sleep(Duration.ofMillis(250).toMillis()); - } catch (InterruptedException e) { - throw new CancelledException("Task Thread was interrupted"); - } - monitor.checkCancelled(); - monitor.setMessage("Checking type info for remaining %s functions".formatted( missing.size())); - DataTypeList newList = service.getApi().getFunctionDataTypes(missing.stream().toList()); - missing.clear(); - for (FunctionDataTypeStatus status : newList.dataTypes()) { - if (status.completed()) { - if (this.callback != null) { - this.callback.dataTypeAvailable(status.functionID(), status); - } - monitor.increment(); - } else { - missing.add(status.functionID()); - } - } - } - } - - public interface DataTypeAvailableCallback { - void dataTypeAvailable(TypedApiInterface.FunctionID functionID, FunctionDataTypeStatus dataTypeStatus); - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/about/AboutDialog.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/about/AboutDialog.java index a6d33907..c7de3e9e 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/about/AboutDialog.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/about/AboutDialog.java @@ -25,12 +25,15 @@ public AboutDialog(PluginTool tool) { private String getPluginVersion() { String pluginVersion = "unknown"; try { - // This file comes from the release.yml running in the CI + // This resource is written by the release workflow, so it is absent from a local build + // and the stream is then null. var inputStream = ResourceManager.getResourceAsStream("reai_ghidra_plugin_version.txt"); - pluginVersion = new String(inputStream.readAllBytes(), StandardCharsets.UTF_8).trim(); - inputStream.close(); + if (inputStream != null) { + pluginVersion = new String(inputStream.readAllBytes(), StandardCharsets.UTF_8).trim(); + inputStream.close(); + } } catch (IOException e) { - + // ignore — fall back to "unknown" } return pluginVersion; diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/aidecompiler/AIDecompilationdWindow.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/aidecompiler/AIDecompilationdWindow.java index 33956ca5..bfb1ec3f 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/aidecompiler/AIDecompilationdWindow.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/aidecompiler/AIDecompilationdWindow.java @@ -1,11 +1,10 @@ package ai.reveng.toolkit.ghidra.binarysimilarity.ui.aidecompiler; import ai.reveng.invoker.ApiException; -import ai.reveng.model.AIDecompFunctionMapping; import ai.reveng.model.DecompilationData; +import ai.reveng.model.GetTokensResponse; import ai.reveng.model.ProgressMessage; -import ai.reveng.model.ReplacementValue; -import ai.reveng.model.TokenisedData; +import ai.reveng.model.RenderedToken; import ai.reveng.model.WorkflowProgress; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; @@ -31,13 +30,13 @@ import javax.swing.*; import javax.swing.text.BadLocationException; -import javax.swing.text.Utilities; import java.awt.*; import java.awt.event.MouseAdapter; import java.awt.event.MouseEvent; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -541,32 +540,79 @@ private void showContextMenu(Point point) { } private String wordAtOffset(int offset) throws BadLocationException { - int start = Utilities.getWordStart(textArea, offset); - int end = Utilities.getWordEnd(textArea, offset); - String word = textArea.getText(start, end - start); - return IDENTIFIER.matcher(word).matches() ? word : null; + int line = textArea.getLineOfOffset(offset); + int lineStart = textArea.getLineStartOffset(line); + String text = textArea.getText(lineStart, textArea.getLineEndOffset(line) - lineStart); + return identifierAt(text, offset - lineStart); + } + + /** + * The identifier in {@code line} spanning {@code index}, or null when that position is not inside + * one. + * + *

Scanned with the same pattern the tokenised text is split into identifiers with, rather than + * through {@code Utilities.getWordStart}/{@code getWordEnd}. Swing breaks a word at an underscore, + * so double-clicking {@code param_1} yielded {@code param} — a word that appears on no line of the + * decompilation, so the rename declined and every name carrying an underscore was unreachable. + */ + static String identifierAt(String line, int index) { + if (line == null || index < 0) { + return null; + } + Matcher matcher = IDENTIFIER.matcher(line); + while (matcher.find()) { + if (index >= matcher.start() && index <= matcher.end()) { + return matcher.group(); + } + } + return null; + } + + /// Say why a rename is not on offer, in the log and to the analyst. The reason is worth writing + /// down: it names which step declined, which is otherwise invisible. + private void declineRename(String word, String reason) { + String message = "'%s' cannot be renamed here: %s.".formatted(word, reason); + var logger = tool.getService(ReaiLoggingService.class); + if (logger != null) { + logger.info(message); + } + SwingUtilities.invokeLater(() -> + Msg.showInfo(AIDecompilationdWindow.this, component, "Rename", message)); } private void handleRename(int displayLine, String word) { RenderModel model = currentRenderModel; Function target = this.function; - if (model == null || target == null || word == null || word.isBlank()) { + if (word == null || word.isBlank()) { + // Not an identifier — a double-click on whitespace or punctuation. Nothing to say. + return; + } + // Every exit below used to be silent, so double-clicking a name that the render model could + // not place did nothing at all: no dialog, no message, no log. That is indistinguishable + // from the feature being broken, which is how it was reported. + if (model == null || target == null) { + declineRename(word, "there is no decompilation on screen to rename in"); return; } if (!model.isCodeLine(displayLine)) { + declineRename(word, "line %d is a comment, not code".formatted(displayLine + 1)); return; } Integer sourceLine = model.sourceLine(displayLine); - if (sourceLine == null) { + if (sourceLine == null || sourceLine < 1 || sourceLine > model.codeLines.size()) { + declineRename(word, "display line %d does not map onto the decompilation" + .formatted(displayLine + 1)); return; } String codeLine = model.codeLines.get(sourceLine - 1); int identIndex = indexOfIdentifier(codeLine, word); if (identIndex < 0) { + declineRename(word, "it is not on source line %d (\"%s\")".formatted(sourceLine, codeLine)); return; } FunctionID functionID = resolveFunctionId(target); if (functionID == null) { + declineRename(word, "%s is not a function RevEng.AI knows".formatted(target.getName())); return; } @@ -586,11 +632,15 @@ private void handleRename(int displayLine, String word) { @Override public void run(TaskMonitor monitor) { try { - TokenisedData tokenised = service.getApi().getAIDecompilationTokenised(functionID); - String token = resolveToken(tokenised, sourceIndex, identIndex, word); + GetTokensResponse tokenValues = service.getApi().getAIDecompilationTokens(functionID); + String token = resolveToken(tokenValues, sourceIndex, identIndex, word); if (token == null) { - SwingUtilities.invokeLater(() -> Msg.showInfo(AIDecompilationdWindow.this, component, - "Rename", "'%s' is not a renameable variable or type.".formatted(word))); + declineRename(word, "the decompilation carries no token for it"); + return; + } + if (!isRenameable(tokenValues, token)) { + declineRename(word, "it is a data type or a function, which is renamed on the type " + + "or the function itself rather than here"); return; } service.getApi().applyAIDecompilationOverrides(functionID, Map.of(token, newName)); @@ -725,71 +775,92 @@ private static List identifiers(String line) { /** * Resolve a displayed identifier to the token to override, mirroring the IDA plugin's * {@code resolve_token}: prefer the token at the same identifier position in the tokenised line, - * and fall back to a unique match across the renameable categories by effective value. + * and fall back to a unique match by effective value across every token the server rendered. */ - static String resolveToken(TokenisedData tokenised, int sourceIndex, int identIndex, String oldIdent) { - if (tokenised == null) { - return null; - } - AIDecompFunctionMapping mapping = tokenised.getFunctionMapping(); - if (mapping == null) { + static String resolveToken(GetTokensResponse tokenValues, int sourceIndex, int identIndex, String oldIdent) { + if (tokenValues == null) { return null; } + Map effectiveValues = effectiveValues(tokenValues); - String tokenisedText = tokenised.getTokenisedDecompilation(); + String tokenisedText = tokenValues.getAiDecomp(); String[] tokenisedLines = (tokenisedText == null ? "" : tokenisedText).split("\n", -1); if (sourceIndex >= 0 && sourceIndex < tokenisedLines.length) { var tokenIdentifiers = identifiers(tokenisedLines[sourceIndex]); if (identIndex >= 0 && identIndex < tokenIdentifiers.size()) { String candidate = tokenIdentifiers.get(identIndex); - for (TokenEntry entry : renameableTokens(mapping)) { - if (entry.token().equals(candidate) - && oldIdent.equals(effectiveValue(mapping, entry.token(), entry.replacement()))) { - return candidate; - } + if (namesToken(oldIdent, effectiveValues.get(candidate))) { + return candidate; } } } String uniqueMatch = null; - for (TokenEntry entry : renameableTokens(mapping)) { - if (oldIdent.equals(effectiveValue(mapping, entry.token(), entry.replacement()))) { + for (var entry : effectiveValues.entrySet()) { + if (namesToken(oldIdent, entry.getValue())) { if (uniqueMatch != null) { return null; } - uniqueMatch = entry.token(); + uniqueMatch = entry.getKey(); } } return uniqueMatch; } - private record TokenEntry(String token, ReplacementValue replacement) {} - - private static List renameableTokens(AIDecompFunctionMapping mapping) { - var entries = new ArrayList(); - addTokens(entries, mapping.getUnmatchedVars()); - addTokens(entries, mapping.getUnmatchedGlobalVars()); - addTokens(entries, mapping.getUnmatchedExternalVars()); - addTokens(entries, mapping.getUnmatchedCustomTypes()); - addTokens(entries, mapping.getUnmatchedEnums()); - return entries; + /** + * Whether a double-clicked identifier names the token rendered as {@code renderedValue}. + * + *

A rendered value is not always a bare identifier: a Rust generic renders as + * {@code lang_start<()>} and a C++ method as {@code Foo::bar}, while a double-click yields only + * the identifier under the cursor. Comparing the two directly never matched, which put every such + * token permanently out of reach of a rename, so the identifiers within the value count too. + */ + private static boolean namesToken(String oldIdent, String renderedValue) { + return renderedValue != null + && (oldIdent.equals(renderedValue) || identifiers(renderedValue).contains(oldIdent)); } - private static void addTokens(List entries, Map category) { - if (category == null) { - return; - } - for (var e : category.entrySet()) { - entries.add(new TokenEntry(e.getKey(), e.getValue())); + /** + * Whether the token the double-click resolved to can be renamed through the overrides endpoint. + * + *

The test is whether the token carries an id. A token with a {@code data_type_id}, + * {@code function_id} or {@code imported_function_id} is a reference to something that lives + * outside this decompilation and is named there — a data type in the analysis' catalogue, a + * function in the analysis — and renaming it is a different call. The endpoint says as much for a + * function: {@code 400 BAD_REQUEST}, "Functions are renamed on the function itself, not in the + * decompilation". A token with no id is a name the decompilation invented, a parameter or a local, + * and the override is the only place it exists. + */ + static boolean isRenameable(GetTokensResponse tokenValues, String token) { + if (tokenValues == null || tokenValues.getPlaceholderToRenderedToken() == null) { + return false; } + RenderedToken rendered = tokenValues.getPlaceholderToRenderedToken().get(token); + return rendered != null + && rendered.getDataTypeId() == null + && rendered.getFunctionId() == null + && rendered.getImportedFunctionId() == null; } - private static String effectiveValue(AIDecompFunctionMapping mapping, String token, ReplacementValue replacement) { - Map overrides = mapping.getUserOverrideMappings(); - if (overrides != null && overrides.containsKey(token)) { - return overrides.get(token); + /** + * Each token mapped to the name currently rendered for it: the caller's own override where one + * exists, otherwise the value the server predicted. The two maps arrive unmerged, so overrides + * are layered on top here. + * + *

Only the rendered value is taken; {@link #isRenameable} reads the kind. TODO: a rendered + * token also carries the data-type/function id behind it, which could drive navigation. + */ + static Map effectiveValues(GetTokensResponse tokenValues) { + var result = new LinkedHashMap(); + if (tokenValues.getPlaceholderToRenderedToken() != null) { + tokenValues.getPlaceholderToRenderedToken() + .forEach((placeholder, token) -> result.put(placeholder, token.getValue())); } - return replacement == null ? null : replacement.getValue(); + if (tokenValues.getPlaceholderToUserOverride() != null) { + tokenValues.getPlaceholderToUserOverride() + .forEach((placeholder, token) -> result.put(placeholder, token.getValue())); + } + return result; } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/analysiscreation/RevEngAIAnalysisOptionsDialog.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/analysiscreation/RevEngAIAnalysisOptionsDialog.java index 08400dcf..79a3b39c 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/analysiscreation/RevEngAIAnalysisOptionsDialog.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/analysiscreation/RevEngAIAnalysisOptionsDialog.java @@ -34,18 +34,12 @@ import java.util.regex.Pattern; public class RevEngAIAnalysisOptionsDialog extends RevEngDialogComponentProvider { - private JCheckBox advancedAnalysisCheckBox; - private JCheckBox dynamicExecutionCheckBox; private final Program program; private final GhidraRevengService service; private JRadioButton privateScope; private JRadioButton publicScope; private JPanel privateScopePanel; private JTextField tagsTextBox; - private JCheckBox scrapeExternalTagsBox; - private JCheckBox identifyCapabilitiesCheckBox; - private JCheckBox identifyCVECheckBox; - private JCheckBox generateSBOMCheckBox; private JComboBox architectureComboBox; private boolean okPressed = false; @@ -92,19 +86,6 @@ private void buildInterface() { fileSizeWarningLabel.setBorder(BorderFactory.createEmptyBorder(5, 5, 5, 5)); workPanel.add(fileSizeWarningLabel); - // Add Platform Drop Down - var platformComboBox = new JComboBox<>(new String[]{ - "Auto", "windows", "linux", - }); - platformComboBox.setEditable(false); - // Center the text - platformComboBox.setAlignmentX(Component.CENTER_ALIGNMENT); - platformComboBox.setMaximumSize(platformComboBox.getPreferredSize()); - var platformLabel = new JLabel("Select Platform"); - platformLabel.setAlignmentX(Component.CENTER_ALIGNMENT); - workPanel.add(platformLabel); - workPanel.add(platformComboBox); - // Add Drop down for AnalysisScope // Currently just public and private, but in the future this will include teams var scopePanel = new JPanel(); @@ -148,40 +129,6 @@ private void buildInterface() { workPanel.add(new JSeparator(SwingConstants.HORIZONTAL)); - // Add Two Check boxes for Dynamic Execution and Advanced Analysis next to each other (horizantally) - var checkBoxPanel = new JPanel(); - checkBoxPanel.setLayout(new GridLayout(0, 2)); - dynamicExecutionCheckBox = new JCheckBox("Dynamic Execution"); - dynamicExecutionCheckBox.setToolTipText("Include Dynamic Execution inside a Sandbox Environment with the Analysis"); - - advancedAnalysisCheckBox = new JCheckBox("Advanced Analysis"); - advancedAnalysisCheckBox.setToolTipText("Run dataflow analysis for advanced analysis. Can increase analysis cost by 500%"); - - - // Add a check box for quick mode - scrapeExternalTagsBox = new JCheckBox("Get External Tags"); - scrapeExternalTagsBox.setToolTipText("Scrape external tags from VirusTotal (requires configured API key)"); - - // Add check box for identifiying capabilities - identifyCapabilitiesCheckBox = new JCheckBox("Identify Capabilities"); - identifyCapabilitiesCheckBox.setToolTipText("Identify capabilities of the binary"); - - // Add Check box for identifying CVEs - identifyCVECheckBox = new JCheckBox("Identify CVEs"); - identifyCVECheckBox.setToolTipText("Identify CVEs in the binary"); - - // Add Check box for generating the SBOM - generateSBOMCheckBox = new JCheckBox("Generate SBOM"); - generateSBOMCheckBox.setToolTipText("Generate a Software Bill of Materials (SBOM) for the binary"); - -// checkBoxPanel.add(dynamicExecutionCheckBox); -// checkBoxPanel.add(advancedAnalysisCheckBox); -// checkBoxPanel.add(scrapeExternalTagsBox); -// checkBoxPanel.add(identifyCapabilitiesCheckBox); -// checkBoxPanel.add(identifyCVECheckBox); -// checkBoxPanel.add(generateSBOMCheckBox); -// workPanel.add(checkBoxPanel); - // Add custom tags field tagsTextBox = new JTextField(); tagsTextBox.setToolTipText("Custom tags for the analysis, as comma separated list"); @@ -222,14 +169,10 @@ private void buildInterface() { var options = AnalysisOptionsBuilder.forProgram(program, function -> includedEntryPoints.contains(function.getEntryPoint())); - options.skipScraping(!scrapeExternalTagsBox.isSelected()); - options.skipCapabilities(!identifyCapabilitiesCheckBox.isSelected()); - - options.skipSBOM(!generateSBOMCheckBox.isSelected()); - options.skipCVE(!identifyCVECheckBox.isSelected()); - - options.advancedAnalysis(advancedAnalysisCheckBox.isSelected()); - options.dynamicExecution(dynamicExecutionCheckBox.isSelected()); + // Capability generation and advanced analysis are not offered by this dialog and are + // always off for analyses the plugin creates. + options.skipCapabilities(true); + options.advancedAnalysis(false); if (privateScope.isSelected() && privateScope.isEnabled()) { options.scope(AnalysisScope.PRIVATE); diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/components/BinarySelectionPanel.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/components/BinarySelectionPanel.java index 405d1e03..ddb4caf5 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/components/BinarySelectionPanel.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/components/BinarySelectionPanel.java @@ -23,13 +23,6 @@ public BinarySelectionPanel(Function getSelectedBinaries() { - return getSelectedItems(); - } - /** * Gets the IDs of currently selected binaries */ @@ -44,13 +37,6 @@ public Set getSelectedBinaryNames() { return getSelectedItemNames(); } - /** - * Sets the selected binaries - */ - public void setSelectedBinaries(Set binaries) { - setSelectedItems(binaries); - } - /** * Adds a listener for binary selection changes */ diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/components/CollectionSelectionPanel.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/components/CollectionSelectionPanel.java index 52c2689d..957eab07 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/components/CollectionSelectionPanel.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/components/CollectionSelectionPanel.java @@ -23,13 +23,6 @@ public CollectionSelectionPanel(Function getSelectedCollections() { - return getSelectedItems(); - } - /** * Gets the IDs of currently selected collections */ @@ -44,13 +37,6 @@ public Set getSelectedCollectionNames() { return getSelectedItemNames(); } - /** - * Sets the selected collections - */ - public void setSelectedCollections(Set collections) { - setSelectedItems(collections); - } - /** * Adds a listener for collection selection changes */ diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/components/ItemSelectionPanel.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/components/ItemSelectionPanel.java index f09647bf..67343ce8 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/components/ItemSelectionPanel.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/components/ItemSelectionPanel.java @@ -168,25 +168,6 @@ public Set getSelectedItemNames() { .collect(Collectors.toSet()); } - /** - * Sets the selected items - */ - public void setSelectedItems(Set items) { - // Clear existing selections - selectedItems.clear(); - selectedItemsPanel.removeAll(); - - // Add new selections - for (SelectableItem item : items) { - selectedItems.add(item); - addItemTag(item); - } - - selectedItemsPanel.revalidate(); - selectedItemsPanel.repaint(); - notifyListeners(); - } - /** * Adds a listener for item selection changes */ diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/components/SimpleAutocompleteField.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/components/SimpleAutocompleteField.java index 74c21e45..20560e8a 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/components/SimpleAutocompleteField.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/components/SimpleAutocompleteField.java @@ -244,18 +244,10 @@ public void addActionListener(ActionListener listener) { listenerList.add(ActionListener.class, listener); } - public void removeActionListener(ActionListener listener) { - listenerList.remove(ActionListener.class, listener); - } - protected void fireActionPerformed() { ActionEvent event = new ActionEvent(this, ActionEvent.ACTION_PERFORMED, getText()); for (ActionListener listener : listenerList.getListeners(ActionListener.class)) { listener.actionPerformed(event); } } - - public JTextField getTextField() { - return textField; - } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/AbstractFunctionMatchingDialog.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/AbstractFunctionMatchingDialog.java index ec925f6f..8f31caa8 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/AbstractFunctionMatchingDialog.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/AbstractFunctionMatchingDialog.java @@ -11,7 +11,6 @@ import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionMatch; import ai.reveng.toolkit.ghidra.core.services.api.types.GhidraFunctionMatch; import ai.reveng.toolkit.ghidra.core.services.api.types.GhidraFunctionMatchWithSignature; -import com.google.common.collect.BiMap; import ghidra.program.model.listing.Function; import ghidra.util.task.Task; import ghidra.util.task.TaskBuilder; @@ -25,6 +24,7 @@ import java.awt.*; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -56,7 +56,7 @@ public abstract class AbstractFunctionMatchingDialog extends RevEngDialogCompone protected AssemblyDiffPanel assemblyDiffPanel; // Data - protected Basic analysisBasicInfo; + protected AnalysisBasicInfoOutputBody analysisBasicInfo; protected final List functionMatchResults; protected final List filteredFunctionMatchResults; @@ -132,7 +132,7 @@ private void runMatchingLoop() { ? progress.errorMessage() : "Function matching returned an error status"; SwingUtilities.invokeLater(() -> { - taskMonitorComponent.setVisible(false); + statusLabel.setText("Function matching failed"); handleError(errorMsg); }); return; @@ -155,15 +155,27 @@ private void runMatchingLoop() { var matches = fetchMatches(); if (matchingCancelled) return; processFunctionMatchingResults(matches); - SwingUtilities.invokeLater(() -> taskMonitorComponent.setVisible(false)); + // The status label is the only thing that says what this dialog is doing, and nothing + // used to write to it again after the "Loading type information..." above. The work + // finished, the bar went away, and the label sat there claiming to still be loading — + // indistinguishable from a request that never returned. + SwingUtilities.invokeLater(() -> statusLabel.setText( + "Matching complete: %d match(es)".formatted(functionMatchResults.size()))); } catch (InterruptedException e) { // matching was cancelled, nothing to report - } catch (Exception e) { + } catch (Throwable t) { + // Throwable rather than Exception: an Error thrown in here — a clash between a bundled + // jar and Ghidra's own, or a large binary's type closure exhausting the heap — killed + // this thread without a word and left the progress message up for ever. Whatever it is, + // the user is told. SwingUtilities.invokeLater(() -> { - Msg.error(this, "Failed to poll function matching status: " + e.getMessage(), e); - handleError("Failed to poll function matching status: " + e.getMessage()); - taskMonitorComponent.setVisible(false); + Msg.error(this, "Function matching failed: " + t, t); + handleError("Function matching failed: " + t); + statusLabel.setText("Function matching failed"); }); + } finally { + // No path out of here may leave the progress bar spinning. + SwingUtilities.invokeLater(() -> taskMonitorComponent.setVisible(false)); } } @@ -205,7 +217,7 @@ protected static List flattenMatches(List response) { List matches = new ArrayList<>(); - final BiMap functionMap = analyzedProgram.getFunctionMap(); + final Map functionMap = analyzedProgram.getFunctionMap(); response.forEach(matchResult -> { // Retrieve the local function name @@ -269,18 +281,16 @@ protected void updateProgressUI(MatchingProgress progress) { updateResultsTable(); } - protected void updateResultsTable() { - // Determine which results to show based on whether we have an active filter + /// The results that back the table model, in model-row order: all results when no function filter + /// is active, otherwise the filtered subset (which may be empty). + protected List displayedResults() { String filterText = functionFilterField != null ? functionFilterField.getText().trim() : ""; - List resultsToShow; + return filterText.isEmpty() ? functionMatchResults : filteredFunctionMatchResults; + } - if (filterText.isEmpty()) { - // No filter text, show all results - resultsToShow = functionMatchResults; - } else { - // Filter text exists, show filtered results (even if empty) - resultsToShow = filteredFunctionMatchResults; - } + protected void updateResultsTable() { + String filterText = functionFilterField != null ? functionFilterField.getText().trim() : ""; + List resultsToShow = displayedResults(); DefaultTableModel model = new DefaultTableModel(getTableColumnNames(), 0) { @Override @@ -636,10 +646,7 @@ protected void onTableSelectionChanged() { // Convert view index to model index (in case table is sorted) int modelRow = resultsTable.convertRowIndexToModel(selectedRow); - // Get the appropriate results list - String filterText = functionFilterField != null ? functionFilterField.getText().trim() : ""; - List resultsToShow = filterText.isEmpty() ? - functionMatchResults : filteredFunctionMatchResults; + List resultsToShow = displayedResults(); if (modelRow >= resultsToShow.size()) { return; @@ -719,13 +726,7 @@ protected JPanel createThresholdPanel() { thresholdValueLabel = new JLabel("70%", SwingConstants.CENTER); thresholdValueLabel.setFont(thresholdValueLabel.getFont().deriveFont(Font.BOLD, 14f)); - thresholdSlider.addChangeListener(e -> { - int value = thresholdSlider.getValue(); - thresholdValueLabel.setText(value + "%"); - if (!thresholdSlider.getValueIsAdjusting()) { - onThresholdChanged(value); - } - }); + thresholdSlider.addChangeListener(e -> thresholdValueLabel.setText(thresholdSlider.getValue() + "%")); JPanel sliderPanel = new JPanel(new BorderLayout()); sliderPanel.add(thresholdSlider, BorderLayout.CENTER); @@ -747,16 +748,12 @@ protected JPanel createDebugSymbolsPanel() { debugSymbolsCheckBox = new JCheckBox("Only include functions with debug symbols", false); debugSymbolsCheckBox.setAlignmentX(Component.LEFT_ALIGNMENT); - debugSymbolsCheckBox.addActionListener(e -> { - boolean selected = debugSymbolsCheckBox.isSelected(); - userSubmittedDebugSymbolsCheckBox.setVisible(selected); - onDebugSymbolsChanged(selected); - }); + debugSymbolsCheckBox.addActionListener( + e -> userSubmittedDebugSymbolsCheckBox.setVisible(debugSymbolsCheckBox.isSelected())); userSubmittedDebugSymbolsCheckBox = new JCheckBox("Include user submitted debug symbols", false); userSubmittedDebugSymbolsCheckBox.setAlignmentX(Component.LEFT_ALIGNMENT); userSubmittedDebugSymbolsCheckBox.setVisible(false); - userSubmittedDebugSymbolsCheckBox.addActionListener(e -> onUserSubmittedDebugSymbolsChanged(userSubmittedDebugSymbolsCheckBox.isSelected())); JPanel indentedPanel = new JPanel(new FlowLayout(FlowLayout.LEFT, 0, 0)); indentedPanel.setAlignmentX(Component.LEFT_ALIGNMENT); @@ -822,18 +819,6 @@ protected void onBinarySelectionChanged(Set selectedBinaries) { Msg.info(this, "Selected binaries: " + binaryNames + " (IDs: " + binaryIds + ")"); } - protected void onThresholdChanged(int threshold) { - Msg.info(this, "Threshold changed to: " + threshold); - } - - protected void onDebugSymbolsChanged(boolean includeDebugSymbols) { - Msg.info(this, "Debug symbols filter changed to: " + includeDebugSymbols); - } - - protected void onUserSubmittedDebugSymbolsChanged(boolean includeUserSubmittedDebugSymbols) { - Msg.info(this, "User submitted debug symbols filter changed to: " + includeUserSubmittedDebugSymbols); - } - protected void onFunctionFilterChanged() { String filterText = functionFilterField.getText().trim().toLowerCase(); @@ -899,26 +884,31 @@ protected void onRenameAllButtonClicked() { renameInBackground(functionMatchResults); } + /// The results behind the currently selected table rows. + /// + /// The table is sortable, so the view row order need not match the order of the list backing the + /// table model. Every selected view index is therefore converted to a model index before it is + /// used to look up a result. + protected List getSelectedMatches() { + List resultsToShow = displayedResults(); + List selectedMatches = new ArrayList<>(); + for (int viewRow : resultsTable.getSelectedRows()) { + int modelRow = resultsTable.convertRowIndexToModel(viewRow); + if (modelRow >= 0 && modelRow < resultsToShow.size()) { + selectedMatches.add(resultsToShow.get(modelRow)); + } + } + return selectedMatches; + } + protected void onRenameSelectedButtonClicked() { - int[] selectedRows = resultsTable.getSelectedRows(); - if (selectedRows.length == 0) { + if (resultsTable.getSelectedRowCount() == 0) { showError("Please select one or more rows to rename."); return; - } else { - hideError(); - } - - List resultsToShow = filteredFunctionMatchResults.isEmpty() ? - functionMatchResults : filteredFunctionMatchResults; - - List selectedMatches = new ArrayList<>(); - for (int row : selectedRows) { - if (row < resultsToShow.size()) { - selectedMatches.add(resultsToShow.get(row)); - } } + hideError(); - renameInBackground(selectedMatches); + renameInBackground(getSelectedMatches()); } private void renameInBackground(List matches) { diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/AssemblyDiffPanel.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/AssemblyDiffPanel.java index 47bcf4f5..e0551ba3 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/AssemblyDiffPanel.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/AssemblyDiffPanel.java @@ -289,6 +289,10 @@ protected Void doInBackground() { localError = assemblyErrorMessage(e); Msg.error(this, "Failed to fetch local function assembly", e); } + if (localAssembly != null && localAssembly.isEmpty()) { + localAssembly = null; + localError = NO_DISASSEMBLY; + } // Fetch matched function assembly try { @@ -297,6 +301,10 @@ protected Void doInBackground() { matchedError = assemblyErrorMessage(e); Msg.error(this, "Failed to fetch matched function assembly", e); } + if (matchedAssembly != null && matchedAssembly.isEmpty()) { + matchedAssembly = null; + matchedError = NO_DISASSEMBLY; + } // Compute diff if both assemblies were fetched successfully if (localAssembly != null && matchedAssembly != null) { @@ -322,12 +330,20 @@ protected void done() { } } - /// A 404 here just means the function has no stored disassembly (common for symbol-only matches), - /// so surface a short note rather than the raw API error; the full error is still logged. + /// Shown when the function has no stored disassembly, which is common for symbol-only matches. + /// The v3 blocks endpoint reports that as a 200 carrying no blocks — hence the empty-assembly + /// check above — but a 404 still lands here when the function itself cannot be reached. + private static final String NO_DISASSEMBLY = "Disassembly is not available for this function."; + + /// Surface a short note rather than the raw API error for the two outcomes that are not really + /// failures; the full error is still logged. private static String assemblyErrorMessage(Exception e) { Throwable cause = (e.getCause() != null) ? e.getCause() : e; if (cause instanceof ApiException ae && ae.getCode() == 404) { - return "Disassembly is not available for this function."; + return NO_DISASSEMBLY; + } + if (cause instanceof ApiException ae && ae.getCode() == 409) { + return "Disassembly is not ready yet; the analysis is still processing."; } return "Failed to fetch disassembly: " + cause.getMessage(); } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/SimilarFunctionsWindow.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/SimilarFunctionsWindow.java index ef27eb6d..2b7823ac 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/SimilarFunctionsWindow.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/SimilarFunctionsWindow.java @@ -257,14 +257,6 @@ public void componentShown() { } } - /** - * Called when the program is not analyzed with RevEng.AI - */ - public void onNoAnalyzedProgram() { - clear(); - statusLabel.setText("Binary not analyzed with RevEng.AI"); - } - /** * Called when the cursor is not within a function */ diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/help/HelpDialog.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/help/HelpDialog.java index aa1dbea6..6770829d 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/help/HelpDialog.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/help/HelpDialog.java @@ -91,6 +91,27 @@ private JPanel createHelpContent() { null ) ); + panel.add( + createMenuItem( + "Sync With Portal", + """ + Reconcile this program with the attached analysis: pull function names that + changed in the portal into Ghidra, and push local renames and function + signature edits back up. Only available when an analysis is attached and has + completed processing.""", + null + ) + ); + panel.add( + createMenuItem( + "Agent Chat", + """ + Open the Agent Chat window and talk to the RevEng.AI agent about the current + binary. The agent can rename and re-type functions, and its changes are + pulled back into Ghidra.""", + null + ) + ); panel.add(createMenuItem("Configure", "Configure the API endpoint and API key", null)); panel.add(createMenuItem("Help", "Display this page", null)); panel.add(createMenuItem("About", "Display plugin version", null)); @@ -105,7 +126,9 @@ private JPanel createHelpContent() { panel.add(createMenuItem( "AI Decompilation", """ - Decompile function using the RevEng.AI proprietary decompiler. + Decompile function using the RevEng.AI proprietary decompiler, and open the result in + the AI Decompilation window. + Not available for thunks or external functions, which the portal does not support. """, null ) @@ -113,7 +136,8 @@ private JPanel createHelpContent() { panel.add(createMenuItem( "Match function", """ - Run a match against the RevEng.AI API for this function. Only available for non-debug functions. + Run a match against the RevEng.AI API for this function. + Not available for thunks or external functions, which the portal does not support. """, null ) @@ -127,6 +151,40 @@ private JPanel createHelpContent() { ) ); + panel.add(Box.createVerticalStrut(20)); + + panel.add(createSectionHeader("Windows")); + panel.add(createDescription( + "Dockable windows provided by the plugin. All of them can be reopened from Ghidra's Window menu.", + null)); + panel.add(Box.createVerticalStrut(10)); + + panel.add(createMenuItem( + "RevEng.AI: Analysis Log", + "Progress and log output of the analyses this program has been attached to.", + null + )); + panel.add(createMenuItem( + "RevEng.AI: Similar Functions", + """ + Matches for the function under the cursor, updating as you move around the program, + with an assembly diff against the selected match.""", + null + )); + panel.add(createMenuItem( + "RevEng.AI: AI Decompilation", + """ + The AI decompilation of the selected function and an explanation of what it does. + The toolbar can re-pull the decompilation, and send positive feedback or report a + problem with it back to RevEng.AI.""", + null + )); + panel.add(createMenuItem( + "RevEng.AI: Agent Chat", + "Conversation with the RevEng.AI agent about the current binary.", + null + )); + return panel; } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysesTableModel.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysesTableModel.java index c3452d9a..54d11f00 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysesTableModel.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysesTableModel.java @@ -1,9 +1,9 @@ package ai.reveng.toolkit.ghidra.binarysimilarity.ui.recentanalyses; +import ai.reveng.model.AnalysisRecordBody; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.types.*; -import ai.reveng.toolkit.ghidra.core.services.function.export.ExportFunctionBoundariesService; import ai.reveng.toolkit.ghidra.core.services.logging.ReaiLoggingService; import docking.widgets.table.AbstractDynamicTableColumn; import docking.widgets.table.TableColumnDescriptor; @@ -16,7 +16,9 @@ import ghidra.util.exception.CancelledException; import ghidra.util.task.TaskMonitor; -public class RecentAnalysesTableModel extends ThreadedTableModelStub { +import java.time.OffsetDateTime; + +public class RecentAnalysesTableModel extends ThreadedTableModelStub { private final TypedApiInterface.BinaryHash hash; private final Address imageBase; @@ -27,25 +29,24 @@ public RecentAnalysesTableModel(PluginTool tool, TypedApiInterface.BinaryHash ha } @Override - protected void doLoad(Accumulator accumulator, TaskMonitor monitor) throws CancelledException { + protected void doLoad(Accumulator accumulator, TaskMonitor monitor) throws CancelledException { var revEngAIService = serviceProvider.getService(GhidraRevengService.class); - var functionBoundariesService = serviceProvider.getService(ExportFunctionBoundariesService.class); var loggingService = serviceProvider.getService(ReaiLoggingService.class); // The search endpoint only returns analyses we have access to so there is no need to filter them. revEngAIService.searchForHash(hash).forEach( result -> { // Filter out analyses that are not Complete - if (result.status() != AnalysisStatus.Complete) { - loggingService.info("[RevEng] Skipping analysis for " + result.binary_id() + " as status is " + result.status()); + if (!AnalysisStatus.Complete.name().equals(result.getStatus())) { + loggingService.info("[RevEng] Skipping analysis for " + result.getBinaryId() + " as status is " + result.getStatus()); return; } // Filter out analyses where the base address does not match our program - if (result.base_address() != imageBase.getOffset()) { + if (result.getBaseAddress() == null || result.getBaseAddress() != imageBase.getOffset()) { loggingService.info( - "[RevEng] Skipping analysis for " + result.binary_id() + " as base address does not match. Expected " + - imageBase.getOffset() + " but got " + result.base_address()); + "[RevEng] Skipping analysis for " + result.getBinaryId() + " as base address does not match. Expected " + + imageBase.getOffset() + " but got " + result.getBaseAddress()); return; } @@ -55,17 +56,17 @@ protected void doLoad(Accumulator accumulator, TaskMonitor } @Override - protected TableColumnDescriptor createTableColumnDescriptor() { - TableColumnDescriptor descriptor = new TableColumnDescriptor<>(); - descriptor.addVisibleColumn(new AbstractDynamicTableColumn() { + protected TableColumnDescriptor createTableColumnDescriptor() { + TableColumnDescriptor descriptor = new TableColumnDescriptor<>(); + descriptor.addVisibleColumn(new AbstractDynamicTableColumn() { @Override public String getColumnName() { return "Analysis ID"; } @Override - public String getValue(LegacyAnalysisResult rowObject, Settings settings, Object data, ServiceProvider serviceProvider) throws IllegalArgumentException { - return String.valueOf(rowObject.analysis_id().id()); + public String getValue(AnalysisRecordBody rowObject, Settings settings, Object data, ServiceProvider serviceProvider) throws IllegalArgumentException { + return String.valueOf(rowObject.getAnalysisId()); } @Override @@ -73,39 +74,39 @@ public String getColumnDescription() { return "Click to open analysis in RevEng.AI portal"; } }); - descriptor.addVisibleColumn(new AbstractDynamicTableColumn() { + descriptor.addVisibleColumn(new AbstractDynamicTableColumn() { @Override public String getColumnName() { return "Binary Name"; } @Override - public String getValue(LegacyAnalysisResult rowObject, Settings settings, Object data, ServiceProvider serviceProvider) throws IllegalArgumentException { - return rowObject.binary_name(); + public String getValue(AnalysisRecordBody rowObject, Settings settings, Object data, ServiceProvider serviceProvider) throws IllegalArgumentException { + return rowObject.getBinaryName(); } }); - descriptor.addVisibleColumn(new AbstractDynamicTableColumn() { + descriptor.addVisibleColumn(new AbstractDynamicTableColumn() { @Override public String getColumnName() { return "Creation Time"; } @Override - public String getValue(LegacyAnalysisResult rowObject, Settings settings, Object data, ServiceProvider serviceProvider) throws IllegalArgumentException { - return rowObject.creation(); + public OffsetDateTime getValue(AnalysisRecordBody rowObject, Settings settings, Object data, ServiceProvider serviceProvider) throws IllegalArgumentException { + return rowObject.getCreation(); } }); - descriptor.addVisibleColumn(new AbstractDynamicTableColumn() { + descriptor.addVisibleColumn(new AbstractDynamicTableColumn() { @Override public String getColumnName() { return "Status"; } @Override - public AnalysisStatus getValue(LegacyAnalysisResult rowObject, Settings settings, Object data, ServiceProvider serviceProvider) throws IllegalArgumentException { - return rowObject.status(); + public String getValue(AnalysisRecordBody rowObject, Settings settings, Object data, ServiceProvider serviceProvider) throws IllegalArgumentException { + return rowObject.getStatus(); } }); diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysisDialog.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysisDialog.java index 21b65b46..d7f7069e 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysisDialog.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysisDialog.java @@ -4,7 +4,8 @@ import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisStatusChangedEvent; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; -import ai.reveng.toolkit.ghidra.core.services.api.types.LegacyAnalysisResult; +import ai.reveng.model.AnalysisRecordBody; +import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisStatus; import ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage; import ghidra.framework.plugintool.PluginTool; import ghidra.program.model.listing.Program; @@ -21,12 +22,12 @@ /** - * Shows a dialog with a table of {@link LegacyAnalysisResult} for a given {@link TypedApiInterface.BinaryHash}, + * Shows a dialog with a table of {@link AnalysisRecordBody} for a given {@link TypedApiInterface.BinaryHash}, * and fires an event when the user picks an analysis */ public class RecentAnalysisDialog extends RevEngDialogComponentProvider { private final RecentAnalysesTableModel recentAnalysesTableModel; - private final GhidraFilterTable recentAnalysesTable; + private final GhidraFilterTable recentAnalysesTable; private final PluginTool tool; private final Program program; private final GhidraRevengService ghidraRevengService; @@ -65,14 +66,13 @@ public void mouseClicked(MouseEvent e) { // Check if clicked column is "Analysis ID" (column 0) String columnName = recentAnalysesTable.getTable().getColumnName(col); if ("Analysis ID".equals(columnName)) { - LegacyAnalysisResult result = recentAnalysesTable.getModel().getRowObject(row); + AnalysisRecordBody result = recentAnalysesTable.getModel().getRowObject(row); if (result != null) { - var binaryID = result.binary_id(); + var analysisID = new TypedApiInterface.AnalysisID(Math.toIntExact(result.getAnalysisId())); tool.execute(new Task("Open analysis in portal", false, false, false) { @Override public void run(TaskMonitor monitor) { try { - var analysisID = ghidraRevengService.getApi().getAnalysisIDfromBinaryID(binaryID); ghidraRevengService.openPortalFor(analysisID); } catch (Exception ex) { Msg.error(RecentAnalysisDialog.this, "Failed to open analysis in portal: " + ex.getMessage(), ex); @@ -91,7 +91,7 @@ public void run(TaskMonitor monitor) { pickMostRecentButton.setName("Pick most recent"); pickMostRecentButton.addActionListener(e -> { var mostRecent = recentAnalysesTable.getModel().getModelData().stream().max( - Comparator.comparing(LegacyAnalysisResult::creation) + Comparator.comparing(AnalysisRecordBody::getCreation) ).orElseThrow(); pickAnalysis(mostRecent); }); @@ -108,20 +108,21 @@ public void run(TaskMonitor monitor) { addWorkPanel(mainPanel); } - private void pickAnalysis(LegacyAnalysisResult result) { + private void pickAnalysis(AnalysisRecordBody result) { var service = tool.getService(GhidraRevengService.class); tool.execute(new Task("Attach to analysis", true, false, false) { @Override public void run(TaskMonitor monitor) { try { - var analysisID = service.getApi().getAnalysisIDfromBinaryID(result.binary_id()); + var analysisID = new TypedApiInterface.AnalysisID(Math.toIntExact(result.getAnalysisId())); var programWithID = service.registerAnalysisForProgram(program, analysisID); SwingUtilities.invokeLater(() -> { tool.firePluginEvent( new RevEngAIAnalysisStatusChangedEvent( "Recent Analysis Dialog", programWithID, - result.status() + // The table only holds analyses the model filtered to Complete + AnalysisStatus.Complete ) ); close(); diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/settingsdialog/ANNSettingsDialog.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/settingsdialog/ANNSettingsDialog.java deleted file mode 100644 index c8de1374..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/settingsdialog/ANNSettingsDialog.java +++ /dev/null @@ -1,56 +0,0 @@ -package ai.reveng.toolkit.ghidra.binarysimilarity.ui.settingsdialog; - -import docking.DialogComponentProvider; - -import javax.swing.*; -import java.awt.*; - -/** - * Dialog for setting the options for similarity searches: - * - Number of Results - * - Distance - * - */ -public class ANNSettingsDialog extends DialogComponentProvider { - private final JTextField numResultsBox; - private final JTextField similarityBox; - - private double similarity; - private int numResults; - - public ANNSettingsDialog() { - super("ANN Settings", true, false, true, false); - var settingsPanel = new JPanel(); - - settingsPanel.setLayout(new GridLayout(0, 1)); - numResultsBox = new JTextField("10"); - settingsPanel.add(new JLabel("Number of Results")); - settingsPanel.add(numResultsBox); - - settingsPanel.add(new JLabel("Similarity")); - similarityBox = new JTextField("0.9"); - settingsPanel.add(similarityBox); - - addOKButton(); - addCancelButton(); - - addWorkPanel(settingsPanel); - } - - @Override - protected void okCallback() { - numResults = Integer.parseInt(numResultsBox.getText()); - similarity = Double.parseDouble(similarityBox.getText()); - close(); - } - - public int getNumResults(){ - return numResults; - } - - public double getSimilarity(){ - return similarity; - } - - -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/chat/model/ChatEvent.java b/src/main/java/ai/reveng/toolkit/ghidra/chat/model/ChatEvent.java index ac4e45e7..ddb4472c 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/chat/model/ChatEvent.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/chat/model/ChatEvent.java @@ -53,8 +53,6 @@ public record EntityRef(long id, String name, long vaddr) {} public static final Set TERMINAL_EVENTS = Set.of("RUN_FINISHED", "RUN_ERROR", "RUN_CANCELLED"); public static final int ROLE_USER = 2; - public static final int ROLE_SYSTEM = 3; - public static final int ROLE_TOOL = 4; /// Resolve a wire {@code type} (string name or integer 1..17) to its canonical name. public static String resolveType(Object typeField) { diff --git a/src/main/java/ai/reveng/toolkit/ghidra/chat/ui/ChatController.java b/src/main/java/ai/reveng/toolkit/ghidra/chat/ui/ChatController.java index c1f8007f..58a64533 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/chat/ui/ChatController.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/chat/ui/ChatController.java @@ -62,10 +62,6 @@ public ChatController(ChatService service, ChatView view, ReaiLoggingService log this.callbacks = callbacks; } - public ChatState state() { - return state; - } - public void send(String text) { String content = text == null ? "" : text.strip(); if (content.isEmpty() || "running".equals(state.runStatus())) { diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/ImportFunctionBoundariesFromRevEngAnalyzer.java b/src/main/java/ai/reveng/toolkit/ghidra/core/ImportFunctionBoundariesFromRevEngAnalyzer.java deleted file mode 100644 index 5da741b5..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/ImportFunctionBoundariesFromRevEngAnalyzer.java +++ /dev/null @@ -1,21 +0,0 @@ -package ai.reveng.toolkit.ghidra.core; - -import ghidra.app.services.AbstractAnalyzer; -import ghidra.app.services.AnalyzerType; -import ghidra.app.util.importer.MessageLog; -import ghidra.program.model.address.AddressSetView; -import ghidra.program.model.listing.Program; -import ghidra.util.exception.CancelledException; -import ghidra.util.task.TaskMonitor; - -public class ImportFunctionBoundariesFromRevEngAnalyzer extends AbstractAnalyzer { - - public ImportFunctionBoundariesFromRevEngAnalyzer() { - super("Import Function Boundaries From RevEng", "Imports function boundaries from RevEng", AnalyzerType.BYTE_ANALYZER); - } - - @Override - public boolean added(Program program, AddressSetView set, TaskMonitor monitor, MessageLog log) throws CancelledException { - return false; - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/RevEngAIAnalysisStatusChangedEvent.java b/src/main/java/ai/reveng/toolkit/ghidra/core/RevEngAIAnalysisStatusChangedEvent.java index 4bbb7930..795244f0 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/RevEngAIAnalysisStatusChangedEvent.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/RevEngAIAnalysisStatusChangedEvent.java @@ -2,7 +2,6 @@ import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisStatus; -import ai.reveng.toolkit.ghidra.core.services.api.types.BinaryID; import ghidra.framework.plugintool.PluginEvent; import ghidra.program.model.listing.Program; diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/Utils.java b/src/main/java/ai/reveng/toolkit/ghidra/core/Utils.java deleted file mode 100644 index 4299a10b..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/Utils.java +++ /dev/null @@ -1,23 +0,0 @@ -package ai.reveng.toolkit.ghidra.core; - -import ai.reveng.toolkit.ghidra.core.services.api.ModelName; -import ghidra.app.util.opinion.ElfLoader; -import ghidra.app.util.opinion.PeLoader; -import ghidra.program.model.listing.Program; - -import java.util.Collections; -import java.util.List; - -public class Utils { - - public static ModelName getModelNameForProgram(Program program, List models){ - var s = models.stream().map (ModelName::modelName); - var format = program.getOptions("Program Information").getString("Executable Format", null); - if (format.equals(ElfLoader.ELF_NAME)){ - s = s.filter(modelName -> modelName.contains("linux")); - } else if (format.equals(PeLoader.PE_NAME)) { - s = s.filter(modelName -> modelName.contains("windows")); - } - return new ModelName(s.sorted(Collections.reverseOrder()).toList().get(0)); - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/models/ReaiConfig.java b/src/main/java/ai/reveng/toolkit/ghidra/core/models/ReaiConfig.java index 1e56f7dd..c3af1ddf 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/models/ReaiConfig.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/models/ReaiConfig.java @@ -10,24 +10,12 @@ public PluginSettings getPluginSettings() { public void setPluginSettings(PluginSettings pluginSettings) { this.pluginSettings = pluginSettings; } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("REAI Config:\n"); - sb.append("\tPlugin Settings:\n"); - sb.append("\t\tAPI_Key: " + this.pluginSettings.getApiKey() + "\n"); - sb.append("\t\tHostname: " + this.pluginSettings.getHostname() + "\n"); - sb.append("\t\tModel Name: " + this.pluginSettings.getModelName() + "\n"); - return sb.toString(); - } - + public static class PluginSettings { private String apiKey; private String hostname; private String portalHostname; - private String modelName; - + public String getApiKey() { return apiKey; } @@ -51,13 +39,5 @@ public String getPortalHostname() { public void setPortalHostname(String portalHostname) { this.portalHostname = portalHostname; } - - public String getModelName() { - return modelName; - } - - public void setModelName(String modelName) { - this.modelName = modelName; - } } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/APIError.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/APIError.java deleted file mode 100644 index db7e1d33..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/APIError.java +++ /dev/null @@ -1,15 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api; - -import org.json.JSONObject; - -public record APIError( - String code, - String message -) { - public static APIError fromJSONObject(JSONObject json) { - return new APIError( - json.getString("code"), - json.getString("message") - ); - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/APIVersion.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/APIVersion.java deleted file mode 100644 index 92e5ffda..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/APIVersion.java +++ /dev/null @@ -1,5 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api; - -public enum APIVersion { - V2 -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/AnalysisDataTypesService.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/AnalysisDataTypesService.java new file mode 100644 index 00000000..a9e265cb --- /dev/null +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/AnalysisDataTypesService.java @@ -0,0 +1,293 @@ +package ai.reveng.toolkit.ghidra.core.services.api; + +import ai.reveng.invoker.ApiException; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface.AnalysisID; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.ServerDataType; +import ghidra.program.model.data.DataType; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/// Owns an analysis' `data_type_id` namespace. +/// +/// Type ids are only meaningful inside the analysis that minted them, and the server hands them out +/// rather than accepting a name. Anything that needs to talk about a type — resolving a signature's +/// references, or later naming a type to write back — has to go through the analysis' catalogue, +/// which is what this builds and caches. +/// +/// It is also the only thing that mints or mutates ids: {@link #ensure} takes Ghidra types and +/// hands back the analysis' id for each of them, creating what the analysis does not have yet. +/// Everything downstream — a function signature naming its parameter types, say — consumes those +/// ids and never invents one. +public final class AnalysisDataTypesService { + + /// `GET /v3/analyses/{analysis_id}/data-types` is paged; 500 is the largest page the endpoint + /// serves, and a binary's whole type catalogue is routinely in the thousands. + private static final long PAGE_SIZE = 500; + + /// Guards against an endless loop if the server ever stops advancing the offset. + private static final int MAX_PAGES = 200; + + /// The create and update endpoints each cap a request at 100 types. + private static final int WRITE_BATCH_SIZE = 100; + + private final TypedApiInterface api; + private final Map cache = new ConcurrentHashMap<>(); + + public AnalysisDataTypesService(TypedApiInterface api) { + this.api = api; + } + + /// Identifies a type the way a human does — by scope, name and kind — for the cases where an id + /// is not known yet. Two types in one analysis never share all three. + public record TypeKey(String namespace, String name, ServerDataType.Kind kind) { + public static TypeKey of(ServerDataType type) { + return new TypeKey(type.namespace() == null ? "" : type.namespace(), type.name(), type.kind()); + } + } + + /// Every type of one analysis, indexed both ways. + public record Catalogue(Map byId, Map idByKey) { + + public static Catalogue of(Collection types) { + Map byId = new LinkedHashMap<>(); + Map idByKey = new LinkedHashMap<>(); + for (ServerDataType type : types) { + if (byId.putIfAbsent(type.id(), type) == null) { + idByKey.putIfAbsent(TypeKey.of(type), type.id()); + } + } + return new Catalogue(Collections.unmodifiableMap(byId), Collections.unmodifiableMap(idByKey)); + } + + public static Catalogue empty() { + return new Catalogue(Map.of(), Map.of()); + } + + /// The id of `(namespace, name, kind)`, or empty if this analysis has no such type. + public Optional idOf(TypeKey key) { + return Optional.ofNullable(idByKey.get(key)); + } + + public Optional get(long dataTypeId) { + return Optional.ofNullable(byId.get(dataTypeId)); + } + + public int size() { + return byId.size(); + } + + /// The catalogue with `types` folded in, the newer entry winning. Used to fold a write's + /// response back in so the next resolve sees what was just created without re-paging. + public Catalogue with(Collection types) { + if (types.isEmpty()) { + return this; + } + Map mergedById = new LinkedHashMap<>(byId); + Map mergedIdByKey = new LinkedHashMap<>(idByKey); + for (ServerDataType type : types) { + mergedById.put(type.id(), type); + mergedIdByKey.put(TypeKey.of(type), type.id()); + } + return new Catalogue(Collections.unmodifiableMap(mergedById), + Collections.unmodifiableMap(mergedIdByKey)); + } + } + + /// Page the analysis' types in and cache the result. + public Catalogue sync(AnalysisID analysisID) { + List collected = new ArrayList<>(); + for (int page = 0; page < MAX_PAGES; page++) { + List batch = api.listAnalysisDataTypes(analysisID, page * PAGE_SIZE, PAGE_SIZE); + if (batch.isEmpty()) { + break; + } + collected.addAll(batch); + if (batch.size() < PAGE_SIZE) { + break; + } + } + Catalogue catalogue = Catalogue.of(collected); + cache.put(analysisID, catalogue); + return catalogue; + } + + /// The cached catalogue, syncing first if this analysis has not been read yet. + public Catalogue catalogue(AnalysisID analysisID) { + Catalogue cached = cache.get(analysisID); + return cached != null ? cached : sync(analysisID); + } + + /// Resolve `(namespace, name, kind)` to the analysis' id for it. + public Optional idOf(AnalysisID analysisID, TypeKey key) { + return catalogue(analysisID).idOf(key); + } + + public Optional get(AnalysisID analysisID, long dataTypeId) { + return catalogue(analysisID).get(dataTypeId); + } + + /// Make sure the analysis holds every type reachable from `roots`, and answer with the id of + /// each one. + /// + /// Two properties matter here, and both are correctness rather than economy. + /// + /// **Resolve before create.** Every type in the closure is first looked up in the analysis' + /// catalogue by `(namespace, name, kind)`; only the genuine gaps are created. A push is + /// reactive and repeats on every edit, so a version that created unconditionally would fill the + /// analysis with duplicates of the same type. Which namespace a Ghidra type is looked up and + /// filed under is {@link #storageKey}'s decision, and part of the same property: a namespace + /// the analysis has never used is a Ghidra-side scope, not a server one. + /// + /// **Create in two phases.** A `Create*` body carries no `data_type_id`, so nothing in a batch + /// can refer to anything else in that same batch. The gaps are therefore created with empty + /// definitions purely to obtain ids, and every definition — the ones just created and the ones + /// that already existed — is then written in a second request, by which time every reference + /// resolves. Kinds that carry no definition are finished after the first phase. + /// + /// Last write wins: the server no longer versions types for optimistic concurrency, so there is + /// no conflict detection and no retry. Failures surface as {@link ApiException}. + /// + /// The returned map is keyed by the Ghidra-derived {@link GhidraDataTypeEncoder#keyOf} of every + /// type in the closure, so a caller can look an id up with nothing but the Ghidra type in hand. + /// Where a type was filed under a different namespace than its category path implies — see + /// {@link #storageKey} — that key is present too, pointing at the same id. + public Map ensure(AnalysisID analysisID, Collection roots) throws ApiException { + List closure = GhidraDataTypeEncoder.closure(roots); + if (closure.isEmpty()) { + return Map.of(); + } + + Catalogue catalogue = catalogue(analysisID); + Map ids = new LinkedHashMap<>(); + // The key each Ghidra type is stored under, which is its derived key unless that namespace + // turned out to be a purely local one. + Map storage = new LinkedHashMap<>(); + Map missing = new LinkedHashMap<>(); + for (DataType type : closure) { + TypeKey derived = GhidraDataTypeEncoder.keyOf(type); + TypeKey stored = storageKey(catalogue, derived); + storage.put(derived, stored); + catalogue.idOf(stored).ifPresentOrElse( + id -> { + ids.put(derived, id); + ids.putIfAbsent(stored, id); + }, + // Two derived keys can flatten onto one storage key; that is one server type. + () -> missing.putIfAbsent(stored, type)); + } + + if (!missing.isEmpty()) { + for (ServerDataType created : create(analysisID, missing)) { + ids.putIfAbsent(TypeKey.of(created), created.id()); + } + storage.forEach((derived, stored) -> { + Long id = ids.get(stored); + if (id != null) { + ids.putIfAbsent(derived, id); + } + }); + } + + List updates = new ArrayList<>(); + Set written = new HashSet<>(); + for (DataType type : closure) { + TypeKey derived = GhidraDataTypeEncoder.keyOf(type); + Long id = ids.get(derived); + // One id is written once even if several Ghidra types resolved onto it. + if (id != null && written.add(id)) { + GhidraDataTypeEncoder.updateEntry(type, storage.get(derived).namespace(), id, ids::get) + .ifPresent(updates::add); + } + } + update(analysisID, updates); + + return Map.copyOf(ids); + } + + /// The key the analysis should hold a Ghidra type under, given what it already holds. + /// + /// A Ghidra category path is not only ever a server namespace. A type out of one of Ghidra's own + /// data-type archives sits in a category named after that archive — `/windows_vs12_32/DWORD` — + /// and never came from the server; taking that category as a namespace would look for a type the + /// analysis has never heard of and create a duplicate of the `DWORD` it does hold at the root. + /// The rule is that only the server names namespaces: a type the analysis already has under the + /// derived namespace keeps it, and anything else is local and belongs at the root. + /// + /// This cannot conflate two genuinely distinct types that share a name in different server + /// namespaces. Both of those are in the catalogue under their own namespaces, so both take the + /// first branch and resolve to their own ids; the root is only ever fallen back to for a + /// namespace the analysis holds no such type in at all, where there is nothing to be confused + /// with. What it does accept is that a Ghidra archive's `DWORD` and a root `DWORD` of the same + /// kind are one type — which is exactly what already happens to a `DWORD` the analyst declares + /// at the root by hand. + private static TypeKey storageKey(Catalogue catalogue, TypeKey derived) { + if (derived.namespace().isEmpty() || catalogue.idOf(derived).isPresent()) { + return derived; + } + return new TypeKey("", derived.name(), derived.kind()); + } + + /// `POST /v3/analyses/{analysis_id}/data-types` for types the analysis does not have, chunked to + /// the endpoint's batch limit. Returns the created types as the server stored them, ids + /// included. + private List create(AnalysisID analysisID, Map types) throws ApiException { + List entries = types.entrySet().stream() + .map(entry -> GhidraDataTypeEncoder.createEntry(entry.getValue(), entry.getKey().namespace())) + .toList(); + List created = new ArrayList<>(); + for (int start = 0; start < entries.size(); start += WRITE_BATCH_SIZE) { + var body = new ai.reveng.model.CreateAnalysisDataTypesInputBody(); + body.setDataTypes(entries.subList(start, Math.min(start + WRITE_BATCH_SIZE, entries.size()))); + created.addAll(api.createAnalysisDataTypes(analysisID, body)); + } + record(analysisID, created); + return created; + } + + /// `PUT /v3/analyses/{analysis_id}/data-types`, chunked to the endpoint's batch limit. + /// + /// The endpoint replaces a stored type in full, so a caller must send a complete definition — + /// {@link GhidraDataTypeEncoder#updateEntry} declines to build one it cannot fill. + public List update(AnalysisID analysisID, + List updates) throws ApiException { + if (updates.isEmpty()) { + return List.of(); + } + List updated = new ArrayList<>(); + for (int start = 0; start < updates.size(); start += WRITE_BATCH_SIZE) { + var body = new ai.reveng.model.UpdateAnalysisDataTypesInputBody(); + body.setDataTypes(updates.subList(start, Math.min(start + WRITE_BATCH_SIZE, updates.size()))); + updated.addAll(api.updateAnalysisDataTypes(analysisID, body)); + } + record(analysisID, updated); + return updated; + } + + /// Fold a write's response into the cached catalogue, so the next push resolves what this one + /// created instead of creating it again. + private void record(AnalysisID analysisID, Collection types) { + if (types.isEmpty()) { + return; + } + cache.compute(analysisID, (id, current) -> (current == null ? Catalogue.empty() : current).with(types)); + } + + /// Drop a cached catalogue, or all of them when `analysisID` is null. + public void invalidate(@Nullable AnalysisID analysisID) { + if (analysisID == null) { + cache.clear(); + } else { + cache.remove(analysisID); + } + } +} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/AnalysisOptionsBuilder.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/AnalysisOptionsBuilder.java index bac48750..b1a0fc25 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/AnalysisOptionsBuilder.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/AnalysisOptionsBuilder.java @@ -20,7 +20,6 @@ public class AnalysisOptionsBuilder { // Package-private constructor for testing AnalysisOptionsBuilder() { options = new JSONObject(); - options.put("size_in_bytes", 0); options.put("tags", new JSONArray()); } @@ -51,15 +50,6 @@ public AnalysisOptionsBuilder fileName(String name) { return this; } - public AnalysisOptionsBuilder size(long size) { - options.put("size_in_bytes", size); - return this; - } - - public long getSize() { - return options.optLong("size_in_bytes", 0); - } - public AnalysisOptionsBuilder scope(AnalysisScope scope){ options.put("binary_scope", scope.scope); return this; @@ -86,26 +76,6 @@ public static AnalysisOptionsBuilder forProgram(Program program, Predicate readAssembly(Object basicBlocks) { + if (basicBlocks == null) { + return List.of(); + } + JsonElement blocks = JSON.getGson().toJsonTree(basicBlocks); + if (!blocks.isJsonArray()) { + return List.of(); + } + + List ordered = new ArrayList<>(); + for (JsonElement block : blocks.getAsJsonArray()) { + if (block.isJsonObject()) { + ordered.add(block.getAsJsonObject()); + } + } + ordered.sort(Comparator.comparingLong(DisassemblyBlocksReader::startAddress)); + + List assembly = new ArrayList<>(); + for (JsonObject block : ordered) { + appendAssembly(assembly, block); + } + return assembly; + } + + /// Blocks that declare no start address sort last, so the ones that do keep their address order. + private static long startAddress(JsonObject block) { + JsonElement minAddr = block.get("min_addr"); + if (minAddr == null || !minAddr.isJsonPrimitive() || !minAddr.getAsJsonPrimitive().isNumber()) { + return Long.MAX_VALUE; + } + return minAddr.getAsLong(); + } + + private static void appendAssembly(List assembly, JsonObject block) { + JsonElement asm = block.get("asm"); + if (asm == null || !asm.isJsonArray()) { + return; + } + for (JsonElement line : asm.getAsJsonArray()) { + if (line.isJsonPrimitive()) { + assembly.add(line.getAsString()); + } + } + } +} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/FunctionSignatureService.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/FunctionSignatureService.java new file mode 100644 index 00000000..1f013f92 --- /dev/null +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/FunctionSignatureService.java @@ -0,0 +1,105 @@ +package ai.reveng.toolkit.ghidra.core.services.api; + +import ai.reveng.invoker.ApiException; +import ai.reveng.model.BatchFunctionSignatureEntry; +import ai.reveng.model.FunctionSignatureVersion; +import ai.reveng.model.UpdateFunctionSignatureInputBody; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface.AnalysisID; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface.FunctionID; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.FunctionSignatureBatch; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.ServerDataType; +import ghidra.util.Msg; + +import java.util.List; +import java.util.Optional; + +/// Reads function signatures, and the data types they reference, from `/v3/functions/signatures`. +/// +/// The endpoint takes function ids from any number of analyses at once and answers with the +/// signatures plus, optionally, every type those signatures point at — grouped by the analysis that +/// owns the ids. That makes a whole-binary read one round trip per chunk instead of one per +/// function, and removes the old "generate types, then poll until ready" dance: the server derives +/// signatures when the analysis completes. +/// +/// Writing goes the other way and is deliberately singular: {@link #put} is one request for one +/// function. It consumes `data_type_id`s and never creates them — a signature that names a type the +/// analysis does not have yet is the caller's problem to solve first, by going through +/// {@link AnalysisDataTypesService#ensure}, which owns that namespace. +public final class FunctionSignatureService { + + /// The function ids ride in the query string, so a whole-binary request overflows the request + /// URI (HTTP 414) unless it is chunked. + private static final int DATA_TYPES_BATCH_SIZE = 50; + + private final TypedApiInterface api; + + public FunctionSignatureService(TypedApiInterface api) { + this.api = api; + } + + /// One function's signature together with the types it references. + public record Resolved(BatchFunctionSignatureEntry entry, List dataTypes) {} + + /// The signature of a single function, or empty when the server holds none for it. + public Optional get(FunctionID functionID) { + FunctionSignatureBatch batch = getMany(List.of(functionID)); + return batch.items().stream() + .filter(entry -> Boolean.TRUE.equals(entry.getHasSignature())) + .findFirst() + .map(entry -> new Resolved(entry, batch.dataTypesFor(analysisOf(entry)))); + } + + /// Signatures for many functions, with their data types. Ids may span analyses. + public FunctionSignatureBatch getMany(List functionIDs) { + return getMany(functionIDs, true); + } + + /// As {@link #getMany(List)}, but `includeDataTypes` false when the caller only needs to know + /// which functions have a signature at all — that answer is far cheaper without the type + /// closure attached. + public FunctionSignatureBatch getMany(List functionIDs, boolean includeDataTypes) { + if (functionIDs == null || functionIDs.isEmpty()) { + return FunctionSignatureBatch.empty(); + } + List ids = functionIDs.stream().distinct().toList(); + FunctionSignatureBatch merged = FunctionSignatureBatch.empty(); + for (int start = 0; start < ids.size(); start += DATA_TYPES_BATCH_SIZE) { + List chunk = ids.subList(start, Math.min(start + DATA_TYPES_BATCH_SIZE, ids.size())); + merged = merged.merge(api.listFunctionSignatures(chunk, includeDataTypes)); + } + return merged; + } + + /// Write one function's signature. True when the server accepted it. + /// + /// `PUT .../signature` edits an extracted signature and nothing else: a function the server has + /// no signature for — `has_signature` false, which is the normal state of a thunk, an external + /// function, or anything in an analysis where type extraction never ran — is answered with 404. + /// That is not a failure worth telling the user about. The push is reactive on a short debounce, + /// so a warning per keystroke on an unextracted function would be pure noise; it is logged at + /// debug and reported as "not written". + public boolean put(AnalysisID analysisID, FunctionID functionID, + UpdateFunctionSignatureInputBody signature) throws ApiException { + try { + api.updateFunctionSignature(analysisID, functionID, signature); + return true; + } catch (ApiException e) { + if (e.getCode() == 404) { + Msg.debug(FunctionSignatureService.class, + "Skipping signature push for function %d: the server holds no extracted signature for it" + .formatted(functionID.value())); + return false; + } + throw e; + } + } + + /// The recorded versions of one function's signature, newest first as the server orders them. + public List history(AnalysisID analysisID, FunctionID functionID) { + return api.getFunctionSignatureHistory(analysisID, functionID); + } + + private static AnalysisID analysisOf(BatchFunctionSignatureEntry entry) { + return new AnalysisID(Math.toIntExact(entry.getAnalysisId())); + } +} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraDataTypeEncoder.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraDataTypeEncoder.java new file mode 100644 index 00000000..974bb46f --- /dev/null +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraDataTypeEncoder.java @@ -0,0 +1,524 @@ +package ai.reveng.toolkit.ghidra.core.services.api; + +import ai.reveng.toolkit.ghidra.core.services.api.AnalysisDataTypesService.TypeKey; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.ServerDataType.Kind; +import ghidra.program.model.data.Array; +import ghidra.program.model.data.BitFieldDataType; +import ghidra.program.model.data.CategoryPath; +import ghidra.program.model.data.Composite; +import ghidra.program.model.data.DataType; +import ghidra.program.model.data.DataTypeComponent; +import ghidra.program.model.data.Enum; +import ghidra.program.model.data.FunctionDefinition; +import ghidra.program.model.data.ParameterDefinition; +import ghidra.program.model.data.Pointer; +import ghidra.program.model.data.Structure; +import ghidra.program.model.data.TypeDef; +import ghidra.program.model.data.Undefined; +import ghidra.program.model.data.Union; +import ghidra.program.model.listing.Function; +import ghidra.program.model.listing.Parameter; +import ghidra.program.model.listing.Variable; +import ghidra.program.model.listing.VariableStorage; + +import javax.annotation.Nullable; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Deque; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/// Turns Ghidra data types into the v3 create/update bodies, the mirror of +/// {@link ServerDataTypeDecoder}. +/// +/// Decoding resolves references by `data_type_id`, so encoding has to produce them. A Ghidra type +/// carries no server id, so the identity used on the way out is the one the server files types +/// under — `(namespace, name, kind)`, a {@link TypeKey} — and every reference between types is +/// emitted by looking the referenced type's key up in a map of already-known ids. Minting those ids +/// is {@link AnalysisDataTypesService}'s job; this class only ever reads them. +/// +/// `namespace` is the inverse of the decoder's category path: a type at the root category encodes +/// as the empty namespace, which is what a locally authored Ghidra type gets, and a type the plugin +/// pulled from the server round-trips back to the namespace it arrived with. +/// +/// The derived namespace is what a type is *looked up* by. What it is *written* under is a separate +/// decision, because a Ghidra category is not only ever a server namespace — a type out of one of +/// Ghidra's own archives sits in a category named after that archive and never came from the server +/// at all. {@link AnalysisDataTypesService} makes that call and passes the namespace in; the +/// overloads without one keep the derived value. +public final class GhidraDataTypeEncoder { + + /// Upper bound on transitive dependency resolution, so a pathological type graph cannot make a + /// reactive push walk forever. Also keeps a single push well inside the endpoints' batch limits. + private static final int MAX_TYPES = 500; + + /// Ghidra's placeholders for "no calling convention recorded", which the API would rather not + /// have at all than have as a literal. + private static final Set UNSET_CALLING_CONVENTIONS = Set.of( + Function.UNKNOWN_CALLING_CONVENTION_STRING, Function.DEFAULT_CALLING_CONVENTION_STRING); + + private GhidraDataTypeEncoder() {} + + /// Resolves a referenced type to the analysis' id for it. Absent ids encode as an omitted + /// reference, which the API reads as "unresolved" rather than as an error. + @FunctionalInterface + public interface Ids { + @Nullable + Long idOf(TypeKey key); + + static Ids of(Map ids) { + return ids::get; + } + } + + /// Every type reachable from a function's signature and variables: its return type, its + /// parameters, its stack variables, and everything those reach transitively. + /// + /// This is the root set of a type push — the closure has to be resolved to ids before the + /// signature that names them can be written. + public static List reachableTypes(Function function) { + return closure(roots(function)); + } + + /// The names of every type {@link #reachableTypes} finds, for deciding which functions a local + /// edit to a named type affects. + public static Set referencedTypeNames(Function function) { + Set names = new LinkedHashSet<>(); + for (DataType type : reachableTypes(function)) { + names.add(type.getName()); + } + return names; + } + + /// Close `roots` over their dependencies, de-duplicated by {@link TypeKey} because that is the + /// identity the server stores: two Ghidra instances of the same key are one server type. + /// + /// Dependencies come after the type that needs them where the graph allows it, but the order is + /// not load-bearing — references are written as ids, which are all known before any definition + /// is built. + public static List closure(Collection roots) { + Map seen = new LinkedHashMap<>(); + Deque queue = new ArrayDeque<>(); + for (DataType root : roots) { + if (root != null) { + queue.add(root); + } + } + while (!queue.isEmpty() && seen.size() < MAX_TYPES) { + DataType type = queue.poll(); + if (type == null || seen.putIfAbsent(keyOf(type), type) != null) { + continue; + } + queue.addAll(dependenciesOf(type)); + } + return List.copyOf(seen.values()); + } + + /// The server's identity for a Ghidra type. + public static TypeKey keyOf(DataType type) { + return new TypeKey(namespaceOf(type), nameOf(type), kindOf(type)); + } + + /// The `kind` discriminator for a Ghidra type. + /// + /// The structural interfaces are tested before the "no length" fallback because a + /// {@link FunctionDefinition} reports a length of -1 while still being a fully modelled type. + public static Kind kindOf(@Nullable DataType type) { + if (type == null || Undefined.isUndefined(type)) { + return Kind.UNKNOWN; + } + if (type instanceof TypeDef) { + return Kind.TYPEDEF; + } + if (type instanceof Pointer) { + return Kind.POINTER; + } + if (type instanceof Array) { + return Kind.ARRAY; + } + if (type instanceof Enum) { + return Kind.ENUM; + } + if (type instanceof Structure) { + return Kind.STRUCT; + } + if (type instanceof Union) { + return Kind.UNION; + } + if (type instanceof FunctionDefinition) { + return Kind.FUNCTION_DEFINITION; + } + if (type.getLength() < 0) { + return Kind.UNKNOWN; + } + return Kind.BASE; + } + + /// A create body for one type, carrying no definition worth the name. + /// + /// A `Create*` variant has no `data_type_id`, so nothing in the same batch can be pointed at: + /// the definitions are deliberately left empty here and filled in by + /// {@link #updateEntry(DataType, long, Ids)} once the server has assigned ids. + public static ai.reveng.model.CreateDataTypeEntry createEntry(DataType type) { + return createEntry(type, namespaceOf(type)); + } + + /// As {@link #createEntry(DataType)}, but filed under `namespace` rather than under the one the + /// type's category path implies. + public static ai.reveng.model.CreateDataTypeEntry createEntry(DataType type, String namespace) { + Long size = sizeOf(type); + String name = nameOf(type); + return new ai.reveng.model.CreateDataTypeEntry(switch (kindOf(type)) { + case STRUCT -> new ai.reveng.model.CreateStructDataType() + .kind(ai.reveng.model.CreateStructDataType.KindEnum.STRUCT) + .name(name).namespace(namespace).size(size) + .definition(new ai.reveng.model.StructDefinition().members(List.of())); + case UNION -> new ai.reveng.model.CreateUnionDataType() + .kind(ai.reveng.model.CreateUnionDataType.KindEnum.UNION) + .name(name).namespace(namespace).size(size) + .definition(new ai.reveng.model.UnionDefinition().members(List.of())); + case ENUM -> new ai.reveng.model.CreateEnumDataType() + .kind(ai.reveng.model.CreateEnumDataType.KindEnum.ENUM) + .name(name).namespace(namespace).size(size) + .definition(new ai.reveng.model.EnumDefinition().values(List.of())); + case TYPEDEF -> new ai.reveng.model.CreateTypedefDataType() + .kind(ai.reveng.model.CreateTypedefDataType.KindEnum.TYPEDEF) + .name(name).namespace(namespace).size(size) + .definition(new ai.reveng.model.TypedefDefinition()); + case POINTER -> new ai.reveng.model.CreatePointerDataType() + .kind(ai.reveng.model.CreatePointerDataType.KindEnum.POINTER) + .name(name).namespace(namespace).size(size) + .definition(new ai.reveng.model.PointerDefinition()); + case ARRAY -> new ai.reveng.model.CreateArrayDataType() + .kind(ai.reveng.model.CreateArrayDataType.KindEnum.ARRAY) + .name(name).namespace(namespace).size(size) + .definition(new ai.reveng.model.ArrayDefinition()); + case FUNCTION_DEFINITION -> new ai.reveng.model.CreateFunctionDataType() + .kind(ai.reveng.model.CreateFunctionDataType.KindEnum.FUNCTION_DEFINITION) + .name(name).namespace(namespace).size(size) + .definition(new ai.reveng.model.FunctionTypeDefinition().parameters(List.of())); + // These kinds carry no definition at all, so creating them completes in one request. + case BITFIELD -> new ai.reveng.model.CreateBitfieldDataType() + .kind(ai.reveng.model.CreateBitfieldDataType.KindEnum.BITFIELD) + .name(name).namespace(namespace).size(size); + case BASE -> new ai.reveng.model.CreateBaseDataType() + .kind(ai.reveng.model.CreateBaseDataType.KindEnum.BASE) + .name(name).namespace(namespace).size(size); + case UNKNOWN -> new ai.reveng.model.CreateUnknownDataType() + .kind(ai.reveng.model.CreateUnknownDataType.KindEnum.UNKNOWN) + .name(name).namespace(namespace).size(size); + }); + } + + /// An update body for one type, with every reference resolved through `ids`. + /// + /// Empty for a type whose Ghidra form says nothing the server does not already have: the kinds + /// that never carry a definition, and a composite or reference type that is locally a bare + /// placeholder. `PUT` replaces a stored type in full, so writing an empty definition would + /// erase whatever the server extracted; a push that has nothing to say says nothing. + public static Optional updateEntry(DataType type, long id, Ids ids) { + return updateEntry(type, namespaceOf(type), id, ids); + } + + /// As {@link #updateEntry(DataType, long, Ids)}, but filed under `namespace` rather than under + /// the one the type's category path implies. `PUT` replaces a stored type in full, so this has + /// to be the namespace the entry `id` already lives at — otherwise the update would move it. + public static Optional updateEntry(DataType type, + String namespace, + long id, + Ids ids) { + Long size = sizeOf(type); + String name = nameOf(type); + return switch (kindOf(type)) { + case STRUCT -> { + List members = membersOf((Composite) type, false, ids); + yield members.isEmpty() ? Optional.empty() : Optional.of(new ai.reveng.model.UpdateDataTypeEntry( + new ai.reveng.model.UpdateStructDataType() + .kind(ai.reveng.model.UpdateStructDataType.KindEnum.STRUCT) + .dataTypeId(id).name(name).namespace(namespace).size(size) + .definition(new ai.reveng.model.StructDefinition().members(members)))); + } + case UNION -> { + List members = membersOf((Composite) type, true, ids); + yield members.isEmpty() ? Optional.empty() : Optional.of(new ai.reveng.model.UpdateDataTypeEntry( + new ai.reveng.model.UpdateUnionDataType() + .kind(ai.reveng.model.UpdateUnionDataType.KindEnum.UNION) + .dataTypeId(id).name(name).namespace(namespace).size(size) + .definition(new ai.reveng.model.UnionDefinition().members(members)))); + } + case ENUM -> { + List values = valuesOf((Enum) type); + yield values.isEmpty() ? Optional.empty() : Optional.of(new ai.reveng.model.UpdateDataTypeEntry( + new ai.reveng.model.UpdateEnumDataType() + .kind(ai.reveng.model.UpdateEnumDataType.KindEnum.ENUM) + .dataTypeId(id).name(name).namespace(namespace).size(size) + .definition(new ai.reveng.model.EnumDefinition().values(values)))); + } + case TYPEDEF -> { + Long target = ids.idOf(keyOf(((TypeDef) type).getDataType())); + yield target == null ? Optional.empty() : Optional.of(new ai.reveng.model.UpdateDataTypeEntry( + new ai.reveng.model.UpdateTypedefDataType() + .kind(ai.reveng.model.UpdateTypedefDataType.KindEnum.TYPEDEF) + .dataTypeId(id).name(name).namespace(namespace).size(size) + .definition(new ai.reveng.model.TypedefDefinition().targetDataTypeId(target)))); + } + case POINTER -> { + // A null pointee is `void *`, which has nothing to resolve and nothing to write. + DataType pointee = ((Pointer) type).getDataType(); + Long target = pointee == null ? null : ids.idOf(keyOf(pointee)); + yield target == null ? Optional.empty() : Optional.of(new ai.reveng.model.UpdateDataTypeEntry( + new ai.reveng.model.UpdatePointerDataType() + .kind(ai.reveng.model.UpdatePointerDataType.KindEnum.POINTER) + .dataTypeId(id).name(name).namespace(namespace).size(size) + .definition(new ai.reveng.model.PointerDefinition().pointeeDataTypeId(target)))); + } + case ARRAY -> { + Array array = (Array) type; + Long element = ids.idOf(keyOf(array.getDataType())); + yield element == null ? Optional.empty() : Optional.of(new ai.reveng.model.UpdateDataTypeEntry( + new ai.reveng.model.UpdateArrayDataType() + .kind(ai.reveng.model.UpdateArrayDataType.KindEnum.ARRAY) + .dataTypeId(id).name(name).namespace(namespace).size(size) + .definition(new ai.reveng.model.ArrayDefinition() + .count((long) array.getNumElements()) + .elementDataTypeId(element)))); + } + case FUNCTION_DEFINITION -> { + FunctionDefinition definition = (FunctionDefinition) type; + List parameters = parametersOf(definition, ids); + Long returnType = ids.idOf(keyOf(definition.getReturnType())); + yield parameters.isEmpty() && returnType == null + ? Optional.empty() + : Optional.of(new ai.reveng.model.UpdateDataTypeEntry( + new ai.reveng.model.UpdateFunctionDataType() + .kind(ai.reveng.model.UpdateFunctionDataType.KindEnum.FUNCTION_DEFINITION) + .dataTypeId(id).name(name).namespace(namespace).size(size) + .definition(new ai.reveng.model.FunctionTypeDefinition() + .parameters(parameters) + .returnDataTypeId(returnType)))); + } + // Nothing beyond name, namespace and size, all of which the create already carried. + case BASE, BITFIELD, UNKNOWN -> Optional.empty(); + }; + } + + /// The function's local signature as a signature update, with every type named by id. + /// + /// `PUT .../signature` replaces the stored signature in full, so parameter storage is carried + /// over as well: leaving it out would clear the storage the server extracted. + public static ai.reveng.model.UpdateFunctionSignatureInputBody signatureOf(Function function, Ids ids) { + var body = new ai.reveng.model.UpdateFunctionSignatureInputBody(); + + String callingConvention = function.getCallingConventionName(); + if (callingConvention != null && !callingConvention.isBlank() + && !UNSET_CALLING_CONVENTIONS.contains(callingConvention)) { + body.setCallingConvention(callingConvention); + } + + List parameters = new ArrayList<>(); + Parameter[] declared = function.getParameters(); + for (int ordinal = 0; ordinal < declared.length; ordinal++) { + Parameter parameter = declared[ordinal]; + var input = new ai.reveng.model.SignatureParameterInput() + .ordinal((long) ordinal) + .name(parameter.getName()) + .dataTypeId(ids.idOf(keyOf(parameter.getDataType()))) + .bitLength(Math.max(0, parameter.getLength()) * 8L); + storageOf(parameter).ifPresent(input::storage); + parameters.add(input); + } + body.setParameters(parameters); + body.setReturnDataTypeId(ids.idOf(keyOf(function.getReturnType()))); + return body; + } + + /// Convenience overload for callers holding the id map {@link AnalysisDataTypesService#ensure} + /// returned. + public static ai.reveng.model.UpdateFunctionSignatureInputBody signatureOf(Function function, + Map ids) { + return signatureOf(function, Ids.of(ids)); + } + + private static List roots(Function function) { + List roots = new ArrayList<>(); + roots.add(function.getReturnType()); + for (Parameter parameter : function.getParameters()) { + roots.add(parameter.getDataType()); + } + for (Variable variable : function.getLocalVariables()) { + if (variable.isStackVariable()) { + roots.add(variable.getDataType()); + } + } + return roots; + } + + /// The types one type refers to. A pointer's pointee and an array's element count as + /// dependencies because they are types in their own right on the server, each with its own id. + private static List dependenciesOf(DataType type) { + List dependencies = new ArrayList<>(); + switch (kindOf(type)) { + case STRUCT, UNION -> { + for (DataTypeComponent component : ((Composite) type).getDefinedComponents()) { + dependencies.add(memberTypeOf(component)); + } + } + case TYPEDEF -> dependencies.add(((TypeDef) type).getDataType()); + case POINTER -> dependencies.add(((Pointer) type).getDataType()); + case ARRAY -> dependencies.add(((Array) type).getDataType()); + case FUNCTION_DEFINITION -> { + FunctionDefinition definition = (FunctionDefinition) type; + dependencies.add(definition.getReturnType()); + for (ParameterDefinition parameter : definition.getArguments()) { + dependencies.add(parameter.getDataType()); + } + } + default -> { + } + } + dependencies.removeIf(java.util.Objects::isNull); + return dependencies; + } + + /// A bitfield is expressed on the member rather than as a type of its own, so what the member + /// points at is the bitfield's base type. + @Nullable + private static DataType memberTypeOf(DataTypeComponent component) { + if (component.getDataType() instanceof BitFieldDataType bitField) { + return bitField.getBaseDataType(); + } + return component.getDataType(); + } + + private static List membersOf(Composite composite, + boolean union, + Ids ids) { + boolean bigEndian = isBigEndian(composite); + List members = new ArrayList<>(); + for (DataTypeComponent component : composite.getDefinedComponents()) { + DataType memberType = memberTypeOf(component); + var member = new ai.reveng.model.DataTypeMemberEntry() + // A null field name is legal: it is how unnamed padding is reported. + .name(component.getFieldName()) + .offset(union ? 0L : component.getOffset()) + .size((long) component.getLength()) + .dataTypeId(memberType == null ? null : ids.idOf(keyOf(memberType))); + if (component.getDataType() instanceof BitFieldDataType bitField) { + member.isBitfield(true) + .bitOffset(bitOffsetOf(component, bitField, bigEndian)) + .bitSize((long) bitField.getBitSize()); + } else { + member.isBitfield(false); + } + members.add(member); + } + return members; + } + + /// The bit offset of a bitfield member from the start of the containing type, which is what the + /// API asks for. + /// + /// Ghidra reports the offset of the least-significant bit within the component's storage unit, + /// so the two agree only on a little-endian target, where the least-significant bit *is* the + /// first one. Big-endian fills a storage unit from the most-significant end, so the same field + /// has to be counted from the other side of the unit: a big-endian `int a:1` at the start of a + /// struct reports a bit offset of 7, not 0, and successive fields count down rather than up. + private static long bitOffsetOf(DataTypeComponent component, BitFieldDataType bitField, boolean bigEndian) { + long withinUnit = bigEndian + ? component.getLength() * 8L - bitField.getBitOffset() - bitField.getBitSize() + : bitField.getBitOffset(); + return component.getOffset() * 8L + withinUnit; + } + + /// A composite with no manager cannot say what it is laid out for; little-endian is both the + /// commoner case and what Ghidra's own default data organisation assumes. + private static boolean isBigEndian(Composite composite) { + var manager = composite.getDataTypeManager(); + return manager != null && manager.getDataOrganization() != null + && manager.getDataOrganization().isBigEndian(); + } + + /// Enum constants keep their decimal-string form all the way out: a value may be negative or + /// exceed 64 unsigned bits, so it is never parsed into a number on the wire. + private static List valuesOf(Enum enumeration) { + List values = new ArrayList<>(); + for (String name : enumeration.getNames()) { + values.add(new ai.reveng.model.DataTypeEnumValueEntry() + .name(name) + .value(Long.toString(enumeration.getValue(name)))); + } + return values; + } + + private static List parametersOf(FunctionDefinition definition, + Ids ids) { + List parameters = new ArrayList<>(); + ParameterDefinition[] arguments = definition.getArguments(); + for (int ordinal = 0; ordinal < arguments.length; ordinal++) { + ParameterDefinition argument = arguments[ordinal]; + parameters.add(new ai.reveng.model.DataTypeFunctionParameterEntry() + .ordinal((long) ordinal) + .size((long) Math.max(0, argument.getLength())) + .name(argument.getName()) + .dataTypeId(ids.idOf(keyOf(argument.getDataType())))); + } + return parameters; + } + + private static Optional storageOf(Parameter parameter) { + VariableStorage storage = parameter.getVariableStorage(); + if (storage == null || !storage.isValid()) { + return Optional.empty(); + } + if (storage.isRegisterStorage() && storage.getRegister() != null) { + return Optional.of(new ai.reveng.model.SignatureStorageInput() + .kind("reg").location(storage.getRegister().getName())); + } + if (storage.isStackStorage()) { + return Optional.of(new ai.reveng.model.SignatureStorageInput() + .kind("stack").location(Integer.toString(storage.getStackOffset()))); + } + if (storage.isMemoryStorage()) { + return Optional.of(new ai.reveng.model.SignatureStorageInput().kind("mem")); + } + return Optional.empty(); + } + + /// The scope the server files the type under, as the inverse of + /// {@link ServerDataTypeDecoder}'s category path. A locally authored type lives at the root + /// category and so pushes with the empty namespace. + private static String namespaceOf(@Nullable DataType type) { + if (type == null) { + return ""; + } + CategoryPath path = type.getCategoryPath(); + if (path == null || path.isRoot()) { + return ""; + } + return String.join("::", path.getPathElements()); + } + + private static String nameOf(@Nullable DataType type) { + if (type == null) { + return "undefined"; + } + String name = type.getName(); + return name == null || name.isBlank() ? "undefined" : name; + } + + /// Ghidra reports -1 for a type with no meaningful size (a function definition); the API wants + /// the field omitted rather than negative. + @Nullable + private static Long sizeOf(@Nullable DataType type) { + if (type == null) { + return null; + } + int length = type.getLength(); + return length < 0 ? null : (long) length; + } +} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java index dc815a82..000838e4 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java @@ -9,20 +9,18 @@ import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionInfo; import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionMatch; import ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage; -import ai.reveng.toolkit.ghidra.binarysimilarity.ui.aidecompiler.AIDecompilationdWindow; import ai.reveng.toolkit.ghidra.core.services.api.mocks.MockApi; import ai.reveng.toolkit.ghidra.core.services.api.types.*; import ai.reveng.toolkit.ghidra.core.services.logging.ReaiLoggingService; -import ai.reveng.toolkit.ghidra.core.services.api.types.binsync.*; -import ai.reveng.toolkit.ghidra.core.services.api.types.exceptions.APIAuthenticationException; -import com.google.common.collect.BiMap; -import com.google.common.collect.HashBiMap; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.FunctionSignatureBatch; import ghidra.app.cmd.function.ApplyFunctionSignatureCmd; +import ghidra.app.cmd.function.FunctionRenameOption; import ghidra.app.cmd.function.SetFunctionNameCmd; import ghidra.framework.plugintool.PluginTool; import ghidra.program.model.address.Address; import ghidra.program.model.data.*; import ghidra.program.model.data.Structure; +import ghidra.program.model.data.TypedefDataType; import ghidra.program.model.listing.BookmarkManager; import ghidra.program.model.listing.CircularDependencyException; import ghidra.program.model.listing.Function; @@ -34,13 +32,11 @@ import ghidra.util.BrowserLoader; import ghidra.util.InvalidNameException; import ghidra.util.Msg; -import ghidra.util.data.DataTypeParser; import ghidra.util.exception.CancelledException; import ghidra.util.exception.DuplicateNameException; import ghidra.util.exception.InvalidInputException; import ghidra.util.exception.NoValueException; import ghidra.util.task.TaskMonitor; -import org.jetbrains.annotations.NotNull; import javax.annotation.Nullable; import java.awt.*; @@ -91,9 +87,6 @@ public class GhidraRevengService { /// plugin's {@code analysis_sync_service.is_worker_running()} guard. private final AtomicBoolean pushbackSuppressed = new AtomicBoolean(false); - /// Number of times a data-type push is retried when the server reports a version conflict. - private static final int TYPE_PUSH_MAX_RETRIES = 3; - /// dedicated functions on the GhidraRevengService should be used instead to enforce assumptions via /// type level guarantees // @Deprecated @@ -101,6 +94,34 @@ public TypedApiInterface getApi() { return api; } + private FunctionSignatureService signatureService; + private AnalysisDataTypesService analysisDataTypesService; + + /// Reads function signatures and the data types they reference. + public FunctionSignatureService signatures() { + if (signatureService == null) { + signatureService = new FunctionSignatureService(api); + } + return signatureService; + } + + /// Reads an analysis' data-type catalogue, which owns its `data_type_id` namespace. + public AnalysisDataTypesService analysisDataTypes() { + if (analysisDataTypesService == null) { + analysisDataTypesService = new AnalysisDataTypesService(api); + } + return analysisDataTypesService; + } + + /// One decoder per analysis: a `data_type_id` only means something inside the analysis that + /// minted it, so each group of types gets its own {@link DataTypeManager}. + private static ServerDataTypeDecoder decoderFor(Map decoders, + FunctionSignatureBatch batch, + BatchFunctionSignatureEntry entry) { + var analysisID = new TypedApiInterface.AnalysisID(Math.toIntExact(entry.getAnalysisId())); + return decoders.computeIfAbsent(analysisID, id -> ServerDataTypeDecoder.decode(batch.dataTypesFor(id))); + } + public GhidraRevengService(ApiInfo apiInfo){ this.apiInfo = apiInfo; this.api = new TypedApiImplementation(apiInfo); @@ -158,57 +179,7 @@ private Namespace getRevEngAINameSpace(Program program) { } return revengMatchNamespace; } - /** - * Tries to find a BinaryID for a given program - * If the program already has a BinaryID associated with it, it will return that - * If we don't have a BinaryID it will return an empty Optional - * @param program - * @return - */ - @Deprecated - public Optional getBinaryIDFor(Program program) { - return getBinaryIDfromOptions(program); - } - - @SuppressWarnings("deprecation") // Using deprecated method to support legacy BinaryID - private Optional getAnalysisIDFor(Program program){ - var optAnalysisID = getAnalysisIDFromOptions(program); - if (optAnalysisID.isPresent()){ - return optAnalysisID; - } - // Fallback to getting it from the BinaryID, if one exists - var legacyBinaryID = getBinaryIDFor(program); - if (legacyBinaryID.isPresent()) { - // We have a legacy binary ID, upgrade to AnalysisID - var analysisID = api.getAnalysisIDfromBinaryID(legacyBinaryID.get()); - addAnalysisIDtoProgramOptions(program, analysisID); - program.withTransaction("Remove legacy BinaryID from program options", () -> - program.getOptions(ReaiPluginPackage.REAI_OPTIONS_CATEGORY) - .setLong(ReaiPluginPackage.OPTION_KEY_BINID, ReaiPluginPackage.INVALID_BINARY_ID) - ); - return Optional.of(analysisID); - } - return Optional.empty(); - } - - /// This is a helper to get the AnalysisID from the BinaryID, in the rare cases that this is required - /// Currently the only known case is when opening the analysis on the portal in the browser - private Optional getBinaryIDFromAnalysisID(TypedApiInterface.AnalysisID analysisID) { - try { - var info = api.getAnalysisBasicInfo(analysisID); - var results = api.search(new TypedApiInterface.BinaryHash(info.getSha256Hash())); - var binaryId = results.stream().filter( r -> r.analysis_id().equals(analysisID)) - .findAny().map( r -> r.binary_id()); - return binaryId; - - } catch (ApiException e) { - throw new RuntimeException(e); - } - } - - private Optional getAnalysisIDFromOptions( - Program program - ) { + private Optional getAnalysisIDFor(Program program) { long bid = program.getOptions( ReaiPluginPackage.REAI_OPTIONS_CATEGORY).getLong(OPTION_KEY_ANALYSIS_ID, ReaiPluginPackage.INVALID_ANALYSIS_ID); @@ -218,36 +189,6 @@ private Optional getAnalysisIDFromOptions( return Optional.of(new TypedApiInterface.AnalysisID((int) bid)); } - @Deprecated - private Optional getBinaryIDfromOptions( - Program program - ) { - long bid = program.getOptions( - ReaiPluginPackage.REAI_OPTIONS_CATEGORY).getLong(ReaiPluginPackage.OPTION_KEY_BINID, - ReaiPluginPackage.INVALID_BINARY_ID); - if (bid == ReaiPluginPackage.INVALID_BINARY_ID) { - return Optional.empty(); - } - var binID = new BinaryID((int) bid); - // Check that it's really valid in the context of the currently configured API - AnalysisStatus status; - try { - status = api.status(binID); - } catch (APIAuthenticationException | ApiException e) { - Msg.error(this, - ("The Binary ID %s stored in the program options is invalid for the currently configured RevEng.AI server %s. " - + "This could be an intermittent error, or you switched servers") - .formatted(binID, this.apiInfo.hostURI()), e); - return Optional.empty(); - } - var analysisID = api.getAnalysisIDfromBinaryID(binID); - statusCache.put(analysisID, status); - - // Now it's certain that it is a valid binary ID - - return Optional.of(binID); - } - /// Loads the function info into a dedicated user property map. /// This method should only concern itself with associating the FunctionID with the Ghidra Function /// This property is immutable within an Analysis: The function ID will never change unless an entirely different @@ -345,9 +286,6 @@ private void markFunctionAsRevEng(BookmarkManager bookmarkManager, Function func public record RenameResult(Function func, String originalName, String newName) { - public String virtualAddress() { - return func.getEntryPoint().toString(); - } } /// Push a local function rename back to the portal. Returns the namespace-qualified name that was @@ -383,52 +321,69 @@ static String qualifiedServerName(Function function) { return String.join(Namespace.DELIMITER, parts); } - /// Push the local signature and variables of a function back to the portal. Uses optimistic - /// concurrency: the current server version is fetched and sent back, and version conflicts are - /// retried against the latest version. Returns true if the server accepted the update. - /// No-op (returns false) if the function is not known on the server. - public boolean pushFunctionTypes(AnalysedProgram analysedProgram, Function function) throws ApiException { - var withId = analysedProgram.getIDForFunction(function); - if (withId.isEmpty()) { - return false; + /// What a push achieved for one function. + /// + /// The data types and the signature go up in separate requests, and the second one legitimately + /// does nothing: the portal answers a signature write for a function it never extracted a + /// signature for with a 404. Collapsing that into a boolean reported the whole push as failed + /// and hid the types that *were* written, which makes a local type edit look like a no-op. + public enum TypePushOutcome { + /// The function is not part of the analysis, so nothing was sent. + NOT_MATCHED, + /// The data types were written; the portal holds no extracted signature to update. + TYPES_ONLY, + /// The data types and the function's signature were both written. + SIGNATURE_WRITTEN + } + + /// Push the local signature and variables of a function back to the portal. + public TypePushOutcome pushFunctionTypes(AnalysedProgram analysedProgram, Function function) throws ApiException { + if (analysedProgram.getIDForFunction(function).isEmpty()) { + return TypePushOutcome.NOT_MATCHED; } - var functionID = withId.get().functionID(); - long imageBase = analysedProgram.program().getImageBase().getOffset(); - var localTypes = GhidraToServerTypeSerializer.buildFunctionInfo(function, imageBase); - - for (int attempt = 0; attempt < TYPE_PUSH_MAX_RETRIES; attempt++) { - long version = api.getFunctionDataTypesWithVersion(functionID) - .map(TypedApiInterface.VersionedFunctionTypes::version) - .orElse(0L); - var results = api.pushFunctionDataTypes(analysedProgram.analysisID(), - List.of(new TypedApiInterface.FunctionDataTypeUpdate(functionID, localTypes, version))); - if (results.isEmpty()) { - return false; - } - var result = results.get(0); - switch (result.status()) { - case UPDATED -> { - return true; - } - case VERSION_CONFLICT -> { - // Re-fetch the latest version and retry. - } - default -> { - Msg.warn(this, "Failed to push types for function %s: %s" - .formatted(function.getName(), result.error())); - return false; - } + // The function is known to the analysis, so the type pass below ran for it; only the + // signature write can still decline. + return pushFunctionTypes(analysedProgram, List.of(function)) > 0 + ? TypePushOutcome.SIGNATURE_WRITTEN + : TypePushOutcome.TYPES_ONLY; + } + + /// Push the local signatures of several functions, and answer with how many the server took. + /// + /// The two halves of the write path compose here, and only here. Data-type management is a + /// batch affair — the union of everything the functions reach is resolved against the analysis' + /// catalogue and created where it is missing, in one pass — while a signature is written one + /// function at a time. Running the type pass once for the whole set is the point of keeping the + /// two apart: the alternative re-resolves the same closure per function. + public int pushFunctionTypes(AnalysedProgram analysedProgram, Collection functions) throws ApiException { + Map known = new LinkedHashMap<>(); + for (Function function : functions) { + analysedProgram.getIDForFunction(function) + .ifPresent(withId -> known.put(function, withId.functionID())); + } + if (known.isEmpty()) { + return 0; + } + + List roots = new ArrayList<>(); + known.keySet().forEach(function -> roots.addAll(GhidraDataTypeEncoder.reachableTypes(function))); + var ids = analysisDataTypes().ensure(analysedProgram.analysisID(), roots); + + int pushed = 0; + for (var entry : known.entrySet()) { + var signature = GhidraDataTypeEncoder.signatureOf(entry.getKey(), ids); + if (signatures().put(analysedProgram.analysisID(), entry.getValue(), signature)) { + pushed++; } } - Msg.warn(this, "Gave up pushing types for function %s after %d version conflicts" - .formatted(function.getName(), TYPE_PUSH_MAX_RETRIES)); - return false; + return pushed; } /// Breakdown of a bidirectional analysis sync, shown to the user afterwards. public record SyncSummary( int matchedFunctions, int namesModifiedRemotely, + int appliedSignatures, int canonicalizedNames, int dedupedNames, int pushedNames, @@ -442,9 +397,11 @@ private record InvalidRemoteName(Function function, TypedApiInterface.FunctionID /// /// Names: applies remote names locally where the local name is not user-defined, canonicalising /// names Ghidra rejects (via the portal canonify endpoint) and de-duplicating names already used - /// this run; corrected names are pushed back to the portal. Types: pushes local types for matched - /// functions whose remote types are absent or could not be applied. Mirrors the IDA plugin's - /// {@code analysis_sync.py}. + /// this run; corrected names are pushed back to the portal. Signatures: applies the portal's + /// signature, and the data types it names, to every matched function whose local signature the + /// analyst did not write by hand — see {@link #applyRemoteSignatures}. Types: pushes local types + /// for matched functions whose remote types are absent or could not be applied. Mirrors the IDA + /// plugin's {@code analysis_sync.py}. public SyncSummary syncAnalysisUpdates(AnalysedProgram analysedProgram, TaskMonitor monitor, ReaiLoggingService log) throws ApiException { pushbackSuppressed.set(true); try { @@ -552,12 +509,150 @@ private SyncSummary syncAnalysisUpdatesInternal(AnalysedProgram analysedProgram, if (pushedNames > 0) { log.info("Pushed %d corrected function name(s) back to the RevEng.AI portal".formatted(pushedNames)); } + // Pull before pushing, so the back-fill below sees the signatures this sync just applied and + // does not send the local placeholder back up for a function the portal already described. + int appliedSignatures = applyRemoteSignatures(analysedProgram, monitor, log); int pushedTypeSets = pushLocalTypesWhereRemoteMissing(analysedProgram, functionInfoMap.keySet(), monitor, log); - log.info(("Sync complete: %d matched, %d name(s) applied, %d canonicalized, %d de-duplicated, " - + "%d name(s) and %d type set(s) pushed back") - .formatted(matched, namesModifiedRemotely, canonicalizedNames, deduped, pushedNames, pushedTypeSets)); - return new SyncSummary(matched, namesModifiedRemotely, canonicalizedNames, deduped, pushedNames, pushedTypeSets); + log.info(("Sync complete: %d matched, %d name(s) applied, %d signature(s) applied, %d canonicalized, " + + "%d de-duplicated, %d name(s) and %d type set(s) pushed back") + .formatted(matched, namesModifiedRemotely, appliedSignatures, canonicalizedNames, deduped, + pushedNames, pushedTypeSets)); + return new SyncSummary(matched, namesModifiedRemotely, appliedSignatures, canonicalizedNames, deduped, + pushedNames, pushedTypeSets); + } + + /// Apply the signatures — and with them the data types they name — that the portal holds for this + /// analysis' functions, and answer with how many landed. + /// + /// Names are deliberately left alone. {@link #syncAnalysisUpdatesInternal} owns that decision, + /// including canonicalising names Ghidra rejects and de-duplicating the ones it has already + /// applied this run, so the command is given {@link FunctionRenameOption#NO_CHANGE} and nothing + /// here competes with it. + /// + /// A signature is applied unless the analyst wrote the local one by hand; anything Ghidra's own + /// analysis inferred is fair game. That is also what makes a second sync useful: an earlier pull + /// stamps {@link SourceType#ANALYSIS}, so a "default only" guard would turn every pull after the + /// first into a no-op. + private int applyRemoteSignatures(AnalysedProgram analysedProgram, TaskMonitor monitor, ReaiLoggingService log) { + var functionMap = analysedProgram.getFunctionMap(); + if (functionMap.isEmpty()) { + return 0; + } + var batch = signatures().getMany(List.copyOf(functionMap.keySet())); + Map decoders = new HashMap<>(); + var program = analysedProgram.program(); + int applied = 0; + var transactionId = program.startTransaction("RevEng.AI: Apply Portal Signatures"); + try { + for (BatchFunctionSignatureEntry entry : batch.items()) { + if (monitor.isCancelled()) { + break; + } + if (!Boolean.TRUE.equals(entry.getHasSignature())) { + continue; + } + var function = functionMap.get(new TypedApiInterface.FunctionID(entry.getFunctionId())); + if (function == null || function.isExternal() || function.isThunk() + || function.getSignatureSource() == SourceType.USER_DEFINED) { + continue; + } + var signature = getFunctionSignature(entry, decoderFor(decoders, batch, entry)); + // The portal derives a signature for every function when the analysis completes, so + // most of them match what is already on the function. Applying those regardless would + // churn the type manager and fill the undo history on every sync. + if (matchesLocalSignature(function, signature)) { + continue; + } + var application = applyRemoteSignature(program, function, signature, monitor, + FunctionRenameOption.NO_CHANGE); + if (application.success()) { + applied++; + // The prototype is logged because it is the only place the analyst can see what + // the portal actually sent, as against what they edited there. + log.info("Applied the portal's signature for \"%s\" at %s: %s" + .formatted(function.getName(), function.getEntryPoint(), + signature.getPrototypeString())); + } else { + Msg.warn(this, "Failed to apply the portal's signature for %s: %s" + .formatted(function.getName(), application.status())); + } + } + } finally { + program.endTransaction(transactionId, applied > 0 && !monitor.isCancelled()); + } + return applied; + } + + /// Apply one server-sourced signature to a function. + /// + /// The conflict handler is the load-bearing argument. Ghidra's default keeps the local type and + /// files the incoming one beside it as `.conflict`, so a type edited in the portal would + /// gather a fresh copy locally on every pull; {@link DataTypeConflictHandler#REPLACE_HANDLER} + /// updates the local type in place instead. The calling convention is preserved because a server + /// signature carries none — see {@link ServerDataTypeDecoder#signature} — and applying an absent + /// convention would discard whatever Ghidra had worked out. + /// Whether the portal's signature is, in every respect this pull applies, what the function + /// already has. + /// + /// {@link FunctionSignature#isEquivalentSignature} cannot answer that. It compares the function + /// name and the calling convention as well, and this pull applies neither: the name belongs to + /// the name reconciliation ({@link FunctionRenameOption#NO_CHANGE}) and a server signature + /// carries no convention. Every function therefore differed on every sync, was re-applied, and + /// was reported as applied for ever. + /// + /// A parameter the server does not name is not a difference. Ghidra names an unnamed parameter + /// `param_N` when the signature is applied, so treating a missing server name as a difference + /// would make the comparison oscillate and bring the same re-apply loop back. + static boolean matchesLocalSignature(Function function, FunctionDefinitionDataType incoming) { + if (!sameType(function.getReturnType(), incoming.getReturnType())) { + return false; + } + var local = function.getParameters(); + var remote = incoming.getArguments(); + if (local.length != remote.length) { + return false; + } + for (int i = 0; i < local.length; i++) { + String remoteName = remote[i].getName(); + if (remoteName != null && !remoteName.isBlank() && !remoteName.equals(local[i].getName())) { + return false; + } + if (!sameType(local[i].getDataType(), remote[i].getDataType())) { + return false; + } + } + return true; + } + + /// Compared through {@link DataType#isEquivalent} rather than `DataTypeUtilities`, whose package + /// moved between the Ghidra versions this extension is built against. + private static boolean sameType(@Nullable DataType local, @Nullable DataType remote) { + if (local == null || remote == null) { + return local == remote; + } + return local.isEquivalent(remote); + } + + /// @param success whether the signature landed + /// @param status the command's own account of why it did not, for the log + private record SignatureApplication(boolean success, String status) {} + + private static SignatureApplication applyRemoteSignature(Program program, Function function, + FunctionDefinitionDataType signature, + TaskMonitor monitor, + FunctionRenameOption renameOption) { + var command = new ApplyFunctionSignatureCmd( + function.getEntryPoint(), + signature, + SourceType.ANALYSIS, + true, + false, + DataTypeConflictHandler.REPLACE_HANDLER, + renameOption + ); + boolean success = command.applyTo(program, monitor); + return new SignatureApplication(success, success ? "" : String.valueOf(command.getStatusMsg())); } private int pushNameBacks(List namePushbacks) throws ApiException { @@ -583,15 +678,17 @@ private int pushLocalTypesWhereRemoteMissing(AnalysedProgram analysedProgram, Set matchedIds, TaskMonitor monitor, ReaiLoggingService log) { - var remoteItems = api.listFunctionDataTypesForAnalysis(analysedProgram.analysisID()).getItems(); - Set remotePresent = (remoteItems == null ? List.of() : remoteItems).stream() - .filter(item -> "completed".equals(item.getStatus())) - .filter(item -> item.getDataTypes() != null && item.getDataTypes().getFuncTypes() != null) + // Only the matched functions are candidates, and only whether the server holds a signature + // at all matters here, so ask for exactly those ids and skip the type closure. + Set remotePresent = signatures() + .getMany(List.copyOf(matchedIds), false) + .items().stream() + .filter(BatchFunctionSignatureEntry::getHasSignature) .map(item -> new TypedApiInterface.FunctionID(item.getFunctionId())) .collect(Collectors.toSet()); var functionMap = analysedProgram.getFunctionMap(); - int pushed = 0; + List candidates = new ArrayList<>(); for (TypedApiInterface.FunctionID functionID : matchedIds) { if (monitor.isCancelled()) { break; @@ -605,17 +702,21 @@ private int pushLocalTypesWhereRemoteMissing(AnalysedProgram analysedProgram, || function.getSignatureSource() == SourceType.DEFAULT) { continue; } - try { - if (pushFunctionTypes(analysedProgram, function)) { - pushed++; - log.info("Pushed local types for \"%s\" at %s (remote had none)" - .formatted(function.getName(), function.getEntryPoint())); - } - } catch (ApiException e) { - Msg.warn(this, "Failed to push types for %s during sync".formatted(function.getName()), e); - } + candidates.add(function); + } + if (candidates.isEmpty()) { + return 0; + } + try { + // One type pass over the union of every candidate's types, then a signature write each. + int pushed = pushFunctionTypes(analysedProgram, candidates); + log.info("Pushed local types for %d of %d functions the portal had none for" + .formatted(pushed, candidates.size())); + return pushed; + } catch (ApiException e) { + Msg.warn(this, "Failed to push local types during sync", e); + return 0; } - return pushed; } private boolean applyRemoteName(Program program, Function function, Namespace revEngNamespace, String name) { @@ -691,14 +792,19 @@ private List pullFunctionInfoFromAnalysisInternal(AnalysedProgram ); - Map signatureMap = api.listFunctionDataTypesForAnalysis(analysedProgram.analysisID).getItems() + // /v3/functions/signatures is addressed by function id rather than by analysis, so ask for + // the functions this analysis reported. The response carries the types those signatures + // reference alongside them, which is what the decoders below are built from. + var signatureBatch = signatures().getMany(List.copyOf(functionInfoMap.keySet())); + Map decoders = new HashMap<>(); + Map signatureMap = signatureBatch.items() .stream() - .filter(item -> item.getStatus().equals("completed")) - .filter(item -> item.getDataTypes().getFuncTypes() != null) + .filter(BatchFunctionSignatureEntry::getHasSignature) .collect( Collectors.toMap( item -> new TypedApiInterface.FunctionID(item.getFunctionId()), - fdtStatus -> fdtStatus + item -> item, + (existing, replacement) -> existing ) ); @@ -720,7 +826,6 @@ private List pullFunctionInfoFromAnalysisInternal(AnalysedProgram } // Get the current name on the server side -// FunctionDetails details = api.getFunctionDetails(fID.get().functionID); FunctionInfo details = functionInfoMap.get(fID.get().functionID); // Extract the mangled name from Ghidra @@ -733,23 +838,11 @@ private List pullFunctionInfoFromAnalysisInternal(AnalysedProgram continue; } - var sig = Optional.ofNullable(signatureMap.get(fID.get().functionID)); - // Get the type information on the server side - Optional functionSignatureMessageOpt = sig - // Try getting the data types if they are available - // If they are available, try converting them to a Ghidra signature - // If the conversion fails, act like there is no signature available - .flatMap (item -> Optional.ofNullable(item.getDataTypes())) - .flatMap((functionDataTypeMessage -> { - try { - return getFunctionSignature(functionDataTypeMessage); - } catch (DataTypeDependencyException e) { - // Something went wrong loading the data type dependencies - // just skip applying the signature and treat it like none being available - Msg.error(this, "Could not get parse signature for function %s".formatted(function.getName())); - return Optional.empty(); - } - })); + // Get the type information on the server side. Every type the signature refers to is + // resolved by id against its analysis' decoder, so there is nothing left to fail on. + Optional functionSignatureMessageOpt = + Optional.ofNullable(signatureMap.get(fID.get().functionID)) + .map(entry -> getFunctionSignature(entry, decoderFor(decoders, signatureBatch, entry))); analysedProgram.setMangledNameForFunction(function, revEngMangledName); @@ -760,62 +853,53 @@ private List pullFunctionInfoFromAnalysisInternal(AnalysedProgram /// IMPORTED: Information taken from an external source — symbols or signatures imported from a file or database. /// USER_DEFINED: A name or signature explicitly set by the analyst. /// See {@link ghidra.program.model.symbol.SourceType} for more details - if (function.getSymbol().getSource() == SourceType.DEFAULT) { - if (functionSignatureMessageOpt.isEmpty()) { - // We don't have signature information for this function, so we can only try renaming it. - // Skip server-side default names — Ghidra's own "FUN_" and IDA's "sub_" — so we never - // overwrite Ghidra's default placeholder with an IDA-style one. - if (function.getSymbol().getSource() == SourceType.DEFAULT - && !revEngMangledName.startsWith("FUN_") && !revEngMangledName.startsWith("sub_")) { - // The local function has the default name, so we can rename it - // The following check should never fail because it is a default name, - // and we checked above that the server name is not a default name - // but just to be safe and make that assumption explicit we check it explicitly - if (!function.getSymbol().getName(false).equals(revEngDemangledName)) { - Msg.info(this, "Renaming function %s to %s [%s]".formatted(ghidraMangledName, revEngMangledName, revEngDemangledName)); - try { - function.setParentNamespace(revEngNamespace); - } catch (DuplicateNameException | InvalidInputException | CircularDependencyException e) { - throw new RuntimeException(e); - } - var success = new SetFunctionNameCmd(function.getEntryPoint(), revEngDemangledName, SourceType.ANALYSIS) - .applyTo(analysedProgram.program()); - if (success) { - renameResults.add(new RenameResult( - function, - ghidraMangledName, - revEngDemangledName - )); - } else { - failedRenames++; - Msg.error(this, "Failed to rename function %s to %s [%s]".formatted(ghidraMangledName, revEngMangledName, revEngDemangledName)); - } - } + // The name and the signature are separate decisions. Gating the signature on the name + // source hid every type the portal held for a function that was already named — by the + // analyst, or by an earlier run of this same pull — and the signature apply is what + // carries the data types down. + if (functionSignatureMessageOpt.isEmpty()) { + // No signature to apply, so a rename is all that is left. Skip server-side default + // names — Ghidra's own "FUN_" and IDA's "sub_" — so we never overwrite Ghidra's + // default placeholder with an IDA-style one. + if (function.getSymbol().getSource() == SourceType.DEFAULT + && !revEngMangledName.startsWith("FUN_") && !revEngMangledName.startsWith("sub_") + && !function.getSymbol().getName(false).equals(revEngDemangledName)) { + Msg.info(this, "Renaming function %s to %s [%s]".formatted(ghidraMangledName, revEngMangledName, revEngDemangledName)); + try { + function.setParentNamespace(revEngNamespace); + } catch (DuplicateNameException | InvalidInputException | CircularDependencyException e) { + throw new RuntimeException(e); } - - } else { - /// We could use {@link ghidra.program.model.listing.FunctionSignature#isEquivalentSignature(FunctionSignature)} - /// if we expect the server to have changing signatures at any point in time. - /// For now, we only apply signatures to functions that have the default signature - if (function.getSignatureSource() == SourceType.DEFAULT) { - var success = new ApplyFunctionSignatureCmd( - function.getEntryPoint(), - functionSignatureMessageOpt.get(), - SourceType.ANALYSIS - ).applyTo(analysedProgram.program(), monitor); - // For unclear reasons the signature source is not set by the command in Ghidra 11.2.x and lower - if (success) { - renameResults.add(new RenameResult( - function, - ghidraMangledName, - revEngDemangledName - )); - } else { - Msg.error(this, "Failed to apply signature to function %s".formatted(function.getName())); - failedRenames++; - } + var success = new SetFunctionNameCmd(function.getEntryPoint(), revEngDemangledName, SourceType.ANALYSIS) + .applyTo(analysedProgram.program()); + if (success) { + renameResults.add(new RenameResult( + function, + ghidraMangledName, + revEngDemangledName + )); + } else { + failedRenames++; + Msg.error(this, "Failed to rename function %s to %s [%s]".formatted(ghidraMangledName, revEngMangledName, revEngDemangledName)); } } + } else if (function.getSignatureSource() != SourceType.USER_DEFINED) { + // RENAME_IF_DEFAULT preserves the long-standing behaviour that applying a signature + // also names a function still carrying Ghidra's placeholder, and leaves every other + // name alone. + var application = applyRemoteSignature(analysedProgram.program(), function, + functionSignatureMessageOpt.get(), monitor, FunctionRenameOption.RENAME_IF_DEFAULT); + // For unclear reasons the signature source is not set by the command in Ghidra 11.2.x and lower + if (application.success()) { + renameResults.add(new RenameResult( + function, + ghidraMangledName, + revEngDemangledName + )); + } else { + Msg.error(this, "Failed to apply signature to function %s".formatted(function.getName())); + failedRenames++; + } } @@ -831,20 +915,6 @@ private List pullFunctionInfoFromAnalysisInternal(AnalysedProgram return renameResults; } - /** - * Get the FunctionID for a Ghidra Function, if there is one - * There are two cases where a function ID is missing: - * 1. Either the whole program has not been analyzed - * (because its bounds were not included when the analysis was triggered) - * - * @deprecated Use {@link AnalysedProgram#getIDForFunction(Function)} instead. It forces the caller to prove that they know that the {@link Program} is indeed known on the server and associated by having to provide a {@link AnalysedProgram} instance. - */ - @Deprecated - public Optional getFunctionIDFor(Function function){ - return getAnalysedProgram(function.getProgram()) - .flatMap(knownProgram -> knownProgram.getIDForFunction(function).map(fidWithStatus -> fidWithStatus.functionID)); - } - /** * Get the Ghidra Function for a given FunctionInfo if there is one */ @@ -858,7 +928,7 @@ private Optional getFunctionFor(FunctionInfo functionInfo, Program pro } @Deprecated - public List searchForHash(TypedApiInterface.BinaryHash hash){ + public List searchForHash(TypedApiInterface.BinaryHash hash){ return api.search(hash); } @@ -876,8 +946,6 @@ public void removeProgramAssociation(Program program){ revengTag.delete(); } var reaiOptions = program.getOptions(ReaiPluginPackage.REAI_OPTIONS_CATEGORY); - //noinspection deprecation - reaiOptions.setLong(ReaiPluginPackage.OPTION_KEY_BINID, ReaiPluginPackage.INVALID_BINARY_ID); reaiOptions.setLong(OPTION_KEY_ANALYSIS_ID, ReaiPluginPackage.INVALID_ANALYSIS_ID); // Clear the entire cache. Getting the correct ID is not worth the effort in terms of edge cases to handle // because this method should still work even if the analysis ID or binary ID that was associated is invalid @@ -936,7 +1004,6 @@ public TypedApiInterface.BinaryHash upload(Program program) { var hash = api.upload(filePath); if (hash.equals(hashOfProgram(program))){ // TODO: Save the information that this program has been uploaded -// program.getOptions(REAI_OPTIONS_CATEGORY).setBoolean(ReaiPluginPackage.OPTION_KEY_BINID, hash.value()); return hash; } else { // This means the file on disk has @@ -957,20 +1024,6 @@ public TypedApiInterface.BinaryHash upload(Path path) { } } - @Deprecated - public AnalysisStatus pollStatus(BinaryID bid) { - try { - return api.status(bid); - } catch (ApiException e) { - throw new RuntimeException(e); - } - } - - /// Use this method if you just have an AnalysisID and it is not clear yet if it can be accessed - public AnalysisStatus pollStatus(TypedApiInterface.AnalysisID id) throws ApiException { - return api.status(id); - } - /// Current status of the server-side auto-unstrip pass, which runs after the analysis is complete. public TypedApiInterface.AutoUnstripStatus getAutoUnstripStatus(TypedApiInterface.AnalysisID id) throws ApiException { return api.getAutoUnstripStatus(id); @@ -987,45 +1040,6 @@ public AnalysisStatus status(ProgramWithID program) { - public String decompileFunctionViaAI(FunctionWithID functionWithID, TaskMonitor monitor, AIDecompilationdWindow window) { - monitor.setMaximum(100 * 50); - // Check if there is an existing process already, because the trigger API will fail with 400 if there is - var fID = functionWithID.functionID; - var function = functionWithID.function; - if (api.pollAIDecompileStatus(fID).status() == DecompilationData.StatusEnum.UNINITIALISED){ - // Trigger the decompilation - api.triggerAIDecompilationForFunctionID(fID); - } - - while (true) { - if (monitor.isCancelled()) { - return "Decompilation cancelled"; - } - var status = api.pollAIDecompileStatus(fID); - window.setDisplayedValuesBasedOnStatus(function, status); - - switch (status.status()) { - case PENDING: - case RUNNING: - case UNINITIALISED: - try { - Thread.sleep(100); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - break; - case COMPLETED: - monitor.setProgress(monitor.getMaximum()); - window.setDisplayedValuesBasedOnStatus(function, status); - return status.decompilation(); - case FAILED: - return "Decompilation failed: %s".formatted(status.status()); - default: - throw new RuntimeException("Unknown status: %s".formatted(status.status())); - } - } - } - /// This method analyses a program by uploading it (if necessary), triggering an analysis, and _blocking_ /// until the analysis is complete. This is for scripts and tests, and must not be used on the UI thread /// It does not upload the program, this must be done beforehand, and the hash must be associated via {@link AnalysisOptionsBuilder#hash(TypedApiInterface.BinaryHash)} @@ -1061,228 +1075,25 @@ public Optional getAnalysedProgram(Program program) { } /** - * Create a {@link FunctionDefinitionDataType} from a @{@link ai.reveng.model.FunctionInfo} in isolation + * Create a self-contained {@link FunctionDefinitionDataType} from a function's server signature. * - * All the required dependency types will be stored in the DataTypeManager that is associated with this - * FunctionDefinitionDataType + *

Every type the signature refers to is named by a {@code data_type_id}, so the decoder that + * holds this analysis' types resolves the return type and each parameter by lookup. The decoder's + * {@link DataTypeManager} owns the dependencies, which is what makes the result standalone. * - * @param functionDataTypeMessage The message containing the function signature, received from the API + * @param entry the signature as the server reports it + * @param decoder the decoded types of the analysis that {@code entry} belongs to * @return Self-contained signature for the function */ - public static Optional getFunctionSignature(ai.reveng.model.V2FunctionInfo functionDataTypeMessage) throws DataTypeDependencyException { - - // Create Data Type Manager with all dependencies - var d = FunctionDependencies.fromOpenAPI(functionDataTypeMessage.getFuncDeps()); - DataTypeManager tmpDtm = null; - try { - tmpDtm = loadDependencyDataTypes(d); - } catch (EndlessTypeParsingException e) { - Msg.error("getFunctionSignature", null, e); - return Optional.empty(); - } - DataTypeManager dtm = tmpDtm; - - if (functionDataTypeMessage.getFuncTypes() == null){ - return Optional.empty(); - } - var funcName = functionDataTypeMessage.getFuncTypes().getName(); - FunctionDefinitionDataType f = new FunctionDefinitionDataType(funcName, dtm); - - try { - f.setName(funcName); - } catch (InvalidNameException e) { - throw new RuntimeException(e); - } - - ParameterDefinitionImpl[] args = functionDataTypeMessage.getFuncTypes().getHeader().getArgs().values().stream().map( - arg -> { - DataType ghidraType = null; - try { - var scopedName = TypePathAndName.fromString(arg.getType()); - ghidraType = loadDataType(dtm, scopedName); - } catch (DataTypeDependencyException e) { - Msg.error(GhidraRevengService.class, - ("" + - "Couldn't find type '%s' for param of %s").formatted(arg.getType(), funcName) - ); - ghidraType = Undefined.getUndefinedDataType(arg.getSize()); - } - // Add the type to the DataTypeManager - return new ParameterDefinitionImpl(arg.getName(), ghidraType, null); - }).toArray(ParameterDefinitionImpl[]::new); - - f.setArguments(args); - - DataType returnType = null; - returnType = loadDataType(dtm, TypePathAndName.fromString(functionDataTypeMessage.getFuncTypes().getHeader().getType())); - f.setReturnType(returnType); - - - return Optional.of(f); - } - - public static class EndlessTypeParsingException extends Exception { - - public FunctionDependencies deps; - public List remaining; - private EndlessTypeParsingException(FunctionDependencies dependencies, List remainingTypes) { - super("Endless type parsing detected for function dependencies: " + dependencies); - deps = dependencies; - remaining = remainingTypes; - - } - } - - public static DataTypeManager loadDependencyDataTypes(FunctionDependencies dependencies) throws EndlessTypeParsingException{ - DataTypeManager dtm = new StandAloneDataTypeManager("transient"); - - if (dependencies == null){ - return dtm; - } - DataTypeParser dataTypeParser = new DataTypeParser( - dtm, - null, - null, - DataTypeParser.AllowedDataTypes.ALL); - - // We do this in two passes: - - // First add all types as empty placeholders - var transactionId = dtm.startTransaction("Load Dependencies"); - Arrays.stream(dependencies.structs()).forEach( - struct -> { -// CategoryPath path = new CategoryPath(CategoryPath.ROOT, struct.name().split("/")); - var typePathAndName = TypePathAndName.fromString(struct.name()); - StructureDataType structDataType = new StructureDataType( - typePathAndName.toCategoryPath(), - typePathAndName.name(), - struct.size(), - dtm); - dtm.addDataType(structDataType, DataTypeConflictHandler.REPLACE_EMPTY_STRUCTS_OR_RENAME_AND_ADD_HANDLER); - } - ); - // The following would be a lot nicer of BinSync could guarantee us that all dependencies are sorted - // As a workaround we just retry until all types are available - // In some cases (specifically bugs in BinSync when dependencies are missing) this will loop forever by default - // To work around _that_ we have a limit of 1000 retries - Queue typeDefsToAdd = Arrays.stream(dependencies.typedefs()).collect(Collectors.toCollection(LinkedList::new)); - int retries = 0; - while (!typeDefsToAdd.isEmpty()){ - if (retries > 1000){ - dtm.endTransaction(transactionId, false); - dtm.close(); - throw new EndlessTypeParsingException(dependencies, typeDefsToAdd.stream().toList()); - } - var typeDef = typeDefsToAdd.remove(); - var path = TypePathAndName.fromString(typeDef.name()); - DataType type; - try { - var scopedType = TypePathAndName.fromString(typeDef.type()); - type = dataTypeParser.parse(scopedType.name()); - } catch (InvalidDataTypeException e) { - // The type wasn't available in the DataTypeManager yet, try again later - typeDefsToAdd.add(typeDef); - retries++; - continue; - } catch (CancelledException e) { - throw new RuntimeException(e); - } - TypedefDataType typedefDataType = new TypedefDataType(path.toCategoryPath(), path.name(), type, null); - dtm.addDataType(typedefDataType, DataTypeConflictHandler.REPLACE_EMPTY_STRUCTS_OR_RENAME_AND_ADD_HANDLER); - } - - // Now we have all necessary types, we can fill out the structs - Arrays.stream(dependencies.structs()).forEach( - struct -> { - var path = TypePathAndName.fromString(struct.name()); - // Get struct type - var type = dtm.getDataType(path.toCategoryPath(), path.name()); - if (type instanceof Structure structType) { - Arrays.stream(struct.members()).forEach( - binSyncStructMember -> { - DataType fieldType = null; - try { - fieldType = loadDataType(dtm, TypePathAndName.fromString(binSyncStructMember.type())); - } catch (DataTypeDependencyException e) { - Msg.error( - GhidraRevengService.class, - "Couldn't find type '%s' for field of %s".formatted(binSyncStructMember.type(), struct.name()) - ); - fieldType = Undefined.getUndefinedDataType(binSyncStructMember.size()); - } - // The server occasionally reports a member that extends past the - // struct's declared size; grow the struct to fit rather than letting - // replaceAtOffset reject it and abort the whole type load. A member - // that still can't be placed is skipped so one bad field doesn't sink - // the entire function pull. - int end = binSyncStructMember.offset() + Math.max(1, binSyncStructMember.size()); - if (structType.getLength() < end) { - structType.growStructure(end - structType.getLength()); - } - try { - structType.replaceAtOffset( - binSyncStructMember.offset(), - fieldType, - binSyncStructMember.size(), - binSyncStructMember.name(), - null - ); - } catch (IllegalArgumentException e) { - Msg.error( - GhidraRevengService.class, - "Skipping struct member '%s' at offset %d of %s: %s".formatted( - binSyncStructMember.name(), binSyncStructMember.offset(), - struct.name(), e.getMessage()) - ); - } - } - ); - } else { - throw new RuntimeException("Struct type not found: %s".formatted(struct.name())); - } - - } - ); - - dtm.endTransaction(transactionId, true); - return dtm; - } - - private static DataType loadDataType(DataTypeManager dtm, TypePathAndName type) throws DataTypeDependencyException { - DataTypeParser dataTypeParser = new DataTypeParser( - dtm, - null, - null, - DataTypeParser.AllowedDataTypes.ALL); - DataType dataType; - try { - dataType = dataTypeParser.parse(type.name()); - } catch (InvalidDataTypeException e) { - // The type wasn't available in the DataTypeManager, so we have to find it in the dependencies - throw new DataTypeDependencyException("Data type not found in DataTypeManager: %s".formatted(type), e); - } catch (CancelledException e) { - throw new RuntimeException(e); - } - return dataType; + public static FunctionDefinitionDataType getFunctionSignature(BatchFunctionSignatureEntry entry, + ServerDataTypeDecoder decoder) { + return decoder.signature(entry.getFunctionName(), entry.getReturnDataTypeId(), entry.getParameters()); } public String getAnalysisLog(TypedApiInterface.AnalysisID analysisID) { return api.getAnalysisLogs(analysisID); } - /** - * Get the "name score" confidence of a match via the new API. - * The old kind of confidence is now called similarity - * - * @param functionMatch the match to get the confidence for - * @return the confidence of the match - */ - public BoxPlot getNameScoreForMatch(GhidraFunctionMatch functionMatch) { - var functionNameScore = api.getNameScore(functionMatch.functionMatch()); - return functionNameScore.score(); - - } - public void openFunctionInPortal(TypedApiInterface.FunctionID functionID) { var details = api.getFunctionDetails(functionID); openFunctionInPortal(details.analysisId(), functionID); @@ -1299,10 +1110,6 @@ public void openPortalFor(TypedApiInterface.FunctionID f){ openFunctionInPortal(f); } - public void openPortalFor(AnalysisResult analysisResult) { - openPortalFor(analysisResult.analysisID()); - } - public void openPortalFor(ProgramWithID programWithID) { openPortalFor(programWithID.analysisID()); } @@ -1356,15 +1163,13 @@ public AnalysisStatus waitForFinishedAnalysis( AnalysisStatus lastStatus = null; while (true) { AnalysisStatus currentStatus = this.status(programWithID); - if (currentStatus != AnalysisStatus.Queued) { + if (currentStatus != AnalysisStatus.Uploaded && currentStatus != AnalysisStatus.Queued) { // Analysis log endpoint only starts to return data after the analysis is processing String logs = this.getAnalysisLog(programWithID.analysisID()); if (logger != null) { logger.consumeLogs(logs, programWithID); } - var logsLines = logs.lines().toList(); - var lastLine = logsLines.get(logsLines.size() - 1); - monitor.setMessage(lastLine); + logs.lines().reduce((first, second) -> second).ifPresent(monitor::setMessage); } if (currentStatus != lastStatus) { lastStatus = currentStatus; @@ -1392,17 +1197,6 @@ public ProgramWithID startAnalysis(Program program, AnalysisOptionsBuilder analy return addAnalysisIDtoProgramOptions(program, analysisID); } - public Map getNameScores(java.util.Collection values) { - // Get the confidence scores for each match in the input - List r = api.getNameScores(values.stream().map(GhidraFunctionMatch::functionMatch).toList(), false); - // Collect to a Map from the FunctionID to the actual score - Map plots = r.stream().collect(Collectors.toMap(FunctionNameScore::functionID, FunctionNameScore::score)); - return values.stream().collect(Collectors.toMap( - match -> match, - match -> plots.get(match.functionMatch().origin_function_id()) - )); - } - /** * Collects the signatures for the matched functions, if they have already been computed (and finished) * @param values @@ -1411,41 +1205,31 @@ public Map getNameScores(java.util.Collection getSignatures(java.util.Collection values) { - // Get all data type info for the neighbour functions. Several local functions can match the same + // Get all signature info for the neighbour functions. Several local functions can match the same // neighbour, so dedupe the ids before fetching to avoid requesting (and getting back) duplicates. - var dataTypesList = this.api.listFunctionDataTypesForFunctions( + // The neighbours can come from any number of analyses; the response groups their types per + // analysis, which is why each signature is decoded against its own analysis' types. + var batch = signatures().getMany( values.stream().map(GhidraFunctionMatch::nearest_neighbor_id).distinct().toList() ); - // Create a map from FunctionID to FunctionInfo for easy lookup, only for completed signatures. + // Create a map from FunctionID to signature for easy lookup, only where the server has one. // The same neighbour can still appear more than once in the response, so keep the first. - Map signatureMap = dataTypesList.getItems().stream() - // Only keep completed signatures - .filter(FunctionDataTypesListItem::getCompleted) - // Double check that there is a data type available - .filter(functionDataTypesListItem -> functionDataTypesListItem.getDataTypes() != null) + Map signatureMap = batch.items().stream() + .filter(BatchFunctionSignatureEntry::getHasSignature) .collect(Collectors.toMap( item -> new TypedApiInterface.FunctionID(item.getFunctionId()), - FunctionDataTypesListItem::getDataTypes, + item -> item, (existing, replacement) -> existing )); - Map matchMap = values.stream() - .filter(match -> signatureMap.containsKey(match.functionMatch().nearest_neighbor_id())) - .collect(Collectors.toMap( - match -> match, - match -> signatureMap.get(match.functionMatch().nearest_neighbor_id()), - (existing, replacement) -> existing - )); - - // Now parse all signatures + Map decoders = new HashMap<>(); Map result = new HashMap<>(); - for (var entry : matchMap.entrySet()){ - try { - var funcDefOpt = getFunctionSignature(entry.getValue()); - funcDefOpt.ifPresent(funcDef -> result.put(entry.getKey(), funcDef)); - } catch (DataTypeDependencyException e) { - Msg.error(this, "Could not parse signature for function %s".formatted(entry.getKey().functionMatch()), e); + for (GhidraFunctionMatch match : values) { + var entry = signatureMap.get(match.functionMatch().nearest_neighbor_id()); + if (entry == null) { + continue; } + result.put(match, getFunctionSignature(entry, decoderFor(decoders, batch, entry))); } return result; @@ -1501,7 +1285,7 @@ public CompletableFuture> searchBinariesWithIds(String quer }); } - public Basic getBasicDetailsForAnalysis(TypedApiInterface.AnalysisID analysisID) throws ApiException { + public AnalysisBasicInfoOutputBody getBasicDetailsForAnalysis(TypedApiInterface.AnalysisID analysisID) throws ApiException { return api.getAnalysisBasicInfo(analysisID); } @@ -1541,13 +1325,6 @@ public void batchRenamingGhidraMatchesWithSignatures(List functionsList) throws ApiException { - var matches = functionsList.stream() - .map(GhidraFunctionMatch::functionMatch) - .toList(); - batchRenameMatches(matches); - } - public void batchRenameMatches(List functionsList) throws ApiException { var items = functionsList.stream() .map(result -> { @@ -1636,10 +1413,10 @@ public Optional getIDForFunction(Function function) { /// Warning: Using this map means having to verify that the function ID has an associated function /// /// `getFunctionMap.get(functionID)` can return `null` - public BiMap getFunctionMap(){ + public Map getFunctionMap(){ var propMap = getFunctionIDPropertyMap(this); - BiMap functionMap = HashBiMap.create(); + Map functionMap = new HashMap<>(); propMap.getPropertyIterator().forEachRemaining( addr -> { var func = program.getFunctionManager().getFunctionAt(addr); diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraToServerTypeSerializer.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraToServerTypeSerializer.java deleted file mode 100644 index 095f961b..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraToServerTypeSerializer.java +++ /dev/null @@ -1,220 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api; - -import ai.reveng.model.FunctionArgument; -import ai.reveng.model.FunctionDependency; -import ai.reveng.model.FunctionHeader; -import ai.reveng.model.FunctionInfo; -import ai.reveng.model.FunctionStackVariable; -import ai.reveng.model.FunctionType; -import ghidra.program.model.data.Array; -import ghidra.program.model.data.DataType; -import ghidra.program.model.data.DataTypeComponent; -import ghidra.program.model.data.Enum; -import ghidra.program.model.data.Pointer; -import ghidra.program.model.data.Structure; -import ghidra.program.model.data.TypeDef; -import ghidra.program.model.data.Union; -import ghidra.program.model.listing.Function; -import ghidra.program.model.listing.Parameter; -import ghidra.program.model.listing.Variable; - -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Deque; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * Serialises a Ghidra {@link Function}'s signature and variables into the server's data-type blob - * ({@link FunctionInfo}) so local edits can be pushed back to the portal. This is the inverse of - * {@link GhidraRevengService#getFunctionSignature}. - * - *

Mirrors the IDA plugin's {@code variable_sync_service._build_function_info} / - * {@code _collect_func_deps}: the header carries the return type and arguments (keyed by ordinal), - * stack variables are keyed by stack offset, and custom types referenced by any of these are - * emitted as {@link FunctionDependency} entries (structs, unions, enums, typedefs), resolved - * transitively. - */ -public final class GhidraToServerTypeSerializer { - - /// Upper bound on transitive dependency resolution, matching the IDA plugin's guard. - private static final int MAX_DEPENDENCIES = 500; - - private GhidraToServerTypeSerializer() {} - - public static FunctionInfo buildFunctionInfo(Function function, long imageBase) { - long addr = function.getEntryPoint().getOffset() - imageBase; - String returnType = typeName(function.getReturnType()); - - Map args = new LinkedHashMap<>(); - Parameter[] parameters = function.getParameters(); - for (int i = 0; i < parameters.length; i++) { - var parameter = parameters[i]; - long offset = i; - args.put(Long.toHexString(offset), new FunctionArgument() - .offset(offset) - .name(parameter.getName()) - .type(typeName(parameter.getDataType())) - .size((long) parameter.getLength())); - } - - var header = new FunctionHeader() - .name(function.getName()) - .addr(addr) - .type(returnType) - .args(args); - - Map stackVars = new LinkedHashMap<>(); - for (Variable variable : function.getLocalVariables()) { - if (!variable.isStackVariable()) { - continue; - } - long offset = variable.getStackOffset(); - stackVars.put(Long.toHexString(offset), new FunctionStackVariable() - .offset(offset) - .name(variable.getName()) - .type(typeName(variable.getDataType())) - .size((long) variable.getLength()) - .addr(addr)); - } - - var funcType = new FunctionType() - .addr(addr) - .size(function.getBody().getNumAddresses()) - .header(header) - .stackVars(stackVars) - .name(function.getName()) - .type(returnType) - .artifactType("Function"); - - return new FunctionInfo() - .funcTypes(funcType) - .funcDeps(collectDependencies(function)); - } - - /// All type names referenced by the function's signature and variables, resolved transitively - /// through pointers, arrays, struct/union members and typedefs. Used to decide which functions - /// to re-push when a data type is edited. - public static Set referencedTypeNames(Function function) { - Deque queue = new ArrayDeque<>(); - queue.add(function.getReturnType()); - for (Parameter parameter : function.getParameters()) { - queue.add(parameter.getDataType()); - } - for (Variable variable : function.getLocalVariables()) { - queue.add(variable.getDataType()); - } - - Set names = new HashSet<>(); - int guard = 0; - while (!queue.isEmpty() && guard++ < MAX_DEPENDENCIES) { - DataType base = baseType(queue.poll()); - if (base == null || !names.add(base.getName())) { - continue; - } - if (base instanceof Structure || base instanceof Union) { - for (DataTypeComponent component : ((ghidra.program.model.data.Composite) base).getDefinedComponents()) { - queue.add(component.getDataType()); - } - } else if (base instanceof TypeDef typeDef) { - queue.add(typeDef.getDataType()); - } - } - return names; - } - - private static List collectDependencies(Function function) { - Deque queue = new ArrayDeque<>(); - queue.add(function.getReturnType()); - for (Parameter parameter : function.getParameters()) { - queue.add(parameter.getDataType()); - } - for (Variable variable : function.getLocalVariables()) { - if (variable.isStackVariable()) { - queue.add(variable.getDataType()); - } - } - - Map deps = new LinkedHashMap<>(); - int guard = 0; - while (!queue.isEmpty() && guard++ < MAX_DEPENDENCIES) { - DataType base = baseType(queue.poll()); - if (base == null) { - continue; - } - String name = base.getName(); - if (deps.containsKey(name)) { - continue; - } - var dependency = toDependency(base, queue); - if (dependency != null) { - deps.put(name, dependency); - } - } - return new ArrayList<>(deps.values()); - } - - /// Emits a dependency for custom types and enqueues nested types to resolve; returns null for - /// primitives and built-ins, which the server already knows. - private static FunctionDependency toDependency(DataType type, Deque queue) { - if (type instanceof Structure || type instanceof Union) { - var composite = (ghidra.program.model.data.Composite) type; - Map members = new LinkedHashMap<>(); - for (DataTypeComponent component : composite.getDefinedComponents()) { - queue.add(component.getDataType()); - Map member = new LinkedHashMap<>(); - member.put("name", component.getFieldName()); - member.put("offset", (long) component.getOffset()); - member.put("type", typeName(component.getDataType())); - member.put("size", (long) component.getLength()); - members.put(Long.toHexString(component.getOffset()), member); - } - return new FunctionDependency() - .name(type.getName()) - .size((long) type.getLength()) - .members(members) - .artifactType("Struct"); - } - if (type instanceof Enum enumType) { - Map members = new LinkedHashMap<>(); - for (String memberName : enumType.getNames()) { - members.put(memberName, enumType.getValue(memberName)); - } - return new FunctionDependency() - .name(type.getName()) - .size((long) type.getLength()) - .members(members) - .artifactType("Enum"); - } - if (type instanceof TypeDef typeDef) { - queue.add(typeDef.getDataType()); - return new FunctionDependency() - .name(type.getName()) - .type(typeName(typeDef.getDataType())) - .artifactType("Typedef"); - } - return null; - } - - /// Unwraps pointer and array decoration to reach the underlying named type. - static DataType baseType(DataType type) { - DataType current = type; - while (true) { - if (current instanceof Pointer pointer) { - current = pointer.getDataType(); - } else if (current instanceof Array array) { - current = array.getDataType(); - } else { - return current; - } - } - } - - /// Server type string, including pointer/array decoration (e.g. {@code "MyStruct *"}). - static String typeName(DataType type) { - return type == null ? "undefined" : type.getName(); - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/ModelName.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/ModelName.java deleted file mode 100644 index c8e67be2..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/ModelName.java +++ /dev/null @@ -1,8 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api; - -public record ModelName(String modelName) { - @Override - public String toString() { - return modelName; - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/ServerDataTypeDecoder.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/ServerDataTypeDecoder.java new file mode 100644 index 00000000..e915caca --- /dev/null +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/ServerDataTypeDecoder.java @@ -0,0 +1,380 @@ +package ai.reveng.toolkit.ghidra.core.services.api; + +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.ServerDataType; +import ai.reveng.toolkit.ghidra.core.services.api.types.TypePathAndName; +import ghidra.program.model.data.ArrayDataType; +import ghidra.program.model.data.CategoryPath; +import ghidra.program.model.data.DataType; +import ghidra.program.model.data.DataTypeConflictHandler; +import ghidra.program.model.data.DataTypeManager; +import ghidra.program.model.data.Enum; +import ghidra.program.model.data.EnumDataType; +import ghidra.program.model.data.FunctionDefinitionDataType; +import ghidra.program.model.data.InvalidDataTypeException; +import ghidra.program.model.data.ParameterDefinitionImpl; +import ghidra.program.model.data.PointerDataType; +import ghidra.program.model.data.StandAloneDataTypeManager; +import ghidra.program.model.data.Structure; +import ghidra.program.model.data.StructureDataType; +import ghidra.program.model.data.TypedefDataType; +import ghidra.program.model.data.Undefined; +import ghidra.program.model.data.Undefined1DataType; +import ghidra.program.model.data.Union; +import ghidra.program.model.data.UnionDataType; +import ghidra.program.model.data.VoidDataType; +import ghidra.util.InvalidNameException; +import ghidra.util.Msg; +import ghidra.util.data.DataTypeParser; +import ghidra.util.exception.CancelledException; + +import javax.annotation.Nullable; +import java.math.BigInteger; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/// Turns the data types of one analysis into a self-contained Ghidra {@link DataTypeManager}. +/// +/// Every type the server reports carries a `data_type_id` that is unique within its analysis, and +/// every reference between types — a struct member's type, a pointer's pointee, a typedef's target, +/// a parameter's type — is expressed as one of those ids. Decoding therefore never has to resolve a +/// type by name, which is what makes this two straightforward passes: +/// +/// 1. create an empty shell for every composite id (struct, union, enum) and register it; +/// 2. resolve every remaining id by lookup, then fill the shells in. +/// +/// Cycles resolve on their own: any cycle in a well-formed type graph runs through a composite, and +/// composites already exist by the time pass 2 starts. A degenerate cycle that never reaches one is +/// broken by a recursion guard rather than by retrying until the graph settles. +/// +/// The manager and everything in it are transient — they exist to give a decoded signature somewhere +/// to keep its dependencies, exactly as a program's own manager would. +public final class ServerDataTypeDecoder { + + private final StandAloneDataTypeManager dtm; + private final Map source; + private final Map decoded = new HashMap<>(); + private final Set resolving = new HashSet<>(); + private final DataTypeParser parser; + + private ServerDataTypeDecoder(Collection types) { + this.dtm = new StandAloneDataTypeManager("transient"); + this.parser = new DataTypeParser(dtm, null, null, DataTypeParser.AllowedDataTypes.ALL); + this.source = new LinkedHashMap<>(); + for (ServerDataType type : types) { + // A duplicate id can only come from stitching several responses together; the entries + // are then identical, so keeping the first is enough. + source.putIfAbsent(type.id(), type); + } + } + + /// Decode a whole analysis' worth of types. Order of the input does not matter. + public static ServerDataTypeDecoder decode(Collection types) { + ServerDataTypeDecoder decoder = new ServerDataTypeDecoder(types); + decoder.run(); + return decoder; + } + + /// The Ghidra type for a `data_type_id`, or an undefined filler of `fallbackSize` bytes when the + /// id is absent or names a type the server never defined. + public DataType typeFor(@Nullable Long dataTypeId, long fallbackSize) { + if (dataTypeId == null) { + return undefined(fallbackSize); + } + DataType type = decoded.get(dataTypeId); + return type != null ? type : undefined(fallbackSize); + } + + private void run() { + int transaction = dtm.startTransaction("Decode data types"); + try { + // Pass 1: an empty shell per composite id, so pass 2 always has something to point at. + for (ServerDataType type : source.values()) { + switch (type.kind()) { + case STRUCT -> put(type.id(), add(new StructureDataType( + categoryPath(type), leafName(type), (int) clampSize(type.size()), dtm))); + case UNION -> put(type.id(), add(new UnionDataType( + categoryPath(type), leafName(type), dtm))); + case ENUM -> put(type.id(), add(new EnumDataType( + categoryPath(type), leafName(type), enumLength(type.size()), dtm))); + default -> { + } + } + } + + // Pass 2: resolve everything else by id, then populate the shells. + for (Long id : source.keySet()) { + resolve(id); + } + for (ServerDataType type : source.values()) { + fill(type); + } + } finally { + dtm.endTransaction(transaction, true); + } + } + + private DataType resolve(long id) { + DataType existing = decoded.get(id); + if (existing != null) { + return existing; + } + ServerDataType type = source.get(id); + if (type == null) { + // Referenced but not shipped: the server only sends the closure it knows about. + return Undefined1DataType.dataType; + } + if (!resolving.add(id)) { + // Only reachable for a cycle that never passes through a composite, which cannot + // describe a real type. Break it instead of looping. + return Undefined1DataType.dataType; + } + try { + DataType built = build(type); + put(id, built); + return built; + } finally { + resolving.remove(id); + } + } + + private DataType build(ServerDataType type) { + return switch (type.kind()) { + case TYPEDEF -> { + Long target = type.definition() instanceof ServerDataType.TypedefDefinition def + ? def.targetDataTypeId() : null; + DataType targetType = target == null ? undefined(type.size()) : resolve(target); + yield add(new TypedefDataType(categoryPath(type), leafName(type), targetType, dtm)); + } + case POINTER -> { + Long pointee = type.definition() instanceof ServerDataType.PointerDefinition def + ? def.pointeeDataTypeId() : null; + // A null pointee is `void *`; a null length lets Ghidra use the manager's default. + DataType pointeeType = pointee == null ? VoidDataType.dataType : resolve(pointee); + yield add(new PointerDataType(pointeeType, pointerLength(type.size()), dtm)); + } + case ARRAY -> { + ServerDataType.ArrayDefinition def = + type.definition() instanceof ServerDataType.ArrayDefinition array ? array : null; + DataType element = def == null || def.elementDataTypeId() == null + ? Undefined1DataType.dataType : resolve(def.elementDataTypeId()); + int count = def == null || def.count() == null ? 0 : (int) clampSize(def.count()); + int elementLength = Math.max(1, element.getLength()); + yield add(new ArrayDataType(element, Math.max(count, 1), elementLength, dtm)); + } + case FUNCTION_DEFINITION -> { + FunctionDefinitionDataType definition = + new FunctionDefinitionDataType(categoryPath(type), leafName(type), dtm); + // Registered before its parameters are resolved so a self-referential signature + // (a function taking a pointer to its own type) terminates. + put(type.id(), definition); + if (type.definition() instanceof ServerDataType.FunctionTypeDefinition def) { + definition.setArguments(def.parameters().stream() + .map(parameter -> new ParameterDefinitionImpl( + parameter.name(), + typeOrResolve(parameter.dataTypeId(), parameter.size()), + null)) + .toArray(ParameterDefinitionImpl[]::new)); + definition.setReturnType(def.returnDataTypeId() == null + ? VoidDataType.dataType + : resolve(def.returnDataTypeId())); + } + yield definition; + } + // BASE and BITFIELD name a built-in; UNKNOWN is a kind this plugin does not model yet. + case BASE, BITFIELD, UNKNOWN -> builtIn(type); + // Composites were created in pass 1, so this is unreachable in practice. + case STRUCT, UNION, ENUM -> undefined(type.size()); + }; + } + + private void fill(ServerDataType type) { + DataType target = decoded.get(type.id()); + switch (type.kind()) { + case STRUCT -> { + if (target instanceof Structure structure + && type.definition() instanceof ServerDataType.StructDefinition def) { + def.members().forEach(member -> place(structure, type, member)); + } + } + case UNION -> { + if (target instanceof Union union + && type.definition() instanceof ServerDataType.UnionDefinition def) { + def.members().forEach(member -> { + try { + union.add(typeOrResolve(member.dataTypeId(), member.size()), + member.name(), null); + } catch (IllegalArgumentException e) { + Msg.error(ServerDataTypeDecoder.class, "Skipping union member '%s' of %s: %s" + .formatted(member.name(), type.name(), e.getMessage())); + } + }); + } + } + case ENUM -> { + if (target instanceof Enum enumeration + && type.definition() instanceof ServerDataType.EnumDefinition def) { + def.values().forEach(value -> { + try { + enumeration.add(value.name(), toLong(value.value())); + } catch (IllegalArgumentException e) { + Msg.error(ServerDataTypeDecoder.class, "Skipping enum value '%s' of %s: %s" + .formatted(value.name(), type.name(), e.getMessage())); + } + }); + } + } + default -> { + } + } + } + + private void place(Structure structure, ServerDataType owner, ServerDataType.Member member) { + DataType fieldType = typeOrResolve(member.dataTypeId(), member.size()); + // The server occasionally reports a member that extends past the struct's declared size; + // grow the struct to fit rather than letting replaceAtOffset reject it and abort the whole + // type load. A member that still can't be placed is skipped so one bad field doesn't sink + // the entire signature. + int offset = (int) clampSize(member.offset()); + int length = (int) Math.max(1, clampSize(member.size())); + int end = offset + length; + if (structure.getLength() < end) { + structure.growStructure(end - structure.getLength()); + } + try { + structure.replaceAtOffset(offset, fieldType, length, member.name(), null); + } catch (IllegalArgumentException e) { + Msg.error(ServerDataTypeDecoder.class, "Skipping struct member '%s' at offset %d of %s: %s" + .formatted(member.name(), offset, owner.name(), e.getMessage())); + } + } + + /// Look a built-in up by name, e.g. `int`, `char *`, `unsigned long`. These types have no + /// definition of their own, so the name is all the server sends. + private DataType builtIn(ServerDataType type) { + String name = type.name(); + if (name == null || name.isBlank()) { + return undefined(type.size()); + } + try { + DataType parsed = parser.parse(name); + if (parsed != null) { + return parsed; + } + } catch (InvalidDataTypeException e) { + // Not a name Ghidra knows; fall through to a same-sized filler. + } catch (CancelledException e) { + throw new RuntimeException(e); + } + return undefined(type.size()); + } + + private DataType typeOrResolve(@Nullable Long dataTypeId, long fallbackSize) { + return dataTypeId == null ? undefined(fallbackSize) : resolve(dataTypeId); + } + + private DataType add(DataType type) { + return dtm.addDataType(type, DataTypeConflictHandler.REPLACE_EMPTY_STRUCTS_OR_RENAME_AND_ADD_HANDLER); + } + + private void put(long id, DataType type) { + decoded.put(id, type); + } + + private static CategoryPath categoryPath(ServerDataType type) { + String namespace = type.namespace(); + if (namespace == null || namespace.isBlank()) { + return CategoryPath.ROOT; + } + return TypePathAndName.fromString(namespace + "::" + leafName(type)).toCategoryPath(); + } + + private static String leafName(ServerDataType type) { + String name = type.name(); + if (name == null || name.isBlank()) { + return "anon_%d".formatted(type.id()); + } + return TypePathAndName.fromString(name).name(); + } + + /// Ghidra sizes are `int`; the API's are `long`. Anything that does not fit is not a real type. + private static long clampSize(@Nullable Long size) { + if (size == null || size <= 0) { + return 0; + } + return Math.min(size, Integer.MAX_VALUE); + } + + /// Ghidra enums must be 1, 2, 4 or 8 bytes wide. + private static int enumLength(@Nullable Long size) { + long clamped = clampSize(size); + if (clamped >= 8) { + return 8; + } + if (clamped >= 4) { + return 4; + } + if (clamped >= 2) { + return 2; + } + return 1; + } + + /// -1 lets Ghidra use the manager's default pointer size, which is what an unsized pointer means. + private static int pointerLength(@Nullable Long size) { + long clamped = clampSize(size); + return clamped <= 0 || clamped > 8 ? -1 : (int) clamped; + } + + /// Enum values stay strings on the wire because they may be negative or exceed 64 unsigned bits. + /// Ghidra can only hold a `long`, so widen through {@link BigInteger} and take the low 64 bits. + private static long toLong(String value) { + return new BigInteger(value.trim()).longValue(); + } + + /// An exact-width filler for a type we could not build. Ghidra only has undefined1/2/4/8, so + /// anything else becomes an array of undefined1 to keep the surrounding layout intact. + private static DataType undefined(long size) { + long clamped = clampSize(size); + if (clamped == 1 || clamped == 2 || clamped == 4 || clamped == 8) { + return Undefined.getUndefinedDataType((int) clamped); + } + if (clamped <= 0) { + return Undefined1DataType.dataType; + } + return new ArrayDataType(Undefined1DataType.dataType, (int) clamped, 1); + } + + /// Build the signature of a function from its server entry, with every referenced type taken + /// from this decoder's manager so the result is self-contained. + public FunctionDefinitionDataType signature(String functionName, + @Nullable Long returnDataTypeId, + List parameters) { + FunctionDefinitionDataType definition = new FunctionDefinitionDataType(functionName, dtm); + try { + definition.setName(functionName); + } catch (InvalidNameException e) { + throw new RuntimeException(e); + } + if (parameters != null) { + definition.setArguments(parameters.stream() + .map(parameter -> new ParameterDefinitionImpl( + parameter.getName(), + typeFor(parameter.getDataTypeId(), bytesOf(parameter.getBitLength())), + null)) + .toArray(ParameterDefinitionImpl[]::new)); + } + definition.setReturnType(returnDataTypeId == null + ? VoidDataType.dataType + : typeFor(returnDataTypeId, 0)); + return definition; + } + + private static long bytesOf(@Nullable Long bitLength) { + return bitLength == null ? 0 : Math.max(0, bitLength / 8); + } +} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiImplementation.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiImplementation.java index 20778987..6cc5b081 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiImplementation.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiImplementation.java @@ -3,70 +3,63 @@ import ai.reveng.api.*; import ai.reveng.model.*; import ai.reveng.model.ConfigResponse; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.FunctionSignatureBatch; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.ServerDataType; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.ServerDataTypeReader; import ai.reveng.toolkit.ghidra.core.services.api.types.*; import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionInfo; -import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionMatch; -import ai.reveng.toolkit.ghidra.core.services.api.types.exceptions.APIAuthenticationException; -import ai.reveng.toolkit.ghidra.core.services.api.types.exceptions.APIConflictException; import ai.reveng.toolkit.ghidra.core.services.api.types.exceptions.InvalidAPIInfoException; import ghidra.framework.Application; import ghidra.framework.Platform; import ghidra.util.Msg; -import org.json.JSONArray; -import org.json.JSONObject; import resources.ResourceManager; import javax.annotation.Nullable; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.net.http.HttpTimeoutException; import java.nio.charset.StandardCharsets; import java.nio.file.Path; -import java.time.Duration; import java.util.*; +import java.util.stream.Collectors; import ai.reveng.invoker.Configuration; +import ai.reveng.invoker.JSON; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import ai.reveng.invoker.auth.ApiKeyAuth; import ai.reveng.invoker.ApiException; import static ai.reveng.toolkit.ghidra.core.services.api.LoggingInterceptor.*; -import static ai.reveng.toolkit.ghidra.core.services.api.Utils.mapJSONArray; -import static java.net.http.HttpClient.Version.HTTP_1_1; -/// The main implementation of the RevEng HTTP API -/// It partially relies on the old manual implementation, but should be migrated to the OpenAPI generated client over time +/// The main implementation of the RevEng HTTP API, on top of the generated SDK client /// Design notes: /// - every method should correspond to a single API endpoint /// - every method should simply execute the request and return the response /// - i.e. no smart checks relying on other API calls to check if e.g. a binary has already been uploaded public class TypedApiImplementation implements TypedApiInterface { - private final HttpClient httpClient; - private final String baseUrl; - Map headers; + /// /v3/analyses caps page_size at 50 and pages forward with an opaque token. + private static final long ANALYSIS_LIST_PAGE_SIZE = 50; + + /// The maximum the v3 function-list endpoint accepts; a larger value is rejected with a 422. + private static final long FUNCTION_LIST_PAGE_SIZE = 500; + + /// Omitting analysis_scope makes the server default to PRIVATE only. + private static final List ALL_ANALYSIS_SCOPES = List.of("PRIVATE", "TEAM", "PUBLIC"); private final AnalysesCoreApi analysisCoreApi; - private final AnalysesResultsMetadataApi analysesResultsMetadataApi; private final ConfigApi configApi; private final SearchApi searchApi; private final CollectionsApi collectionsApi; private final FunctionsCoreApi functionsCoreApi; private final FunctionsRenamingHistoryApi functionsRenamingHistoryApi; private final FunctionsAiDecompilationApi functionsAiDecompilationApi; - private final FunctionsDataTypesApi functionsDataTypesApi; + private final DataTypesApi dataTypesApi; private final IamUsersApi iamUsersApi; - // Cache for binary ID to analysis ID mappings - @Deprecated - private final Map binaryToAnalysisCache = new HashMap<>(); - - // Cache for analysis basic info to avoid repeated API calls - private final Map analysisBasicInfoCache = new HashMap<>(); + private final Map analysisBasicInfoCache = new HashMap<>(); public TypedApiImplementation(String baseUrl, String apiKey) { var apiClient = Configuration.getDefaultApiClient(); @@ -102,28 +95,14 @@ public TypedApiImplementation(String baseUrl, String apiKey) { APIKey.setApiKey(apiKey); this.analysisCoreApi = new AnalysesCoreApi(apiClient); - this.analysesResultsMetadataApi = new AnalysesResultsMetadataApi(apiClient); this.searchApi = new SearchApi(apiClient); this.collectionsApi = new CollectionsApi(apiClient); this.functionsCoreApi = new FunctionsCoreApi(apiClient); this.functionsRenamingHistoryApi = new FunctionsRenamingHistoryApi(apiClient); this.functionsAiDecompilationApi = new FunctionsAiDecompilationApi(apiClient); - this.functionsDataTypesApi = new FunctionsDataTypesApi(apiClient); + this.dataTypesApi = new DataTypesApi(apiClient); this.configApi = new ConfigApi(apiClient); this.iamUsersApi = new IamUsersApi(apiClient); - - this.baseUrl = baseUrl + "/"; - this.httpClient = HttpClient.newBuilder() - .connectTimeout(Duration.ofSeconds(5)) - .version(HTTP_1_1) // by default the client would attempt HTTP2.0 which leads to weird issues - .build(); - headers = new HashMap<>(); - headers.put("Authorization", apiKey); - headers.put("User-Agent", userAgent); - headers.put("X-RevEng-Application", userAgent); - - // TODO: Actually implement support for some encodings and then accept them -// headers.put("Accept-Encoding", "gzip, deflate, br"); } @@ -142,67 +121,33 @@ public BinaryHash upload(Path binPath) throws FileNotFoundException, ApiExceptio return new BinaryHash(result.getData().getSha256Hash()); } - /* - Allows you to search for specific analyses and collections. - The query parameter follows a non standard formatting using key-pair comma seperated values. - he base query is formatted as follows: /search?search=sha_256_hash:,binary_name:,tags=,collection_name:. - Not all parameters are required, for example /search?search=sha_256_hash: only searches for binaries and collection with hashes like . - - */ + /// GET /v3/analyses, filtered to one binary hash and paged to exhaustion. + /// + /// All three analysis scopes are requested explicitly because the endpoint narrows to PRIVATE + /// when the parameter is absent. @Deprecated - public List search(BinaryHash hash) { - Map params = new HashMap<>(); - params.put("sha256_hash", hash.sha256()); - - JSONObject json = sendRequest( - requestBuilderForEndpoint("analyses", "list", queryParams(params)) - .GET() - .header("Content-Type", "application/json" ) - .build()); - - return mapJSONArray(json.getJSONObject("data").getJSONArray("results"), LegacyAnalysisResult::fromJSONObject); - } - - private V2Response sendVersion2Request(HttpRequest request){ - return V2Response.fromJSONObject(sendRequest(request)); - } - - private JSONObject sendRequest(HttpRequest request) throws APIAuthenticationException { - Msg.info(this, "Sending request to: " + request.uri()); - HttpResponse response = null; - - var retryAttempts = 3; - while (response == null && retryAttempts > 0) { - try { - response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); - } catch (HttpTimeoutException timeout) { - // Sometimes the API hangs, and works again shortly after, so we just try again - Msg.info(this, "Timed out waiting for response from: " + request.uri()); - Msg.info(this, "Trying again: " + request.uri()); - retryAttempts--; - } catch (IOException e) { - throw new RuntimeException(e); - } catch (InterruptedException e) { - throw new RuntimeException(e); + public List search(BinaryHash hash) { + List results = new ArrayList<>(); + String pageToken = null; + try { + while (true) { + ListAnalysesOutputBody page = analysisCoreApi.v3ListAnalyses( + null, ALL_ANALYSIS_SCOPES, null, null, null, hash.sha256(), + ANALYSIS_LIST_PAGE_SIZE, pageToken, null, null); + var records = page.getResults(); + if (records == null || records.isEmpty()) { + break; + } + results.addAll(records); + pageToken = page.getNextPageToken(); + if (pageToken == null || pageToken.isBlank()) { + break; + } } + } catch (ApiException e) { + throw new RuntimeException(describeApiException(e), e); } - - switch (response.statusCode()){ - case 200: - case 201: - Msg.info(this, "Request to %s succeeded with status code: %s".formatted(request.uri(), response.statusCode())); - return new JSONObject(response.body()); - case 404: - return new JSONObject(response.body()); - case 401: - throw new APIAuthenticationException(response.body()); - case 409: - throw new APIConflictException(response.body()); - default: - var errorMsg = "Request to %s failed with status code: %s and message: %s".formatted(request.uri(), response.statusCode(), response.body()); - Msg.showError(this, null, "Request failed with status code: " + response.statusCode(), errorMsg); - throw new RuntimeException(errorMsg); - } + return results; } @Override @@ -212,202 +157,217 @@ public AnalysisID analyse(AnalysisOptionsBuilder options) throws ApiException { return new AnalysisID(result.getData().getAnalysisId()); } - @Deprecated - @Override - public AnalysisStatus status(BinaryID binaryID) throws ApiException { - var analysisID = this.getAnalysisIDfromBinaryID(binaryID); - - var status = this.analysisCoreApi.getAnalysisStatus(analysisID.id()); - - return AnalysisStatus.valueOf(status.getData().getAnalysisStatus()); - } - @Override public AnalysisStatus status(AnalysisID analysisID) throws ApiException { var status = analysisCoreApi.getAnalysisStatus(analysisID.id()); - return AnalysisStatus.valueOf(status.getData().getAnalysisStatus()); + return AnalysisStatus.fromApiValue(status.getData().getAnalysisStatus()); } + /** + * The endpoint is paginated by offset and limit, and reports the unpaginated population size as + * {@code total_count}, so paging walks the offset forward until that many entries have arrived. + * The offset advances by the number of entries actually returned rather than by the requested + * limit, so a server-side cap below {@code limit} neither skips nor repeats entries. + */ @Override public List getFunctionInfo(AnalysisID analysisID) { - // The server caps page_size at 1000, so paginate until every function is retrieved. - int pageSize = 1000; + long limit = FUNCTION_LIST_PAGE_SIZE; List functions = new ArrayList<>(); - int page = 1; + long offset = 0; while (true) { - BaseResponseAnalysisFunctions response; + ListAnalysisFunctionsOutputBody response; try { - response = this.analysesResultsMetadataApi.getFunctionsList( - analysisID.id(), null, null, null, false, page, pageSize); + response = this.functionsCoreApi.listAnalysisFunctions((long) analysisID.id(), offset, limit); } catch (ApiException e) { - throw new RuntimeException("Could not find analysis with ID: " + analysisID.id(), e); + throw new RuntimeException( + "Could not list functions for analysis " + analysisID.id() + ": " + describeApiException(e), e); } - response.getData().getFunctions().stream().map(f -> ( + var page = response.getFunctions(); + if (page == null || page.isEmpty()) { + break; + } + + page.stream().map(f -> ( new FunctionInfo( new FunctionID(f.getFunctionId()), f.getFunctionName(), - f.getFunctionMangledName(), + // The mangled name is optional here; an unmangled symbol carries none, + // and callers rely on this field being populated. + f.getMangledName() != null ? f.getMangledName() : f.getFunctionName(), f.getFunctionVaddr(), - f.getFunctionSize() + Math.toIntExact(f.getFunctionSize()) ) )).forEach(functions::add); - var pagination = response.getMeta() != null ? response.getMeta().getPagination() : null; - if (pagination == null || !Boolean.TRUE.equals(pagination.getHasNextPage())) { + offset += page.size(); + Long totalCount = response.getTotalCount(); + if (totalCount == null || offset >= totalCount) { break; } - page++; } return functions; } - private String queryParams(Map params){ - return "?" + params.entrySet().stream() - .filter(e -> e.getValue() != null) - .map(e -> e.getKey() + "=" + e.getValue()) - .reduce((a, b) -> a + "&" + b) - .orElse(""); - } + /// GET /v3/analyses/{analysis_id}/logs + /// + /// v3 answers with structured entries where v2 answered with one preformatted blob, so the lines + /// are rendered here into the single string the log view and the progress monitor consume. @Override public String getAnalysisLogs(AnalysisID analysisID) { - var request = requestBuilderForEndpoint("analyses", String.valueOf(analysisID.id()), "logs") - .build(); - JSONObject response = sendVersion2Request(request).getJsonData(); - return response.getString("logs"); + List entries; + try { + entries = analysisCoreApi.v3GetAnalysisLogs((long) analysisID.id()).getEntries(); + } catch (ApiException e) { + throw new RuntimeException(describeApiException(e), e); + } + if (entries == null || entries.isEmpty()) { + return ""; + } + return entries.stream() + .map(TypedApiImplementation::renderLogEntry) + .collect(Collectors.joining("\n")); } - private HttpRequest.Builder requestBuilderForEndpoint(String... endpointPaths){ - URI uri; - String apiVersionPath = "v2"; - String endpoint = String.join("/", endpointPaths).replace("/?", "?").replace("?/", "?"); + private static String renderLogEntry(AnalysisLogEntry entry) { + StringBuilder line = new StringBuilder(); + if (entry.getTimestamp() != null) { + line.append(entry.getTimestamp()).append(' '); + } + if (entry.getLevel() != null) { + line.append(entry.getLevel().getValue()).append(' '); + } + if (entry.getSource() != null && !entry.getSource().isBlank()) { + line.append('[').append(entry.getSource()).append("] "); + } + if (entry.getText() != null) { + line.append(entry.getText()); + } + return line.toString(); + } + /// GET /v3/functions/signatures + /// + /// Read through the generated call rather than the generated response model: the response + /// embeds `DataTypeEntry`, whose generated deserialiser picks a variant by counting matching + /// fields instead of reading the `kind` discriminator, and every variant carries the same + /// required fields. The call still builds the request — path, query, auth — exactly as the SDK + /// would; only the body is read by {@link ServerDataTypeReader}. + @Override + public FunctionSignatureBatch listFunctionSignatures(List functionIDs, boolean includeDataTypes) { try { - uri = new URI(baseUrl + apiVersionPath + "/" + endpoint); - } catch (URISyntaxException e) { + var call = dataTypesApi.v3ListFunctionSignaturesCall( + functionIDs.stream().map(FunctionID::value).toList(), includeDataTypes, null); + JsonObject body = executeForJsonObject(call, "list function signatures"); + + List items = new ArrayList<>(); + JsonArray rawItems = body.getAsJsonArray("items"); + if (rawItems != null) { + for (JsonElement item : rawItems) { + items.add(JSON.getGson().fromJson(item, BatchFunctionSignatureEntry.class)); + } + } + + Map> dataTypes = new LinkedHashMap<>(); + JsonArray groups = body.getAsJsonArray("data_types"); + if (groups != null) { + for (JsonElement group : groups) { + if (!group.isJsonObject()) { + continue; + } + JsonElement analysisId = group.getAsJsonObject().get("analysis_id"); + if (analysisId == null || analysisId.isJsonNull()) { + continue; + } + dataTypes.computeIfAbsent(new AnalysisID(analysisId.getAsInt()), ignored -> new ArrayList<>()) + .addAll(ServerDataTypeReader.readEntries(group, "items")); + } + } + return new FunctionSignatureBatch(items, dataTypes); + } catch (ApiException e) { throw new RuntimeException(e); } - var requestBuilder = HttpRequest.newBuilder(uri); - headers.forEach(requestBuilder::header); - requestBuilder.timeout(Duration.ofSeconds(20)); - return requestBuilder; } - /** - * ... - * - * The mapping never changes so we can cache it to avoid repeated requests. - * - * @param binaryID the binary id to look up - * @return the analysis id - */ + /// GET /v3/analyses/{analysis_id}/data-types + /// + /// Read through the generated call for the same reason as + /// {@link #listFunctionSignatures(List, boolean)}. @Override - @Deprecated - public AnalysisID getAnalysisIDfromBinaryID(BinaryID binaryID){ - // Check cache first - AnalysisID cachedResult = binaryToAnalysisCache.get(binaryID); - if (cachedResult != null) { - return cachedResult; + public List listAnalysisDataTypes(AnalysisID analysisID, long offset, long limit) { + try { + var call = dataTypesApi.v3ListAnalysisDataTypesCall( + (long) analysisID.id(), offset, limit, null, null, null, null, null, null, null); + return ServerDataTypeReader.readEntries( + executeForJsonObject(call, "list analysis data types"), "items"); + } catch (ApiException e) { + throw new RuntimeException(e); } - - // If not in cache, make HTTP request - JSONObject response = sendRequest(requestBuilderForEndpoint("analyses/lookup/" + binaryID.value()) - .GET() - .build()); - - AnalysisID analysisID = new AnalysisID(response.getInt("analysis_id")); - - // Cache the result - binaryToAnalysisCache.put(binaryID, analysisID); - - return analysisID; } - /** - * Triggers the generation of function data types for a provided list of functions - * ... - * https://api.reveng.ai/v2/analyses/{analysis_id}/info/functions/data_types - * @param functionIDS - * @return - */ + /// POST /v3/analyses/{analysis_id}/data-types + /// + /// Written through the generated call for the same reason the reads are: the 201 body embeds + /// `DataTypeEntry`. The request body is a generated model, which serialises correctly — only + /// the deserialiser is unusable. @Override - public DataTypeList generateFunctionDataTypes(AnalysisID analysisID, List functionIDS) throws APIConflictException{ - JSONObject params = new JSONObject(); - params.put("function_ids", functionIDS.stream().map(FunctionID::value).toList()); - - var request = requestBuilderForEndpoint("analyses/%s/info/functions/data_types".formatted(analysisID.id())) - .POST(HttpRequest.BodyPublishers.ofString(params.toString())) - .header("Content-Type", "application/json" ) - .build(); - - var response = sendVersion2Request(request); - return DataTypeList.fromJson(response.getJsonData().getJSONObject("data_types_list")); + public List createAnalysisDataTypes(AnalysisID analysisID, + CreateAnalysisDataTypesInputBody request) throws ApiException { + var call = dataTypesApi.v3CreateAnalysisDataTypesCall((long) analysisID.id(), request, null); + return ServerDataTypeReader.readEntries( + executeForJsonObject(call, "create analysis data types"), "data_types"); } + /// PUT /v3/analyses/{analysis_id}/data-types @Override - public DataTypeList getFunctionDataTypes(List functionIDS) { - String queryString = functionIDS.stream().map( f -> "function_ids=" + f.value() ).reduce((a, b) -> a + "&" + b).orElseThrow(); - var request = requestBuilderForEndpoint("functions", "data_types?", queryString) - .GET() - .header("Content-Type", "application/json" ) - .build(); - var response = sendVersion2Request(request); - return DataTypeList.fromJson(response.getJsonData()); + public List updateAnalysisDataTypes(AnalysisID analysisID, + UpdateAnalysisDataTypesInputBody request) throws ApiException { + var call = dataTypesApi.v3UpdateAnalysisDataTypesCall((long) analysisID.id(), request, null); + return ServerDataTypeReader.readEntries( + executeForJsonObject(call, "update analysis data types"), "data_types"); } - public FunctionDataTypesList listFunctionDataTypesForAnalysis(AnalysisID id, List ids) { - try { - List functionIds = null; - if (ids == null) { - functionIds = null; - } else { - functionIds = ids.stream().map(FunctionID::value).map(Long::intValue).toList(); - } - var r = functionsDataTypesApi.listFunctionDataTypesForAnalysis(id.id(), functionIds); - var data = r.getData(); - return data; - } catch (ApiException e) { - throw new RuntimeException(e); - } + /// PUT /v3/analyses/{analysis_id}/functions/{function_id}/signature + /// + /// The response holds no `DataTypeEntry`, so the generated model reads it fine — and going + /// through it keeps the status code on the {@link ApiException}, which is how a function + /// without an extracted signature is told apart from a real failure. + @Override + public void updateFunctionSignature(AnalysisID analysisID, FunctionID functionID, + UpdateFunctionSignatureInputBody signature) throws ApiException { + dataTypesApi.v3UpdateFunctionSignature((long) analysisID.id(), functionID.value(), signature); } - /// GET /v2/functions/data_types carries the function ids as a query parameter. A whole-binary match - /// resolves type info for every matched function at once, so the id list overflows the request URI - /// (HTTP 414) unless it is chunked. - private static final int DATA_TYPES_BATCH_SIZE = 50; - + /// GET /v3/analyses/{analysis_id}/functions/{function_id}/signature/history + /// + /// The history body holds no `DataTypeEntry`, so the generated model reads it fine. @Override - public FunctionDataTypesList listFunctionDataTypesForFunctions(List functionIDs) { + public List getFunctionSignatureHistory(AnalysisID analysisID, FunctionID functionID) { try { - List ids = functionIDs.stream().map(FunctionID::value).map(Long::intValue).toList(); - var merged = new FunctionDataTypesList(); - merged.setItems(new ArrayList<>()); - for (int i = 0; i < ids.size(); i += DATA_TYPES_BATCH_SIZE) { - var batch = ids.subList(i, Math.min(i + DATA_TYPES_BATCH_SIZE, ids.size())); - var data = functionsDataTypesApi.listFunctionDataTypesForFunctions(batch).getData(); - if (data != null && data.getItems() != null) { - merged.getItems().addAll(data.getItems()); - } - } - return merged; + var versions = dataTypesApi + .v3GetFunctionSignatureHistory((long) analysisID.id(), functionID.value()) + .getVersions(); + return versions == null ? List.of() : versions; } catch (ApiException e) { throw new RuntimeException(e); } } - @Override - public Optional getFunctionDataTypes(AnalysisID analysisID, FunctionID functionID) { - // https://api.reveng.ai/v2/analyses/{analysis_id}/info/functions/{function_id}/data_types - var request = requestBuilderForEndpoint("analyses/%s/info/functions/%s/data_types".formatted(analysisID.id(), functionID.value())) - .GET() - .header("Content-Type", "application/json" ) - .build(); - var response = sendVersion2Request(request); - if (response.errors() == null){ - return Optional.of(FunctionDataTypeStatus.fromJson(response.getJsonData())); - } else { - return Optional.empty(); + private static JsonObject executeForJsonObject(okhttp3.Call call, String what) throws ApiException { + try (okhttp3.Response response = call.execute()) { + okhttp3.ResponseBody responseBody = response.body(); + String text = responseBody == null ? "" : responseBody.string(); + if (!response.isSuccessful()) { + throw new ApiException(response.code(), "Failed to %s: HTTP %d".formatted(what, response.code())); + } + JsonElement parsed = JsonParser.parseString(text); + if (!parsed.isJsonObject()) { + throw new ApiException("Failed to %s: response was not a JSON object".formatted(what)); + } + return parsed.getAsJsonObject(); + } catch (IOException e) { + throw new ApiException(e); } } @@ -415,7 +375,9 @@ public Optional getFunctionDataTypes(AnalysisID analysis public boolean triggerAIDecompilationForFunctionID(FunctionID functionID) { try { // POST /v3/functions/{function_id}/ai-decompilation - var result = functionsAiDecompilationApi.createAiDecompilation(functionID.value(), false, null); + // The context_aware flag was removed from the API with no replacement; temperature is + // left null so the server applies its own default. + var result = functionsAiDecompilationApi.createAiDecompilation(functionID.value(), null); return Boolean.TRUE.equals(result.getStatus()); } catch (ApiException e) { throw new RuntimeException("Failed to trigger AI decompilation", e); @@ -428,6 +390,10 @@ public AIDecompilationStatus pollAIDecompileStatus(FunctionID functionID) { // GET /v3/functions/{function_id}/ai-decompilation DecompilationData data = functionsAiDecompilationApi.getAiDecompilation(functionID.value()); String summary = null; + // TODO: no v3 endpoint currently returns a predicted function name. It used to ride on + // the removed /ai-decompilation/tokenised response; neither /token-values nor + // /line-attributions carries it, so the predicted-name panel stays hidden until the API + // offers it again. String predictedFunctionName = null; WorkflowProgress.StatusEnum summaryStatus = null; WorkflowProgress.StatusEnum inlineCommentsStatus = null; @@ -457,13 +423,6 @@ public AIDecompilationStatus pollAIDecompileStatus(FunctionID functionID) { } catch (ApiException e) { Msg.info(this, "Decompilation completed but summary not yet available for function " + functionID.value()); } - try { - // GET /v3/functions/{function_id}/ai-decompilation/tokenised — carries the predicted name - TokenisedData tokenised = functionsAiDecompilationApi.getAiDecompilationTokenised(functionID.value()); - predictedFunctionName = tokenised.getPredictedFunctionName(); - } catch (ApiException e) { - Msg.info(this, "Could not fetch predicted function name for function " + functionID.value() + ": " + e.getMessage()); - } try { // GET /v3/functions/{function_id}/ai-decompilation/inline-comments/status WorkflowProgress commentsProgress = functionsAiDecompilationApi.getAiDecompilationInlineCommentsStatus(functionID.value()); @@ -527,16 +486,18 @@ public void triggerAIDecompilationSummary(FunctionID functionID) { } @Override - public TokenisedData getAIDecompilationTokenised(FunctionID functionID) throws ApiException { - // GET /v3/functions/{function_id}/ai-decompilation/tokenised - return functionsAiDecompilationApi.getAiDecompilationTokenised(functionID.value()); + public GetTokensResponse getAIDecompilationTokens(FunctionID functionID) throws ApiException { + // GET /v3/functions/{function_id}/ai-decompilation/tokens + return functionsAiDecompilationApi.v3GetAiDecompilationTokens(functionID.value()); } @Override public UpsertOverridesData applyAIDecompilationOverrides(FunctionID functionID, java.util.Map overrides) throws ApiException { // PUT /v3/functions/{function_id}/ai-decompilation/overrides - var body = new UpsertOverridesInputBody().overrides(overrides); - return functionsAiDecompilationApi.upsertAiDecompilationOverrides(functionID.value(), body); + var wrapped = new java.util.LinkedHashMap(); + overrides.forEach((token, value) -> wrapped.put(token, new Token().value(value))); + var body = new UpsertOverridesInputBody().overrides(wrapped); + return functionsAiDecompilationApi.v3UpsertAiDecompilationOverrides(functionID.value(), body); } @Override @@ -558,92 +519,37 @@ private static String describeApiException(ApiException e) { return "HTTP " + e.getCode() + " — " + (e.getResponseBody() != null ? e.getResponseBody() : e.getMessage()); } - /** - * https://api.reveng.ai/v2/docs#tag/Functions-overview/operation/rename_function_id_v2_functions_rename__function_id__post - * - * @param id - * @param newName - * @param newNameMangled - */ + /// POST /v3/functions/rename, with a one-item body: v3 has no per-function rename route. + /// The endpoint answers 200 with the number of functions it renamed, so a count of zero is + /// raised rather than passed off to the caller as a successful rename. @Override public void renameFunction(FunctionID id, String newName, String newNameMangled) { - var fn = new FunctionRename(); - fn.setNewName(newName); - fn.setNewMangledName(newNameMangled); + var item = new BatchRenameItem(); + item.setFunctionId(id.value()); + item.setNewName(newName); + item.setNewMangledName(newNameMangled); + var request = new BatchRenameInputBody(); + request.setFunctions(List.of(item)); + BatchRenameOutputBody response; try { - functionsRenamingHistoryApi.renameFunctionId((int) id.value(), fn); + response = functionsRenamingHistoryApi.batchRenameFunctions(request); } catch (ApiException e) { throw new RuntimeException(e); } - } - - @Override - public FunctionNameScore getNameScore(FunctionMatch match) { - return getNameScores(List.of(match), false).get(0); - } - - /** - * https://api.reveng.ai/v2/docs#tag/Confidence-Scores/operation/function_threat_score_v2_confidence_functions_threat_score_post - */ - @Override - public List getNameScores(List matches, Boolean isDebug) { - JSONObject params = new JSONObject(); - params.put("is_debug", isDebug); - var functions = new ArrayList(); - for (var match : matches){ - functions.add(new JSONObject() - // The id of the original function that matches were searched for - .put("function_id", match.origin_function_id().value()) - // The name of the nearest neighbor function for which we want the score - .put("function_name_mangled", match.nearest_neighbor_function_name())); + Long renamedCount = response == null ? null : response.getRenamedCount(); + if (renamedCount == null || renamedCount < 1) { + throw new RuntimeException("Server did not rename function " + id.value() + " to " + newName + + " (renamed_count: " + renamedCount + ")"); } - params.put("functions", functions); - - HttpRequest request = requestBuilderForEndpoint("confidence", "functions", "name_score") - .POST(HttpRequest.BodyPublishers.ofString(params.toString())) - .header("Content-Type", "application/json" ) - .build(); - JSONArray responseData = (JSONArray) sendVersion2Request(request).data(); - return mapJSONArray(responseData, FunctionNameScore::fromJSONObject); } - /** - * - * @param id - * @return - */ - @Override - public AnalysisResult getInfoForAnalysis(AnalysisID id) { - try { - var response = analysisCoreApi.getAnalysisBasicInfo(id.id()); - var data = response.getData(); - if (data == null) { - throw new RuntimeException("Unexpected null data for analysis ID: " + id.id()); - } - return new AnalysisResult( - id, - data - ); - } catch (ApiException e) { - throw new IllegalArgumentException("Could not find analysis with ID: " + id.id()); - } - } - - /** - * https://api.reveng.ai/redoc#tag/Functions-overview/operation/function_detail_v2_functions__function_id__get - * @param id - * @return - */ @Override public FunctionDetails getFunctionDetails(FunctionID id) { - BaseResponseFunctionsDetailResponse dets = null; try { - dets = functionsCoreApi.getFunctionDetails((int) id.value()); + return FunctionDetails.fromServerResponse(functionsCoreApi.getFunctionDetails_0(id.value())); } catch (ApiException e) { throw new RuntimeException(e); } - return FunctionDetails.fromServerResponse(dets.getData()); - } @Override @@ -684,23 +590,20 @@ public List searchCollections(String partialCollectionNa @Override public List searchBinaries(String partialBinaryName, String modelName) throws ApiException { - return this.searchApi.searchBinaries(1, 10, partialBinaryName, null, null, modelName, null, null).getData().getResults(); + return this.searchApi.searchBinaries(1, 10, partialBinaryName, null, null, modelName, null, null, null).getData().getResults(); } @Override - public ai.reveng.model.Basic getAnalysisBasicInfo(AnalysisID analysisID) throws ApiException { - // Check cache first - ai.reveng.model.Basic cachedResult = analysisBasicInfoCache.get(analysisID); + public AnalysisBasicInfoOutputBody getAnalysisBasicInfo(AnalysisID analysisID) throws ApiException { + AnalysisBasicInfoOutputBody cachedResult = analysisBasicInfoCache.get(analysisID); if (cachedResult != null) { Msg.info(this, "Returning cached analysis basic info for analysis ID: " + analysisID.id()); return cachedResult; } - // If not in cache, make API call Msg.info(this, "Fetching analysis basic info from API for analysis ID: " + analysisID.id()); - ai.reveng.model.Basic result = this.analysisCoreApi.getAnalysisBasicInfo(analysisID.id()).getData(); + AnalysisBasicInfoOutputBody result = this.analysisCoreApi.getAnalysisBasicInfo_0((long) analysisID.id()); - // Cache the result for future requests analysisBasicInfoCache.put(analysisID, result); return result; @@ -741,23 +644,21 @@ public void batchRenameFunctions(BatchRenameInputBody request) throws ApiExcepti this.functionsRenamingHistoryApi.batchRenameFunctions(request); } + /// GET /v3/functions/{function_id}/blocks + /// + /// Returns the function's assembly in address order, or an empty list when the function carries + /// no stored disassembly: v3 reports that as a 200 whose block fields are simply absent, where + /// the deprecated v2 endpoint answered 404. A 404 from v3 means the function itself could not be + /// reached, and a 409 that the analysis is not ready yet; both stay on the {@link ApiException} + /// so the caller can tell them apart by status code. @Override public List getAssembly(FunctionID id) { - - FunctionBlocksResponse blocks; - List result = new ArrayList<>(); try { - blocks = this.functionsCoreApi.getFunctionBlocks(id.asInteger()).getData(); + DisassemblyOutputBody disassembly = this.functionsCoreApi.getFunctionBlocks_0(id.value()); + return DisassemblyBlocksReader.readAssembly(disassembly.getBasicBlocks()); } catch (ApiException e) { throw new RuntimeException(e); } - blocks.getBlocks().stream() - .sorted( (b1, b2) -> b1.getMinAddr().compareTo(b2.getMinAddr()) ) - .forEach(block -> { - result.addAll(block.getAsm()); - }); - - return result; } @Override @@ -775,52 +676,12 @@ public java.util.Map canonicalizeFunctionNames(List name return mapping; } - @Override - public Optional getFunctionDataTypesWithVersion(FunctionID functionID) throws ApiException { - var data = functionsDataTypesApi.listFunctionDataTypesForFunctions(List.of(functionID.asInteger())).getData(); - if (data == null || data.getItems() == null) { - return Optional.empty(); - } - return data.getItems().stream() - .filter(item -> item.getFunctionId() != null && item.getFunctionId() == functionID.value()) - .findFirst() - .map(item -> new VersionedFunctionTypes( - item.getDataTypes(), - item.getDataTypesVersion() == null ? 0L : item.getDataTypesVersion().longValue())); - } - - @Override - public List pushFunctionDataTypes(AnalysisID analysisID, List updates) throws ApiException { - var items = updates.stream() - .map(update -> new BatchUpdateDataTypesItem() - .functionId(update.functionID().value()) - .dataTypes(update.dataTypes()) - .dataTypesVersion(update.version())) - .toList(); - var body = new BatchUpdateDataTypesInputBody().functions(items); - var response = functionsDataTypesApi.batchUpdateFunctionDataTypes((long) analysisID.id(), body); - if (response.getResults() == null) { - return List.of(); - } - return response.getResults().stream() - .map(result -> new DataTypePushResult( - new FunctionID(result.getFunctionId()), - mapPushStatus(result.getStatus()), - result.getError())) - .toList(); - } - - private static DataTypePushStatus mapPushStatus(BatchUpdateDataTypesResult.StatusEnum status) { - if (status == null) { - return DataTypePushStatus.UNKNOWN; - } - return switch (status) { - case UPDATED -> DataTypePushStatus.UPDATED; - case VERSION_CONFLICT -> DataTypePushStatus.VERSION_CONFLICT; - case ERROR -> DataTypePushStatus.ERROR; - default -> DataTypePushStatus.UNKNOWN; - }; - } + // TODO: getFunctionDataTypesWithVersion / pushFunctionDataTypes / mapPushStatus were removed + // here. They pushed a whole v2 data-type blob per function under optimistic concurrency, and + // neither those endpoints nor their models exist in the v3 API. The replacement writes types + // and signatures separately — POST/PATCH /v3/analyses/{analysis_id}/data-types to mint or + // update a type and get its data_type_id back, then PUT the signature that refers to it — and + // lands in a follow-up. @Override public ConfigResponse getConfig() { diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiInterface.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiInterface.java index c77e9f62..789ab84e 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiInterface.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiInterface.java @@ -6,6 +6,8 @@ import java.util.Optional; import ai.reveng.model.*; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.FunctionSignatureBatch; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.ServerDataType; import ai.reveng.toolkit.ghidra.core.services.api.types.*; import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionInfo; import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionMatch; @@ -30,11 +32,7 @@ public interface TypedApiInterface { /// Data type to represent the RevEng.AI API concept of a function ID - record FunctionID(long value){ - public Integer asInteger() { - return Math.toIntExact(value); - } - } + record FunctionID(long value){} /// This is a special box type for an analysis ID /// It enforces that the integer is specifically an analysis ID, @@ -58,21 +56,9 @@ default List getFunctionInfo(AnalysisID analysisID) { throw new UnsupportedOperationException("getFunctionInfo not implemented yet"); } + /// GET /v3/analyses, filtered to one binary hash and paged to exhaustion. @Deprecated - default List getFunctionInfo(BinaryID binID) throws ApiException { - return getFunctionInfo(getAnalysisIDfromBinaryID(binID)); - } - - @Deprecated - default AnalysisStatus status(BinaryID binID) throws ApiException { - throw new UnsupportedOperationException("status not implemented yet"); - }; - - /** - * https://docs.reveng.ai/#/Utility/get_search - */ - @Deprecated - default List search(BinaryHash hash) { + default List search(BinaryHash hash) { throw new UnsupportedOperationException("search not implemented yet"); } @@ -84,40 +70,28 @@ default BinaryHash upload(Path binPath) throws FileNotFoundException, ai.reveng. String getAnalysisLogs(AnalysisID analysisID); - default DataTypeList generateFunctionDataTypes(AnalysisID analysisID, List functionIDS) { - throw new UnsupportedOperationException("generateFunctionDataTypes not implemented yet"); - } - - default DataTypeList getFunctionDataTypes(List functionIDS) { - throw new UnsupportedOperationException("getFunctionDataTypes not implemented yet"); - } - - default Optional getFunctionDataTypes(AnalysisID analysisID, FunctionID functionID) { - throw new UnsupportedOperationException("getFunctionDataTypes not implemented yet"); - } - - default FunctionDataTypesList listFunctionDataTypesForAnalysis(AnalysisID analysisID) { - return listFunctionDataTypesForAnalysis(analysisID, null); - } - - default FunctionDataTypesList listFunctionDataTypesForAnalysis(AnalysisID analysisID, @Nullable List ids) { - throw new UnsupportedOperationException("listFunctionDataTypesForAnalysis not implemented yet"); - } - - default FunctionDataTypesList listFunctionDataTypesForFunctions(List functionIDs) { - throw new UnsupportedOperationException("listFunctionDataTypesForFunctions not implemented yet"); + /// GET /v3/functions/signatures + /// + /// Signatures for the given functions, which may belong to different analyses, plus — when + /// `includeDataTypes` is set — every data type those signatures reference, grouped by owning + /// analysis. Callers should go through {@link FunctionSignatureService}, which chunks the ids. + default FunctionSignatureBatch listFunctionSignatures(List functionIDs, boolean includeDataTypes) { + throw new UnsupportedOperationException("listFunctionSignatures not implemented yet"); } - @Deprecated - default AnalysisID getAnalysisIDfromBinaryID(BinaryID binaryID) { - throw new UnsupportedOperationException("getAnalysisIDfromBinaryID not implemented yet"); + /// GET /v3/analyses/{analysis_id}/data-types + /// + /// One page of an analysis' data types. Callers should go through + /// {@link AnalysisDataTypesService}, which pages this into a catalogue. + default List listAnalysisDataTypes(AnalysisID analysisID, long offset, long limit) { + throw new UnsupportedOperationException("listAnalysisDataTypes not implemented yet"); } - default AnalysisResult getInfoForAnalysis(AnalysisID id) { - throw new UnsupportedOperationException("getInfoForAnalysis not implemented yet"); + /// GET /v3/analyses/{analysis_id}/functions/{function_id}/signature/history + default List getFunctionSignatureHistory(AnalysisID analysisID, FunctionID functionID) { + throw new UnsupportedOperationException("getFunctionSignatureHistory not implemented yet"); } - default boolean triggerAIDecompilationForFunctionID(FunctionID functionID) { throw new UnsupportedOperationException("triggerAIDecompilationForFunctionID not implemented yet"); } @@ -137,10 +111,11 @@ default void triggerAIDecompilationSummary(FunctionID functionID) { /** * Tokenised view of an AI decompilation. The tokenised text mirrors the human-readable * decompilation but with renameable identifiers replaced by stable tokens, and carries the - * mapping used to resolve a displayed name back to the token to override. + * value each token renders as, plus the caller's own overrides as a separate map, which is + * how a displayed name is resolved back to the token to override. */ - default TokenisedData getAIDecompilationTokenised(FunctionID functionID) throws ApiException { - throw new UnsupportedOperationException("getAIDecompilationTokenised not implemented yet"); + default GetTokensResponse getAIDecompilationTokens(FunctionID functionID) throws ApiException { + throw new UnsupportedOperationException("getAIDecompilationTokens not implemented yet"); } /** @@ -173,43 +148,38 @@ default java.util.Map canonicalizeFunctionNames(List nam throw new UnsupportedOperationException("canonicalizeFunctionNames not implemented yet"); } - /// The server's data-type blob for a function together with its optimistic-concurrency version. - record VersionedFunctionTypes(ai.reveng.model.V2FunctionInfo dataTypes, long version) {} - - /** - * Fetch the current server-side data types for a function and the version to send back on update. - * Empty if the server has no data types for the function yet. - */ - default Optional getFunctionDataTypesWithVersion(FunctionID functionID) throws ApiException { - throw new UnsupportedOperationException("getFunctionDataTypesWithVersion not implemented yet"); + /// POST /v3/analyses/{analysis_id}/data-types + /// + /// Create types the analysis does not have. The bodies carry no `data_type_id`; the server + /// assigns one to each and returns the stored types. Callers should go through + /// {@link AnalysisDataTypesService}, which resolves against the catalogue first and chunks the + /// batch. + default List createAnalysisDataTypes(AnalysisID analysisID, + CreateAnalysisDataTypesInputBody request) throws ApiException { + throw new UnsupportedOperationException("createAnalysisDataTypes not implemented yet"); } - /// Outcome of a single data-type push, mirroring the server status values. - enum DataTypePushStatus { UPDATED, VERSION_CONFLICT, ERROR, UNKNOWN } - - /// A local data-type blob to push for a function, carrying the version it was based on. - record FunctionDataTypeUpdate(FunctionID functionID, ai.reveng.model.FunctionInfo dataTypes, long version) {} - - /// Per-function outcome of a data-type push. - record DataTypePushResult(FunctionID functionID, DataTypePushStatus status, @Nullable String error) {} + /// PUT /v3/analyses/{analysis_id}/data-types + /// + /// Replace stored types in full — a field left out of the request is cleared. Every body must + /// name the `data_type_id` it replaces. + default List updateAnalysisDataTypes(AnalysisID analysisID, + UpdateAnalysisDataTypesInputBody request) throws ApiException { + throw new UnsupportedOperationException("updateAnalysisDataTypes not implemented yet"); + } - /** - * Push local data-type blobs back to the portal for the given analysis. Version conflicts are - * reported per function so the caller can re-fetch and retry. - */ - default List pushFunctionDataTypes(AnalysisID analysisID, List updates) throws ApiException { - throw new UnsupportedOperationException("pushFunctionDataTypes not implemented yet"); + /// PUT /v3/analyses/{analysis_id}/functions/{function_id}/signature + /// + /// Replace one function's parameters, return type and calling convention. Edit-only: a function + /// the server has no extracted signature for is answered with 404. Callers should go through + /// {@link FunctionSignatureService#put}, which treats that 404 as "nothing to edit". + default void updateFunctionSignature(AnalysisID analysisID, FunctionID functionID, + UpdateFunctionSignatureInputBody signature) throws ApiException { + throw new UnsupportedOperationException("updateFunctionSignature not implemented yet"); } void renameFunction(FunctionID id, String newName, String newNameMangled); - default FunctionNameScore getNameScore(FunctionMatch match) { - throw new UnsupportedOperationException("getNameScore not implemented yet"); - } - default List getNameScores(List matches, Boolean isDebug) { - throw new UnsupportedOperationException("getNameScores not implemented yet"); - } - default FunctionDetails getFunctionDetails(FunctionID id) { throw new UnsupportedOperationException("getFunctionInfo not implemented yet"); } @@ -239,7 +209,7 @@ default List searchBinaries(String partialCollectionName, St throw new UnsupportedOperationException("searchBinaries not implemented yet"); } - default ai.reveng.model.Basic getAnalysisBasicInfo(AnalysisID analysisID) throws ApiException { + default AnalysisBasicInfoOutputBody getAnalysisBasicInfo(AnalysisID analysisID) throws ApiException { throw new UnsupportedOperationException("getAnalysisBasicInfo not implemented yet"); } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/Utils.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/Utils.java deleted file mode 100644 index 497af6be..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/Utils.java +++ /dev/null @@ -1,17 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api; - -import org.json.JSONArray; -import org.json.JSONObject; - -import java.util.ArrayList; -import java.util.List; -import java.util.function.Function; - -public class Utils { - - public static List mapJSONArray(JSONArray jsonArray, Function mapper) { - var result = new ArrayList(); - jsonArray.forEach(o -> result.add(mapper.apply((JSONObject) o))); - return result; - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/V2Response.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/V2Response.java deleted file mode 100644 index 6005f04a..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/V2Response.java +++ /dev/null @@ -1,50 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api; - - -import org.json.JSONObject; - -import java.util.List; - -import static ai.reveng.toolkit.ghidra.core.services.api.Utils.mapJSONArray; - -/** - * Structured Response from any V2 Endpoint - * { - * "status": true, - * "data": { - * "queued": true, - * "reference": "404f60e6-7b1d-4adf-951c-710925422bd8" - * }, - * "message": null, - * "errors": null, - * "meta": { - * "pagination": null - * } - * } - * - */ -public record V2Response( - boolean status, - // Either a JSONObject or JSONArray - Object data, - String message, - List errors, - JSONObject meta - -) { - - - public static V2Response fromJSONObject(JSONObject json) { - return new V2Response( - json.getBoolean("status"), - !json.isNull("data") ? json.get("data") : null, - !json.isNull("message") ? json.getString("message") : null, - !json.isNull("errors") ? mapJSONArray(json.getJSONArray("errors"), APIError::fromJSONObject) : null, - json.getJSONObject("meta") - ); - } - - public JSONObject getJsonData() { - return (JSONObject) data; - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/datatypes/FunctionSignatureBatch.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/datatypes/FunctionSignatureBatch.java new file mode 100644 index 00000000..9a4b0a91 --- /dev/null +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/datatypes/FunctionSignatureBatch.java @@ -0,0 +1,46 @@ +package ai.reveng.toolkit.ghidra.core.services.api.datatypes; + +import ai.reveng.model.BatchFunctionSignatureEntry; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface.AnalysisID; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/// One `GET /v3/functions/signatures` response. +/// +/// The endpoint accepts function ids across analyses, so the data types come back grouped by the +/// analysis that owns them: `data_type_id` is only unique within an analysis, and a signature's +/// `return_data_type_id` / parameter `data_type_id` are resolved against the group named by the +/// entry's own `analysis_id`. +/// +/// @param items one entry per requested function id the caller may see. `has_signature` says +/// whether the server actually holds a signature for it. +/// @param dataTypes every type referenced by the entries above, keyed by owning analysis. Empty +/// when the request did not ask for data types. +public record FunctionSignatureBatch( + List items, + Map> dataTypes) { + + public static FunctionSignatureBatch empty() { + return new FunctionSignatureBatch(List.of(), Map.of()); + } + + /// Combine two responses. Used to stitch the chunks of a batched request back together. + public FunctionSignatureBatch merge(FunctionSignatureBatch other) { + List mergedItems = new ArrayList<>(items); + mergedItems.addAll(other.items); + + Map> mergedTypes = new LinkedHashMap<>(); + dataTypes.forEach((analysis, types) -> mergedTypes.put(analysis, new ArrayList<>(types))); + other.dataTypes.forEach((analysis, types) -> + mergedTypes.computeIfAbsent(analysis, ignored -> new ArrayList<>()).addAll(types)); + + return new FunctionSignatureBatch(mergedItems, mergedTypes); + } + + public List dataTypesFor(AnalysisID analysisID) { + return dataTypes.getOrDefault(analysisID, List.of()); + } +} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/datatypes/ServerDataType.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/datatypes/ServerDataType.java new file mode 100644 index 00000000..ada00f03 --- /dev/null +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/datatypes/ServerDataType.java @@ -0,0 +1,78 @@ +package ai.reveng.toolkit.ghidra.core.services.api.datatypes; + +import javax.annotation.Nullable; +import java.util.List; + +/** + * A v3 data type as the portal reports it, flattened. + * + *

The spec models this as {@code DataTypeEntry}, a ten-way {@code oneOf} discriminated by + * {@code kind}. All ten variants carry an identical set of fields and differ only in the optional + * {@code definition} object, so the union is really one record with a variant payload — which is + * what this is. The plugin never handles the generated union; see {@link ServerDataTypeReader}. + * + * @param id {@code data_type_id}; identifies the type within its analysis. 0 is valid. + * @param hasDefinition whether the server says this type carries a definition. Distinguishes a + * kind that never has one from a type referenced but never defined. + * @param sourceFunctionId set when the type was transferred from another function rather than + * extracted. + * @param createdAt raw ISO-8601 timestamp, left unparsed. + * @param definition null for {@code BASE}, {@code BITFIELD} and {@code UNKNOWN}, which never + * carry one, and for a type that was referenced but never defined. + */ +public record ServerDataType( + long id, + String namespace, + String name, + Kind kind, + @Nullable Long size, + String sourceType, + boolean hasDefinition, + @Nullable Long sourceFunctionId, + @Nullable String createdAt, + @Nullable Definition definition) { + + /// The {@code kind} discriminator. Order matches the spec's discriminator mapping. + public enum Kind { + STRUCT, UNION, ENUM, TYPEDEF, POINTER, ARRAY, FUNCTION_DEFINITION, BITFIELD, BASE, UNKNOWN; + + /// Kinds the server may add later map to {@link #UNKNOWN} rather than failing the read. + public static Kind fromJson(@Nullable String value) { + for (Kind kind : values()) { + if (kind.name().equals(value)) { + return kind; + } + } + return UNKNOWN; + } + } + + /// The kind-specific payload. Sealed; every implementation lives in this file. + public sealed interface Definition {} + + /// A struct or union field. Every id/offset/size in the v3 family is a 64-bit integer. + public record Member(@Nullable String name, long offset, long size, @Nullable Long dataTypeId, + boolean isBitfield, @Nullable Long bitOffset, @Nullable Long bitSize) {} + + /// An enum constant. {@code value} stays a decimal string: it may be negative or exceed 64 + /// unsigned bits, which no Java integer type nor a JSON number can carry safely. + public record EnumValue(String name, String value) {} + + /// A parameter of a function-definition type. + public record Parameter(@Nullable String name, long ordinal, long size, @Nullable Long dataTypeId) {} + + public record StructDefinition(List members) implements Definition {} + + public record UnionDefinition(List members) implements Definition {} + + public record EnumDefinition(List values) implements Definition {} + + public record TypedefDefinition(@Nullable Long targetDataTypeId) implements Definition {} + + public record PointerDefinition(@Nullable Long pointeeDataTypeId) implements Definition {} + + public record ArrayDefinition(@Nullable Long count, @Nullable Long elementDataTypeId) implements Definition {} + + public record FunctionTypeDefinition(@Nullable Long returnDataTypeId, List parameters) + implements Definition {} +} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/datatypes/ServerDataTypeReader.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/datatypes/ServerDataTypeReader.java new file mode 100644 index 00000000..372628ca --- /dev/null +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/datatypes/ServerDataTypeReader.java @@ -0,0 +1,128 @@ +package ai.reveng.toolkit.ghidra.core.services.api.datatypes; + +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.ServerDataType.*; +import com.google.gson.*; +import com.google.gson.reflect.TypeToken; + +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.List; + +/** + * Reads the spec's {@code DataTypeEntry} JSON straight into the flattened {@link ServerDataType}, + * switching on the {@code kind} discriminator. + * + *

Why this exists. The generated {@code ai.reveng.model.DataTypeEntry} cannot deserialise: + * its gson adapter match-counts the payload against all ten variants instead of using the + * discriminator, and since every variant shares the same required fields several always match. That + * is not fixable from the plugin side by registering an adapter for {@code DataTypeEntry}, because + * the container models call the static {@code DataTypeEntry.validateJsonElement} before + * delegating to any adapter — so a factory registered for {@code DataTypeEntry} is never reached. + * + *

How the read path uses it. Call the generated {@code DataTypesApi} {@code ...Call(...)} + * form, which still builds every request (path, query, auth) exactly as the SDK would, then hand the + * response body here instead of to the generated deserialiser. {@code ConversationsApiChatService} + * uses the same {@code okhttp3.Call} escape hatch for SSE. Nothing needs registering on the shared + * {@code ApiClient}: no generated model the plugin deserialises embeds a {@code DataTypeEntry}. + * + *

{@link #readEntries} covers every container in one call — each of them holds its entries in a + * single named array ({@code items} for {@code ListAnalysisDataTypesOutputBody} and + * {@code AnalysisDataTypesOutputBody}, {@code data_types} for {@code FunctionSignatureBody}) — so no + * container needs its own model or adapter. + */ +public final class ServerDataTypeReader implements JsonDeserializer { + + private static final Type MEMBERS = new TypeToken>() {}.getType(); + private static final Type VALUES = new TypeToken>() {}.getType(); + private static final Type PARAMETERS = new TypeToken>() {}.getType(); + + /// Snake-case naming binds the nested entries' fields ({@code data_type_id}, {@code is_bitfield}, + /// {@code bit_offset}, ...) reflectively; the entry itself is read by hand below. + private static final Gson GSON = new GsonBuilder() + .setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES) + .registerTypeAdapter(ServerDataType.class, new ServerDataTypeReader()) + .create(); + + /// Read one {@code DataTypeEntry}. + public static ServerDataType readEntry(JsonElement entry) { + return GSON.fromJson(entry, ServerDataType.class); + } + + /// Read the {@code DataTypeEntry} array held in {@code field} of a container body. Returns empty + /// when the field is absent or JSON null, which the spec allows for every such array. + public static List readEntries(JsonElement body, String field) { + if (body == null || !body.isJsonObject()) { + return List.of(); + } + JsonElement array = body.getAsJsonObject().get(field); + if (array == null || !array.isJsonArray()) { + return List.of(); + } + List entries = new ArrayList<>(); + for (JsonElement entry : array.getAsJsonArray()) { + entries.add(readEntry(entry)); + } + return entries; + } + + @Override + public ServerDataType deserialize(JsonElement json, Type type, JsonDeserializationContext context) { + JsonObject entry = json.getAsJsonObject(); + Kind kind = Kind.fromJson(string(entry, "kind")); + Long id = number(entry, "data_type_id"); + return new ServerDataType( + id == null ? 0L : id, + string(entry, "namespace"), + string(entry, "name"), + kind, + number(entry, "size"), + string(entry, "source_type"), + Boolean.TRUE.equals(bool(entry, "has_definition")), + number(entry, "source_function_id"), + string(entry, "created_at"), + definition(kind, entry.get("definition"), context)); + } + + private static Definition definition(Kind kind, JsonElement raw, JsonDeserializationContext context) { + if (raw == null || !raw.isJsonObject()) { + return null; + } + JsonObject def = raw.getAsJsonObject(); + return switch (kind) { + case STRUCT -> new StructDefinition(list(def, "members", MEMBERS, context)); + case UNION -> new UnionDefinition(list(def, "members", MEMBERS, context)); + case ENUM -> new EnumDefinition(list(def, "values", VALUES, context)); + case TYPEDEF -> new TypedefDefinition(number(def, "target_data_type_id")); + case POINTER -> new PointerDefinition(number(def, "pointee_data_type_id")); + case ARRAY -> new ArrayDefinition(number(def, "count"), number(def, "element_data_type_id")); + case FUNCTION_DEFINITION -> new FunctionTypeDefinition( + number(def, "return_data_type_id"), list(def, "parameters", PARAMETERS, context)); + // These kinds never carry a definition; ignore one if the server ever sends it. + case BASE, BITFIELD, UNKNOWN -> null; + }; + } + + private static List list(JsonObject owner, String field, Type type, JsonDeserializationContext context) { + JsonElement array = owner.get(field); + if (array == null || !array.isJsonArray()) { + return List.of(); + } + List items = context.deserialize(array, type); + return items == null ? List.of() : items; + } + + private static String string(JsonObject owner, String field) { + JsonElement value = owner.get(field); + return value == null || value.isJsonNull() ? null : value.getAsString(); + } + + private static Long number(JsonObject owner, String field) { + JsonElement value = owner.get(field); + return value == null || value.isJsonNull() ? null : value.getAsLong(); + } + + private static Boolean bool(JsonObject owner, String field) { + JsonElement value = owner.get(field); + return value == null || value.isJsonNull() ? null : value.getAsBoolean(); + } +} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/MockApi.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/MockApi.java index 59dcc567..50697e70 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/MockApi.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/MockApi.java @@ -1,12 +1,13 @@ package ai.reveng.toolkit.ghidra.core.services.api.mocks; +import ai.reveng.model.AnalysisRecordBody; import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.types.*; -import ai.reveng.toolkit.ghidra.core.services.api.types.exceptions.APIAuthenticationException; import org.json.JSONObject; import java.io.FileNotFoundException; import java.nio.file.Path; +import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.List; @@ -19,29 +20,23 @@ public BinaryHash upload(Path binPath) throws FileNotFoundException { @Override @Deprecated - public List search(BinaryHash hash) { + public List search(BinaryHash hash) { if (hash.equals(new BinaryHash("b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6"))) { - return List.of(new LegacyAnalysisResult( - new AnalysisID(1234), - new BinaryID(17920), - "true", - "no creation date", - 1, - "model name", - hash, - AnalysisStatus.Complete, - 123456, - "b48f61e85bcbc7866d78a8f0b72acd8c0c177ebd15cea466d1edb67409fca269" - )); + return List.of(new AnalysisRecordBody() + .analysisId(1234L) + .binaryId(17920L) + .binaryName("true") + .creation(OffsetDateTime.parse("2024-04-19T08:57:18Z")) + .modelId(1L) + .modelName("model name") + .sha256Hash(hash.sha256()) + .status(AnalysisStatus.Complete.name()) + .baseAddress(123456L) + .functionBoundariesHash("b48f61e85bcbc7866d78a8f0b72acd8c0c177ebd15cea466d1edb67409fca269")); } return List.of(); } - @Override - public AnalysisStatus status(BinaryID binID) { - return AnalysisStatus.Complete; - } - @Override public String getAnalysisLogs(AnalysisID analysisID) { return ""; diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/UnimplementedAPI.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/UnimplementedAPI.java index e20684e8..383598a5 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/UnimplementedAPI.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/UnimplementedAPI.java @@ -1,9 +1,13 @@ package ai.reveng.toolkit.ghidra.core.services.api.mocks; -import ai.reveng.model.FunctionDataTypesList; +import ai.reveng.invoker.ApiException; +import ai.reveng.model.CreateAnalysisDataTypesInputBody; +import ai.reveng.model.UpdateAnalysisDataTypesInputBody; +import ai.reveng.model.UpdateFunctionSignatureInputBody; import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.FunctionSignatureBatch; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.ServerDataType; import ai.reveng.toolkit.ghidra.core.services.api.types.*; -import org.jetbrains.annotations.Nullable; import java.io.IOException; import java.nio.file.Files; @@ -18,10 +22,12 @@ public class UnimplementedAPI implements TypedApiInterface { protected AnalysisStatus getNextStatus(AnalysisStatus previousStatus) { Objects.requireNonNull(previousStatus); return switch (previousStatus) { + case Uploaded -> AnalysisStatus.Queued; case Queued -> AnalysisStatus.Processing; case Processing -> AnalysisStatus.Complete; case Complete -> AnalysisStatus.Complete; case Error -> AnalysisStatus.Error; + case Unknown -> AnalysisStatus.Unknown; }; } @@ -52,7 +58,33 @@ public BinaryHash upload(Path binPath) { /// This gets called when registering the initial mock analysis /// it just pretends that there is no type info available @Override - public FunctionDataTypesList listFunctionDataTypesForAnalysis(AnalysisID analysisID, @Nullable List ids) { - return new FunctionDataTypesList(); + public FunctionSignatureBatch listFunctionSignatures(List functionIDs, boolean includeDataTypes) { + return FunctionSignatureBatch.empty(); + } + + /// The write path is answered rather than refused, so a test exercising a push does not have to + /// stub all three endpoints just to get past them. An empty catalogue that accepts everything and + /// remembers nothing: the analysis has no types, creating some reports none back, and a signature + /// write succeeds silently. Tests that care about what was written override these. + @Override + public List listAnalysisDataTypes(AnalysisID analysisID, long offset, long limit) { + return List.of(); + } + + @Override + public List createAnalysisDataTypes(AnalysisID analysisID, + CreateAnalysisDataTypesInputBody request) { + return List.of(); + } + + @Override + public List updateAnalysisDataTypes(AnalysisID analysisID, + UpdateAnalysisDataTypesInputBody request) { + return List.of(); + } + + @Override + public void updateFunctionSignature(AnalysisID analysisID, FunctionID functionID, + UpdateFunctionSignatureInputBody signature) throws ApiException { } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisResult.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisResult.java deleted file mode 100644 index 639c490e..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisResult.java +++ /dev/null @@ -1,20 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types; - - -import ai.reveng.model.Basic; -import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; - -/// This is a remnant of an older class that contained the analysis result data directly. -/// Now it's a wrapper around the generated Basic class with some shim methods for convenience. -public record AnalysisResult( - TypedApiInterface.AnalysisID analysisID, - Basic base_response_basic -) { - public TypedApiInterface.BinaryHash sha_256_hash() { - return new TypedApiInterface.BinaryHash(base_response_basic().getSha256Hash()); - } - - public String binary_name() { - return base_response_basic.getBinaryName(); - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisStatus.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisStatus.java index ec34835a..b410cb0f 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisStatus.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisStatus.java @@ -1,14 +1,36 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; +/** + * The lifecycle states an analysis is reported to be in by the API. + * + *

A freshly created analysis reports {@link #Uploaded} before it is queued. The server may also + * report values this plugin does not model, so statuses are resolved with {@link #fromApiValue} + * rather than {@link #valueOf}: an unrecognised value becomes {@link #Unknown} instead of throwing. + * + *

The constant names are the wire values, so {@code name()} can be compared against a raw status + * string from the API. + */ public enum AnalysisStatus { - Complete("Complete"), - Error("Error"), - Processing("Processing"), - Queued("Queued"); + Uploaded, + Queued, + Processing, + Complete, + Error, + /** Any status the server reported that is not modelled above. */ + Unknown; - private final String status; - - AnalysisStatus(final String status) { - this.status = status; + /** + * Resolves a status string from the API, mapping null and unrecognised values to {@link #Unknown}. + */ + public static AnalysisStatus fromApiValue(String value) { + if (value == null) { + return Unknown; + } + for (AnalysisStatus status : values()) { + if (status.name().equalsIgnoreCase(value)) { + return status; + } + } + return Unknown; } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/BinaryID.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/BinaryID.java deleted file mode 100644 index 17a6174b..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/BinaryID.java +++ /dev/null @@ -1,26 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types; - - -/** - * Data type for all reveng API responses or parameters that are a binary ID - * They are called binary ID in the API doc, but they should be thought of as _analysis_ ids - * for a single binary (identified by hash), there can be multiple analyses, which are distinguished by this ID - */ -@Deprecated -public record BinaryID(int value) implements Comparable { - public BinaryID { - if (value < 0) { - throw new IllegalArgumentException("BinaryID must be positive"); - } - } - - @Override - public int compareTo(BinaryID binaryID) { - return Integer.compare(value, binaryID.value); - } - - @Override - public String toString() { - return "BinaryID[" + value + ']'; - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/BoxPlot.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/BoxPlot.java deleted file mode 100644 index 74a9aa2e..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/BoxPlot.java +++ /dev/null @@ -1,28 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types; - -import org.json.JSONObject; - -/** - * The BoxPlot object returned by https://api.reveng.ai/v2/docs#tag/Confidence-Scores/operation/function_name_score_v2_confidence_functions_name_score_post - */ -public record BoxPlot( - double min, - double max, - double average, - double upper_quartile, - double lower_quartile, - int positive_count, - int negative_count -) { - public static BoxPlot fromJSONObject(JSONObject boxplotJson) { - return new BoxPlot( - boxplotJson.getDouble("min"), - boxplotJson.getDouble("max"), - boxplotJson.getDouble("average"), - boxplotJson.getDouble("upper_quartile"), - boxplotJson.getDouble("lower_quartile"), - boxplotJson.getInt("positive_count"), - boxplotJson.getInt("negative_count") - ); - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/DataTypeList.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/DataTypeList.java deleted file mode 100644 index a2efa258..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/DataTypeList.java +++ /dev/null @@ -1,37 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types; - -import ai.reveng.model.FunctionDataTypesList; -import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; -import org.json.JSONObject; - -/** - * Example in data_types_batch_response.json - * @deprecated Use OpenAPI {@link FunctionDataTypesList} - */ -@Deprecated -public record DataTypeList( - int totalCount, - int totalDataTypesCount, - FunctionDataTypeStatus[] dataTypes -) { - - public static DataTypeList fromJson(JSONObject json) { - int totalCount = json.getInt("total_count"); - int totalDataTypesCount = json.getInt("total_data_types_count"); - var dataTypesJson = json.getJSONArray("items"); - FunctionDataTypeStatus[] dataTypes = new FunctionDataTypeStatus[dataTypesJson.length()]; - for (int i = 0; i < dataTypesJson.length(); i++) { - dataTypes[i] = FunctionDataTypeStatus.fromJson(dataTypesJson.getJSONObject(i)); - } - return new DataTypeList(totalCount, totalDataTypesCount, dataTypes); - } - - public FunctionDataTypeStatus statusForFunction(TypedApiInterface.FunctionID functionID) { - for (FunctionDataTypeStatus status : dataTypes) { - if (status.functionID().equals(functionID)) { - return status; - } - } - throw new IllegalArgumentException("FunctionID not found in data types list: " + functionID); - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionBoundary.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionBoundary.java index 3c99ba67..a6eea264 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionBoundary.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionBoundary.java @@ -21,13 +21,4 @@ public JSONObject toJSON() { obj.put("include_in_analysis", includeInAnalysis); return obj; } - - public static FunctionBoundary fromJSON(JSONObject json) { - return new FunctionBoundary( - json.getString("mangled_name"), - json.getLong("start_addr"), - json.getLong("end_addr"), - json.optBoolean("include_in_analysis", true) - ); - } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDataTypeStatus.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDataTypeStatus.java deleted file mode 100644 index e2ca65cf..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDataTypeStatus.java +++ /dev/null @@ -1,77 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types; - -import ai.reveng.model.FunctionDataTypes; -import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; -import ai.reveng.toolkit.ghidra.core.services.api.types.binsync.FunctionDataTypeMessage; -import org.json.JSONObject; - -import javax.annotation.Nullable; -import java.util.Optional; - -/** - * { - * "completed": true, - * "data_types": { - * "func_types": { - * "stack_vars": null, - * "size": 107, - * "last_change": null, - * "name": "FUN_0010203b", - * "header": { - * "args": { - * "0x0": { - * "offset": 0, - * "size": 8, - * "last_change": null, - * "name": "param_1", - * "type": "long *" - * }, - * "0x1": { - * "offset": 1, - * "size": 8, - * "last_change": null, - * "name": "param_2", - * "type": "char * *" - * } - * }, - * "last_change": null, - * "name": "FUN_0010203b", - * "addr": 8251, - * "type": "int" - * }, - * "addr": 8251, - * "type": "int" - * }, - * "func_deps": [] - * }, - * "status": "completed" - * } - * @deprecated Use {@link FunctionDataTypes} instead - */ -@Deprecated -public record FunctionDataTypeStatus( - boolean completed, - Optional data_types, -// JSONObject data_types, - String status, - @Nullable Integer dataTypesVersion, - @Nullable TypedApiInterface.FunctionID functionID - ) { - - public static FunctionDataTypeStatus fromJson(JSONObject json) { - Integer dataTypesVersion; - if (json.has("data_types_version") && !json.isNull("data_types_version")) { - dataTypesVersion = json.getInt("data_types_version"); - } else { - dataTypesVersion = null; - } - return new FunctionDataTypeStatus( - json.getBoolean("completed"), - // Can be null if the function is not completed yet - !json.isNull("data_types") ? Optional.of(FunctionDataTypeMessage.fromJsonObject(json.getJSONObject("data_types"))) : Optional.empty(), - json.getString("status"), - dataTypesVersion, - json.has("function_id") ? new TypedApiInterface.FunctionID(json.getInt("function_id")) : null - ); - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDetails.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDetails.java index b1c7414c..9fe62c20 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDetails.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDetails.java @@ -1,6 +1,6 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; -import ai.reveng.model.FunctionsDetailResponse; +import ai.reveng.model.FunctionDetailsOutputBody; import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; /** @@ -12,21 +12,17 @@ public record FunctionDetails( Long functionVaddr, Long functionSize, TypedApiInterface.AnalysisID analysisId, - String binaryName, - TypedApiInterface.BinaryHash sha256Hash, String demangledName ) { - public static FunctionDetails fromServerResponse(FunctionsDetailResponse response) { + public static FunctionDetails fromServerResponse(FunctionDetailsOutputBody response) { return new FunctionDetails( new TypedApiInterface.FunctionID(response.getFunctionId()), - response.getFunctionNameMangled(), + response.getMangledName(), response.getFunctionVaddr(), - response.getFunctionSize().longValue(), - new TypedApiInterface.AnalysisID(response.getAnalysisId()), - response.getBinaryName(), - new TypedApiInterface.BinaryHash(response.getSha256Hash()), + response.getFunctionSize(), + new TypedApiInterface.AnalysisID(response.getAnalysisId().intValue()), response.getFunctionName() ); } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionNameScore.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionNameScore.java deleted file mode 100644 index 837a5286..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionNameScore.java +++ /dev/null @@ -1,17 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types; - -import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; -import org.json.JSONObject; - -public record FunctionNameScore( - TypedApiInterface.FunctionID functionID, - BoxPlot score -) { - public static FunctionNameScore fromJSONObject(JSONObject jsonObject) { - var boxplotJson = jsonObject.getJSONObject("box_plot"); - return new FunctionNameScore( - new TypedApiInterface.FunctionID(jsonObject.getInt("function_id")), - BoxPlot.fromJSONObject(boxplotJson) - ); - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/GhidraFunctionInfo.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/GhidraFunctionInfo.java deleted file mode 100644 index a89d49f7..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/GhidraFunctionInfo.java +++ /dev/null @@ -1,15 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types; - -import ghidra.program.model.listing.Function; - - -/** - * Combined record of a Ghidra Function and its corresponding FunctionInfo from - * @param functionInfo - * @param function - */ -public record GhidraFunctionInfo( - FunctionInfo functionInfo, - Function function -) { -} \ No newline at end of file diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/GhidraFunctionMatchWithSignature.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/GhidraFunctionMatchWithSignature.java index e31f7a9f..99b75705 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/GhidraFunctionMatchWithSignature.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/GhidraFunctionMatchWithSignature.java @@ -1,7 +1,6 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; -import ai.reveng.toolkit.ghidra.core.services.api.types.binsync.FunctionDataTypeMessage; import ghidra.program.model.data.FunctionDefinitionDataType; import ghidra.program.model.listing.Function; diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/LegacyAnalysisResult.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/LegacyAnalysisResult.java deleted file mode 100644 index 568c0eb5..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/LegacyAnalysisResult.java +++ /dev/null @@ -1,38 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types; - -import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; -import org.json.JSONObject; - - -/// {"analyses":[ -/// {"analysis_scope":"PRIVATE","binary_id":27665,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:57:18 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27664,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:55:28 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27663,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:53:33 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27662,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:32:34 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27661,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:24:44 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27660,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:23:54 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27633,"binary_name":"true","creation":"Thu, 18 Apr 2024 17:34:42 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27632,"binary_name":"true","creation":"Thu, 18 Apr 2024 17:33:36 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27631,"binary_name":"ls","creation":"Thu, 18 Apr 2024 17:17:48 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"2e7ef3da2b295c77820a0782b00c9d607cd48d5c8f7458a76b0921bec20a30ae","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27630,"binary_name":"ls","creation":"Thu, 18 Apr 2024 17:16:48 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"2e7ef3da2b295c77820a0782b00c9d607cd48d5c8f7458a76b0921bec20a30ae","status":"Error"}]} -/// @deprecated Use {@link AnalysisResult)} -@Deprecated -public record LegacyAnalysisResult( - TypedApiInterface.AnalysisID analysis_id, - @Deprecated - BinaryID binary_id, - String binary_name, - String creation, - int model_id, - String model_name, - TypedApiInterface.BinaryHash sha_256_hash, - AnalysisStatus status, - long base_address, - String function_boundaries_hash -) { - public static LegacyAnalysisResult fromJSONObject(JSONObject json) { - return new LegacyAnalysisResult( - new TypedApiInterface.AnalysisID(json.getInt("analysis_id")), - new BinaryID(json.getInt("binary_id")), - json.getString("binary_name"), - json.getString("creation"), - json.getInt("model_id"), - json.getString("model_name"), - new TypedApiInterface.BinaryHash(json.getString("sha_256_hash")), - AnalysisStatus.valueOf(json.getString("status")), - json.getLong("base_address"), - json.getString("function_boundaries_hash") - ); - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/OrderDirection.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/OrderDirection.java deleted file mode 100644 index 6e68b7e5..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/OrderDirection.java +++ /dev/null @@ -1,6 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types; - -public enum OrderDirection { - ASC, - DESC -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/TypePathAndName.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/TypePathAndName.java similarity index 70% rename from src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/TypePathAndName.java rename to src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/TypePathAndName.java index 699c302e..5a572744 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/TypePathAndName.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/TypePathAndName.java @@ -1,22 +1,22 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types.binsync; +package ai.reveng.toolkit.ghidra.core.services.api.types; import ghidra.program.model.data.CategoryPath; +/// A scoped type name split into its Ghidra {@link CategoryPath} and its leaf name. +/// +/// The server reports a type's scope in a `namespace` field, using `::` as the separator, e.g. +/// `stdint`, `DWARF::stdio.h` or the empty string for the root scope. Ghidra models the same idea +/// as a {@link CategoryPath}, so this splits one into the other. public record TypePathAndName( String name, String[] path ) { - - - /// based on `ArtifactLifter.parse_scoped_type` from binsync /// Takes strings like: /// /// - "uint32_t" /// - "stdint::uint32_t" /// - "DWARF::stdio.h::off_t" - /// @param str - /// @return public static TypePathAndName fromString(String str){ // split into path and name on "::" if (str.contains("::")) { diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/Artifact.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/Artifact.java deleted file mode 100644 index b393df2e..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/Artifact.java +++ /dev/null @@ -1,4 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types.binsync; - -public class Artifact { -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionArgument.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionArgument.java deleted file mode 100644 index 1c946b67..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionArgument.java +++ /dev/null @@ -1,32 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types.binsync; - - -import org.json.JSONObject; - -/** - * { - * * "offset": 0, - * * "size": 8, - * * "last_change": null, - * * "name": "param_1", - * * "type": "char *" - * * }, - */ -public record FunctionArgument( - int offset, - int size, - String last_change, - String name, - String type -) { - - public static FunctionArgument fromJsonObject(JSONObject arg) { - return new FunctionArgument( - arg.getInt("offset"), - arg.getInt("size"), - !arg.isNull("last_change") ? arg.getString("last_change") : null, - arg.getString("name"), - arg.getString("type") - ); - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionArtifact.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionArtifact.java deleted file mode 100644 index 6cb1e6d5..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionArtifact.java +++ /dev/null @@ -1,59 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types.binsync; - -import org.json.JSONObject; - -import java.util.ArrayList; -import java.util.List; - -/** - * The Function class describes a Function found a decompiler. There are three components to a function: - * 1. Metadata - * 2. Header - * 3. Stack Vars - * - * The metadata contains info on changes and size. The header holds the return type, - * and arguments (including their types). The stack vars contain StackVariables. - */ -public record FunctionArtifact( - long addr, - int size, - FunctionHeader header, - StackVariable[] stack_vars -) { - public static FunctionArtifact fromJsonObject(JSONObject func) { - - List stackVars = new ArrayList<>(); - if (!func.isNull("stack_vars")) { - JSONObject stackVarsJson = func.getJSONObject("stack_vars"); - for (String key : stackVarsJson.keySet()) { - stackVars.add(StackVariable.fromJsonObject(stackVarsJson.getJSONObject(key))); - } - } - return new FunctionArtifact( - func.getLong("addr"), - func.getInt("size"), - FunctionHeader.fromJsonObject(func.getJSONObject("header")), - stackVars.toArray(new StackVariable[0]) - ); - } - - /** - * Creates a C signature for the function, based on the return type, name and arguments - * @return - */ - public String getSignature() { - StringBuilder signature = new StringBuilder(); - signature.append(header.type()).append(" ").append(header.name()).append("("); - for (int i = 0; i < header.args().length; i++) { - FunctionArgument arg = header.args()[i]; - signature.append(arg.type()).append(" ").append(arg.name()); - if (i < header.args().length - 1) { - signature.append(", "); - } - } - signature.append(")"); - return signature.toString(); - } - -} - diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionDataTypeMessage.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionDataTypeMessage.java deleted file mode 100644 index 8007336d..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionDataTypeMessage.java +++ /dev/null @@ -1,254 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types.binsync; - -import ai.reveng.model.FunctionInfo; -import org.json.JSONObject; - -/** - * { - * "func_types": { - * "stack_vars": null, - * "size": 107, - * "last_change": null, - * "name": "FUN_0010203b", - * "header": { - * "args": { - * "0x0": { - * "offset": 0, - * "size": 8, - * "last_change": null, - * "name": "param_1", - * "type": "long *" - * }, - * "0x1": { - * "offset": 1, - * "size": 8, - * "last_change": null, - * "name": "param_2", - * "type": "char * *" - * } - * }, - * "last_change": null, - * "name": "FUN_0010203b", - * "addr": 8251, - * "type": "int" - * }, - * "addr": 8251, - * "type": "int" - * }, - * "func_deps": [ - * { - * "last_change": null, - * "name": "stat.h/stat64", - * "size": 144, - * "members": { - * "0x0": { - * "last_change": null, - * "name": "st_dev", - * "offset": 0, - * "type": "__dev_t", - * "size": 8 - * }, - * "0x8": { - * "last_change": null, - * "name": "st_ino", - * "offset": 8, - * "type": "__ino64_t", - * "size": 8 - * }, - * "0x10": { - * "last_change": null, - * "name": "st_nlink", - * "offset": 16, - * "type": "__nlink_t", - * "size": 8 - * }, - * "0x18": { - * "last_change": null, - * "name": "st_mode", - * "offset": 24, - * "type": "__mode_t", - * "size": 4 - * }, - * "0x1c": { - * "last_change": null, - * "name": "st_uid", - * "offset": 28, - * "type": "__uid_t", - * "size": 4 - * }, - * "0x20": { - * "last_change": null, - * "name": "st_gid", - * "offset": 32, - * "type": "__gid_t", - * "size": 4 - * }, - * "0x24": { - * "last_change": null, - * "name": "__pad0", - * "offset": 36, - * "type": "int", - * "size": 4 - * }, - * "0x28": { - * "last_change": null, - * "name": "st_rdev", - * "offset": 40, - * "type": "__dev_t", - * "size": 8 - * }, - * "0x30": { - * "last_change": null, - * "name": "st_size", - * "offset": 48, - * "type": "__off_t", - * "size": 8 - * }, - * "0x38": { - * "last_change": null, - * "name": "st_blksize", - * "offset": 56, - * "type": "__blksize_t", - * "size": 8 - * }, - * "0x40": { - * "last_change": null, - * "name": "st_blocks", - * "offset": 64, - * "type": "__blkcnt64_t", - * "size": 8 - * }, - * "0x48": { - * "last_change": null, - * "name": "st_atim", - * "offset": 72, - * "type": "timespec", - * "size": 16 - * }, - * "0x58": { - * "last_change": null, - * "name": "st_mtim", - * "offset": 88, - * "type": "timespec", - * "size": 16 - * }, - * "0x68": { - * "last_change": null, - * "name": "st_ctim", - * "offset": 104, - * "type": "timespec", - * "size": 16 - * }, - * "0x78": { - * "last_change": null, - * "name": "__unused", - * "offset": 120, - * "type": "long[3]", - * "size": 24 - * } - * } - * }, - * { - * "last_change": null, - * "name": "time.h/timespec", - * "size": 16, - * "members": { - * "0x0": { - * "last_change": null, - * "name": "tv_sec", - * "offset": 0, - * "type": "__time_t", - * "size": 8 - * }, - * "0x8": { - * "last_change": null, - * "name": "tv_nsec", - * "offset": 8, - * "type": "long", - * "size": 8 - * } - * } - * }, - * { - * "last_change": null, - * "name": "types.h/__mode_t", - * "type": "uint" - * }, - * { - * "last_change": null, - * "name": "types.h/__gid_t", - * "type": "uint" - * }, - * { - * "last_change": null, - * "name": "types.h/__off_t", - * "type": "long" - * }, - * { - * "last_change": null, - * "name": "types.h/__uid_t", - * "type": "uint" - * }, - * { - * "last_change": null, - * "name": "types.h/__time_t", - * "type": "long" - * }, - * { - * "last_change": null, - * "name": "types.h/__dev_t", - * "type": "ulong" - * }, - * { - * "last_change": null, - * "name": "types.h/__blksize_t", - * "type": "long" - * }, - * { - * "last_change": null, - * "name": "types.h/__nlink_t", - * "type": "ulong" - * }, - * { - * "last_change": null, - * "name": "types.h/__blkcnt64_t", - * "type": "long" - * }, - * { - * "last_change": null, - * "name": "types.h/__ino64_t", - * "type": "ulong" - * } - * ] - * }, - * - * Consists of: - * - a regular BinSync Function Artifact in the `func_types` field - * - an array of unclear types in the `func_deps` field - * - * This object isn't part of the BinSync types - * The func_deps members are either typedefs or structures - * - * @deprecated see {@link FunctionInfo} - */ -@Deprecated -public record FunctionDataTypeMessage( - FunctionArtifact func_types, - FunctionDependencies func_deps -) { - public static FunctionDataTypeMessage fromJsonObject(JSONObject dataTypes) { - return new FunctionDataTypeMessage( - FunctionArtifact.fromJsonObject(dataTypes. getJSONObject("func_types")), - FunctionDependencies.fromJsonObject(dataTypes.getJSONArray("func_deps")) - - ); - } - public boolean hasDependencies() { - return func_deps != null; - } - - public String functionName() { - return func_types.header().name(); - } - -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionDependencies.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionDependencies.java deleted file mode 100644 index c7a620d1..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionDependencies.java +++ /dev/null @@ -1,94 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types.binsync; - -import ai.reveng.model.*; -import ghidra.util.Msg; -import org.json.JSONArray; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; - -/** - * Describes the dependencies of a function. - * Has no corresponding artifact in BinSync or the RevEng.AI OpenAPI spec. In the RevEng.AI response it's just a list of - * type artifacts - * For ease of use we split it into its own record, and split the type of artifacts into their own - * arrays. This needs to happen for deserialization anyway - */ -public record FunctionDependencies( - Typedef[] typedefs, - Struct[] structs -) { - - public static FunctionDependencies fromOpenAPI(List deps) { - if (deps.isEmpty()) { - return null; - } - var typedefs = new ArrayList(); - var structs = new ArrayList(); - - for (var dep : deps) { - var instance = dep.getActualInstance(); - switch (instance) { - case TypeDefinition typedef -> typedefs.add(Typedef.fromOpenAPI(typedef)); - case Structure struct -> structs.add(Struct.fromOpenAPI(struct)); - case Enumeration enumeration -> { - // We don't handle enums for now - } - case GlobalVariable globalVariable -> { - // We don't handle global variables for now - } - default ->{ - Msg.error(FunctionDependencies.class, "Unexpected type dependency: " + instance); - } - - - } - } - - return new FunctionDependencies( - typedefs.toArray(new Typedef[0]), - structs.toArray(new Struct[0]) - ); - - } - - public static FunctionDependencies fromJsonObject(JSONArray funcDeps) { - // We need to distinguish the object type somehow - if (funcDeps.isEmpty()) { - return null; - } - var typedefs = new ArrayList(); - var structs = new ArrayList(); - - // For each element in the array check if it's a Typedef or a Struct - for (int i = 0; i < funcDeps.length(); i++) { - // If it's a Typedef - if (Typedef.matches(funcDeps.getJSONObject(i))) { - typedefs.add(Typedef.fromJsonObject(funcDeps.getJSONObject(i))); - } - // If it's a Struct - if (Struct.matches(funcDeps.getJSONObject(i))) { - structs.add(Struct.fromJsonObject(funcDeps.getJSONObject(i))); - } - } - - return new FunctionDependencies( - typedefs.toArray(new Typedef[0]), - structs.toArray(new Struct[0]) - ); - } - - public Optional findTypeDef(String name) { - return Arrays.stream(typedefs) - .filter(t -> t.name().equals(name)) - .findFirst(); - } - public Optional findStruct(String name) { - return Arrays.stream(structs) - .filter(t -> t.name().equals(name)) - .findFirst(); - } - -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionHeader.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionHeader.java deleted file mode 100644 index c7642b4b..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionHeader.java +++ /dev/null @@ -1,57 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types.binsync; - -import org.json.JSONObject; - -import java.util.ArrayList; -import java.util.List; - -/** - * "header": { - * "args": { - * "0x0": { - * "offset": 0, - * "size": 8, - * "last_change": null, - * "name": "param_1", - * "type": "char *" - * }, - * "0x1": { - * "offset": 1, - * "size": 8, - * "last_change": null, - * "name": "param_2", - * "type": "char *" - * } - * }, - * "last_change": null, - * "name": "FUN_00101fca", - * "addr": 8138, - * "type": "uint" - * }, - */ -public record FunctionHeader( - String last_change, - String name, - long addr, - String type, - FunctionArgument[] args - -) { - public static FunctionHeader fromJsonObject(JSONObject header) { - // Create function argument array - - List args = new ArrayList<>(); - JSONObject argsJson = header.getJSONObject("args"); - for (String key : argsJson.keySet()) { - args.add(FunctionArgument.fromJsonObject(argsJson.getJSONObject(key))); - } - - return new FunctionHeader( - !header.isNull("last_change") ? header.getString("last_change") : null, - header.getString("name"), - header.getLong("addr"), - header.getString("type"), - args.toArray(new FunctionArgument[0]) - ); - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/StackVariable.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/StackVariable.java deleted file mode 100644 index 838c91bc..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/StackVariable.java +++ /dev/null @@ -1,38 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types.binsync; - -import org.json.JSONArray; -import org.json.JSONObject; - -/** - * - { - "offset": -152, - "size": 144, - "last_change": null, - "name": "local_98", - "type": "stat64", - "addr": 8138 - } - */ -public record StackVariable( - String last_change, int offset, - String name, String type, int size, - int addr -) { - - public static StackVariable fromJsonObject(JSONObject stackVar) { - return new StackVariable( - !stackVar.isNull("last_change") ? stackVar.getString("last_change") : null, stackVar.getInt("offset"), - stackVar.getString("name"), stackVar.getString("type"), stackVar.getInt("size"), - stackVar.getInt("addr") - ); - } - - public static StackVariable[] fromJsonArray(JSONArray stackVars) { - StackVariable[] stackVariables = new StackVariable[stackVars.length()]; - for (int i = 0; i < stackVars.length(); i++) { - stackVariables[i] = fromJsonObject(stackVars.getJSONObject(i)); - } - return stackVariables; - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/Struct.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/Struct.java deleted file mode 100644 index 76101147..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/Struct.java +++ /dev/null @@ -1,51 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types.binsync; - -import ai.reveng.model.Structure; -import org.json.JSONObject; - -/** - * Describes a struct. - * All members are stored by their byte offset from the start of the struct. - * - * based on the Struct artifact from BinSync. - */ -public record Struct( - String last_change, - String name, - int size, - StructMember[] members -) { - public static Struct fromJsonObject(JSONObject jsonObject) { - JSONObject jsonMembers = jsonObject.getJSONObject("members"); - // Handle members - StructMember[] members = jsonMembers.keySet().stream() - .map(jsonMembers::getJSONObject) - .map(StructMember::fromJsonObject) - .toList().toArray(new StructMember[0]); - - return new Struct( - !jsonObject.isNull("last_change") ? jsonObject.getString("last_change") : null, - jsonObject.getString("name"), - jsonObject.getInt("size"), - members - ); - } - - public static boolean matches(JSONObject jsonObject) { - return jsonObject.has("name") && jsonObject.has("size") && jsonObject.has("members"); - } - - - public static Struct fromOpenAPI(Structure struct) { - StructMember[] members = struct.getMembers().values().stream() - .map(StructMember::fromOpenAPI) - .toList().toArray(new StructMember[0]); - - return new Struct( - struct.getLastChange(), - struct.getName(), - struct.getSize(), - members - ); - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/StructMember.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/StructMember.java deleted file mode 100644 index d89905ba..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/StructMember.java +++ /dev/null @@ -1,36 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types.binsync; - -import ai.reveng.model.StructureMember; -import org.json.JSONObject; - -/** - * Based directly on the StructMember artifact from BinSync. - */ -public record StructMember( - String last_change, - String name, - int offset, - String type, - int size -) { - - public static StructMember fromJsonObject(JSONObject jsonObject) { - return new StructMember( - !jsonObject.isNull("last_change") ? jsonObject.getString("last_change") : null, - jsonObject.getString("name"), - jsonObject.getInt("offset"), - jsonObject.getString("type"), - jsonObject.getInt("size") - ); - } - - public static StructMember fromOpenAPI(StructureMember structureMember) { - return new StructMember( - structureMember.getLastChange(), - structureMember.getName(), - structureMember.getOffset(), - structureMember.getType(), - structureMember.getSize() - ); - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/Typedef.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/Typedef.java deleted file mode 100644 index 57671e3f..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/Typedef.java +++ /dev/null @@ -1,38 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types.binsync; - -import ai.reveng.model.TypeDefinition; -import org.json.JSONObject; - -import java.util.Set; - -/** - * Based on the Typedef artifact from BinSync. - */ -public record Typedef( - String last_change, - String name, - String type -) { - - public static Typedef fromJsonObject(JSONObject obj) { - return new Typedef( - !obj.isNull("last_change") ? obj.getString("last_change") : null, - obj.getString("name"), - obj.getString("type") - ); - } - - - public static boolean matches(JSONObject obj) { - return obj.keySet().equals(Set.of("last_change", "name", "type")); -// return obj.has("type") && obj.has("name"); - } - - public static Typedef fromOpenAPI(TypeDefinition typedef) { - return new Typedef( - typedef.getLastChange(), - typedef.getName(), - typedef.getType() - ); - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/exceptions/APIAuthenticationException.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/exceptions/APIAuthenticationException.java deleted file mode 100644 index 4a760bc6..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/exceptions/APIAuthenticationException.java +++ /dev/null @@ -1,11 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types.exceptions; - -/** - * This exception indicates an unexpected case of the API returning an authentication error - * this can happen when attempting to retrieve information about a private analysis ID owned by a different account - */ -public class APIAuthenticationException extends RuntimeException{ - public APIAuthenticationException(String message) { - super(message); - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/exceptions/APIConflictException.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/exceptions/APIConflictException.java deleted file mode 100644 index 7ea405ee..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/exceptions/APIConflictException.java +++ /dev/null @@ -1,7 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types.exceptions; - -public class APIConflictException extends RuntimeException{ - public APIConflictException(String message) { - super(message); - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/configuration/ConfigurationService.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/configuration/ConfigurationService.java deleted file mode 100644 index 2b2e2947..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/configuration/ConfigurationService.java +++ /dev/null @@ -1,12 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.configuration; - -public interface ConfigurationService { - public String getApiKey(); - public void setApiKey(String apiKey); - - public String getHostname(); - public void setHostname(String hostname); - - public String getModel(); - public void setModel(String model); -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/configuration/ModelInfo.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/configuration/ModelInfo.java deleted file mode 100644 index 0f7c5ac1..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/configuration/ModelInfo.java +++ /dev/null @@ -1,79 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.configuration; - -import java.util.regex.Pattern; - -/** - * Object that models a RevEng.ai model - * - * TODO Create either an enum or method that checks for valid names and version - */ -public class ModelInfo { - private String name; - private int majVersion; - private int minVersion; - - /** - * Create a new model using a separate name and version - * - * @param name model name, e.g. "binnet" - * @param majVersion major version of model - * @param minVersion minor version of model - * - */ - public ModelInfo(String name, int majVersion, int minVersion) { - this.name = name; - this.majVersion = majVersion; - this.minVersion = minVersion; - } - - /** - * Create a new model from a string - * - * @param modelString model identifier in the form - - */ - public ModelInfo(String modelString) { - // TODO check string is in valid format - this.name = modelString.split("-")[0]; - try { - this.majVersion = Integer.parseInt(modelString.split("-")[1].split(Pattern.quote("."))[0]); - } catch (IndexOutOfBoundsException e) { - System.err.println("No major version provided"); - this.majVersion = 0; - } - try { - this.minVersion = Integer.parseInt(modelString.split("-")[1].split(Pattern.quote("."))[1]); - } catch (IndexOutOfBoundsException e) { - System.err.println("No minor version provided"); - this.minVersion = 0; - } - - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public int getMajVersion() { - return this.majVersion; - } - - public void setMajVersion(int majVersion) { - this.majVersion = majVersion; - } - - public int getMinVersion() { - return this.minVersion; - } - - public void setMinVersion(int minVersion) { - this.minVersion = minVersion; - } - - public String toString() { - return this.name + "-" + this.majVersion + "." + this.minVersion; - } -} \ No newline at end of file diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/function/export/ExportFunctionBoundariesService.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/function/export/ExportFunctionBoundariesService.java deleted file mode 100644 index 1752bcab..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/function/export/ExportFunctionBoundariesService.java +++ /dev/null @@ -1,32 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.function.export; - -import org.json.JSONArray; -import org.json.JSONObject; - -import ai.reveng.toolkit.ghidra.plugins.AnalysisManagementPlugin; -import ghidra.framework.plugintool.ServiceInfo; -import ghidra.program.model.address.Address; - -@ServiceInfo(defaultProvider = AnalysisManagementPlugin.class, description = "Export Function Boundaries for passing to the binary analysis server") -public interface ExportFunctionBoundariesService { - /** - * Return the boundaries for a single function - * - * @param entry - * @return - */ - public JSONObject getFunctionAt(Address entry); - - /** - * Return a list of function boundary info objects for the whole binary - * - * @return - */ - public JSONObject getFunctions(); - - /** - * Return an array of functions boundaries for insertion to a symbols object - * @return - */ - public JSONArray getFunctionsArray(); -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/function/export/ExportFunctionBoundariesServiceImpl.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/function/export/ExportFunctionBoundariesServiceImpl.java deleted file mode 100644 index 97dc3f22..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/function/export/ExportFunctionBoundariesServiceImpl.java +++ /dev/null @@ -1,83 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.function.export; - -import org.json.JSONObject; -import org.json.JSONArray; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; - -import ghidra.app.services.ProgramManager; -import ghidra.framework.plugintool.PluginTool; -import ghidra.program.model.address.Address; -import ghidra.program.model.listing.Function; -import ghidra.program.model.listing.FunctionManager; -import ghidra.program.model.listing.Program; - -public class ExportFunctionBoundariesServiceImpl implements ExportFunctionBoundariesService { - - private PluginTool tool; - private FunctionManager fm; - - private boolean isReady; - - public ExportFunctionBoundariesServiceImpl(PluginTool tool) { - this.tool = tool; - isReady = false; - } - - /** - * This is done separately to the constructor as current program will be null if - * the plugin is being configured without a binary loaded - */ - private void init() { - ProgramManager programManager = tool.getService(ProgramManager.class); - Program currentProgram = programManager.getCurrentProgram(); - fm = currentProgram.getFunctionManager(); - isReady = true; - } - - @Override - public JSONObject getFunctionAt(Address entry) { - if (!isReady) - init(); - - Function f = fm.getFunctionAt(entry); - - JSONObject jFunctionBoundaries = new JSONObject(); - jFunctionBoundaries.put("name", f.getName()); - jFunctionBoundaries.put("start_addr", f.getEntryPoint().toString("0x")); - jFunctionBoundaries.put("end_addr", f.getBody().getMaxAddress().toString("0x")); - - return jFunctionBoundaries; - } - - @Override - public JSONObject getFunctions() { - if (!isReady) - init(); - - JSONObject jFunctions = new JSONObject(); - - JSONArray fArray = new JSONArray(); - for (Function f : fm.getFunctions(true)) { - fArray.put(getFunctionAt(f.getEntryPoint())); - } - - jFunctions.put("functions", fArray); - return jFunctions; - } - - @Override - public JSONArray getFunctionsArray() { - if (!isReady) - init(); - - JSONArray fArray = new JSONArray(); - for (Function f : fm.getFunctions(true)) { - fArray.put(getFunctionAt(f.getEntryPoint())); - } - return fArray; - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/logging/ReaiLoggingService.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/logging/ReaiLoggingService.java index 50c68479..6c7884b1 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/logging/ReaiLoggingService.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/logging/ReaiLoggingService.java @@ -2,10 +2,9 @@ import ghidra.framework.plugintool.ServiceInfo; -@ServiceInfo(description = "Service for writing plugin messages to a logfile that can then be exported by a user for debuging") +@ServiceInfo(description = "Service for writing plugin messages to the Ghidra console") public interface ReaiLoggingService { public void info(String message); public void warn(String message); public void error(String message); - public void export(String targetDirectoryPath, String exportedFileName); } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/logging/ReaiLoggingServiceImpl.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/logging/ReaiLoggingServiceImpl.java deleted file mode 100644 index d74e95a9..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/logging/ReaiLoggingServiceImpl.java +++ /dev/null @@ -1,89 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.logging; - -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.StandardCopyOption; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.logging.FileHandler; -import java.util.logging.Logger; -import java.util.logging.SimpleFormatter; - -import ghidra.util.Msg; - -/** - * Very simple logging service to enable us to export error logs - */ -public class ReaiLoggingServiceImpl implements ReaiLoggingService { - private static final Logger logger = Logger.getLogger("REAIPlugin"); - private static FileHandler fileHandler; - private static String uHome = System.getProperty("user.home"); - private static Path logDir = Paths.get(uHome, ".reai/logs"); - private static Path logFilePath = Paths.get(logDir.toString(), "ReaiLogFile.txt"); - - static { - createLogsDirectory(); - - try { - fileHandler = new FileHandler(logFilePath.toString(), true); - logger.addHandler(fileHandler); - - SimpleFormatter formatter = new SimpleFormatter(); - fileHandler.setFormatter(formatter); - } catch (SecurityException | IOException e) { - Msg.error(ReaiLoggingServiceImpl.class, "Cannot create logfile: " + e.getMessage()); - } - } - - private static void createLogsDirectory() { - try { - Files.createDirectories(logDir); - } catch (IOException e) { - Msg.error(ReaiLoggingServiceImpl.class, "Unable to create logs directory: " + e.getMessage()); - } - } - - @Override - public void info(String message) { - logger.info(message); - } - - @Override - public void warn(String message) { - logger.warning(message); - } - - @Override - public void error(String message) { - logger.severe(message); - } - - @Override - public void export(String targetDirectoryPath, String exportedFileName) { - Path targetPath = Paths.get(targetDirectoryPath, exportedFileName); - - try { - Files.walkFileTree(logDir, new SimpleFileVisitor() { - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - Path targetDir = targetPath.resolve(logDir.relativize(dir)); - Files.createDirectories(targetDir); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - Files.copy(file, targetPath.resolve(logDir.relativize(file)), StandardCopyOption.REPLACE_EXISTING); - return FileVisitResult.CONTINUE; - } - }); - - Msg.info(this.getClass(), "Log directory successfully exported to: " + targetPath); - } catch (IOException e) { - Msg.error(this.getClass(), "Unable to export log directory: " + e.getMessage()); - } - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/logging/ReaiLoggingToConsole.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/logging/ReaiLoggingToConsole.java index 41b44f7d..1e4929f3 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/logging/ReaiLoggingToConsole.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/logging/ReaiLoggingToConsole.java @@ -43,12 +43,6 @@ public void error(String message) { } } - @Override - public void export(String targetDirectoryPath, String exportedFileName) { - throw new UnsupportedOperationException("Not implemented for console logger"); - - } - public void setConsoleService(ConsoleService service) { this.consoleService = service; for (String message : logBuffer) { diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/sync/AutoUnstripSyncService.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/sync/AutoUnstripSyncService.java index 865ad7e0..4c518469 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/sync/AutoUnstripSyncService.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/sync/AutoUnstripSyncService.java @@ -121,8 +121,10 @@ private void runSync(AnalysedProgram analysedProgram) { announce("RevEng.AI: auto-unstrip finished; syncing recovered function names and data types…", false); try { var summary = revengService.syncAnalysisUpdates(analysedProgram, TaskMonitor.DUMMY, loggingService); - announce("RevEng.AI: auto-unstrip sync applied %d recovered names and pushed %d local type sets." - .formatted(summary.namesModifiedRemotely(), summary.pushedTypeSets()), false); + announce(("RevEng.AI: auto-unstrip sync applied %d recovered name(s) and %d signature(s), " + + "and pushed %d local type set(s).") + .formatted(summary.namesModifiedRemotely(), summary.appliedSignatures(), + summary.pushedTypeSets()), false); } catch (Exception e) { Msg.warn(this, "Failed to sync analysis after auto-unstrip", e); announce("RevEng.AI: failed to sync analysis after auto-unstrip: " + e.getMessage(), true); diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/sync/LocalEditSyncService.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/sync/LocalEditSyncService.java index 156d0b35..1f19e354 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/sync/LocalEditSyncService.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/sync/LocalEditSyncService.java @@ -1,13 +1,13 @@ package ai.reveng.toolkit.ghidra.core.services.sync; import ai.reveng.invoker.ApiException; +import ai.reveng.toolkit.ghidra.core.services.api.GhidraDataTypeEncoder; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService.AnalysedProgram; import ai.reveng.toolkit.ghidra.core.services.logging.ReaiLoggingService; import ghidra.framework.model.DomainObjectChangeRecord; import ghidra.framework.model.DomainObjectListener; import ghidra.framework.model.DomainObjectListenerBuilder; -import ai.reveng.toolkit.ghidra.core.services.api.GhidraToServerTypeSerializer; import ghidra.program.model.address.Address; import ghidra.program.model.data.DataType; import ghidra.program.model.listing.Function; @@ -197,7 +197,12 @@ private void schedule(Map> pending, K key, Runnable ta } /// Push every server-known function that references the edited type, so a type edit is - /// propagated to the portal (which only stores types inside each function's data-types blob). + /// propagated to the portal. + /// + /// A data type is not attached to any one function, so an edit to it is turned back into + /// function pushes: each affected function's own push re-resolves the type and writes its new + /// definition. Rescheduling rather than pushing directly means the per-function debounce still + /// applies, so editing several members of a struct collapses into one push per function. private void pushFunctionsReferencingType(Program program, String typeName) { var analysedProgram = revengService.getAnalysedProgram(program); if (analysedProgram.isEmpty()) { @@ -205,7 +210,7 @@ private void pushFunctionsReferencingType(Program program, String typeName) { } for (Function function : analysedProgram.get().getFunctionMap().values()) { if (isSyncable(function) - && GhidraToServerTypeSerializer.referencedTypeNames(function).contains(typeName)) { + && GhidraDataTypeEncoder.referencedTypeNames(function).contains(typeName)) { scheduleTypes(program, function.getEntryPoint()); } } @@ -226,9 +231,20 @@ private void pushRename(Program program, Address entryPoint) { private void pushTypes(Program program, Address entryPoint) { withAnalysedFunction(program, entryPoint, (analysedProgram, function) -> { try { - if (revengService.pushFunctionTypes(analysedProgram, function)) { - loggingService.info("Pushed types for function \"%s\" at %s to the RevEng.AI portal" - .formatted(function.getName(), entryPoint)); + switch (revengService.pushFunctionTypes(analysedProgram, function)) { + case SIGNATURE_WRITTEN -> loggingService.info( + "Pushed types for function \"%s\" at %s to the RevEng.AI portal" + .formatted(function.getName(), entryPoint)); + // The data types did reach the portal; only the signature had nothing to update. + // Saying so matters, because otherwise editing a type on a function the portal + // never extracted a signature for looks like it did nothing at all. + case TYPES_ONLY -> loggingService.info( + ("Pushed the data types for function \"%s\" at %s; the portal holds no extracted " + + "signature for it, so its signature was left unchanged") + .formatted(function.getName(), entryPoint)); + case NOT_MATCHED -> { + // Not part of the analysis, so there was nothing to push. + } } } catch (ApiException e) { Msg.warn(this, "Failed to push types for %s to portal".formatted(function.getName()), e); diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/ui/wizard/SetupWizardManager.java b/src/main/java/ai/reveng/toolkit/ghidra/core/ui/wizard/SetupWizardManager.java index aaef0c29..31b3bd57 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/ui/wizard/SetupWizardManager.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/ui/wizard/SetupWizardManager.java @@ -46,12 +46,10 @@ protected void doFinish() throws IllegalPanelStateException { String apiKey = (String) getState().get(SetupWizardStateKey.API_KEY); String hostname = (String) getState().get(SetupWizardStateKey.HOSTNAME); String portalHostname = (String) getState().get(SetupWizardStateKey.PORTAL_HOSTNAME); - String model = (String) getState().get(SetupWizardStateKey.MODEL); - + tool.getOptions(REAI_OPTIONS_CATEGORY).setString(ReaiPluginPackage.OPTION_KEY_APIKEY, apiKey); tool.getOptions(REAI_OPTIONS_CATEGORY).setString(ReaiPluginPackage.OPTION_KEY_HOSTNAME, hostname); tool.getOptions(REAI_OPTIONS_CATEGORY).setString(ReaiPluginPackage.OPTION_KEY_PORTAL_HOSTNAME, portalHostname); - tool.getOptions(REAI_OPTIONS_CATEGORY).setString(ReaiPluginPackage.OPTION_KEY_MODEL, model); tool.getOptions(REAI_OPTIONS_CATEGORY).setString(REAI_WIZARD_RUN_PREF, "true"); String configFileOverride = (String) getState().get(SetupWizardStateKey.CONFIGFILE); @@ -78,7 +76,6 @@ protected void doFinish() throws IllegalPanelStateException { pluginSettings.setApiKey(apiKey); pluginSettings.setHostname(hostname); pluginSettings.setPortalHostname(portalHostname); - pluginSettings.setModelName(model); config.setPluginSettings(pluginSettings); Gson gson = new GsonBuilder().setPrettyPrinting().create(); diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/ui/wizard/SetupWizardStateKey.java b/src/main/java/ai/reveng/toolkit/ghidra/core/ui/wizard/SetupWizardStateKey.java index d7c5c379..466332f3 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/ui/wizard/SetupWizardStateKey.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/ui/wizard/SetupWizardStateKey.java @@ -5,7 +5,6 @@ public enum SetupWizardStateKey { HOSTNAME, PORTAL_HOSTNAME, CREDENTIALS_VALIDATED, - MODEL, CONFIGFILE, CREDENTIAL_VALIDATOR, } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/plugins/AnalysisManagementPlugin.java b/src/main/java/ai/reveng/toolkit/ghidra/plugins/AnalysisManagementPlugin.java index edd069e5..821ba032 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/plugins/AnalysisManagementPlugin.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/plugins/AnalysisManagementPlugin.java @@ -25,11 +25,8 @@ import ai.reveng.toolkit.ghidra.core.services.sync.AutoUnstripSyncService; import ai.reveng.toolkit.ghidra.core.services.sync.LocalEditSyncService; -import ai.reveng.toolkit.ghidra.core.services.function.export.ExportFunctionBoundariesService; -import ai.reveng.toolkit.ghidra.core.services.function.export.ExportFunctionBoundariesServiceImpl; import ai.reveng.toolkit.ghidra.core.services.logging.ReaiLoggingService; import ai.reveng.toolkit.ghidra.core.tasks.StartAnalysisTask; -import docking.action.DockingAction; import docking.action.builder.ActionBuilder; import docking.widgets.OptionDialog; import ghidra.app.plugin.PluginCategoryNames; @@ -72,24 +69,14 @@ shortDescription = "Toolkit for using the RevEng.AI API", description = "Toolkit for using RevEng.AI API", servicesRequired = { OptionsService.class, ReaiLoggingService.class, GhidraRevengService.class}, - servicesProvided = { ExportFunctionBoundariesService.class }, eventsConsumed = { RevEngAIAnalysisStatusChangedEvent.class} ) //@formatter:on public class AnalysisManagementPlugin extends ProgramPlugin { private static final String REAI_ANALYSIS_MANAGEMENT_MENU_GROUP = "RevEng.AI Analysis Management"; - private static final String REAI_PLUGIN_PORTAL_MENU_GROUP = "RevEng.AI Portal"; private static final Logger log = LoggerFactory.getLogger(AnalysisManagementPlugin.class); - // Store references to actions that need to be refreshed - private DockingAction createNewAction; - private DockingAction attachToExistingAction; - private DockingAction detachAction; - private DockingAction checkStatusAction; - private DockingAction viewInPortalAction; - private GhidraRevengService revengService; - private ExportFunctionBoundariesService exportFunctionBoundariesService; private AnalysisLogComponent analysisLogComponent; private LocalEditSyncService localEditSyncService; private AutoUnstripSyncService autoUnstripSyncService; @@ -101,11 +88,6 @@ public AnalysisManagementPlugin(PluginTool tool) { super(tool); this.tool = tool; - - - exportFunctionBoundariesService = new ExportFunctionBoundariesServiceImpl(tool); - registerServiceProvided(ExportFunctionBoundariesService.class, exportFunctionBoundariesService); - } @Override @@ -132,7 +114,7 @@ private void setupActions() { - createNewAction = new ActionBuilder("Create new", this.getName()) + new ActionBuilder("Create new", this.getName()) .enabledWhen(context -> { var currentProgram = tool.getService(ProgramManager.class).getCurrentProgram(); if (currentProgram == null) { @@ -178,7 +160,7 @@ private void setupActions() { .popupMenuIcon(ReaiPluginPackage.REVENG_16) .buildAndInstall(tool); - attachToExistingAction = new ActionBuilder("Attach to existing", this.toString()) + new ActionBuilder("Attach to existing", this.toString()) .enabledWhen(c -> { var currentProgram = tool.getService(ProgramManager.class).getCurrentProgram(); if (currentProgram == null) { @@ -199,7 +181,7 @@ private void setupActions() { .popupMenuIcon(ReaiPluginPackage.REVENG_16) .buildAndInstall(tool); - detachAction = new ActionBuilder("Detach", this.toString()) + new ActionBuilder("Detach", this.toString()) .enabledWhen(c -> { var currentProgram = tool.getService(ProgramManager.class).getCurrentProgram(); if (currentProgram == null) { @@ -234,7 +216,7 @@ private void setupActions() { .menuGroup(REAI_ANALYSIS_MANAGEMENT_MENU_GROUP, "300") .buildAndInstall(tool); - checkStatusAction = new ActionBuilder("Check status", this.getName()) + new ActionBuilder("Check status", this.getName()) .enabledWhen(context -> { var currentProgram = tool.getService(ProgramManager.class).getCurrentProgram(); if (currentProgram == null) { @@ -268,7 +250,7 @@ public void run(TaskMonitor monitor) { .menuGroup(REAI_ANALYSIS_MANAGEMENT_MENU_GROUP, "400") .buildAndInstall(tool); - viewInPortalAction = new ActionBuilder("View in portal", this.getName()) + new ActionBuilder("View in portal", this.getName()) .enabledWhen(context -> { var currentProgram = tool.getService(ProgramManager.class).getCurrentProgram(); if (currentProgram == null) { diff --git a/src/main/java/ai/reveng/toolkit/ghidra/plugins/BinarySimilarityPlugin.java b/src/main/java/ai/reveng/toolkit/ghidra/plugins/BinarySimilarityPlugin.java index ab5c5621..2c5e3ff1 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/plugins/BinarySimilarityPlugin.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/plugins/BinarySimilarityPlugin.java @@ -22,7 +22,6 @@ import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisResultsLoaded; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ai.reveng.toolkit.ghidra.core.services.api.types.*; -import ai.reveng.toolkit.ghidra.core.services.function.export.ExportFunctionBoundariesService; import ai.reveng.toolkit.ghidra.core.services.logging.ReaiLoggingService; import docking.action.builder.ActionBuilder; import ghidra.app.context.ProgramLocationActionContext; @@ -53,7 +52,7 @@ category = PluginCategoryNames.COMMON, shortDescription = "Support for Binary Similarity Features of RevEng.AI Toolkit.", description = "Enable features that support binary similarity operations, including binary upload, and auto-renaming", - servicesRequired = { GhidraRevengService.class, ProgramManager.class, ExportFunctionBoundariesService.class, ReaiLoggingService.class }, + servicesRequired = { GhidraRevengService.class, ProgramManager.class, ReaiLoggingService.class }, eventsConsumed = { RevEngAIAnalysisResultsLoaded.class, } ) //@formatter:on @@ -285,10 +284,11 @@ public void run(TaskMonitor monitor) { private static String formatSyncSummary(GhidraRevengService.SyncSummary summary) { return ("Synced %d matched function(s) with the portal.\n" + - "Applied %d remote name(s); canonicalized %d and de-duplicated %d.\n" + + "Applied %d remote name(s) and %d remote signature(s); canonicalized %d and de-duplicated %d.\n" + "Pushed %d name(s) and %d type set(s) back to the portal.").formatted( summary.matchedFunctions(), summary.namesModifiedRemotely(), + summary.appliedSignatures(), summary.canonicalizedNames(), summary.dedupedNames(), summary.pushedNames(), diff --git a/src/main/java/ai/reveng/toolkit/ghidra/plugins/ReaiPluginPackage.java b/src/main/java/ai/reveng/toolkit/ghidra/plugins/ReaiPluginPackage.java index 8e1e78a8..d83404bb 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/plugins/ReaiPluginPackage.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/plugins/ReaiPluginPackage.java @@ -21,9 +21,7 @@ public class ReaiPluginPackage extends PluginPackage { public static final String OPTION_KEY_APIKEY = PREFIX + "API Key"; public static final String OPTION_KEY_HOSTNAME = PREFIX + "Hostname"; public static final String OPTION_KEY_PORTAL_HOSTNAME = PREFIX + "Portal Hostname"; - public static final String OPTION_KEY_MODEL = PREFIX + "Model"; @Deprecated - public static final String OPTION_KEY_BINID = PREFIX + "Binary ID"; public static final String OPTION_KEY_ANALYSIS_ID = PREFIX + "Analysis ID"; public static final String REAI_OPTIONS_CATEGORY = "RevEngAI Options"; @@ -32,7 +30,6 @@ public class ReaiPluginPackage extends PluginPackage { @Deprecated - public static final Integer INVALID_BINARY_ID = -1; public static final Integer INVALID_ANALYSIS_ID = -1; public static final Icon REVENG_16 = ResourceManager.loadImage("images/reveng_16.png"); diff --git a/src/test/java/AbstractRevEngIntegrationTest.java b/src/test/java/AbstractRevEngIntegrationTest.java deleted file mode 100644 index 40d9b2f2..00000000 --- a/src/test/java/AbstractRevEngIntegrationTest.java +++ /dev/null @@ -1,20 +0,0 @@ -import ai.reveng.toolkit.ghidra.core.services.api.V2Response; -import ghidra.test.AbstractGhidraHeadedIntegrationTest; -import ghidra.test.AbstractGhidraHeadlessIntegrationTest; -import org.json.JSONObject; - -import java.io.IOException; - -abstract class AbstractRevEngIntegrationTest extends AbstractGhidraHeadedIntegrationTest { - protected V2Response getMockResponseFromFile(String filename) { - String json = null; - try { - json = new String(getClass().getClassLoader().getResourceAsStream(filename).readAllBytes()); - } catch (IOException e) { - throw new RuntimeException(e); - } - JSONObject jsonObject = new JSONObject(json); - return V2Response.fromJSONObject(jsonObject); - - } -} diff --git a/src/test/java/ConvertBinSyncArtifactTests.java b/src/test/java/ConvertBinSyncArtifactTests.java deleted file mode 100644 index a654755c..00000000 --- a/src/test/java/ConvertBinSyncArtifactTests.java +++ /dev/null @@ -1,229 +0,0 @@ -import ai.reveng.model.FunctionDataTypes; -import ai.reveng.model.V2FunctionInfo; -import ai.reveng.toolkit.ghidra.binarysimilarity.cmds.ComputeTypeInfoTask; -import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; -import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; -import ai.reveng.toolkit.ghidra.core.services.api.V2Response; -import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; -import ai.reveng.toolkit.ghidra.core.services.api.types.*; - -import ghidra.program.model.data.CategoryPath; -import ghidra.program.model.data.DataType; -import ghidra.program.model.data.DataTypeDependencyException; -import ghidra.program.model.data.Structure; -import ghidra.util.Msg; -import ghidra.util.exception.CancelledException; -import ghidra.util.task.TaskMonitor; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.IOException; -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -@Ignore("Integration tests that rely on mock data from files") -public class ConvertBinSyncArtifactTests extends AbstractRevEngIntegrationTest { - - TypedApiInterface.AnalysisID analysisID = new TypedApiInterface.AnalysisID(1337); - - - @Test - public void testSimpleGhidraSignatureGeneration() throws DataTypeDependencyException, IOException { - V2Response mockResponse = getMockResponseFromFile("main_fdupes_77846709.json"); - -// FunctionDataTypeStatus functionDataTypeStatus = FunctionDataTypeStatus.fromJson(mockResponse.getJsonData()); - var funcInfo = FunctionDataTypes.fromJson(mockResponse.getJsonData().toString()); - var signature = GhidraRevengService.getFunctionSignature(funcInfo.getDataTypes()).orElseThrow(); - - assert signature.getName().equals("main"); - assert signature.getReturnType().getName().equals("int"); - assert signature.getArguments().length == 2; - - assert signature.getArguments()[0].getName().equals("argc"); - assert signature.getArguments()[0].getDataType().getName().equals("int"); - - assert signature.getArguments()[1].getName().equals("argv"); - assert signature.getArguments()[1].getDataType().getName().equals("char * *"); - } - - - - - @Test - public void testDependencyToDtm() throws GhidraRevengService.EndlessTypeParsingException { - var mockResponse = getMockResponseFromFile("confirmmatch_fdupes_77846700.json"); - FunctionDataTypeStatus functionDataTypeStatus = FunctionDataTypeStatus.fromJson(mockResponse.getJsonData()); - var dtm = GhidraRevengService.loadDependencyDataTypes(functionDataTypeStatus.data_types().get().func_deps()); - - - // Print all datatypes for debugging - for (Iterator it = dtm.getAllDataTypes(); it.hasNext(); ) { - var ty = it.next(); - Msg.info(this, ty.getCategoryPath()); - Msg.info(this, ty.getName()); - } - // There should be a typedef FILE in the folder DWARF/stdio.h/ for - var fileType = dtm.getDataType(new CategoryPath(CategoryPath.ROOT, "DWARF", "stdio.h"), "FILE"); - List results = new ArrayList(); - dtm.findDataTypes("FILE", results); - assert fileType != null; - - } - - @Test - public void testComplexGhidraSignatureGeneration() throws DataTypeDependencyException, IOException { - var mockResponse = getMockResponseFromFile("confirmmatch_fdupes_77846700.json"); - - var funcInfo = V2FunctionInfo.fromJson(mockResponse.getJsonData().toString()); - var signature = GhidraRevengService.getFunctionSignature(funcInfo).orElseThrow(); - - assert signature.getName().equals("confirmmatch"); - Msg.info(this, signature); - } - - - @Test - public void testComplexGhidraSignatureGeneration2() throws DataTypeDependencyException, IOException { - var mockResponse = getMockResponseFromFile("summarizematches_fdupes.json"); - - FunctionDataTypeStatus functionDataTypeStatus = FunctionDataTypeStatus.fromJson(mockResponse.getJsonData()); - var funcInfo = V2FunctionInfo.fromJson(mockResponse.getJsonData().toString()); - var signature = GhidraRevengService.getFunctionSignature(funcInfo).orElseThrow(); - - assert signature.getName().equals("summarizematches"); - Msg.info(this, signature); - } - - @Test - public void testComplexGhidraSignatureGeneration3() throws DataTypeDependencyException, IOException { - var mockResponse = getMockResponseFromFile("md5_process_fdupes.json"); - - FunctionDataTypeStatus functionDataTypeStatus = FunctionDataTypeStatus.fromJson(mockResponse.getJsonData()); - var funcInfo = V2FunctionInfo.fromJson(mockResponse.getJsonData().toString()); - var signature = GhidraRevengService.getFunctionSignature(funcInfo).orElseThrow(); - - var dtm = signature.getDataTypeManager(); - assert signature.getName().equals("md5_process"); - - Structure stateType = (Structure) dtm.getDataType("/DWARF/md5.h/md5_state_s"); - assert stateType.getLength() == 88; - assert stateType.getNumComponents() == 3; - - } - - - /** - * This test is to ensure that the function signature generation does not loop infinitely - */ - @Ignore("Ignored until it can properly distinguish an infinite loop and an exception") - @Test - public void testNoLoopForBrokenDeps() throws DataTypeDependencyException, IOException { - var mockResponse = getMockResponseFromFile("errormsg.json"); - - var funcInfo = V2FunctionInfo.fromJson(mockResponse.getJsonData().toString()); - var signature = GhidraRevengService.getFunctionSignature(funcInfo).orElseThrow(); - - assert signature.getName().equals("md5_process"); - Msg.info(this, signature); - } - - /** - * This function takes a function pointer as an argument - * BinSync doesn't serialize them by default, and this specific example JSON is missing it - * The function is `registerpair` from `fdupes`: registerpair - */ - @Test - public void testFunctionPointerArgument() throws DataTypeDependencyException, IOException { - var mockResponse = getMockResponseFromFile("complex_pointer.json"); - var signature = GhidraRevengService.getFunctionSignature( - V2FunctionInfo.fromJson(mockResponse.getJsonData().toString()) - ); - } - - @Test - public void testPendingStatus() { - var mockResponse = getMockResponseFromFile("pending.json"); - FunctionDataTypeStatus functionDataTypeStatus = FunctionDataTypeStatus.fromJson(mockResponse.getJsonData()); - assert !functionDataTypeStatus.completed(); - assert functionDataTypeStatus.data_types().isEmpty(); - assert functionDataTypeStatus.status().equals("pending"); - } - - @Test - public void testBatchResponse() { - var mockResponse = getMockResponseFromFile("data_types_batch_response.json"); - DataTypeList batchResponse = DataTypeList.fromJson(mockResponse.getJsonData()); - - var r1 = batchResponse.statusForFunction(new TypedApiInterface.FunctionID(266294328)); - assert r1.data_types().orElseThrow().functionName().equals("sort_pairs_by_mtime"); - } - - @Test - public void testDataTypeGenerationTask() throws CancelledException { - var mockApi = new TypeGenerationMock(); - var task = new ComputeTypeInfoTask( - new GhidraRevengService(mockApi), - IntStream.range(0, 5).boxed().map(TypedApiInterface.FunctionID::new).collect(Collectors.toList()), null - ); - task.run(TaskMonitor.DUMMY); - - } - - public static class TypeGenerationMock extends UnimplementedAPI { - - Set generatedFunctions = new HashSet<>(); - @Override - public DataTypeList generateFunctionDataTypes(AnalysisID analysisID, List functionIDS) { - var statuses = functionIDS.stream() - .map(id -> new FunctionDataTypeStatus( - false, - Optional.empty(), - "UNKNOWN", - null, - id - )) - .toList(); - return new DataTypeList( - functionIDS.size(), 0, statuses.toArray(new FunctionDataTypeStatus[0]) - ); - } - - @Override - public DataTypeList getFunctionDataTypes(List functionIDS) { - for (FunctionID functionID : functionIDS) { - if (generatedFunctions.contains(functionID)) continue; - generatedFunctions.add(functionID); - break; - } - - var statuses = functionIDS.stream() - .map(id -> new FunctionDataTypeStatus( - generatedFunctions.contains(id), - Optional.empty(), - generatedFunctions.contains(id) ? "completed" : "UNKNOWN", - null, - id - )) - .toList(); - - return new DataTypeList( - functionIDS.size(), 0, statuses.toArray(new FunctionDataTypeStatus[0]) - ); - } - - @Override - public FunctionDetails getFunctionDetails(FunctionID id) { - return new FunctionDetails( - id, - "placeholder_for_%s".formatted(id), - 0L, - 10L, - new AnalysisID(1337), - "placeholder_for_%s".formatted(id), - new BinaryHash("placeholder_for_%s".formatted(id)), - "demangled_placeholder_for_%s".formatted(id) - ); - } - } -} diff --git a/src/test/java/HelperTests.java b/src/test/java/HelperTests.java deleted file mode 100644 index 17895b9c..00000000 --- a/src/test/java/HelperTests.java +++ /dev/null @@ -1,22 +0,0 @@ -import ai.reveng.toolkit.ghidra.core.services.api.types.binsync.TypePathAndName; -import org.junit.Test; - -public class HelperTests { - - - @Test - public void testPathSplitting(){ - var path = TypePathAndName.fromString("a::b::c"); - assert path.name().equals("c"); - assert path.path().length == 2; - assert path.path()[0].equals("a"); - assert path.path()[1].equals("b"); - } - - @Test - public void testPathSplittingNoPath(){ - var path = TypePathAndName.fromString("PlainName"); - assert path.name().equals("PlainName"); - assert path.path().length == 0; - } -} diff --git a/src/test/java/ai/reveng/AIDecompilerComponentTest.java b/src/test/java/ai/reveng/AIDecompilerComponentTest.java index cdc69d81..684ef2cb 100644 --- a/src/test/java/ai/reveng/AIDecompilerComponentTest.java +++ b/src/test/java/ai/reveng/AIDecompilerComponentTest.java @@ -14,7 +14,6 @@ import ai.reveng.toolkit.ghidra.plugins.BinarySimilarityPlugin; import docking.widgets.dialogs.InputDialog; import ghidra.app.context.ProgramLocationActionContext; -import ghidra.program.database.ProgramBuilder; import ghidra.program.model.data.Undefined; import ghidra.program.model.listing.Function; import ghidra.program.util.ProgramLocation; @@ -102,7 +101,7 @@ public boolean triggerAIDecompilationForFunctionID(FunctionID functionID) { var binarySimilarityPlugin = env.addPlugin(BinarySimilarityPlugin.class); - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); + var builder = newX64Program(); var func1 = builder.createEmptyFunction(null, "0x1000", 10, Undefined.getUndefinedDataType(4)); var func2 = builder.createEmptyFunction(null, "0x2000", 10, Undefined.getUndefinedDataType(4)); @@ -160,7 +159,7 @@ public void testAIDecompFeedbackMechanism() throws Exception { var service = addMockedService(tool, ratingsAPI); var binarySimilarityPlugin = env.addPlugin(BinarySimilarityPlugin.class); - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); + var builder = newX64Program(); var func1 = builder.createEmptyFunction(null, "0x1000", 10, Undefined.getUndefinedDataType(4)); var func2 = builder.createEmptyFunction(null, "0x2000", 10, Undefined.getUndefinedDataType(4)); @@ -202,7 +201,7 @@ public void testFeedbackDoesNotBlockSwingThread() throws Exception { var service = addMockedService(tool, ratingsAPI); env.addPlugin(BinarySimilarityPlugin.class); - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); + var builder = newX64Program(); var func1 = builder.createEmptyFunction(null, "0x1000", 10, Undefined.getUndefinedDataType(4)); builder.createEmptyFunction(null, "0x2000", 10, Undefined.getUndefinedDataType(4)); @@ -253,7 +252,7 @@ public AIDecompilationStatus pollAIDecompileStatus(FunctionID functionID) { }); env.addPlugin(BinarySimilarityPlugin.class); - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); + var builder = newX64Program(); var func1 = builder.createEmptyFunction(null, "0x1000", 10, Undefined.getUndefinedDataType(4)); var programWithID = service.analyse(builder.getProgram(), null, TaskMonitor.DUMMY); env.showTool(programWithID.program()); @@ -317,7 +316,7 @@ public boolean triggerAIDecompilationForFunctionID(FunctionID functionID) { }); env.addPlugin(BinarySimilarityPlugin.class); - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); + var builder = newX64Program(); var func1 = builder.createEmptyFunction(null, "0x1000", 10, Undefined.getUndefinedDataType(4)); var programWithID = service.analyse(builder.getProgram(), null, TaskMonitor.DUMMY); env.showTool(programWithID.program()); diff --git a/src/test/java/ai/reveng/AgentChatWindowTest.java b/src/test/java/ai/reveng/AgentChatWindowTest.java index c4565e2d..10f02c58 100644 --- a/src/test/java/ai/reveng/AgentChatWindowTest.java +++ b/src/test/java/ai/reveng/AgentChatWindowTest.java @@ -12,7 +12,6 @@ import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisStatus; import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionInfo; import ai.reveng.toolkit.ghidra.plugins.AgentChatPlugin; -import ghidra.program.database.ProgramBuilder; import ghidra.program.model.data.Undefined; import ghidra.program.model.listing.Function; import ghidra.program.model.listing.Program; @@ -59,7 +58,7 @@ public List getFunctionInfo(AnalysisID analysisID) { }); env.addPlugin(AgentChatPlugin.class); - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); + var builder = newX64Program(); var func1 = builder.createEmptyFunction(null, "0x1000", 10, Undefined.getUndefinedDataType(4)); var programWithID = service.analyse(builder.getProgram(), null, TaskMonitor.DUMMY); env.showTool(programWithID.program()); @@ -140,7 +139,7 @@ public List getFunctionInfo(AnalysisID analysisID) { var service = addMockedService(tool, api); env.addPlugin(AgentChatPlugin.class); - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); + var builder = newX64Program(); var func = builder.createEmptyFunction(null, "0x1000", 10, Undefined.getUndefinedDataType(4)); var programWithID = service.analyse(builder.getProgram(), null, TaskMonitor.DUMMY); Program program = programWithID.program(); diff --git a/src/test/java/ai/reveng/AnalysisOptionsDialogTest.java b/src/test/java/ai/reveng/AnalysisOptionsDialogTest.java index 48069517..af3c4fda 100644 --- a/src/test/java/ai/reveng/AnalysisOptionsDialogTest.java +++ b/src/test/java/ai/reveng/AnalysisOptionsDialogTest.java @@ -22,6 +22,7 @@ import javax.swing.*; +import ai.reveng.model.AnalysisCreateRequest; import ai.reveng.model.User; import ai.reveng.toolkit.ghidra.binarysimilarity.ui.analysiscreation.RevEngAIAnalysisOptionsDialog; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; @@ -31,7 +32,6 @@ import ghidra.framework.main.FrontEndTool; import org.junit.*; -import ghidra.program.database.ProgramBuilder; import ghidra.test.TestEnv; public class AnalysisOptionsDialogTest extends RevEngMockableHeadedIntegrationTest { @@ -47,7 +47,7 @@ public AnalysisOptionsDialogTest() { public void testBasicOptionsDialog() throws Exception { var reService = new GhidraRevengService( new MockApi() {}); - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); + var builder = newX64Program(); var program = builder.getProgram(); var dialog = RevEngAIAnalysisOptionsDialog.withModelsFromServer(program, reService); SwingUtilities.invokeLater(() -> { @@ -69,6 +69,46 @@ public void testBasicOptionsDialog() throws Exception { assertNotNull(options); } + /** + * Pins the analysis configuration the dialog submits. The dialog offers no controls for + * capability generation, advanced analysis, third-party scraping or sandbox execution, so + * every analysis it creates must carry these fixed values. + */ + @Test + public void testSubmittedAnalysisConfig() throws Exception { + var reService = new GhidraRevengService(new MockApi() {}); + var builder = newX64Program(); + var program = builder.getProgram(); + var dialog = RevEngAIAnalysisOptionsDialog.withModelsFromServer(program, reService); + SwingUtilities.invokeLater(() -> { + DockingWindowManager.showDialog(null, dialog); + }); + waitForSwing(); + waitFor(() -> { + JButton okButton = (JButton) getInstanceField("okButton", dialog); + return okButton.isEnabled(); + }); + runSwing(() -> { + JButton okButton = (JButton) getInstanceField("okButton", dialog); + okButton.doClick(); + }); + + var options = dialog.getOptionsFromUI(); + assertNotNull(options); + AnalysisCreateRequest request = options.toAnalysisCreateRequest(); + + var analysisConfig = request.getAnalysisConfig(); + assertNotNull("The request must always carry an analysis config", analysisConfig); + assertEquals("Capabilities are never generated", Boolean.FALSE, analysisConfig.getGenerateCapabilities()); + assertEquals("Advanced analysis is never requested", Boolean.FALSE, analysisConfig.getAdvancedAnalysis()); + assertNull("Third-party scraping is never requested", analysisConfig.getScrapeThirdPartyConfig()); + assertNull("Sandbox execution is never requested", analysisConfig.getSandboxConfig()); + + var binaryConfig = request.getBinaryConfig(); + assertNotNull("The request must always carry a binary config", binaryConfig); + assertNull("The default 'Auto' architecture leaves the ISA unset", binaryConfig.getIsa()); + } + @Test public void testPrivateScopeDisabledForEnthusiast() throws Exception { var reService = new GhidraRevengService(new MockApi() { @@ -77,7 +117,7 @@ public User getMe() { return new User().tier(User.TierEnum.ENTHUSIAST); } }); - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); + var builder = newX64Program(); var program = builder.getProgram(); var dialog = RevEngAIAnalysisOptionsDialog.withModelsFromServer(program, reService); SwingUtilities.invokeLater(() -> { @@ -115,7 +155,7 @@ public User getMe() { return new User().tier(User.TierEnum.REVERSER); } }); - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); + var builder = newX64Program(); var program = builder.getProgram(); var dialog = RevEngAIAnalysisOptionsDialog.withModelsFromServer(program, reService); SwingUtilities.invokeLater(() -> { @@ -149,7 +189,7 @@ public User getMe() { throw new RuntimeException("tier lookup failed"); } }); - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); + var builder = newX64Program(); var program = builder.getProgram(); var dialog = RevEngAIAnalysisOptionsDialog.withModelsFromServer(program, reService); SwingUtilities.invokeLater(() -> { @@ -195,7 +235,7 @@ public User getMe() { return new User().tier(User.TierEnum.REVERSER); } }); - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); + var builder = newX64Program(); var program = builder.getProgram(); var dialog = RevEngAIAnalysisOptionsDialog.withModelsFromServer(program, reService); SwingUtilities.invokeLater(() -> { @@ -232,7 +272,7 @@ public User getMe() { return new User().tier(User.TierEnum.ENTHUSIAST); } }); - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); + var builder = newX64Program(); var program = builder.getProgram(); var dialog = RevEngAIAnalysisOptionsDialog.withModelsFromServer(program, reService); SwingUtilities.invokeLater(() -> { diff --git a/src/test/java/ai/reveng/ApplyMatchCmdTest.java b/src/test/java/ai/reveng/ApplyMatchCmdTest.java index 10eb4653..bade3af4 100644 --- a/src/test/java/ai/reveng/ApplyMatchCmdTest.java +++ b/src/test/java/ai/reveng/ApplyMatchCmdTest.java @@ -11,7 +11,6 @@ import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionInfo; import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionMatch; import ai.reveng.toolkit.ghidra.core.services.api.types.GhidraFunctionMatchWithSignature; -import ghidra.program.database.ProgramBuilder; import ghidra.program.model.data.Undefined; import ghidra.util.task.TaskMonitor; import org.junit.Test; @@ -29,7 +28,7 @@ public void testFailedServerRenameDoesNotLeakTransaction() throws Exception { var tool = env.getTool(); var service = addMockedService(tool, new RenameFailsAPI()); - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); + var builder = newX64Program(); var func1 = builder.createEmptyFunction(null, "0x1000", 10, Undefined.getUndefinedDataType(4)); var programWithID = service.analyse(builder.getProgram(), null, TaskMonitor.DUMMY); var program = programWithID.program(); diff --git a/src/test/java/ai/reveng/AutoUnstripSyncTest.java b/src/test/java/ai/reveng/AutoUnstripSyncTest.java index b49c8630..5385195c 100644 --- a/src/test/java/ai/reveng/AutoUnstripSyncTest.java +++ b/src/test/java/ai/reveng/AutoUnstripSyncTest.java @@ -1,7 +1,6 @@ package ai.reveng; import ai.reveng.model.BatchRenameInputBody; -import ai.reveng.model.FunctionDataTypesList; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface.AutoUnstripStatus; @@ -10,15 +9,12 @@ import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionInfo; import ai.reveng.toolkit.ghidra.core.services.logging.ReaiLoggingService; import ai.reveng.toolkit.ghidra.core.services.sync.AutoUnstripSyncService; -import ghidra.program.database.ProgramBuilder; import ghidra.program.model.data.Undefined; import ghidra.program.model.listing.Function; import ghidra.program.model.listing.Program; import ghidra.util.task.TaskMonitor; -import org.jetbrains.annotations.Nullable; import org.junit.Test; -import java.io.IOException; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Deque; @@ -26,6 +22,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.FunctionSignatureBatch; /** * Integration tests for the post-auto-unstrip sync (PLU-300): once the server-side auto-unstrip pass @@ -37,7 +34,6 @@ public class AutoUnstripSyncTest extends RevEngMockableHeadedIntegrationTest { @Override public void info(String message) {} @Override public void warn(String message) {} @Override public void error(String message) {} - @Override public void export(String targetDirectoryPath, String exportedFileName) {} }; /// Mock API that scripts auto-unstrip status responses and records rename calls. @@ -59,12 +55,9 @@ public List getFunctionInfo(TypedApiInterface.AnalysisID analysisI } @Override - public FunctionDataTypesList listFunctionDataTypesForAnalysis(TypedApiInterface.AnalysisID analysisID, @Nullable List ids) { - try { - return FunctionDataTypesList.fromJson("{\"total_count\":0,\"total_data_types_count\":0,\"items\":[]}"); - } catch (IOException e) { - throw new RuntimeException(e); - } + public FunctionSignatureBatch listFunctionSignatures(List functionIDs, + boolean includeDataTypes) { + return FunctionSignatureBatch.empty(); } @Override @@ -84,7 +77,7 @@ private Fixture setUp(UnstripApi api) throws Exception { api.functions = List.of(new FunctionInfo(new TypedApiInterface.FunctionID(7), "recovered_name", "recovered_name", 0x4000L, 0x100)); var service = new GhidraRevengService(api); - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); + var builder = newX64Program(); builder.createMemory("mem", "0x4000", 0x100); Function function = builder.createEmptyFunction(null, "0x4000", 0x100, Undefined.getUndefinedDataType(8)); var program = builder.getProgram(); diff --git a/src/test/java/ai/reveng/BidirectionalSyncTest.java b/src/test/java/ai/reveng/BidirectionalSyncTest.java deleted file mode 100644 index 3ee03119..00000000 --- a/src/test/java/ai/reveng/BidirectionalSyncTest.java +++ /dev/null @@ -1,304 +0,0 @@ -package ai.reveng; - -import ai.reveng.model.BatchRenameInputBody; -import ai.reveng.model.FunctionDataTypesList; -import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; -import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; -import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface.DataTypePushResult; -import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface.DataTypePushStatus; -import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface.FunctionDataTypeUpdate; -import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; -import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisStatus; -import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionInfo; -import ai.reveng.toolkit.ghidra.core.services.logging.ReaiLoggingService; -import ai.reveng.toolkit.ghidra.core.services.sync.LocalEditSyncService; -import ghidra.program.database.ProgramBuilder; -import ghidra.program.model.data.DataType; -import ghidra.program.model.data.IntegerDataType; -import ghidra.program.model.data.PointerDataType; -import ghidra.program.model.data.StructureDataType; -import ghidra.program.model.data.Undefined; -import ghidra.program.model.listing.Function; -import ghidra.program.model.listing.LocalVariableImpl; -import ghidra.program.model.listing.Parameter; -import ghidra.program.model.listing.ParameterImpl; -import ghidra.program.model.listing.Program; -import ghidra.program.model.symbol.SourceType; -import ghidra.util.task.TaskMonitor; -import org.jetbrains.annotations.Nullable; -import org.junit.Test; - -import java.io.IOException; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Deque; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * Integration tests for the bidirectional push-back of local edits (PLU-322): reactive rename and - * type pushes, and the analysis-sync name reconciliation (canonify + push). - */ -public class BidirectionalSyncTest extends RevEngMockableHeadedIntegrationTest { - - /// Mock API that records push-back calls and can script data-type push outcomes. - private static class CapturingApi extends UnimplementedAPI { - final List renameCalls = new ArrayList<>(); - final List> typePushCalls = new ArrayList<>(); - final Deque scriptedStatuses = new ArrayDeque<>(); - long currentVersion = 0; - Map canonicalMapping = Map.of(); - List functions = List.of(); - - @Override - public AnalysisStatus status(TypedApiInterface.AnalysisID analysisID) { - return AnalysisStatus.Complete; - } - - @Override - public List getFunctionInfo(TypedApiInterface.AnalysisID analysisID) { - return functions; - } - - @Override - public FunctionDataTypesList listFunctionDataTypesForAnalysis(TypedApiInterface.AnalysisID analysisID, @Nullable List ids) { - try { - return FunctionDataTypesList.fromJson("{\"total_count\":0,\"total_data_types_count\":0,\"items\":[]}"); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public void batchRenameFunctions(BatchRenameInputBody request) { - renameCalls.add(request); - } - - @Override - public Optional getFunctionDataTypesWithVersion(TypedApiInterface.FunctionID functionID) { - return Optional.of(new VersionedFunctionTypes(null, currentVersion)); - } - - @Override - public List pushFunctionDataTypes(TypedApiInterface.AnalysisID analysisID, List updates) { - typePushCalls.add(updates); - var status = scriptedStatuses.isEmpty() ? DataTypePushStatus.UPDATED : scriptedStatuses.poll(); - return updates.stream() - .map(u -> new DataTypePushResult(u.functionID(), status, null)) - .toList(); - } - - @Override - public Map canonicalizeFunctionNames(List names) { - return canonicalMapping; - } - } - - private GhidraRevengService.AnalysedProgram register(GhidraRevengService service, Program program) throws Exception { - var programWithID = service.registerAnalysisForProgram(program, new TypedApiInterface.AnalysisID(1)); - service.registerFinishedAnalysisForProgram(programWithID, TaskMonitor.DUMMY); - return service.getAnalysedProgram(program).orElseThrow(); - } - - @Test - public void pushFunctionRename_sendsLocalNameToPortal() throws Exception { - var api = new CapturingApi(); - api.functions = List.of(new FunctionInfo(new TypedApiInterface.FunctionID(7), "orig", "orig", 0x4000L, 0x100)); - var service = new GhidraRevengService(api); - - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); - builder.createMemory("mem", "0x4000", 0x100); - Function function = builder.createEmptyFunction(null, "0x4000", 0x100, Undefined.getUndefinedDataType(8)); - var program = builder.getProgram(); - var analysed = register(service, program); - - program.withTransaction("rename", () -> { - try { - function.setName("user_chosen_name", SourceType.USER_DEFINED); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - - service.pushFunctionRename(analysed, function); - - assertEquals(1, api.renameCalls.size()); - var item = api.renameCalls.get(0).getFunctions().get(0); - assertEquals(7L, item.getFunctionId().longValue()); - assertEquals("user_chosen_name", item.getNewName()); - } - - @Test - public void pushFunctionRename_qualifiesNameWithNamespace() throws Exception { - var api = new CapturingApi(); - api.functions = List.of(new FunctionInfo(new TypedApiInterface.FunctionID(7), "orig", "orig", 0x4000L, 0x100)); - var service = new GhidraRevengService(api); - - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); - builder.createMemory("mem", "0x4000", 0x100); - Function function = builder.createEmptyFunction(null, "0x4000", 0x100, Undefined.getUndefinedDataType(8)); - var program = builder.getProgram(); - var analysed = register(service, program); - - program.withTransaction("rename in namespace", () -> { - try { - var namespace = program.getSymbolTable().createNameSpace( - program.getGlobalNamespace(), "MyClass", SourceType.USER_DEFINED); - function.setParentNamespace(namespace); - function.setName("method", SourceType.USER_DEFINED); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - - service.pushFunctionRename(analysed, function); - - var item = api.renameCalls.get(0).getFunctions().get(0); - assertEquals("MyClass::method", item.getNewName()); - } - - @Test - public void pushFunctionTypes_retriesOnVersionConflictWithLatestVersion() throws Exception { - var api = new CapturingApi(); - api.functions = List.of(new FunctionInfo(new TypedApiInterface.FunctionID(7), "orig", "orig", 0x4000L, 0x100)); - api.currentVersion = 42; - api.scriptedStatuses.add(DataTypePushStatus.VERSION_CONFLICT); - api.scriptedStatuses.add(DataTypePushStatus.UPDATED); - var service = new GhidraRevengService(api); - - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); - builder.createMemory("mem", "0x4000", 0x100); - Function function = builder.createEmptyFunction(null, "0x4000", 0x100, Undefined.getUndefinedDataType(8)); - var program = builder.getProgram(); - var analysed = register(service, program); - - boolean pushed = service.pushFunctionTypes(analysed, function); - - assertTrue("push should succeed after retrying the conflict", pushed); - assertEquals("one conflicting attempt then one successful attempt", 2, api.typePushCalls.size()); - assertEquals(7L, api.typePushCalls.get(0).get(0).functionID().value()); - assertEquals("version fetched before each attempt", 42L, api.typePushCalls.get(0).get(0).version()); - } - - private static final ReaiLoggingService NOOP_LOG = new ReaiLoggingService() { - @Override public void info(String message) {} - @Override public void warn(String message) {} - @Override public void error(String message) {} - @Override public void export(String targetDirectoryPath, String exportedFileName) {} - }; - - @Test - public void reactiveListener_pushesTypesWhenLocalVariableEdited() throws Exception { - var api = new CapturingApi(); - api.functions = List.of(new FunctionInfo(new TypedApiInterface.FunctionID(7), "orig", "orig", 0x4000L, 0x100)); - var service = new GhidraRevengService(api); - - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); - builder.createMemory("mem", "0x4000", 0x100); - Function function = builder.createEmptyFunction(null, "0x4000", 0x100, Undefined.getUndefinedDataType(8)); - var program = builder.getProgram(); - register(service, program); - - var syncService = new LocalEditSyncService(service, NOOP_LOG); - try { - syncService.attach(program); - - program.withTransaction("add local variable", () -> { - try { - function.addLocalVariable( - new LocalVariableImpl("renamed_local", IntegerDataType.dataType, -0x8, program), - SourceType.USER_DEFINED); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - program.flushEvents(); - waitForSwing(); - - long deadline = System.currentTimeMillis() + 5000; - while (api.typePushCalls.isEmpty() && System.currentTimeMillis() < deadline) { - Thread.sleep(50); - } - - assertEquals("editing a variable pushes the function's types once", 1, api.typePushCalls.size()); - assertEquals(7L, api.typePushCalls.get(0).get(0).functionID().value()); - } finally { - syncService.dispose(); - } - } - - @Test - public void reactiveListener_pushesTypesWhenReferencedDataTypeEdited() throws Exception { - var api = new CapturingApi(); - api.functions = List.of(new FunctionInfo(new TypedApiInterface.FunctionID(7), "orig", "orig", 0x4000L, 0x100)); - var service = new GhidraRevengService(api); - - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); - builder.createMemory("mem", "0x4000", 0x100); - var struct = new StructureDataType("MyStruct", 0); - struct.add(IntegerDataType.dataType, "field0", null); - builder.addDataType(struct); - Parameter param = new ParameterImpl("arg", new PointerDataType(struct), builder.getProgram()); - builder.createEmptyFunction(null, "0x4000", 0x100, Undefined.getUndefinedDataType(8), param); - var program = builder.getProgram(); - register(service, program); - - var syncService = new LocalEditSyncService(service, NOOP_LOG); - try { - syncService.attach(program); - - DataType resolved = program.getDataTypeManager().getDataType("/MyStruct"); - program.withTransaction("rename data type", () -> { - try { - resolved.setName("RenamedStruct"); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - program.flushEvents(); - waitForSwing(); - - long deadline = System.currentTimeMillis() + 5000; - while (api.typePushCalls.isEmpty() && System.currentTimeMillis() < deadline) { - Thread.sleep(50); - } - - assertEquals("editing a referenced data type pushes the function that uses it", - 7L, api.typePushCalls.get(0).get(0).functionID().value()); - } finally { - syncService.dispose(); - } - } - - @Test - public void syncAnalysisUpdates_canonicalizesInvalidRemoteNameAndPushesItBack() throws Exception { - var api = new CapturingApi(); - api.functions = List.of( - new FunctionInfo(new TypedApiInterface.FunctionID(1), "valid_name", "valid_name", 0x4000L, 0x100), - new FunctionInfo(new TypedApiInterface.FunctionID(2), "bad name!", "bad name!", 0x5000L, 0x100)); - api.canonicalMapping = Map.of("bad name!", "bad_name"); - var service = new GhidraRevengService(api); - - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); - builder.createMemory("a", "0x4000", 0x100); - builder.createMemory("b", "0x5000", 0x100); - builder.createEmptyFunction(null, "0x4000", 0x100, Undefined.getUndefinedDataType(8)); - Function invalidNamed = builder.createEmptyFunction(null, "0x5000", 0x100, Undefined.getUndefinedDataType(8)); - var program = builder.getProgram(); - var analysed = register(service, program); - - var summary = service.syncAnalysisUpdates(analysed, TaskMonitor.DUMMY, NOOP_LOG); - - assertEquals("invalid remote name is canonicalized locally", "bad_name", invalidNamed.getName()); - assertEquals(1, summary.canonicalizedNames()); - - boolean canonicalPushedBack = api.renameCalls.stream() - .flatMap(call -> call.getFunctions().stream()) - .anyMatch(item -> item.getFunctionId() == 2L && "bad_name".equals(item.getNewName())); - assertTrue("the canonicalized name is pushed back to the portal", canonicalPushedBack); - } -} diff --git a/src/test/java/ai/reveng/DependencyDataTypeLoadingTest.java b/src/test/java/ai/reveng/DependencyDataTypeLoadingTest.java deleted file mode 100644 index 23d9a645..00000000 --- a/src/test/java/ai/reveng/DependencyDataTypeLoadingTest.java +++ /dev/null @@ -1,38 +0,0 @@ -package ai.reveng; - -import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; -import ai.reveng.toolkit.ghidra.core.services.api.types.binsync.FunctionDependencies; -import ai.reveng.toolkit.ghidra.core.services.api.types.binsync.Struct; -import ai.reveng.toolkit.ghidra.core.services.api.types.binsync.StructMember; -import ai.reveng.toolkit.ghidra.core.services.api.types.binsync.Typedef; -import ghidra.program.model.data.DataTypeManager; -import ghidra.program.model.data.Structure; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * Regression test for {@link GhidraRevengService#loadDependencyDataTypes}: a struct member reported - * beyond the struct's declared size must not abort the whole type load (the crash seen when the - * agent's decompile tool result triggered a function-info pull). - */ -public class DependencyDataTypeLoadingTest extends RevEngMockableHeadedIntegrationTest { - - @Test - public void growsStructToFitMemberBeyondDeclaredSize() throws Exception { - // Declared size 32, but a member sits at offset 32 (== size) — replaceAtOffset would reject it. - var head = new StructMember(null, "head", 0, "int_type_missing", 4); - var tail = new StructMember(null, "tail", 32, "ptr_type_missing", 8); - var struct = new Struct(null, "OversizedStruct", 32, new StructMember[]{head, tail}); - var deps = new FunctionDependencies(new Typedef[0], new Struct[]{struct}); - - DataTypeManager dtm = GhidraRevengService.loadDependencyDataTypes(deps); - - Structure loaded = (Structure) dtm.getDataType("/OversizedStruct"); - assertTrue("struct should have grown to fit the trailing member, length was " + loaded.getLength(), - loaded.getLength() >= 40); - assertEquals("tail", loaded.getComponentAt(32).getFieldName()); - assertEquals("head", loaded.getComponentAt(0).getFieldName()); - } -} diff --git a/src/test/java/ai/reveng/DetachProgramAssociationTest.java b/src/test/java/ai/reveng/DetachProgramAssociationTest.java index 073b4528..0f840816 100644 --- a/src/test/java/ai/reveng/DetachProgramAssociationTest.java +++ b/src/test/java/ai/reveng/DetachProgramAssociationTest.java @@ -1,17 +1,25 @@ package ai.reveng; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; -import ghidra.program.database.ProgramBuilder; import org.junit.Test; +import static org.junit.Assert.assertTrue; + public class DetachProgramAssociationTest extends RevEngMockableHeadedIntegrationTest { @Test public void testDetachWithoutMarkedFunctions() throws Exception { var service = addMockedService(env.getTool(), new UnimplementedAPI()); - var builder = new ProgramBuilder("detach-test", ProgramBuilder._X64, this); + var builder = newX64Program("detach-test"); var program = builder.getProgram(); + service.registerAnalysisForProgram(program, new TypedApiInterface.AnalysisID(1)); + assertTrue("the program should be associated before detaching", + service.getKnownProgram(program).isPresent()); program.withTransaction("Undo binary association", () -> service.removeProgramAssociation(program)); + + assertTrue("detaching should clear the association even with no marked functions", + service.getKnownProgram(program).isEmpty()); } } diff --git a/src/test/java/ai/reveng/FunctionLevelFunctionMatchingDialogTest.java b/src/test/java/ai/reveng/FunctionLevelFunctionMatchingDialogTest.java index 0d42845e..50a7f74b 100644 --- a/src/test/java/ai/reveng/FunctionLevelFunctionMatchingDialogTest.java +++ b/src/test/java/ai/reveng/FunctionLevelFunctionMatchingDialogTest.java @@ -7,17 +7,19 @@ import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisStatus; import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionInfo; +import ai.reveng.toolkit.ghidra.core.services.api.types.GhidraFunctionMatchWithSignature; import ai.reveng.toolkit.ghidra.plugins.BinarySimilarityPlugin; import docking.DockingWindowManager; -import ghidra.program.database.ProgramBuilder; import ghidra.program.model.data.Undefined; import ghidra.util.task.TaskMonitor; +import ghidra.util.task.TaskMonitorComponent; import org.junit.Test; import javax.swing.*; import java.util.List; import static org.junit.Assert.*; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.FunctionSignatureBatch; /** * Integration tests for the FunctionLevelFunctionMatchingDialog. @@ -37,7 +39,7 @@ public void testDialogOpensWithMockedService() throws Exception { env.addPlugin(BinarySimilarityPlugin.class); // Create a test program with a function - var builder = new ProgramBuilder("test_binary", ProgramBuilder._X64, this); + var builder = newX64Program("test_binary"); var testFunction = builder.createEmptyFunction("test_function", "0x1000", 50, Undefined.getUndefinedDataType(4)); // Register the program as analyzed (this triggers associateFunctionInfo internally) @@ -87,7 +89,7 @@ public void testDialogHasResultsTableConfigured() throws Exception { env.addPlugin(BinarySimilarityPlugin.class); - var builder = new ProgramBuilder("test_binary", ProgramBuilder._X64, this); + var builder = newX64Program("test_binary"); var testFunction = builder.createEmptyFunction("test_function", "0x1000", 50, Undefined.getUndefinedDataType(4)); var analysedProgram = service.analyse(builder.getProgram(), null, TaskMonitor.DUMMY); @@ -123,7 +125,7 @@ public void testAssemblyComparisonPanelExists() throws Exception { env.addPlugin(BinarySimilarityPlugin.class); - var builder = new ProgramBuilder("test_binary", ProgramBuilder._X64, this); + var builder = newX64Program("test_binary"); var testFunction = builder.createEmptyFunction("test_function", "0x1000", 50, Undefined.getUndefinedDataType(4)); var analysedProgram = service.analyse(builder.getProgram(), null, TaskMonitor.DUMMY); @@ -157,7 +159,7 @@ public void testFunctionMatchingTriggersAPICall() throws Exception { env.addPlugin(BinarySimilarityPlugin.class); - var builder = new ProgramBuilder("test_binary", ProgramBuilder._X64, this); + var builder = newX64Program("test_binary"); var testFunction = builder.createEmptyFunction("test_function", "0x1000", 50, Undefined.getUndefinedDataType(4)); var analysedProgram = service.analyse(builder.getProgram(), null, TaskMonitor.DUMMY); @@ -193,7 +195,7 @@ public void testClickMatchButtonPopulatesResultsTable() throws Exception { env.addPlugin(BinarySimilarityPlugin.class); - var builder = new ProgramBuilder("test_binary", ProgramBuilder._X64, this); + var builder = newX64Program("test_binary"); var testFunction = builder.createEmptyFunction("test_function", "0x1000", 50, Undefined.getUndefinedDataType(4)); var analysedProgram = service.analyse(builder.getProgram(), null, TaskMonitor.DUMMY); @@ -235,11 +237,135 @@ public void testClickMatchButtonPopulatesResultsTable() throws Exception { // Verify the API was actually called assertTrue("Function matching API should have been called", mockApi.functionMatchingCalled); + // Once the results are in, the status must say so. Nothing used to write to the label after + // "Loading type information...", so a completed match looked exactly like a stuck one. + JLabel statusLabel = (JLabel) getInstanceField("statusLabel", foundDialog); + waitForCondition(() -> !statusLabel.getText().contains("Loading"), + "Status label should stop saying it is loading once matching has finished"); + assertTrue("The finished status should report the matches, was: " + statusLabel.getText(), + statusLabel.getText().contains("Matching complete")); + TaskMonitorComponent monitor = + (TaskMonitorComponent) getInstanceField("taskMonitorComponent", foundDialog); + assertFalse("The progress bar should not be left spinning", monitor.isVisible()); + // Close the dialog close(foundDialog); waitForSwing(); } + /** + * The results table is sortable, so a selected view row need not be the same row in the list + * backing the table model. Selecting a row after sorting must still pick the match that is + * actually displayed on that row, otherwise "Rename Selected" renames an unrelated function. + */ + @Test + public void testSelectedMatchFollowsSortOrderNotModelOrder() throws Exception { + var tool = env.getTool(); + + var mockApi = new MultiMatchMockApi(); + var service = addMockedService(tool, mockApi); + + env.addPlugin(BinarySimilarityPlugin.class); + + var builder = newX64Program("test_binary"); + var testFunction = builder.createEmptyFunction("test_function", "0x1000", 50, Undefined.getUndefinedDataType(4)); + + var analysedProgram = service.analyse(builder.getProgram(), null, TaskMonitor.DUMMY); + env.showTool(analysedProgram.program()); + waitForSwing(); + + FunctionLevelFunctionMatchingDialog dialog = runSwing(() -> + new FunctionLevelFunctionMatchingDialog(tool, analysedProgram, testFunction) + ); + + runSwing(() -> DockingWindowManager.showDialog(null, dialog), false); + var foundDialog = waitForDialogComponent(FunctionLevelFunctionMatchingDialog.class); + assertNotNull("Dialog should be shown", foundDialog); + + JTable resultsTable = (JTable) getInstanceField("resultsTable", foundDialog); + + pressButton(findButtonByText(foundDialog.getComponent(), "Match Functions")); + waitForTasks(); + waitForSwing(); + waitForCondition(() -> resultsTable.getRowCount() == 3, + "Results table should hold all three mocked matches"); + + // The model holds the matches in the order the server returned them + assertEquals("zeta_match", resultsTable.getModel().getValueAt(0, 0)); + assertEquals("alpha_match", resultsTable.getModel().getValueAt(1, 0)); + assertEquals("mid_match", resultsTable.getModel().getValueAt(2, 0)); + + // Sort ascending by matched function name, which reorders the view against the model + runSwing(() -> resultsTable.getRowSorter().toggleSortOrder(0)); + waitForSwing(); + assertEquals("Sorting should have reordered the view", 1, resultsTable.convertRowIndexToModel(0)); + + // Select the first row as displayed, which is no longer the first row of the model + runSwing(() -> resultsTable.setRowSelectionInterval(0, 0)); + waitForSwing(); + + List selected = (List) invokeInstanceMethod("getSelectedMatches", foundDialog); + assertEquals("Exactly one match should be selected", 1, selected.size()); + assertEquals("The match on the selected view row must be the one that gets renamed", + "alpha_match", + ((GhidraFunctionMatchWithSignature) selected.get(0)).functionMatch().name()); + + // And the last displayed row maps back to the first row of the model + runSwing(() -> resultsTable.setRowSelectionInterval(2, 2)); + waitForSwing(); + selected = (List) invokeInstanceMethod("getSelectedMatches", foundDialog); + assertEquals(1, selected.size()); + assertEquals("zeta_match", + ((GhidraFunctionMatchWithSignature) selected.get(0)).functionMatch().name()); + + close(foundDialog); + waitForSwing(); + } + + /** + * Returns several matches whose server order differs from their alphabetical order, so that + * sorting the results table genuinely separates view indices from model indices. + */ + static class MultiMatchMockApi extends FunctionMatchingMockApi { + @Override + public GetMatchesOutputBody getFunctionsMatches(List functionIds) { + var response = new GetMatchesOutputBody(); + response.setStatus(GetMatchesOutputBody.StatusEnum.COMPLETED); + + var functionMatch = new ai.reveng.model.FunctionMatch(); + functionMatch.setFunctionId(100L); + functionMatch.setMatchedFunctions(List.of( + matchedFunction(200L, "zeta_match", 0.50), + matchedFunction(201L, "alpha_match", 0.95), + matchedFunction(202L, "mid_match", 0.72) + )); + response.setMatches(List.of(functionMatch)); + + return response; + } + + private static MatchedFunction matchedFunction(long id, String name, double similarity) { + var matchedFunc = new MatchedFunction(); + matchedFunc.setFunctionId(id); + matchedFunc.setFunctionName(name); + matchedFunc.setMangledName(name); + matchedFunc.setSha256Hash(Long.toString(id).repeat(64).substring(0, 64)); + matchedFunc.setBinaryName("libc.so"); + matchedFunc.setBinaryId(1L); + matchedFunc.setFunctionVaddr(0x2000L + id); + matchedFunc.setAnalysisId(12345L); + matchedFunc.setDebug(false); + matchedFunc.setSimilarity(similarity); + matchedFunc.setConfidence(similarity); + return matchedFunc; + } + + @Override + public List getAssembly(TypedApiInterface.FunctionID functionID) { + return List.of("push rbp", "mov rbp, rsp", "ret"); + } + } + /** * Mock API implementation for function matching dialog tests. * Provides necessary responses for the dialog to function without a real server. @@ -273,10 +399,9 @@ public List getFunctionInfo(TypedApiInterface.AnalysisID analysisI } @Override - public Basic getAnalysisBasicInfo(TypedApiInterface.AnalysisID analysisID) { - // Create a Basic object with required fields - var basic = new Basic(); - basic.setModelId(1); + public AnalysisBasicInfoOutputBody getAnalysisBasicInfo(TypedApiInterface.AnalysisID analysisID) { + var basic = new AnalysisBasicInfoOutputBody(); + basic.setModelId(1L); basic.setModelName("test-model"); basic.setBinaryName("test_binary"); basic.setSha256Hash("0".repeat(64)); @@ -358,9 +483,10 @@ public List getAssembly(TypedApiInterface.FunctionID functionID) { } @Override - public FunctionDataTypesList listFunctionDataTypesForFunctions(List functionIDs) { - // Return empty list - no type info available - return new FunctionDataTypesList(); + public FunctionSignatureBatch listFunctionSignatures(List functionIDs, + boolean includeDataTypes) { + // No type info available + return FunctionSignatureBatch.empty(); } } } \ No newline at end of file diff --git a/src/test/java/ai/reveng/GhidraDataTypeEncoderTest.java b/src/test/java/ai/reveng/GhidraDataTypeEncoderTest.java new file mode 100644 index 00000000..d036c6bf --- /dev/null +++ b/src/test/java/ai/reveng/GhidraDataTypeEncoderTest.java @@ -0,0 +1,353 @@ +package ai.reveng; + +import ai.reveng.toolkit.ghidra.core.services.api.AnalysisDataTypesService.TypeKey; +import ai.reveng.toolkit.ghidra.core.services.api.GhidraDataTypeEncoder; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.ServerDataType.Kind; +import ghidra.program.model.data.ArrayDataType; +import ghidra.program.model.data.CategoryPath; +import ghidra.program.model.data.CharDataType; +import ghidra.program.model.data.EnumDataType; +import ghidra.program.model.data.FunctionDefinitionDataType; +import ghidra.program.model.data.IntegerDataType; +import ghidra.program.model.data.ParameterDefinitionImpl; +import ghidra.program.model.data.PointerDataType; +import ghidra.program.model.data.Structure; +import ghidra.program.model.data.StructureDataType; +import ghidra.program.model.data.TypedefDataType; +import ghidra.program.model.data.UnionDataType; +import ghidra.program.model.data.UnsignedLongLongDataType; +import ghidra.program.model.data.VoidDataType; +import org.junit.Test; + +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +/// Tests for {@link GhidraDataTypeEncoder}, which turns Ghidra types into the v3 create/update +/// bodies. The mirror of {@link ai.reveng.toolkit.ghidra.core.services.api.ServerDataTypeDecoder}. +public class GhidraDataTypeEncoderTest extends ghidra.test.AbstractGhidraHeadlessIntegrationTest { + + /// Resolves every key to a distinct id, in the order the keys are first asked for, which is + /// what the create/update pass would have produced. + private static GhidraDataTypeEncoder.Ids idsFor(List closure) { + Map ids = new java.util.LinkedHashMap<>(); + long next = 1; + for (var type : closure) { + ids.putIfAbsent(GhidraDataTypeEncoder.keyOf(type), next++); + } + return ids::get; + } + + private static Object instanceOfCreate(ghidra.program.model.data.DataType type) { + return GhidraDataTypeEncoder.createEntry(type).getActualInstance(); + } + + private static Object instanceOfUpdate(ghidra.program.model.data.DataType type, + GhidraDataTypeEncoder.Ids ids) { + return GhidraDataTypeEncoder.updateEntry(type, 1L, ids).orElseThrow().getActualInstance(); + } + + @Test + public void mapsEveryGhidraKindToItsVariant() { + var struct = new StructureDataType("S", 0); + struct.add(new IntegerDataType(), "a", null); + var union = new UnionDataType("U"); + union.add(new IntegerDataType(), "a", null); + var enumeration = new EnumDataType("E", 4); + enumeration.add("A", 1); + var typedef = new TypedefDataType("T", new IntegerDataType()); + var pointer = new PointerDataType(struct); + var array = new ArrayDataType(new CharDataType(), 16, 1); + var functionType = new FunctionDefinitionDataType("F"); + functionType.setReturnType(new IntegerDataType()); + + assertEquals(Kind.STRUCT, GhidraDataTypeEncoder.kindOf(struct)); + assertEquals(Kind.UNION, GhidraDataTypeEncoder.kindOf(union)); + assertEquals(Kind.ENUM, GhidraDataTypeEncoder.kindOf(enumeration)); + assertEquals(Kind.TYPEDEF, GhidraDataTypeEncoder.kindOf(typedef)); + assertEquals(Kind.POINTER, GhidraDataTypeEncoder.kindOf(pointer)); + assertEquals(Kind.ARRAY, GhidraDataTypeEncoder.kindOf(array)); + assertEquals(Kind.FUNCTION_DEFINITION, GhidraDataTypeEncoder.kindOf(functionType)); + assertEquals(Kind.BASE, GhidraDataTypeEncoder.kindOf(new IntegerDataType())); + assertEquals(Kind.UNKNOWN, + GhidraDataTypeEncoder.kindOf(ghidra.program.model.data.Undefined1DataType.dataType)); + + assertTrue(instanceOfCreate(struct) instanceof ai.reveng.model.CreateStructDataType); + assertTrue(instanceOfCreate(union) instanceof ai.reveng.model.CreateUnionDataType); + assertTrue(instanceOfCreate(enumeration) instanceof ai.reveng.model.CreateEnumDataType); + assertTrue(instanceOfCreate(typedef) instanceof ai.reveng.model.CreateTypedefDataType); + assertTrue(instanceOfCreate(pointer) instanceof ai.reveng.model.CreatePointerDataType); + assertTrue(instanceOfCreate(array) instanceof ai.reveng.model.CreateArrayDataType); + assertTrue(instanceOfCreate(functionType) instanceof ai.reveng.model.CreateFunctionDataType); + assertTrue(instanceOfCreate(new IntegerDataType()) instanceof ai.reveng.model.CreateBaseDataType); + assertTrue(instanceOfCreate(ghidra.program.model.data.Undefined1DataType.dataType) + instanceof ai.reveng.model.CreateUnknownDataType); + } + + /// A `Create*` body carries no `data_type_id` and cannot point at anything created alongside it, + /// so the definitions it does carry have to be empty. + @Test + public void createBodiesCarryEmptyDefinitions() { + var struct = new StructureDataType("S", 0); + struct.add(new IntegerDataType(), "a", null); + + var created = (ai.reveng.model.CreateStructDataType) instanceOfCreate(struct); + assertEquals("S", created.getName()); + assertEquals("", created.getNamespace()); + assertEquals(Long.valueOf(struct.getLength()), created.getSize()); + assertTrue("the definition is filled in by the update pass", created.getDefinition().getMembers().isEmpty()); + + var pointer = (ai.reveng.model.CreatePointerDataType) instanceOfCreate(new PointerDataType(struct)); + assertNull(pointer.getDefinition().getPointeeDataTypeId()); + + var functionType = new FunctionDefinitionDataType("F"); + functionType.setArguments(new ParameterDefinitionImpl("a", new IntegerDataType(), null)); + var created3 = (ai.reveng.model.CreateFunctionDataType) instanceOfCreate(functionType); + assertTrue(created3.getDefinition().getParameters().isEmpty()); + } + + @Test + public void structMembersEncodeWithOffsetSizeAndTypeId() { + var inner = new StructureDataType("Inner", 0); + inner.add(new IntegerDataType(), "x", null); + var outer = new StructureDataType("Outer", 0); + outer.add(new IntegerDataType(), "count", null); + outer.add(inner, "body", null); + // Unnamed padding is legal and must survive as a null name. + outer.add(new CharDataType(), null, null); + + var closure = GhidraDataTypeEncoder.closure(List.of(outer)); + var ids = idsFor(closure); + var updated = (ai.reveng.model.UpdateStructDataType) instanceOfUpdate(outer, ids); + + var members = updated.getDefinition().getMembers(); + assertEquals(3, members.size()); + assertEquals("count", members.get(0).getName()); + assertEquals(Long.valueOf(0), members.get(0).getOffset()); + assertEquals(Long.valueOf(4), members.get(0).getSize()); + assertEquals(Boolean.FALSE, members.get(0).getIsBitfield()); + assertEquals("body", members.get(1).getName()); + assertEquals(Long.valueOf(4), members.get(1).getOffset()); + assertEquals("the member points at the inner struct's id", + ids.idOf(GhidraDataTypeEncoder.keyOf(inner)), members.get(1).getDataTypeId()); + assertNull("unnamed padding keeps a null name", members.get(2).getName()); + } + + /// A bitfield is a property of the member that holds it, not a type of its own: the member + /// points at the bitfield's base type and carries the bit geometry itself. + @Test + public void bitfieldsEncodeOnTheMemberNotAsAType() throws Exception { + var flags = new StructureDataType("Flags", 0); + flags.setPackingEnabled(true); + flags.addBitField(new IntegerDataType(), 1, "enabled", null); + flags.addBitField(new IntegerDataType(), 3, "level", null); + + var closure = GhidraDataTypeEncoder.closure(List.of(flags)); + assertFalse("no standalone BITFIELD type is invented", + closure.stream().anyMatch(type -> GhidraDataTypeEncoder.kindOf(type) == Kind.BITFIELD)); + assertTrue("the member's base type is what gets an id", + closure.stream().anyMatch(type -> "int".equals(type.getName()))); + + var updated = (ai.reveng.model.UpdateStructDataType) instanceOfUpdate(flags, idsFor(closure)); + var members = updated.getDefinition().getMembers(); + assertEquals(2, members.size()); + assertEquals(Boolean.TRUE, members.get(0).getIsBitfield()); + assertEquals(Long.valueOf(1), members.get(0).getBitSize()); + assertEquals(Boolean.TRUE, members.get(1).getIsBitfield()); + assertEquals(Long.valueOf(3), members.get(1).getBitSize()); + } + + /// The API wants a bitfield's offset from the start of the containing type. Ghidra reports the + /// offset of the least-significant bit within the component's storage unit, and on a big-endian + /// target that is counted from the far end of the unit: the same four fields report 7, 4, 0 and + /// 0 rather than 0, 1, 4 and 0. Taking that at face value would put the first field of a + /// big-endian struct at bit 7 and then walk backwards. + @Test + public void bitfieldOffsetsAreCountedFromTheStartOfTheTypeOnEitherEndianness() throws Exception { + assertEquals("little-endian offsets run 0, 1, 4, 8", + List.of(0L, 1L, 4L, 8L), bitOffsetsOfPackedFlags(false)); + assertEquals("and big-endian offsets have to run the same way", + List.of(0L, 1L, 4L, 8L), bitOffsetsOfPackedFlags(true)); + } + + /// `int a:1; int b:3; int c:4; int d:8;` packed into a manager of the given endianness. + private static List bitOffsetsOfPackedFlags(boolean bigEndian) throws Exception { + var organization = ghidra.program.model.data.DataOrganizationImpl.getDefaultOrganization(null); + organization.setBigEndian(bigEndian); + var dtm = new ghidra.program.model.data.StandAloneDataTypeManager("endianness", organization); + int transaction = dtm.startTransaction("build"); + try { + var flags = new StructureDataType("Flags", 0, dtm); + flags.setPackingEnabled(true); + flags.addBitField(new IntegerDataType(dtm), 1, "a", null); + flags.addBitField(new IntegerDataType(dtm), 3, "b", null); + flags.addBitField(new IntegerDataType(dtm), 4, "c", null); + flags.addBitField(new IntegerDataType(dtm), 8, "d", null); + + var updated = (ai.reveng.model.UpdateStructDataType) instanceOfUpdate(flags, key -> 1L); + return updated.getDefinition().getMembers().stream() + .map(ai.reveng.model.DataTypeMemberEntry::getBitOffset) + .toList(); + } finally { + dtm.endTransaction(transaction, true); + dtm.close(); + } + } + + @Test + public void unionMembersAllSitAtOffsetZero() { + var union = new UnionDataType("U"); + union.add(new IntegerDataType(), "asInt", null); + union.add(new ArrayDataType(new CharDataType(), 4, 1), "asBytes", null); + + var closure = GhidraDataTypeEncoder.closure(List.of(union)); + var updated = (ai.reveng.model.UpdateUnionDataType) instanceOfUpdate(union, idsFor(closure)); + + assertEquals(2, updated.getDefinition().getMembers().size()); + updated.getDefinition().getMembers() + .forEach(member -> assertEquals(Long.valueOf(0), member.getOffset())); + } + + /// Enum values stay decimal strings on the wire because they may be negative or exceed 64 + /// unsigned bits, which no JSON number and no Java integer type can carry safely. Ghidra can + /// only hold a signed long, so what the encoder has to get right is that it never turns the + /// value back into a number — negatives keep their sign, and the field stays wide enough to + /// carry a value Ghidra itself could not have produced. + @Test + public void enumValuesSurviveAsStringsIncludingNegativeAndAbove64Bits() { + var enumeration = new EnumDataType("E", 8); + enumeration.add("NEGATIVE", -1); + enumeration.add("ZERO", 0); + enumeration.add("MAX_SIGNED", Long.MAX_VALUE); + + var updated = (ai.reveng.model.UpdateEnumDataType) instanceOfUpdate(enumeration, key -> null); + var byName = updated.getDefinition().getValues().stream() + .collect(java.util.stream.Collectors.toMap( + ai.reveng.model.DataTypeEnumValueEntry::getName, + ai.reveng.model.DataTypeEnumValueEntry::getValue)); + + assertEquals(3, byName.size()); + assertEquals("-1", byName.get("NEGATIVE")); + assertEquals("0", byName.get("ZERO")); + assertEquals("9223372036854775807", byName.get("MAX_SIGNED")); + + // The wire field is a string end to end, so a value past what any Java integer type holds + // round-trips unchanged rather than overflowing on the way through. + String aboveUnsigned64 = "18446744073709551616"; + var carried = new ai.reveng.model.DataTypeEnumValueEntry().name("HUGE").value(aboveUnsigned64); + assertEquals(aboveUnsigned64, carried.getValue()); + } + + @Test + public void pointersAndArraysGetTheirOwnDerivedNames() { + var struct = new StructureDataType("Foo", 0); + struct.add(new IntegerDataType(), "a", null); + var pointer = new PointerDataType(struct); + var array = new ArrayDataType(new CharDataType(), 16, 1); + + assertEquals("Foo *", GhidraDataTypeEncoder.keyOf(pointer).name()); + assertEquals(Kind.POINTER, GhidraDataTypeEncoder.keyOf(pointer).kind()); + assertEquals("char[16]", GhidraDataTypeEncoder.keyOf(array).name()); + assertEquals(Kind.ARRAY, GhidraDataTypeEncoder.keyOf(array).kind()); + + var closure = GhidraDataTypeEncoder.closure(List.of(pointer, array)); + var ids = idsFor(closure); + var encodedPointer = (ai.reveng.model.UpdatePointerDataType) instanceOfUpdate(pointer, ids); + assertEquals("the pointee gets an id of its own", + ids.idOf(GhidraDataTypeEncoder.keyOf(struct)), + encodedPointer.getDefinition().getPointeeDataTypeId()); + + var encodedArray = (ai.reveng.model.UpdateArrayDataType) instanceOfUpdate(array, ids); + assertEquals(Long.valueOf(16), encodedArray.getDefinition().getCount()); + assertEquals(ids.idOf(GhidraDataTypeEncoder.keyOf(new CharDataType())), + encodedArray.getDefinition().getElementDataTypeId()); + } + + @Test + public void typedefsAndFunctionTypesResolveTheirTargets() { + var typedef = new TypedefDataType("size_t", new UnsignedLongLongDataType()); + var functionType = new FunctionDefinitionDataType("callback"); + functionType.setReturnType(new IntegerDataType()); + functionType.setArguments( + new ParameterDefinitionImpl("ctx", new PointerDataType(VoidDataType.dataType), null), + new ParameterDefinitionImpl("n", typedef, null)); + + var closure = GhidraDataTypeEncoder.closure(List.of(typedef, functionType)); + var ids = idsFor(closure); + + var encodedTypedef = (ai.reveng.model.UpdateTypedefDataType) instanceOfUpdate(typedef, ids); + assertEquals(ids.idOf(GhidraDataTypeEncoder.keyOf(new UnsignedLongLongDataType())), + encodedTypedef.getDefinition().getTargetDataTypeId()); + + var encoded = (ai.reveng.model.UpdateFunctionDataType) instanceOfUpdate(functionType, ids); + var parameters = encoded.getDefinition().getParameters(); + assertEquals(2, parameters.size()); + assertEquals(Long.valueOf(0), parameters.get(0).getOrdinal()); + assertEquals("ctx", parameters.get(0).getName()); + assertEquals(Long.valueOf(1), parameters.get(1).getOrdinal()); + assertEquals(ids.idOf(GhidraDataTypeEncoder.keyOf(typedef)), parameters.get(1).getDataTypeId()); + assertEquals(ids.idOf(GhidraDataTypeEncoder.keyOf(new IntegerDataType())), + encoded.getDefinition().getReturnDataTypeId()); + } + + /// `PUT` replaces a stored type in full, so a Ghidra type with nothing to say must not be + /// allowed to write an empty definition over whatever the server extracted. + @Test + public void typesWithNothingToSayProduceNoUpdate() { + assertTrue("a base type has no definition to write", + GhidraDataTypeEncoder.updateEntry(new IntegerDataType(), 1L, key -> null).isEmpty()); + assertTrue("an empty local struct must not clear a populated server one", + GhidraDataTypeEncoder.updateEntry(new StructureDataType("Empty", 0), 1L, key -> null).isEmpty()); + assertTrue("a pointee that resolved to nothing leaves the stored pointer alone", + GhidraDataTypeEncoder.updateEntry( + new PointerDataType(new StructureDataType("Unknown", 4)), 1L, key -> null).isEmpty()); + } + + /// The namespace round-trips through the category path, which is how a type the plugin pulled + /// from the server resolves back to the same entry instead of being created again. + @Test + public void namespaceIsTheInverseOfTheDecodersCategoryPath() { + var local = new StructureDataType("Local", 0); + assertEquals("", GhidraDataTypeEncoder.keyOf(local).namespace()); + + var scoped = new StructureDataType(new CategoryPath("/DWARF/stdio.h"), "FILE", 0); + assertEquals("DWARF::stdio.h", GhidraDataTypeEncoder.keyOf(scoped).namespace()); + assertEquals("FILE", GhidraDataTypeEncoder.keyOf(scoped).name()); + } + + /// Two Ghidra instances of the same type are one server type, so the closure de-duplicates on + /// the key rather than on object identity. + @Test + public void closureDedupesByServerIdentityAndTerminatesOnCycles() { + var node = new StructureDataType("Node", 0); + node.add(new PointerDataType(node), "next", null); + node.add(new PointerDataType(node), "prev", null); + node.add(new IntegerDataType(), "value", null); + + var closure = GhidraDataTypeEncoder.closure(List.of(node, node)); + var keys = closure.stream().map(GhidraDataTypeEncoder::keyOf).toList(); + + assertEquals("no key appears twice", keys.size(), keys.stream().distinct().count()); + assertTrue(keys.contains(new TypeKey("", "Node", Kind.STRUCT))); + assertTrue(keys.contains(new TypeKey("", "Node *", Kind.POINTER))); + assertTrue(keys.contains(new TypeKey("", "int", Kind.BASE))); + } + + @Test + public void structsKeepGrowingMembersInOffsetOrder() { + Structure struct = new StructureDataType("Ordered", 0); + struct.add(new CharDataType(), "a", null); + struct.add(new IntegerDataType(), "b", null); + + var closure = GhidraDataTypeEncoder.closure(List.of(struct)); + var updated = (ai.reveng.model.UpdateStructDataType) instanceOfUpdate(struct, idsFor(closure)); + var members = updated.getDefinition().getMembers(); + + assertEquals(Long.valueOf(0), members.get(0).getOffset()); + assertEquals(Long.valueOf(1), members.get(1).getOffset()); + } +} diff --git a/src/test/java/ai/reveng/GhidraRevengServiceTest.java b/src/test/java/ai/reveng/GhidraRevengServiceTest.java index 12fb1599..113f3f5f 100644 --- a/src/test/java/ai/reveng/GhidraRevengServiceTest.java +++ b/src/test/java/ai/reveng/GhidraRevengServiceTest.java @@ -8,7 +8,6 @@ import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisStatus; import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionInfo; -import ghidra.program.database.ProgramBuilder; import ghidra.program.model.data.Undefined; import ghidra.util.task.TaskMonitor; import org.junit.Test; @@ -26,7 +25,7 @@ public void testKnownProgramLookupIsNetworkFree() throws Exception { var mock = new OfflineAfterSetupAPI(); var service = addMockedService(tool, mock); - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); + var builder = newX64Program(); builder.createEmptyFunction(null, "0x1000", 10, Undefined.getUndefinedDataType(4)); var programWithID = service.analyse(builder.getProgram(), null, TaskMonitor.DUMMY); var program = programWithID.program(); diff --git a/src/test/java/ai/reveng/GhidraToServerTypeSerializerTest.java b/src/test/java/ai/reveng/GhidraToServerTypeSerializerTest.java deleted file mode 100644 index b4e6491e..00000000 --- a/src/test/java/ai/reveng/GhidraToServerTypeSerializerTest.java +++ /dev/null @@ -1,70 +0,0 @@ -package ai.reveng; - -import ai.reveng.toolkit.ghidra.core.services.api.GhidraToServerTypeSerializer; -import ghidra.program.database.ProgramBuilder; -import ghidra.program.model.data.CharDataType; -import ghidra.program.model.data.IntegerDataType; -import ghidra.program.model.data.PointerDataType; -import ghidra.program.model.data.StructureDataType; -import ghidra.program.model.listing.Function; -import ghidra.program.model.listing.LocalVariableImpl; -import ghidra.program.model.listing.Parameter; -import ghidra.program.model.listing.ParameterImpl; -import ghidra.program.model.symbol.SourceType; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * Integration test for {@link GhidraToServerTypeSerializer}: verifies a Ghidra function's signature, - * variables, and referenced custom types are serialised into the server's data-type blob for - * push-back (PLU-322). - */ -public class GhidraToServerTypeSerializerTest extends RevEngMockableHeadedIntegrationTest { - - @Test - public void serialisesSignatureVariablesAndStructDependency() throws Exception { - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); - builder.createMemory("mem", "0x4000", 0x100); - var program = builder.getProgram(); - - var intType = IntegerDataType.dataType; - var charType = CharDataType.dataType; - - var struct = new StructureDataType("MyStruct", 0); - struct.add(intType, "field0", null); - builder.addDataType(struct); - - Parameter count = new ParameterImpl("count", intType, program); - Parameter buffer = new ParameterImpl("buffer", new PointerDataType(struct), program); - Function function = builder.createEmptyFunction("process_input", "0x4000", 0x40, intType, count, buffer); - - int txId = program.startTransaction("add local"); - function.addLocalVariable(new LocalVariableImpl("tmp", charType, -0x8, program), SourceType.USER_DEFINED); - program.endTransaction(txId, true); - - long imageBase = program.getImageBase().getOffset(); - var info = GhidraToServerTypeSerializer.buildFunctionInfo(function, imageBase); - - var funcTypes = info.getFuncTypes(); - assertEquals("process_input", funcTypes.getName()); - assertEquals("Function", funcTypes.getArtifactType()); - assertEquals("addr is relative to the image base", - function.getEntryPoint().getOffset() - imageBase, funcTypes.getAddr().longValue()); - - var header = funcTypes.getHeader(); - assertEquals("int", header.getType()); - assertEquals(2, header.getArgs().size()); - assertEquals("count", header.getArgs().get("0").getName()); - assertEquals("int", header.getArgs().get("0").getType()); - assertEquals("buffer", header.getArgs().get("1").getName()); - - assertTrue("stack variable is serialised", - funcTypes.getStackVars().values().stream().anyMatch(v -> v.getName().equals("tmp"))); - - assertTrue("referenced struct is emitted as a dependency", - info.getFuncDeps().stream() - .anyMatch(dep -> dep.getName().equals("MyStruct") && "Struct".equals(dep.getArtifactType()))); - } -} diff --git a/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java b/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java index 81a58219..016b1fd8 100644 --- a/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java +++ b/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java @@ -1,29 +1,29 @@ package ai.reveng; import ai.reveng.invoker.ApiException; -import ai.reveng.model.FunctionDataTypesList; -import ai.reveng.model.FunctionDataTypesListItem; +import ai.reveng.model.BatchFunctionSignatureEntry; +import ai.reveng.model.SignatureParameterEntry; import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisResultsLoaded; import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisStatusChangedEvent; import ai.reveng.toolkit.ghidra.core.services.api.AnalysisOptionsBuilder; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.FunctionSignatureBatch; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.ServerDataTypeReader; import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; import ai.reveng.toolkit.ghidra.core.services.api.types.*; -import ai.reveng.toolkit.ghidra.core.services.api.types.binsync.*; +import com.google.gson.JsonParser; import ai.reveng.toolkit.ghidra.plugins.AnalysisManagementPlugin; import ghidra.framework.Application; import ghidra.framework.ApplicationVersion; -import ghidra.program.database.ProgramBuilder; import ghidra.program.model.data.Undefined; import ghidra.program.model.symbol.SourceType; import ghidra.util.task.TaskMonitor; -import org.jetbrains.annotations.Nullable; import org.junit.Assert; import org.junit.Test; -import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; @@ -49,64 +49,73 @@ public List getFunctionInfo(AnalysisID analysisID) { } @Override - public FunctionDataTypesList listFunctionDataTypesForAnalysis(AnalysisID analysisID, @Nullable List ids) { - - try { - var list = FunctionDataTypesList.fromJson( - """ - { - "total_count": 1, - "total_data_types_count": 1, - "items": [ - { - "completed": true, - "status": "completed", - "data_types": { - "func_types": { - "addr": 1052960, - "size": 22, - "header": { - "name": "portal_name_demangled", - "addr": 1052960, - "type": "int", - "args": { - "0x0": { - "offset": 0, - "name": "ctx", - "type": "ossl_typ.h::EVP_PKEY_CTX *", - "size": 1 - } - } - }, - "name": "portal_name_demangled", - "type": "int", - "artifact_type": "Function" - }, - "func_deps": [ - { - "name": "evp_pkey_ctx_st", - "size": 0, - "members": {}, - "artifact_type": "Struct" - }, - { - "name": "EVP_PKEY_CTX", - "type": "ossl_typ.h::evp_pkey_ctx_st", - "artifact_type": "Typedef" - } - ] - }, - "function_id": 1 - } - ] - } - """ - ); - return list; - - } catch (IOException e) { - throw new RuntimeException(e); - } + public FunctionSignatureBatch listFunctionSignatures(List functionIDs, + boolean includeDataTypes) { + // int portal_name_demangled(EVP_PKEY_CTX *ctx), where EVP_PKEY_CTX is a typedef in + // the "ossl_typ.h" namespace for an (empty) struct. Every reference between the + // types is by data_type_id, which is what the decoder resolves. + var dataTypes = ServerDataTypeReader.readEntries(JsonParser.parseString( + """ + { + "items": [ + { + "data_type_id": 10, + "namespace": "", + "name": "int", + "kind": "BASE", + "size": 4, + "source_type": "AUTO", + "has_definition": false + }, + { + "data_type_id": 11, + "namespace": "ossl_typ.h", + "name": "evp_pkey_ctx_st", + "kind": "STRUCT", + "size": 0, + "source_type": "AUTO", + "has_definition": true, + "definition": { "members": [] } + }, + { + "data_type_id": 12, + "namespace": "ossl_typ.h", + "name": "EVP_PKEY_CTX", + "kind": "TYPEDEF", + "size": 0, + "source_type": "AUTO", + "has_definition": true, + "definition": { "target_data_type_id": 11 } + }, + { + "data_type_id": 13, + "namespace": "", + "name": "EVP_PKEY_CTX *", + "kind": "POINTER", + "size": 8, + "source_type": "AUTO", + "has_definition": true, + "definition": { "pointee_data_type_id": 12 } + } + ] + } + """), "items"); + + var parameter = new SignatureParameterEntry(); + parameter.setName("ctx"); + parameter.setOrdinal(0L); + parameter.setDataTypeId(13L); + parameter.setBitLength(64L); + + var entry = new BatchFunctionSignatureEntry(); + entry.setAnalysisId(1L); + entry.setFunctionId(1L); + entry.setFunctionName("portal_name_demangled"); + entry.setHasSignature(true); + entry.setReturnDataTypeId(10L); + entry.setParameters(List.of(parameter)); + + return new FunctionSignatureBatch(List.of(entry), Map.of(new AnalysisID(1), dataTypes)); } @Override @@ -122,8 +131,6 @@ public FunctionDetails getFunctionDetails(FunctionID id) { 0x4000L, 0x100L, new AnalysisID(1), - "binary_name", - new BinaryHash("dummyhash"), "portal_name_demangled" ); } @@ -133,7 +140,7 @@ public AnalysisID analyse(AnalysisOptionsBuilder options) throws ApiException { return new AnalysisID(1); } }); - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); + var builder = newX64Program(); // Add an example function var exampleFunc = builder.createEmptyFunction(null, "0x4000", 0x100, Undefined.getUndefinedDataType(8)); /// Tell Ghidra that the function signature source is just default, @@ -145,9 +152,9 @@ public AnalysisID analyse(AnalysisOptionsBuilder options) throws ApiException { // We need to also create the memory where the function lives, `getFunctions` doesn't find it otherwise builder.createMemory("test", "0x4000", 0x100); Assert.assertNotNull(builder.getProgram().getFunctionManager().getFunctionAt(exampleFunc.getEntryPoint())); - assert builder.getProgram().getFunctionManager().getFunctionCount() == 1; - assert builder.getProgram().getFunctionManager().getFunctionAt(exampleFunc.getEntryPoint()) != null; - assert builder.getProgram().getFunctionManager().getFunctions(true).hasNext(); + assertEquals(1, builder.getProgram().getFunctionManager().getFunctionCount()); + assertTrue("the created function should be reachable by iteration", + builder.getProgram().getFunctionManager().getFunctions(true).hasNext()); var program = builder.getProgram(); var defaultTool = env.showTool(program); @@ -160,8 +167,10 @@ public AnalysisID analyse(AnalysisOptionsBuilder options) throws ApiException { // We start an analysis to get an Analysis ID associated with the program var id = service.startAnalysis(program, null); - assert service.getKnownProgram(program).isPresent(); - assert service.getAnalysedProgram(program).isEmpty(); + assertTrue("starting an analysis should associate it with the program", + service.getKnownProgram(program).isPresent()); + assertTrue("results should not be available before the analysis completes", + service.getAnalysedProgram(program).isEmpty()); // Register a listener for the results loaded event, to verify that has been fired later AtomicBoolean receivedResultsLoadedEvent = new AtomicBoolean(false); @@ -186,7 +195,8 @@ public AnalysisID analyse(AnalysisOptionsBuilder options) throws ApiException { assertTrue(receivedResultsLoadedEvent.get()); // Check that an analysed program is now known - assert service.getAnalysedProgram(program).isPresent(); + assertTrue("results should be available once the analysis completes", + service.getAnalysedProgram(program).isPresent()); var analyzedProgram = service.getAnalysedProgram(program).get(); // Check that the function names have been updated to the one returned by the portal diff --git a/src/test/java/ai/reveng/PullSignaturesOnSyncTest.java b/src/test/java/ai/reveng/PullSignaturesOnSyncTest.java new file mode 100644 index 00000000..09131892 --- /dev/null +++ b/src/test/java/ai/reveng/PullSignaturesOnSyncTest.java @@ -0,0 +1,218 @@ +package ai.reveng; + +import ai.reveng.model.BatchFunctionSignatureEntry; +import ai.reveng.model.BatchRenameInputBody; +import ai.reveng.model.SignatureParameterEntry; +import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface.AnalysisID; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface.FunctionID; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.FunctionSignatureBatch; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.ServerDataType; +import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; +import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisStatus; +import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionInfo; +import ai.reveng.toolkit.ghidra.core.services.logging.ReaiLoggingService; +import ghidra.program.model.data.CharDataType; +import ghidra.program.model.data.Undefined; +import ghidra.program.model.listing.Function; +import ghidra.program.model.symbol.SourceType; +import ghidra.util.task.TaskMonitor; +import org.junit.Test; + +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/// The portal -> Ghidra half of "Sync With Portal": the signature the portal holds, and the data +/// types it names, land on the matched function. +/// +/// The case that matters most here is the second sync. An applied signature is stamped +/// {@link SourceType#ANALYSIS}, so a guard of "apply only to a default signature" made every sync +/// after the first a no-op and a portal-side edit could never arrive. +public class PullSignaturesOnSyncTest extends RevEngMockableHeadedIntegrationTest { + + private static final long ADDRESS = 0x4000L; + private static final long FUNCTION_ID = 7; + /// The local function carries the portal's name from the start, so the name reconciliation in + /// sync has nothing to do and only the signature half of it is under test. + private static final String FUNCTION_NAME = "target"; + + private static final ReaiLoggingService NOOP_LOG = new ReaiLoggingService() { + @Override public void info(String message) {} + @Override public void warn(String message) {} + @Override public void error(String message) {} + }; + + /// Serves whatever signature the test currently wants the portal to hold. + private static class SignatureApi extends UnimplementedAPI { + /// The name of the return type the portal reports, or null when it holds no signature. + String remoteReturnType; + /// The function name carried on the signature entry, which need not be the local name. + String remoteSignatureName = FUNCTION_NAME; + /// The name the portal gives the single parameter, or null for a parameter it does not name. + String remoteParameterName = "count"; + + @Override + public AnalysisStatus status(AnalysisID analysisID) { + return AnalysisStatus.Complete; + } + + @Override + public List getFunctionInfo(AnalysisID analysisID) { + return List.of(new FunctionInfo( + new FunctionID(FUNCTION_ID), FUNCTION_NAME, FUNCTION_NAME, ADDRESS, 0x100)); + } + + @Override + public FunctionSignatureBatch listFunctionSignatures(List functionIDs, + boolean includeDataTypes) { + if (remoteReturnType == null) { + return FunctionSignatureBatch.empty(); + } + var entry = new BatchFunctionSignatureEntry(); + entry.setAnalysisId(1L); + entry.setFunctionId(FUNCTION_ID); + entry.setFunctionName(remoteSignatureName); + entry.setHasSignature(true); + entry.setReturnDataTypeId(1L); + var parameter = new SignatureParameterEntry(); + parameter.setOrdinal(0L); + parameter.setName(remoteParameterName); + parameter.setDataTypeId(1L); + parameter.setBitLength(32L); + entry.setParameters(List.of(parameter)); + var returnType = new ServerDataType(1L, "", remoteReturnType, ServerDataType.Kind.BASE, + 4L, "AUTO", false, null, null, null); + return new FunctionSignatureBatch(List.of(entry), + Map.of(new AnalysisID(1), List.of(returnType))); + } + + @Override + public void batchRenameFunctions(BatchRenameInputBody request) { + // A name pushback is not what these tests are about; accept and ignore it. + } + } + + private record Fixture(GhidraRevengService service, GhidraRevengService.AnalysedProgram analysed, + Function function) {} + + /// Attach an analysis that holds no signature yet, so the attach-time pull leaves the local + /// signature alone and each test can decide what the portal gains afterwards. + private Fixture attachWithoutRemoteSignature(SignatureApi api) throws Exception { + var service = new GhidraRevengService(api); + var builder = newX64Program(); + builder.createMemory("mem", "0x4000", 0x100); + Function function = builder.createEmptyFunction( + FUNCTION_NAME, "0x4000", 0x100, Undefined.getUndefinedDataType(8)); + var program = builder.getProgram(); + // ProgramBuilder stamps a new function's signature USER_DEFINED, which the pull is required + // to leave alone. A real stripped binary's signature comes from Ghidra's own analysis, and + // that is the case under test: it is neither hand-written nor default, so the old + // "default signatures only" guard skipped exactly these. + program.withTransaction("mark the signature analysis-derived", () -> + function.setSignatureSource(SourceType.ANALYSIS)); + + var programWithID = service.registerAnalysisForProgram(program, new AnalysisID(1)); + service.registerFinishedAnalysisForProgram(programWithID, TaskMonitor.DUMMY); + return new Fixture(service, service.getAnalysedProgram(program).orElseThrow(), function); + } + + @Test + public void appliesThePortalSignature() throws Exception { + var api = new SignatureApi(); + var fixture = attachWithoutRemoteSignature(api); + + api.remoteReturnType = "int"; + var summary = fixture.service().syncAnalysisUpdates(fixture.analysed(), TaskMonitor.DUMMY, NOOP_LOG); + + assertEquals("sync should report the signature it applied", 1, summary.appliedSignatures()); + assertEquals("the portal's return type should be on the local function", + "int", fixture.function().getReturnType().getName()); + } + + @Test + public void appliesTheNewSignatureWhenThePortalChangesItAfterAnEarlierSync() throws Exception { + var api = new SignatureApi(); + var fixture = attachWithoutRemoteSignature(api); + + api.remoteReturnType = "int"; + fixture.service().syncAnalysisUpdates(fixture.analysed(), TaskMonitor.DUMMY, NOOP_LOG); + assertEquals("int", fixture.function().getReturnType().getName()); + + // The analyst edits the signature in the portal; a second sync has to bring that down even + // though the first sync already put a signature on the function. + api.remoteReturnType = "char"; + var summary = fixture.service().syncAnalysisUpdates(fixture.analysed(), TaskMonitor.DUMMY, NOOP_LOG); + + assertEquals("the changed signature should be applied by the second sync", + 1, summary.appliedSignatures()); + assertEquals("char", fixture.function().getReturnType().getName()); + } + + @Test + public void appliesNothingWhenThePortalSignatureIsAlreadyTheLocalOne() throws Exception { + var api = new SignatureApi(); + var fixture = attachWithoutRemoteSignature(api); + + api.remoteReturnType = "int"; + fixture.service().syncAnalysisUpdates(fixture.analysed(), TaskMonitor.DUMMY, NOOP_LOG); + var summary = fixture.service().syncAnalysisUpdates(fixture.analysed(), TaskMonitor.DUMMY, NOOP_LOG); + + assertEquals("an unchanged signature should not be re-applied", 0, summary.appliedSignatures()); + assertEquals("int", fixture.function().getReturnType().getName()); + } + + /// The portal names a function that Ghidra still calls `FUN_...`, and sync applies names in a + /// separate pass. If the signature comparison counts that name difference, every function differs + /// on every sync, is re-applied, and is reported for ever. + @Test + public void appliesNothingASecondTimeWhenOnlyTheSignaturesFunctionNameDiffers() throws Exception { + var api = new SignatureApi(); + var fixture = attachWithoutRemoteSignature(api); + + api.remoteSignatureName = "a_name_ghidra_does_not_have"; + api.remoteReturnType = "int"; + var first = fixture.service().syncAnalysisUpdates(fixture.analysed(), TaskMonitor.DUMMY, NOOP_LOG); + var second = fixture.service().syncAnalysisUpdates(fixture.analysed(), TaskMonitor.DUMMY, NOOP_LOG); + + assertEquals("the changed return type is applied once", 1, first.appliedSignatures()); + assertEquals("and not again, because only the signature's function name differs", + 0, second.appliedSignatures()); + } + + /// A parameter the portal does not name must not read as a difference either: Ghidra names an + /// unnamed parameter `param_N` on apply, so counting that would reintroduce the same loop. + @Test + public void appliesNothingASecondTimeWhenThePortalLeavesAParameterUnnamed() throws Exception { + var api = new SignatureApi(); + var fixture = attachWithoutRemoteSignature(api); + + api.remoteReturnType = "int"; + api.remoteParameterName = null; + fixture.service().syncAnalysisUpdates(fixture.analysed(), TaskMonitor.DUMMY, NOOP_LOG); + var second = fixture.service().syncAnalysisUpdates(fixture.analysed(), TaskMonitor.DUMMY, NOOP_LOG); + + assertEquals(0, second.appliedSignatures()); + } + + @Test + public void leavesASignatureTheAnalystWroteAlone() throws Exception { + var api = new SignatureApi(); + var fixture = attachWithoutRemoteSignature(api); + var function = fixture.function(); + var program = fixture.analysed().program(); + + program.withTransaction("set a user-defined signature", () -> { + function.setReturnType(new CharDataType(), SourceType.USER_DEFINED); + function.setSignatureSource(SourceType.USER_DEFINED); + }); + + api.remoteReturnType = "int"; + var summary = fixture.service().syncAnalysisUpdates(fixture.analysed(), TaskMonitor.DUMMY, NOOP_LOG); + + assertEquals("a signature the analyst set must not be overwritten by the portal's", + 0, summary.appliedSignatures()); + assertEquals("char", function.getReturnType().getName()); + } +} diff --git a/src/test/java/ai/reveng/PushFunctionTypesTest.java b/src/test/java/ai/reveng/PushFunctionTypesTest.java new file mode 100644 index 00000000..9f1cf827 --- /dev/null +++ b/src/test/java/ai/reveng/PushFunctionTypesTest.java @@ -0,0 +1,236 @@ +package ai.reveng; + +import ai.reveng.invoker.ApiException; +import ai.reveng.model.CreateAnalysisDataTypesInputBody; +import ai.reveng.model.UpdateAnalysisDataTypesInputBody; +import ai.reveng.model.UpdateFunctionSignatureInputBody; +import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface.AnalysisID; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface.FunctionID; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.ServerDataType; +import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; +import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisStatus; +import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionInfo; +import ghidra.app.cmd.function.CreateFunctionCmd; +import ghidra.program.model.data.IntegerDataType; +import ghidra.program.model.data.PointerDataType; +import ghidra.program.model.data.StructureDataType; +import ghidra.program.model.listing.Function; +import ghidra.program.model.listing.ParameterImpl; +import ghidra.program.model.symbol.SourceType; +import ghidra.util.task.TaskMonitor; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/// Tests how {@link GhidraRevengService#pushFunctionTypes} composes the two halves of the write +/// path: batch data-type management, then one signature write per function. +public class PushFunctionTypesTest extends RevEngMockableHeadedIntegrationTest { + + private static final long FIRST_ADDRESS = 0x4000L; + private static final long SECOND_ADDRESS = 0x4200L; + + /// Records every write and assigns ids to created types the way the server does. + private static class RecordingApi extends UnimplementedAPI { + final List calls = new ArrayList<>(); + final List signatures = new ArrayList<>(); + final List signedFunctions = new ArrayList<>(); + private long nextId = 100; + + @Override + public AnalysisStatus status(AnalysisID analysisID) { + return AnalysisStatus.Complete; + } + + @Override + public List getFunctionInfo(AnalysisID analysisID) { + return List.of( + new FunctionInfo(new FunctionID(1), "first", "first", FIRST_ADDRESS, 0x100), + new FunctionInfo(new FunctionID(2), "second", "second", SECOND_ADDRESS, 0x100)); + } + + @Override + public List listAnalysisDataTypes(AnalysisID analysisID, long offset, long limit) { + calls.add("list"); + return List.of(); + } + + @Override + public List createAnalysisDataTypes(AnalysisID analysisID, + CreateAnalysisDataTypesInputBody request) { + calls.add("create"); + List created = new ArrayList<>(); + for (var entry : request.getDataTypes()) { + Object instance = entry.getActualInstance(); + created.add(new ServerDataType(nextId++, string(instance, "getNamespace"), + string(instance, "getName"), + ServerDataType.Kind.fromJson(string(instance, "getKind")), + null, "USER", false, null, null, null)); + } + return created; + } + + @Override + public List updateAnalysisDataTypes(AnalysisID analysisID, + UpdateAnalysisDataTypesInputBody request) { + calls.add("update"); + return List.of(); + } + + /// When true, answer a signature write the way the portal answers one for a function it never + /// extracted a signature for. + boolean noExtractedSignature = false; + + @Override + public void updateFunctionSignature(AnalysisID analysisID, FunctionID functionID, + UpdateFunctionSignatureInputBody signature) throws ApiException { + calls.add("signature"); + if (noExtractedSignature) { + throw new ApiException(404, "no signature to edit"); + } + signedFunctions.add(functionID); + signatures.add(signature); + } + + private static String string(Object target, String method) { + try { + Object value = target.getClass().getMethod(method).invoke(target); + return value == null ? null : value.toString(); + } catch (ReflectiveOperationException e) { + throw new RuntimeException(e); + } + } + } + + private record Fixture(GhidraRevengService service, + RecordingApi api, + GhidraRevengService.AnalysedProgram analysedProgram, + List functions) {} + + /// Two functions sharing a struct, so the union of their closures is smaller than the sum. + private Fixture twoFunctionsSharingAStruct() throws Exception { + var api = new RecordingApi(); + var service = new GhidraRevengService(api); + + var builder = newX64Program("push"); + builder.createMemory("code", "0x4000", 0x400); + var program = builder.getProgram(); + + var shared = new StructureDataType("shared_state", 0); + shared.add(new IntegerDataType(), "count", null); + + Function first = builder.createEmptyFunction("first", "0x4000", 0x100, new IntegerDataType(), + new ParameterImpl("state", new PointerDataType(shared), program)); + Function second = builder.createEmptyFunction("second", "0x4200", 0x100, new IntegerDataType(), + new ParameterImpl("state", new PointerDataType(shared), program)); + + var programWithID = service.registerAnalysisForProgram(program, new AnalysisID(1)); + var analysedProgram = service.registerFinishedAnalysisForProgram(programWithID, TaskMonitor.DUMMY); + + // Only what the push itself does is of interest, not what attaching the analysis did. + api.calls.clear(); + return new Fixture(service, api, analysedProgram, List.of(first, second)); + } + + /// The type pass is a batch and the signature write is not, which is the whole reason the two + /// services are separate: several functions cost one resolve over the union of their types and + /// then one write each. + @Test + public void multiFunctionPushResolvesTypesOnceThenWritesEachSignature() throws Exception { + var fixture = twoFunctionsSharingAStruct(); + + int pushed = fixture.service().pushFunctionTypes(fixture.analysedProgram(), fixture.functions()); + + assertEquals("both signatures written", 2, pushed); + assertEquals("one catalogue read for the whole push", 1, count(fixture.api(), "list")); + assertEquals("one create pass over the union of both closures", 1, count(fixture.api(), "create")); + assertEquals("one definition pass", 1, count(fixture.api(), "update")); + assertEquals("and one signature write per function", 2, count(fixture.api(), "signature")); + assertEquals(List.of(new FunctionID(1), new FunctionID(2)), fixture.api().signedFunctions); + + assertEquals("every type is resolved before any signature names one", + List.of("list", "create", "update", "signature", "signature"), fixture.api().calls); + } + + /// The signature names ids the type pass just minted, not names. + @Test + public void writtenSignaturesNameTheIdsTheTypePassMinted() throws Exception { + var fixture = twoFunctionsSharingAStruct(); + fixture.service().pushFunctionTypes(fixture.analysedProgram(), fixture.functions()); + + var signature = fixture.api().signatures.get(0); + assertEquals(1, signature.getParameters().size()); + assertEquals(Long.valueOf(0), signature.getParameters().get(0).getOrdinal()); + assertEquals("state", signature.getParameters().get(0).getName()); + assertNotNull("the parameter's type resolved to an id", + signature.getParameters().get(0).getDataTypeId()); + assertNotNull("so did the return type", signature.getReturnDataTypeId()); + } + + /// A second push of the same functions must resolve everything it created the first time round + /// rather than creating it again — this is the reactive case, which repeats on every edit. + @Test + public void pushingTwiceCreatesNothingTheSecondTime() throws Exception { + var fixture = twoFunctionsSharingAStruct(); + + fixture.service().pushFunctionTypes(fixture.analysedProgram(), fixture.functions()); + int createsAfterFirst = count(fixture.api(), "create"); + fixture.service().pushFunctionTypes(fixture.analysedProgram(), fixture.functions()); + + assertTrue("the first push created the analysis' types", createsAfterFirst > 0); + assertEquals("the second push resolved them", createsAfterFirst, count(fixture.api(), "create")); + } + + /// The reactive push reports what it achieved, because the two halves fail independently: the + /// portal declines a signature write for a function it never extracted one for, and the types + /// have gone up regardless by then. + @Test + public void singleFunctionPushReportsTheSignatureWrite() throws Exception { + var fixture = twoFunctionsSharingAStruct(); + + var outcome = fixture.service().pushFunctionTypes( + fixture.analysedProgram(), fixture.functions().get(0)); + + assertEquals(GhidraRevengService.TypePushOutcome.SIGNATURE_WRITTEN, outcome); + } + + @Test + public void singleFunctionPushReportsTypesOnlyWhenThePortalHasNoExtractedSignature() throws Exception { + var fixture = twoFunctionsSharingAStruct(); + fixture.api().noExtractedSignature = true; + + var outcome = fixture.service().pushFunctionTypes( + fixture.analysedProgram(), fixture.functions().get(0)); + + assertEquals("a declined signature write must not hide the types that were written", + GhidraRevengService.TypePushOutcome.TYPES_ONLY, outcome); + assertTrue("the types still went up", count(fixture.api(), "create") > 0); + } + + @Test + public void singleFunctionPushReportsAFunctionTheAnalysisDoesNotKnow() throws Exception { + var fixture = twoFunctionsSharingAStruct(); + // The mock analysis reports only the two functions the fixture matched, so a third one — + // an unanalysed function, or one Ghidra found and the server did not — has no FunctionID. + var program = fixture.analysedProgram().program(); + var address = program.getAddressFactory().getDefaultAddressSpace().getAddress(0x4300); + Function unmatched = program.withTransaction("add an unmatched function", () -> { + new CreateFunctionCmd("unmatched", address, null, SourceType.USER_DEFINED).applyTo(program); + return program.getFunctionManager().getFunctionAt(address); + }); + assertNotNull("the fixture needs a function outside the analysis", unmatched); + + var outcome = fixture.service().pushFunctionTypes(fixture.analysedProgram(), unmatched); + + assertEquals(GhidraRevengService.TypePushOutcome.NOT_MATCHED, outcome); + } + + private static int count(RecordingApi api, String call) { + return (int) api.calls.stream().filter(call::equals).count(); + } +} diff --git a/src/test/java/ai/reveng/RecentAnalysisDialogTest.java b/src/test/java/ai/reveng/RecentAnalysisDialogTest.java index 46715d3b..aae9693b 100644 --- a/src/test/java/ai/reveng/RecentAnalysisDialogTest.java +++ b/src/test/java/ai/reveng/RecentAnalysisDialogTest.java @@ -6,10 +6,8 @@ import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisStatus; -import ai.reveng.toolkit.ghidra.core.services.api.types.BinaryID; -import ai.reveng.toolkit.ghidra.core.services.api.types.LegacyAnalysisResult; +import ai.reveng.model.AnalysisRecordBody; import docking.DockingWindowManager; -import ghidra.program.database.ProgramBuilder; import org.junit.Test; import javax.swing.*; @@ -34,7 +32,7 @@ public void testSelectRecentAnalysisFiresEventAndUpdatesKnownProgram() throws Ex var service = addMockedService(tool, mockApi); // Create a test program with matching hash - var builder = new ProgramBuilder("test_binary", ProgramBuilder._X64, this); + var builder = newX64Program("test_binary"); builder.createMemory("test", "0x1000", 100); var program = builder.getProgram(); @@ -68,7 +66,7 @@ public void testSelectRecentAnalysisFiresEventAndUpdatesKnownProgram() throws Ex var tableModelField = getInstanceField("recentAnalysesTableModel", foundDialog); assertNotNull("Table model should exist in dialog", tableModelField); @SuppressWarnings("unchecked") - var tableModel = (docking.widgets.table.threaded.ThreadedTableModel) tableModelField; + var tableModel = (docking.widgets.table.threaded.ThreadedTableModel) tableModelField; // Wait for the threaded table model to finish loading waitForTableModel(tableModel); @@ -116,7 +114,7 @@ public void testDialogShowsRecentAnalyses() throws Exception { var mockApi = new RecentAnalysesMockApi(); addMockedService(tool, mockApi); - var builder = new ProgramBuilder("test_binary", ProgramBuilder._X64, this); + var builder = newX64Program("test_binary"); builder.createMemory("test", "0x1000", 100); var program = builder.getProgram(); @@ -156,30 +154,23 @@ static class RecentAnalysesMockApi extends UnimplementedAPI { static final int MOCK_BINARY_ID = 88888; @Override - public List search(TypedApiInterface.BinaryHash hash) { + public List search(TypedApiInterface.BinaryHash hash) { // Return a single recent analysis result return List.of( - new LegacyAnalysisResult( - new TypedApiInterface.AnalysisID(MOCK_ANALYSIS_ID), - new BinaryID(MOCK_BINARY_ID), - "test_binary", - "2024-01-15 10:00:00", - 1, - "binnet-0.2-x86-linux", - hash, - AnalysisStatus.Complete, - 0x0L, // Default image base for x64 programs - "abc123hash" - ) + new AnalysisRecordBody() + .analysisId((long) MOCK_ANALYSIS_ID) + .binaryId((long) MOCK_BINARY_ID) + .binaryName("test_binary") + .creation(java.time.OffsetDateTime.parse("2024-01-15T10:00:00Z")) + .modelId(1L) + .modelName("binnet-0.2-x86-linux") + .sha256Hash(hash.sha256()) + .status(AnalysisStatus.Complete.name()) + .baseAddress(0x0L) // Default image base for x64 programs + .functionBoundariesHash("abc123hash") ); } - @Override - public TypedApiInterface.AnalysisID getAnalysisIDfromBinaryID(BinaryID binaryID) { - assertEquals("Binary ID should match mock data", MOCK_BINARY_ID, binaryID.value()); - return new TypedApiInterface.AnalysisID(MOCK_ANALYSIS_ID); - } - @Override public AnalysisStatus status(TypedApiInterface.AnalysisID analysisID) { assertEquals("Analysis ID should match mock data", MOCK_ANALYSIS_ID, analysisID.id()); diff --git a/src/test/java/ai/reveng/RevEngMockableHeadedIntegrationTest.java b/src/test/java/ai/reveng/RevEngMockableHeadedIntegrationTest.java index 650d9ae4..9f25a0fb 100644 --- a/src/test/java/ai/reveng/RevEngMockableHeadedIntegrationTest.java +++ b/src/test/java/ai/reveng/RevEngMockableHeadedIntegrationTest.java @@ -6,6 +6,7 @@ import ghidra.framework.plugintool.PluginTool; import ghidra.framework.plugintool.mgr.ServiceManager; import ghidra.framework.plugintool.util.PluginException; +import ghidra.program.database.ProgramBuilder; import ghidra.test.AbstractGhidraHeadedIntegrationTest; import ghidra.test.TestEnv; import org.junit.After; @@ -33,6 +34,16 @@ public void tearDown() throws Exception { env.dispose(); } + /// An empty x86-64 program owned by this test, for tests that don't care what it is called. + protected ProgramBuilder newX64Program() throws Exception { + return newX64Program("mock"); + } + + /// An empty x86-64 program owned by this test, for tests whose program name is load bearing. + protected ProgramBuilder newX64Program(String name) throws Exception { + return new ProgramBuilder(name, ProgramBuilder._X64, this); + } + /// This method adds a provided mocked service to the tool diff --git a/src/test/java/ai/reveng/ServerDataTypeDecoderTest.java b/src/test/java/ai/reveng/ServerDataTypeDecoderTest.java new file mode 100644 index 00000000..6772a52a --- /dev/null +++ b/src/test/java/ai/reveng/ServerDataTypeDecoderTest.java @@ -0,0 +1,221 @@ +package ai.reveng; + +import ai.reveng.toolkit.ghidra.core.services.api.GhidraDataTypeEncoder; +import ai.reveng.toolkit.ghidra.core.services.api.ServerDataTypeDecoder; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.ServerDataType; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.ServerDataTypeReader; +import com.google.gson.JsonParser; +import ghidra.program.model.data.DataType; +import ghidra.program.model.data.Enum; +import ghidra.program.model.data.Pointer; +import ghidra.program.model.data.Structure; +import ghidra.program.model.data.TypeDef; +import ghidra.program.model.data.Union; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/// Tests for {@link ServerDataTypeDecoder}, which resolves the server's data types by +/// `data_type_id` rather than by name. +public class ServerDataTypeDecoderTest extends RevEngMockableHeadedIntegrationTest { + + private static List types(String json) { + return ServerDataTypeReader.readEntries(JsonParser.parseString(json), "items"); + } + + /// A struct member reported beyond the struct's declared size must not abort the whole type + /// load: the struct grows to fit instead. + @Test + public void growsStructToFitMemberBeyondDeclaredSize() { + var decoder = ServerDataTypeDecoder.decode(types(""" + {"items": [ + {"data_type_id": 1, "namespace": "", "name": "OversizedStruct", "kind": "STRUCT", + "size": 32, "source_type": "AUTO", "has_definition": true, + "definition": {"members": [ + {"name": "head", "offset": 0, "size": 4, "data_type_id": null, "is_bitfield": false}, + {"name": "tail", "offset": 32, "size": 8, "data_type_id": null, "is_bitfield": false} + ]}} + ]} + """)); + + Structure loaded = (Structure) decoder.typeFor(1L, 0); + assertTrue("struct should have grown to fit the trailing member, length was " + loaded.getLength(), + loaded.getLength() >= 40); + assertEquals("head", loaded.getComponentAt(0).getFieldName()); + assertEquals("tail", loaded.getComponentAt(32).getFieldName()); + } + + /// A struct holding a pointer to itself used to need the dependency list to arrive in a usable + /// order. Resolving by id closes the cycle without any retrying. + @Test + public void resolvesSelfReferentialStruct() { + var decoder = ServerDataTypeDecoder.decode(types(""" + {"items": [ + {"data_type_id": 1, "namespace": "", "name": "Node", "kind": "STRUCT", + "size": 8, "source_type": "AUTO", "has_definition": true, + "definition": {"members": [ + {"name": "next", "offset": 0, "size": 8, "data_type_id": 2, "is_bitfield": false} + ]}}, + {"data_type_id": 2, "namespace": "", "name": "Node *", "kind": "POINTER", + "size": 8, "source_type": "AUTO", "has_definition": true, + "definition": {"pointee_data_type_id": 1}} + ]} + """)); + + Structure node = (Structure) decoder.typeFor(1L, 0); + DataType next = node.getComponentAt(0).getDataType(); + assertTrue("member should be a pointer, was " + next.getClass(), next instanceof Pointer); + assertEquals(node, ((Pointer) next).getDataType()); + } + + /// The types arrive in whatever order the server lists them, so a typedef may be read before its + /// target. Both passes work off the id map, so the order does not matter. + @Test + public void resolvesTypedefDeclaredBeforeItsTarget() { + var decoder = ServerDataTypeDecoder.decode(types(""" + {"items": [ + {"data_type_id": 5, "namespace": "sys", "name": "handle_t", "kind": "TYPEDEF", + "size": 8, "source_type": "AUTO", "has_definition": true, + "definition": {"target_data_type_id": 6}}, + {"data_type_id": 6, "namespace": "sys", "name": "handle_s", "kind": "STRUCT", + "size": 4, "source_type": "AUTO", "has_definition": true, + "definition": {"members": [ + {"name": "fd", "offset": 0, "size": 4, "data_type_id": null, "is_bitfield": false} + ]}} + ]} + """)); + + DataType typedef = decoder.typeFor(5L, 0); + assertTrue("expected a typedef, got " + typedef.getClass(), typedef instanceof TypeDef); + assertEquals("handle_t", typedef.getName()); + assertEquals("/sys", typedef.getCategoryPath().getPath()); + assertEquals(decoder.typeFor(6L, 0), ((TypeDef) typedef).getDataType()); + } + + /// Enum constants stay strings on the wire because they can be negative or exceed what a signed + /// 64-bit value holds. + @Test + public void decodesEnumValuesIncludingNegativeAndUnsigned64() { + var decoder = ServerDataTypeDecoder.decode(types(""" + {"items": [ + {"data_type_id": 3, "namespace": "", "name": "Flags", "kind": "ENUM", + "size": 8, "source_type": "AUTO", "has_definition": true, + "definition": {"values": [ + {"name": "NEG", "value": "-1"}, + {"name": "ZERO", "value": "0"}, + {"name": "MAX_U64", "value": "18446744073709551615"} + ]}} + ]} + """)); + + Enum flags = (Enum) decoder.typeFor(3L, 0); + assertEquals(-1L, flags.getValue("NEG")); + assertEquals(0L, flags.getValue("ZERO")); + assertEquals(-1L, flags.getValue("MAX_U64")); + } + + @Test + public void decodesUnionMembers() { + var decoder = ServerDataTypeDecoder.decode(types(""" + {"items": [ + {"data_type_id": 7, "namespace": "", "name": "Value", "kind": "UNION", + "size": 8, "source_type": "AUTO", "has_definition": true, + "definition": {"members": [ + {"name": "as_int", "offset": 0, "size": 4, "data_type_id": null, "is_bitfield": false}, + {"name": "as_ptr", "offset": 0, "size": 8, "data_type_id": null, "is_bitfield": false} + ]}} + ]} + """)); + + Union value = (Union) decoder.typeFor(7L, 0); + assertEquals(2, value.getNumComponents()); + assertEquals("as_int", value.getComponent(0).getFieldName()); + assertEquals("as_ptr", value.getComponent(1).getFieldName()); + } + + /// The server only ships the type closure it knows about, so a reference can dangle. That has to + /// degrade to a filler rather than fail the decode. + @Test + public void referencedButUndefinedTypeBecomesUndefinedFiller() { + var decoder = ServerDataTypeDecoder.decode(types(""" + {"items": [ + {"data_type_id": 1, "namespace": "", "name": "Holder", "kind": "STRUCT", + "size": 4, "source_type": "AUTO", "has_definition": true, + "definition": {"members": [ + {"name": "missing", "offset": 0, "size": 4, "data_type_id": 999, "is_bitfield": false} + ]}} + ]} + """)); + + Structure holder = (Structure) decoder.typeFor(1L, 0); + assertNotNull(holder.getComponentAt(0)); + assertEquals("missing", holder.getComponentAt(0).getFieldName()); + // An id nobody defined is still answered, with a same-sized placeholder. + assertEquals(4, decoder.typeFor(999L, 4).getLength()); + } + + /// What lets the write path keep a namespace it found on a Ghidra category path is that a type + /// pulled from server namespace `X` lands in a category the encoder maps back to exactly `X`. + /// + /// That holds for the kinds the decoder builds with an explicit category — struct, union, enum, + /// typedef and function definition. It does not for pointers and arrays, which Ghidra derives + /// from their pointee and element and which therefore carry that type's category rather than + /// their own, nor for base types, which are looked up as Ghidra built-ins by name. Those kinds + /// never round-trip a namespace at all, which is why the write path treats a namespace the + /// analysis does not already use as a local one. + @Test + public void namedKindsRoundTripTheirNamespaceThroughTheCategoryPath() { + var entries = types(""" + {"items": [ + {"data_type_id": 1, "namespace": "DWARF::stdio.h", "name": "FILE", "kind": "STRUCT", + "size": 8, "source_type": "AUTO", "has_definition": true, + "definition": {"members": [ + {"name": "fd", "offset": 0, "size": 4, "data_type_id": null, "is_bitfield": false}]}}, + {"data_type_id": 2, "namespace": "sys", "name": "handle_t", "kind": "TYPEDEF", + "size": 8, "source_type": "AUTO", "has_definition": true, + "definition": {"target_data_type_id": 1}}, + {"data_type_id": 3, "namespace": "flags", "name": "Level", "kind": "ENUM", + "size": 4, "source_type": "AUTO", "has_definition": true, + "definition": {"values": [{"name": "LOW", "value": "1"}]}}, + {"data_type_id": 4, "namespace": "u", "name": "Value", "kind": "UNION", + "size": 8, "source_type": "AUTO", "has_definition": true, + "definition": {"members": [ + {"name": "as_int", "offset": 0, "size": 4, "data_type_id": null, "is_bitfield": false}]}}, + {"data_type_id": 5, "namespace": "api", "name": "callback", "kind": "FUNCTION_DEFINITION", + "size": 0, "source_type": "AUTO", "has_definition": true, + "definition": {"return_data_type_id": null, "parameters": []}}, + {"data_type_id": 6, "namespace": "", "name": "Local", "kind": "STRUCT", + "size": 4, "source_type": "USER", "has_definition": true, + "definition": {"members": [ + {"name": "x", "offset": 0, "size": 4, "data_type_id": null, "is_bitfield": false}]}} + ]} + """); + var decoder = ServerDataTypeDecoder.decode(entries); + + for (ServerDataType entry : entries) { + DataType decoded = decoder.typeFor(entry.id(), 0); + assertEquals("namespace of " + entry.name() + " must survive the round trip", + entry.namespace(), + GhidraDataTypeEncoder.keyOf(decoded).namespace()); + assertEquals("and so must its name", + entry.name(), GhidraDataTypeEncoder.keyOf(decoded).name()); + } + } + + /// BASE types carry no definition — only a name, which has to resolve to a Ghidra built-in. + @Test + public void resolvesBaseTypesByName() { + var decoder = ServerDataTypeDecoder.decode(types(""" + {"items": [ + {"data_type_id": 1, "namespace": "", "name": "int", "kind": "BASE", + "size": 4, "source_type": "AUTO", "has_definition": false} + ]} + """)); + + assertEquals("int", decoder.typeFor(1L, 0).getName()); + } +} diff --git a/src/test/java/ai/reveng/SimilarFunctionsWindowTest.java b/src/test/java/ai/reveng/SimilarFunctionsWindowTest.java index 29e2334b..6a48a642 100644 --- a/src/test/java/ai/reveng/SimilarFunctionsWindowTest.java +++ b/src/test/java/ai/reveng/SimilarFunctionsWindowTest.java @@ -10,7 +10,6 @@ import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionInfo; import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionMatch; import ai.reveng.toolkit.ghidra.plugins.BinarySimilarityPlugin; -import ghidra.program.database.ProgramBuilder; import ghidra.program.model.data.Undefined; import ghidra.program.model.listing.Function; import ghidra.util.task.TaskMonitor; @@ -21,6 +20,7 @@ import java.util.Map; import static org.junit.Assert.*; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.FunctionSignatureBatch; public class SimilarFunctionsWindowTest extends RevEngMockableHeadedIntegrationTest { @@ -34,7 +34,7 @@ public void testSimilarFunctionsWindowBasics() throws Exception { var binarySimilarityPlugin = env.addPlugin(BinarySimilarityPlugin.class); // Create a program with two functions - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); + var builder = newX64Program(); var func1 = builder.createEmptyFunction(null, "0x1000", 10, Undefined.getUndefinedDataType(4)); var func2 = builder.createEmptyFunction(null, "0x2000", 10, Undefined.getUndefinedDataType(4)); @@ -96,7 +96,7 @@ public void testSimilarFunctionsWindowCaching() throws Exception { env.addPlugin(BinarySimilarityPlugin.class); - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); + var builder = newX64Program(); var func1 = builder.createEmptyFunction(null, "0x1000", 10, Undefined.getUndefinedDataType(4)); var func2 = builder.createEmptyFunction(null, "0x2000", 10, Undefined.getUndefinedDataType(4)); @@ -141,7 +141,7 @@ public void testSimilarFunctionsWindowTableSelection() throws Exception { env.addPlugin(BinarySimilarityPlugin.class); - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); + var builder = newX64Program(); var func1 = builder.createEmptyFunction(null, "0x1000", 10, Undefined.getUndefinedDataType(4)); var programWithID = service.analyse(builder.getProgram(), null, TaskMonitor.DUMMY); @@ -203,9 +203,9 @@ public List getFunctionInfo(TypedApiInterface.AnalysisID analysisI } @Override - public Basic getAnalysisBasicInfo(TypedApiInterface.AnalysisID analysisID) throws ApiException { - var basic = new Basic(); - basic.setModelId(1); + public AnalysisBasicInfoOutputBody getAnalysisBasicInfo(TypedApiInterface.AnalysisID analysisID) throws ApiException { + var basic = new AnalysisBasicInfoOutputBody(); + basic.setModelId(1L); basic.setSha256Hash("abc123"); basic.setBinaryName("test_binary"); return basic; @@ -282,11 +282,10 @@ public List getAssembly(TypedApiInterface.FunctionID functionID) throws } @Override - public FunctionDataTypesList listFunctionDataTypesForFunctions(List functionIDs) { - // Return empty list - no signatures available in mock - var result = new FunctionDataTypesList(); - result.setItems(List.of()); - return result; + public FunctionSignatureBatch listFunctionSignatures(List functionIDs, + boolean includeDataTypes) { + // No signatures available in mock + return FunctionSignatureBatch.empty(); } } } diff --git a/src/test/java/ai/reveng/SyncMarkingTest.java b/src/test/java/ai/reveng/SyncMarkingTest.java index 961e05b2..1485439d 100644 --- a/src/test/java/ai/reveng/SyncMarkingTest.java +++ b/src/test/java/ai/reveng/SyncMarkingTest.java @@ -1,23 +1,20 @@ package ai.reveng; -import ai.reveng.model.FunctionDataTypesList; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisStatus; import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionInfo; -import ghidra.program.database.ProgramBuilder; import ghidra.program.model.data.Undefined; import ghidra.program.model.listing.Function; -import org.jetbrains.annotations.Nullable; import org.junit.Test; -import java.io.IOException; import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.FunctionSignatureBatch; public class SyncMarkingTest extends RevEngMockableHeadedIntegrationTest { @@ -41,16 +38,13 @@ public List getFunctionInfo(TypedApiInterface.AnalysisID analysisI } @Override - public FunctionDataTypesList listFunctionDataTypesForAnalysis(TypedApiInterface.AnalysisID analysisID, @Nullable List ids) { - try { - return FunctionDataTypesList.fromJson("{\"total_count\":0,\"total_data_types_count\":0,\"items\":[]}"); - } catch (IOException e) { - throw new RuntimeException(e); - } + public FunctionSignatureBatch listFunctionSignatures(List functionIDs, + boolean includeDataTypes) { + return FunctionSignatureBatch.empty(); } }); - var builder = new ProgramBuilder("mock", ProgramBuilder._X64, this); + var builder = newX64Program(); builder.createMemory("matched", "0x4000", 0x100); builder.createMemory("unmatched", "0x5000", 0x100); Function matchedFunc = builder.createEmptyFunction(null, "0x4000", 0x100, Undefined.getUndefinedDataType(8)); diff --git a/src/test/java/ai/reveng/TestAnalysisLogComponent.java b/src/test/java/ai/reveng/TestAnalysisLogComponent.java index 0e65ee6c..a8c80870 100644 --- a/src/test/java/ai/reveng/TestAnalysisLogComponent.java +++ b/src/test/java/ai/reveng/TestAnalysisLogComponent.java @@ -7,7 +7,6 @@ import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisStatusChangedEvent; import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; import ai.reveng.toolkit.ghidra.core.services.api.types.*; -import ghidra.program.database.ProgramBuilder; import ghidra.program.model.listing.Program; import ghidra.util.task.Task; import ghidra.util.task.TaskMonitorComponent; @@ -22,7 +21,7 @@ public class TestAnalysisLogComponent extends RevEngMockableHeadedIntegrationTest { private GhidraRevengService.ProgramWithID getPlaceHolderID() throws Exception{ - var builder = new ghidra.program.database.ProgramBuilder("mock", ProgramBuilder._X64, this); + var builder = newX64Program(); // Add an example function var program = builder.getProgram(); return new GhidraRevengService.ProgramWithID( diff --git a/src/test/java/ai/reveng/TestUpgradeFromBinaryID.java b/src/test/java/ai/reveng/TestUpgradeFromBinaryID.java deleted file mode 100644 index 705ae2a6..00000000 --- a/src/test/java/ai/reveng/TestUpgradeFromBinaryID.java +++ /dev/null @@ -1,64 +0,0 @@ -package ai.reveng; - -import ai.reveng.invoker.ApiException; -import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; -import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; -import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisStatus; -import ai.reveng.toolkit.ghidra.core.services.api.types.BinaryID; -import ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage; -import ghidra.program.database.ProgramBuilder; -import org.junit.Assert; -import org.junit.Test; - -import java.util.Optional; - -@SuppressWarnings("deprecation") // This test ensures upgrade from deprecated Binary ID usage -public class TestUpgradeFromBinaryID extends RevEngMockableHeadedIntegrationTest { - - /// Tests the logic for handling a program that has only a binary ID stored in its properties - @Test - public void test() throws Exception { - var builder = new ProgramBuilder("upgrade-test", ProgramBuilder._X64, this); - var tId = builder.getProgram().startTransaction("Set Binary ID"); - builder.getProgram() - .getOptions(ReaiPluginPackage.REAI_OPTIONS_CATEGORY) - .setLong(ReaiPluginPackage.OPTION_KEY_BINID, 1); - builder.getProgram().endTransaction(tId, true); - addMockedService(env.getTool(), new UnimplementedAPI() { - @Override - public AnalysisID getAnalysisIDfromBinaryID(BinaryID binaryID) { - if (binaryID.value() == 1) { - return new AnalysisID(42); - } - return null; - } - - @Override - public AnalysisStatus status(BinaryID binID) throws ApiException { - if (binID.value() == 1) { - return AnalysisStatus.Complete; - } - return AnalysisStatus.Error; - } - }); - var program = builder.getProgram(); - - env.open(program); - - var service = env.getTool().getService(GhidraRevengService.class); - Optional analysisID = service.getKnownProgram(program); - Assert.assertTrue(analysisID.isPresent()); - Assert.assertEquals(42, analysisID.get().analysisID().id()); - // Verify that after opening, the program has the Analysis ID set and the Binary ID removed - Assert.assertEquals(-1, program.getOptions(ReaiPluginPackage.REAI_OPTIONS_CATEGORY).getLong(ReaiPluginPackage.OPTION_KEY_BINID, -1)); - Assert.assertEquals(42, program.getOptions(ReaiPluginPackage.REAI_OPTIONS_CATEGORY).getLong(ReaiPluginPackage.OPTION_KEY_ANALYSIS_ID, -1)); - - - - - - - - - } -} diff --git a/src/test/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/aidecompiler/AIDecompTokenResolutionTest.java b/src/test/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/aidecompiler/AIDecompTokenResolutionTest.java index 9e999b9f..6a944233 100644 --- a/src/test/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/aidecompiler/AIDecompTokenResolutionTest.java +++ b/src/test/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/aidecompiler/AIDecompTokenResolutionTest.java @@ -1,15 +1,17 @@ package ai.reveng.toolkit.ghidra.binarysimilarity.ui.aidecompiler; -import ai.reveng.model.AIDecompFunctionMapping; -import ai.reveng.model.ReplacementValue; -import ai.reveng.model.TokenisedData; +import ai.reveng.model.GetTokensResponse; +import ai.reveng.model.RenderedToken; +import ai.reveng.model.Token; import org.junit.Test; import java.util.LinkedHashMap; import java.util.Map; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; /** * Unit tests for the identifier → token resolution that backs the "rename variable/type" edit, @@ -17,6 +19,45 @@ */ public class AIDecompTokenResolutionTest { + /// A double-click has to yield the whole identifier. Swing's word iterator breaks at an + /// underscore, so it answered "param" for "param_1" — a word on no line of the decompilation, so + /// the rename declined and every name carrying an underscore was unreachable. + @Test + public void identifierAt_takesTheWholeIdentifierIncludingUnderscores() { + String line = " int param_1,"; + + assertEquals("param_1", AIDecompilationdWindow.identifierAt(line, 8)); // first character + assertEquals("param_1", AIDecompilationdWindow.identifierAt(line, 11)); // inside + assertEquals("param_1", AIDecompilationdWindow.identifierAt(line, 13)); // the underscore + assertEquals("param_1", AIDecompilationdWindow.identifierAt(line, 14)); // the digit + assertEquals("int", AIDecompilationdWindow.identifierAt(line, 5)); + } + + @Test + public void identifierAt_handlesALeadingUnderscoreRun() { + String line = " return f(__rustc_debug_gdb_scripts_section__, 0);"; + + assertEquals("__rustc_debug_gdb_scripts_section__", + AIDecompilationdWindow.identifierAt(line, 20)); + } + + /// The identifier stops where the rendered name's punctuation begins, which is what makes + /// "lang_start<()>" reachable by its identifier alone. + @Test + public void identifierAt_stopsAtPunctuation() { + String line = " return lang_start<()>(main, 0);"; + + assertEquals("lang_start", AIDecompilationdWindow.identifierAt(line, 13)); + assertEquals("main", AIDecompilationdWindow.identifierAt(line, 26)); + } + + @Test + public void identifierAt_returnsNullOffAnIdentifier() { + assertNull(AIDecompilationdWindow.identifierAt(" ", 3)); + assertNull(AIDecompilationdWindow.identifierAt(" int x;", -1)); + assertNull(AIDecompilationdWindow.identifierAt(null, 0)); + } + @Test public void indexOfIdentifier_returnsPositionAmongIdentifiers() { String line = "int result = compute(value);"; @@ -36,90 +77,245 @@ public void indexOfIdentifier_missingWordReturnsMinusOne() { assertEquals(-1, AIDecompilationdWindow.indexOfIdentifier("int result = 1;", "missing")); } + @Test + public void effectiveValues_overrideWinsOverPredictedValue() { + var tokenValues = tokenValues("int TOKEN_A = TOKEN_B;", + Map.of("TOKEN_A", "result", "TOKEN_B", "value"), + Map.of("TOKEN_A", "myResult")); + + assertEquals(Map.of("TOKEN_A", "myResult", "TOKEN_B", "value"), + AIDecompilationdWindow.effectiveValues(tokenValues)); + } + + @Test + public void effectiveValues_unwrapsARenderedTokenToItsValue() { + var data = new GetTokensResponse(); + data.setAiDecomp("int TOKEN_A = 1;"); + data.setPlaceholderToRenderedToken(Map.of("TOKEN_A", new RenderedToken() + .value("result") + .kind(RenderedToken.KindEnum.LOCAL) + .dataTypeId(42L) + .vaddr(0x1000L))); + + // Only the rendered value is taken; the kind and ids the token also carries are not used. + assertEquals(Map.of("TOKEN_A", "result"), AIDecompilationdWindow.effectiveValues(data)); + } + + @Test + public void effectiveValues_toleratesTheNullMapsReturnedBeforeARunSucceeds() { + var tokenValues = new GetTokensResponse(); + tokenValues.setAiDecomp(""); + assertTrue(AIDecompilationdWindow.effectiveValues(tokenValues).isEmpty()); + } + + @Test + public void effectiveValues_keepsAnOverrideForATokenMissingFromTheRenderedMap() { + var tokenValues = tokenValues("int TOKEN_A = 1;", Map.of(), Map.of("TOKEN_A", "myResult")); + + assertEquals(Map.of("TOKEN_A", "myResult"), AIDecompilationdWindow.effectiveValues(tokenValues)); + } + @Test public void resolveToken_matchesTokenAtSameIdentifierPosition() { - var mapping = new AIDecompFunctionMapping(); - mapping.setUnmatchedVars(vars(Map.of( - "TOKEN_A", "result", - "TOKEN_B", "value"))); - var tokenised = tokenised("int TOKEN_A = compute(TOKEN_B);", mapping); + var tokenValues = tokenValues("int TOKEN_A = compute(TOKEN_B);", + Map.of("TOKEN_A", "result", "TOKEN_B", "value"), + Map.of()); // "result" is the identifier at index 1 in the source line. - assertEquals("TOKEN_A", AIDecompilationdWindow.resolveToken(tokenised, 0, 1, "result")); + assertEquals("TOKEN_A", AIDecompilationdWindow.resolveToken(tokenValues, 0, 1, "result")); // "value" is the identifier at index 3. - assertEquals("TOKEN_B", AIDecompilationdWindow.resolveToken(tokenised, 0, 3, "value")); + assertEquals("TOKEN_B", AIDecompilationdWindow.resolveToken(tokenValues, 0, 3, "value")); } @Test public void resolveToken_userOverrideTakesPrecedenceOverPredictedValue() { - var mapping = new AIDecompFunctionMapping(); - mapping.setUnmatchedVars(vars(Map.of("TOKEN_A", "result"))); - mapping.setUserOverrideMappings(Map.of("TOKEN_A", "myResult")); - var tokenised = tokenised("int TOKEN_A = 1;", mapping); + var tokenValues = tokenValues("int TOKEN_A = 1;", + Map.of("TOKEN_A", "result"), + Map.of("TOKEN_A", "myResult")); // The displayed name is the override, so that is what the user double-clicks. - assertEquals("TOKEN_A", AIDecompilationdWindow.resolveToken(tokenised, 0, 1, "myResult")); + assertEquals("TOKEN_A", AIDecompilationdWindow.resolveToken(tokenValues, 0, 1, "myResult")); // The stale predicted value no longer resolves. - assertNull(AIDecompilationdWindow.resolveToken(tokenised, 0, 1, "result")); + assertNull(AIDecompilationdWindow.resolveToken(tokenValues, 0, 1, "result")); } @Test - public void resolveToken_resolvesTypeCategory() { - var mapping = new AIDecompFunctionMapping(); - mapping.setUnmatchedCustomTypes(vars(Map.of("TOKEN_T", "MyStruct"))); - var tokenised = tokenised("TOKEN_T *p = 0;", mapping); + public void resolveToken_resolvesTypeToken() { + var tokenValues = tokenValues("TOKEN_T *p = 0;", Map.of("TOKEN_T", "MyStruct"), Map.of()); - assertEquals("TOKEN_T", AIDecompilationdWindow.resolveToken(tokenised, 0, 0, "MyStruct")); + assertEquals("TOKEN_T", AIDecompilationdWindow.resolveToken(tokenValues, 0, 0, "MyStruct")); } @Test public void resolveToken_fallsBackToUniqueValueMatchWhenPositionMisses() { - var mapping = new AIDecompFunctionMapping(); - mapping.setUnmatchedVars(vars(Map.of("TOKEN_X", "foo"))); // Position lookup misses (identIndex out of range for the tokenised line), but there is // exactly one token whose effective value is "foo", so it still resolves. - var tokenised = tokenised("return 0;", mapping); + var tokenValues = tokenValues("return 0;", Map.of("TOKEN_X", "foo"), Map.of()); + + assertEquals("TOKEN_X", AIDecompilationdWindow.resolveToken(tokenValues, 0, 99, "foo")); + } + + @Test + public void resolveToken_fallbackUsesOverriddenValueNotPredictedValue() { + var tokenValues = tokenValues("return 0;", + Map.of("TOKEN_X", "foo"), + Map.of("TOKEN_X", "bar")); - assertEquals("TOKEN_X", AIDecompilationdWindow.resolveToken(tokenised, 0, 99, "foo")); + assertEquals("TOKEN_X", AIDecompilationdWindow.resolveToken(tokenValues, 0, 99, "bar")); + assertNull(AIDecompilationdWindow.resolveToken(tokenValues, 0, 99, "foo")); } @Test public void resolveToken_ambiguousValueMatchReturnsNull() { - var mapping = new AIDecompFunctionMapping(); - mapping.setUnmatchedVars(vars(Map.of( - "TOKEN_X", "foo", - "TOKEN_Y", "foo"))); - var tokenised = tokenised("return 0;", mapping); + var tokenValues = tokenValues("return 0;", + Map.of("TOKEN_X", "foo", "TOKEN_Y", "foo"), + Map.of()); - assertNull(AIDecompilationdWindow.resolveToken(tokenised, 0, 99, "foo")); + assertNull(AIDecompilationdWindow.resolveToken(tokenValues, 0, 99, "foo")); } @Test public void resolveToken_unknownIdentifierReturnsNull() { - var mapping = new AIDecompFunctionMapping(); - mapping.setUnmatchedVars(vars(Map.of("TOKEN_A", "result"))); - var tokenised = tokenised("int TOKEN_A = 1;", mapping); + var tokenValues = tokenValues("int TOKEN_A = 1;", Map.of("TOKEN_A", "result"), Map.of()); - assertNull(AIDecompilationdWindow.resolveToken(tokenised, 0, 0, "int")); + assertNull(AIDecompilationdWindow.resolveToken(tokenValues, 0, 0, "int")); } @Test - public void resolveToken_nullMappingReturnsNull() { - var tokenised = new TokenisedData(); - tokenised.setTokenisedDecompilation("int TOKEN_A = 1;"); - assertNull(AIDecompilationdWindow.resolveToken(tokenised, 0, 1, "result")); + public void resolveToken_noTokenValuesReturnsNull() { + var tokenValues = new GetTokensResponse(); + tokenValues.setAiDecomp("int TOKEN_A = 1;"); + assertNull(AIDecompilationdWindow.resolveToken(tokenValues, 0, 1, "result")); } - private static Map vars(Map tokenToValue) { - var result = new LinkedHashMap(); - tokenToValue.forEach((token, value) -> result.put(token, new ReplacementValue().value(value))); - return result; + /// A real document from the tokens endpoint: a Rust `main`, with two parameters, an invented type + /// name, the function itself, and two called functions. Every identifier the analyst can + /// double-click in it is accounted for here. + private static GetTokensResponse rustMainTokens() { + var data = new GetTokensResponse(); + data.setAiDecomp("int\n_FUNC0_(\n int _PARAM0_,\n _TYPE0_ *_PARAM1_\n)\n{\n" + + " return _FCN0_(_FCN1_, __rustc_debug_gdb_scripts_section__, _PARAM0_, _PARAM1_, 0);\n}"); + var rendered = new LinkedHashMap(); + rendered.put("_FCN0_", new RenderedToken().value("lang_start<()>").functionId(1380166L)); + rendered.put("_FCN1_", new RenderedToken().value("main").functionId(1380659L)); + rendered.put("_FUNC0_", new RenderedToken().value("main").functionId(1380664L)); + rendered.put("_PARAM0_", new RenderedToken().value("param_1")); + rendered.put("_PARAM1_", new RenderedToken().value("param_2")); + rendered.put("_TYPE0_", new RenderedToken().value("Type_1")); + data.setPlaceholderToRenderedToken(rendered); + return data; + } + + /// The parameter is the case that has to work: " int param_1," is the third line of the + /// decompilation, and "param_1" is the second identifier on it. + @Test + public void realDocument_resolvesAndAllowsAParameter() { + var tokens = rustMainTokens(); + + assertEquals("_PARAM0_", AIDecompilationdWindow.resolveToken(tokens, 2, 1, "param_1")); + assertTrue("a parameter carries no id, so the override owns its name", + AIDecompilationdWindow.isRenameable(tokens, "_PARAM0_")); + + // And from the body: "return" is an identifier too, so on + // "return lang_start<()>(main, __rustc..., param_1, param_2, 0)" param_1 is the fifth. + assertEquals("_PARAM0_", AIDecompilationdWindow.resolveToken(tokens, 6, 4, "param_1")); + } + + /// The type name the decompilation invented has no data_type_id, so it is renameable too. + @Test + public void realDocument_resolvesAndAllowsAnInventedTypeName() { + var tokens = rustMainTokens(); + + assertEquals("_TYPE0_", AIDecompilationdWindow.resolveToken(tokens, 3, 0, "Type_1")); + assertTrue(AIDecompilationdWindow.isRenameable(tokens, "_TYPE0_")); + } + + /// A called function resolves, and is then refused: this is the token that produced the 400. + @Test + public void realDocument_refusesACalledFunction() { + var tokens = rustMainTokens(); + + assertEquals("_FCN0_", AIDecompilationdWindow.resolveToken(tokens, 6, 1, "lang_start")); + assertFalse("it carries a function_id, so it is renamed on the function", + AIDecompilationdWindow.isRenameable(tokens, "_FCN0_")); + } + + /// Two tokens render as "main" — the function itself and a call to it — so the position on the + /// line is what tells them apart, and it is consulted before the ambiguity check. Either way the + /// gate then refuses both, because a function is renamed on the function. + @Test + public void realDocument_distinguishesTheFunctionFromTheCallToItByPosition() { + var tokens = rustMainTokens(); + + assertEquals("the signature line names the function itself", + "_FUNC0_", AIDecompilationdWindow.resolveToken(tokens, 1, 0, "main")); + assertEquals("the call in the body names the callee", + "_FCN1_", AIDecompilationdWindow.resolveToken(tokens, 6, 2, "main")); + assertFalse(AIDecompilationdWindow.isRenameable(tokens, "_FUNC0_")); + assertFalse(AIDecompilationdWindow.isRenameable(tokens, "_FCN1_")); + } + + /// A keyword is no token at all, and nothing is invented for it. + @Test + public void realDocument_declinesAKeyword() { + assertNull(AIDecompilationdWindow.resolveToken(rustMainTokens(), 2, 0, "int")); + } + + /// A token with no id is a name the decompilation invented — a parameter, a local — and the + /// override endpoint is the only place it exists. + @Test + public void isRenameable_allowsATokenThatCarriesNoId() { + assertTrue(AIDecompilationdWindow.isRenameable( + tokenWithIds(null, null, null), "TOKEN_A")); + } + + /// A token that carries an id refers to something named outside this decompilation, and renaming + /// it is a different call. The overrides endpoint answers one for a function with a 400. + @Test + public void isRenameable_refusesATokenThatCarriesAnId() { + assertFalse("a data type is renamed on the type", + AIDecompilationdWindow.isRenameable(tokenWithIds(42L, null, null), "TOKEN_A")); + assertFalse("a function is renamed on the function", + AIDecompilationdWindow.isRenameable(tokenWithIds(null, 7L, null), "TOKEN_A")); + assertFalse("and so is an imported one", + AIDecompilationdWindow.isRenameable(tokenWithIds(null, null, 9L), "TOKEN_A")); + } + + @Test + public void isRenameable_refusesATokenTheResponseNeverMentioned() { + assertFalse(AIDecompilationdWindow.isRenameable( + tokenWithIds(null, null, null), "TOKEN_MISSING")); + assertFalse("no rendered tokens at all", + AIDecompilationdWindow.isRenameable(new GetTokensResponse(), "TOKEN_A")); + } + + /// One rendered token, TOKEN_A, carrying the given ids. + private static GetTokensResponse tokenWithIds(Long dataTypeId, Long functionId, Long importedFunctionId) { + var data = new GetTokensResponse(); + data.setAiDecomp("TOKEN_A = 1;"); + var rendered = new LinkedHashMap(); + rendered.put("TOKEN_A", new RenderedToken() + .value("name") + .dataTypeId(dataTypeId) + .functionId(functionId) + .importedFunctionId(importedFunctionId)); + data.setPlaceholderToRenderedToken(rendered); + return data; } - private static TokenisedData tokenised(String tokenisedDecompilation, AIDecompFunctionMapping mapping) { - var data = new TokenisedData(); - data.setTokenisedDecompilation(tokenisedDecompilation); - data.setFunctionMapping(mapping); + private static GetTokensResponse tokenValues(String aiDecomp, + Map renderedValues, + Map userOverrides) { + var data = new GetTokensResponse(); + data.setAiDecomp(aiDecomp); + var rendered = new LinkedHashMap(); + renderedValues.forEach((placeholder, value) -> + rendered.put(placeholder, new RenderedToken().value(value))); + data.setPlaceholderToRenderedToken(rendered); + var overrides = new LinkedHashMap(); + userOverrides.forEach((placeholder, value) -> + overrides.put(placeholder, new Token().value(value))); + data.setPlaceholderToUserOverride(overrides); return data; } } diff --git a/src/test/java/ai/reveng/toolkit/ghidra/chat/ui/ChatControllerTest.java b/src/test/java/ai/reveng/toolkit/ghidra/chat/ui/ChatControllerTest.java index e3f87eec..362a61f8 100644 --- a/src/test/java/ai/reveng/toolkit/ghidra/chat/ui/ChatControllerTest.java +++ b/src/test/java/ai/reveng/toolkit/ghidra/chat/ui/ChatControllerTest.java @@ -31,7 +31,6 @@ public class ChatControllerTest { @Override public void info(String message) {} @Override public void warn(String message) {} @Override public void error(String message) {} - @Override public void export(String targetDirectoryPath, String exportedFileName) {} }; private static ChatEvent event(String type, Map data) { diff --git a/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/AbstractStubServerTest.java b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/AbstractStubServerTest.java new file mode 100644 index 00000000..efeb5a0e --- /dev/null +++ b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/AbstractStubServerTest.java @@ -0,0 +1,63 @@ +package ai.reveng.toolkit.ghidra.core.services.api; + +import ai.reveng.invoker.ApiClient; +import ai.reveng.invoker.Configuration; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import ghidra.test.AbstractGhidraHeadlessIntegrationTest; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; + +/// Base for tests that exercise {@link TypedApiImplementation} against a stub HTTP server bound to +/// an ephemeral loopback port. Subclasses only register their handlers in {@link #configureStubs} +/// and reach the client under test through {@link #api()}. +public abstract class AbstractStubServerTest extends AbstractGhidraHeadlessIntegrationTest { + + protected HttpServer server; + private ApiClient originalApiClient; + + @Before + public void startStubServer() throws Exception { + // TypedApiImplementation mutates the ApiClient that Configuration holds as a process-global + // singleton (base path, stacked interceptors). The test task forks in parallel and runs + // several classes per fork, so without this save/restore one class leaks its client into + // whichever class runs next in the same fork. + originalApiClient = Configuration.getDefaultApiClient(); + Configuration.setDefaultApiClient(new ApiClient()); + + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + configureStubs(server); + server.start(); + } + + @After + public void stopStubServer() { + if (server != null) { + server.stop(0); + } + if (originalApiClient != null) { + Configuration.setDefaultApiClient(originalApiClient); + } + } + + /// Registers the contexts this test serves. Called before the server is started. + protected abstract void configureStubs(HttpServer server); + + protected TypedApiImplementation api() { + return new TypedApiImplementation("http://127.0.0.1:" + server.getAddress().getPort(), "test-key"); + } + + protected static void respondJson(HttpExchange exchange, String body) throws IOException { + byte[] bytes = body.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(200, bytes.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(bytes); + } + } +} diff --git a/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/AnalysisDataTypesServiceTest.java b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/AnalysisDataTypesServiceTest.java new file mode 100644 index 00000000..31e2fe3c --- /dev/null +++ b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/AnalysisDataTypesServiceTest.java @@ -0,0 +1,417 @@ +package ai.reveng.toolkit.ghidra.core.services.api; + +import ai.reveng.model.CreateAnalysisDataTypesInputBody; +import ai.reveng.model.UpdateAnalysisDataTypesInputBody; +import ai.reveng.toolkit.ghidra.core.services.api.AnalysisDataTypesService.TypeKey; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface.AnalysisID; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.ServerDataType; +import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; +import ghidra.program.model.data.CategoryPath; +import ghidra.program.model.data.CharDataType; +import ghidra.program.model.data.IntegerDataType; +import ghidra.program.model.data.StructureDataType; +import ghidra.program.model.data.TypedefDataType; +import ghidra.program.model.data.UnsignedIntegerDataType; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.IntStream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/// Tests for {@link AnalysisDataTypesService}, which owns an analysis' `data_type_id` namespace. +public class AnalysisDataTypesServiceTest { + + /// Serves `total` types, one page at a time, and records the paging it was asked for. + private static class PagingApi extends UnimplementedAPI { + final List pages = new ArrayList<>(); + private final int total; + int calls = 0; + + PagingApi(int total) { + this.total = total; + } + + @Override + public List listAnalysisDataTypes(AnalysisID analysisID, long offset, long limit) { + calls++; + pages.add(new long[]{offset, limit}); + return IntStream.range(0, (int) Math.max(0, Math.min(limit, total - offset))) + .mapToObj(i -> type(offset + i)) + .toList(); + } + + private static ServerDataType type(long id) { + return new ServerDataType(id, "ns", "T" + id, ServerDataType.Kind.STRUCT, + 8L, "AUTO", true, null, null, new ServerDataType.StructDefinition(List.of())); + } + } + + @Test + public void pagesUntilTheServerRunsOut() { + var api = new PagingApi(1200); + var catalogue = new AnalysisDataTypesService(api).sync(new AnalysisID(1)); + + assertEquals(1200, catalogue.size()); + assertEquals(3, api.calls); + assertEquals(0, api.pages.get(0)[0]); + assertEquals(500, api.pages.get(0)[1]); + assertEquals(500, api.pages.get(1)[0]); + assertEquals(1000, api.pages.get(2)[0]); + } + + /// A final page that exactly fills the limit still needs one more request to learn it was the + /// last one. + @Test + public void stopsOnTheFirstEmptyPage() { + var api = new PagingApi(1000); + var catalogue = new AnalysisDataTypesService(api).sync(new AnalysisID(1)); + + assertEquals(1000, catalogue.size()); + assertEquals(3, api.calls); + } + + @Test + public void resolvesNamespaceNameAndKindToAnId() { + var service = new AnalysisDataTypesService(new PagingApi(3)); + var analysis = new AnalysisID(1); + + assertEquals(java.util.Optional.of(2L), + service.idOf(analysis, new TypeKey("ns", "T2", ServerDataType.Kind.STRUCT))); + // Kind is part of the identity: the same name of another kind is a different type. + assertTrue(service.idOf(analysis, new TypeKey("ns", "T2", ServerDataType.Kind.UNION)).isEmpty()); + assertTrue(service.idOf(analysis, new TypeKey("other", "T2", ServerDataType.Kind.STRUCT)).isEmpty()); + } + + @Test + public void cachesTheCatalogueUntilInvalidated() { + var api = new PagingApi(3); + var service = new AnalysisDataTypesService(api); + var analysis = new AnalysisID(1); + + service.catalogue(analysis); + int afterFirst = api.calls; + service.catalogue(analysis); + assertEquals("second read is served from the cache", afterFirst, api.calls); + + service.invalidate(analysis); + service.catalogue(analysis); + assertTrue("invalidating forces a re-read", api.calls > afterFirst); + } + + @Test + public void looksTypesUpById() { + var service = new AnalysisDataTypesService(new PagingApi(3)); + var type = service.get(new AnalysisID(1), 1L); + + assertTrue(type.isPresent()); + assertEquals("T1", type.get().name()); + assertTrue(service.get(new AnalysisID(1), 99L).isEmpty()); + } + + /// Serves a fixed catalogue, records every write, and assigns ids to created types the way the + /// server does. + private static class WritingApi extends UnimplementedAPI { + final List calls = new ArrayList<>(); + final List creates = new ArrayList<>(); + final List updates = new ArrayList<>(); + private final List existing; + private long nextId = 1000; + + WritingApi(List existing) { + this.existing = existing; + } + + @Override + public List listAnalysisDataTypes(AnalysisID analysisID, long offset, long limit) { + calls.add("list"); + return offset == 0 ? existing : List.of(); + } + + @Override + public List createAnalysisDataTypes(AnalysisID analysisID, + CreateAnalysisDataTypesInputBody request) { + calls.add("create"); + creates.add(request); + List created = new ArrayList<>(); + for (var entry : request.getDataTypes()) { + created.add(stored(entry.getActualInstance(), nextId++)); + } + return created; + } + + @Override + public List updateAnalysisDataTypes(AnalysisID analysisID, + UpdateAnalysisDataTypesInputBody request) { + calls.add("update"); + updates.add(request); + return List.of(); + } + + /// Rebuilds the entry the server would have stored, which is all the service reads back. + private static ServerDataType stored(Object created, long id) { + String namespace = invoke(created, "getNamespace"); + String name = invoke(created, "getName"); + String kind = String.valueOf(invokeObject(created, "getKind")); + return new ServerDataType(id, namespace == null ? "" : namespace, name, + ServerDataType.Kind.fromJson(kind), null, "USER", false, null, null, null); + } + + private static String invoke(Object target, String method) { + Object value = invokeObject(target, method); + return value == null ? null : value.toString(); + } + + private static Object invokeObject(Object target, String method) { + try { + return target.getClass().getMethod(method).invoke(target); + } catch (ReflectiveOperationException e) { + throw new RuntimeException(e); + } + } + } + + private static ServerDataType serverType(long id, String namespace, String name, ServerDataType.Kind kind) { + return new ServerDataType(id, namespace, name, kind, null, "AUTO", true, null, null, null); + } + + private static StructureDataType packetHeader() { + var header = new StructureDataType("packet_header", 0); + header.add(new IntegerDataType(), "length", null); + header.add(new CharDataType(), "kind", null); + return header; + } + + /// The heart of it: a reactive push repeats on every edit, so a type the analysis already has + /// must be resolved to its existing id and never posted again. Duplicating a type on each edit + /// would corrupt the analysis, not merely waste a request. + @Test + public void resolvesExistingTypesInsteadOfCreatingThemAgain() throws Exception { + var header = packetHeader(); + var api = new WritingApi(List.of( + serverType(7L, "", "packet_header", ServerDataType.Kind.STRUCT), + serverType(8L, "", "int", ServerDataType.Kind.BASE), + serverType(9L, "", "char", ServerDataType.Kind.BASE))); + var service = new AnalysisDataTypesService(api); + + var ids = service.ensure(new AnalysisID(1), List.of(header)); + + assertFalse("nothing was missing, so nothing may be created", api.calls.contains("create")); + assertEquals(Long.valueOf(7), + ids.get(new TypeKey("", "packet_header", ServerDataType.Kind.STRUCT))); + assertEquals(Long.valueOf(8), ids.get(new TypeKey("", "int", ServerDataType.Kind.BASE))); + } + + /// The same push run twice must not create anything the second time round: the ids the first + /// run minted are folded into the catalogue and resolved from there. + @Test + public void repeatedPushesCreateNothingTheSecondTime() throws Exception { + var api = new WritingApi(List.of()); + var service = new AnalysisDataTypesService(api); + var analysis = new AnalysisID(1); + + var first = service.ensure(analysis, List.of(packetHeader())); + int createsAfterFirst = api.creates.size(); + var second = service.ensure(analysis, List.of(packetHeader())); + + assertTrue("the first push had to create the types", createsAfterFirst > 0); + assertEquals("the second push resolves them instead", createsAfterFirst, api.creates.size()); + assertEquals("and lands on the same ids", first, second); + } + + /// A `Create*` body has no `data_type_id`, so nothing in a batch can point at anything else in + /// it. The types are therefore created empty to obtain ids, and the definitions written after. + @Test + public void createsInTwoPhasesWithTheUpdateCarryingTheAssignedIds() throws Exception { + var api = new WritingApi(List.of()); + var service = new AnalysisDataTypesService(api); + + var ids = service.ensure(new AnalysisID(1), List.of(packetHeader())); + + assertEquals("list, then create, then update", + List.of("list", "create", "update"), api.calls); + + var created = api.creates.get(0).getDataTypes(); + var createdStruct = created.stream() + .map(entry -> entry.getActualInstance()) + .filter(instance -> instance instanceof ai.reveng.model.CreateStructDataType) + .map(instance -> (ai.reveng.model.CreateStructDataType) instance) + .findFirst().orElseThrow(); + assertTrue("phase one carries no members to point at", + createdStruct.getDefinition().getMembers().isEmpty()); + + var updated = api.updates.get(0).getDataTypes().stream() + .map(entry -> entry.getActualInstance()) + .filter(instance -> instance instanceof ai.reveng.model.UpdateStructDataType) + .map(instance -> (ai.reveng.model.UpdateStructDataType) instance) + .findFirst().orElseThrow(); + var structId = ids.get(new TypeKey("", "packet_header", ServerDataType.Kind.STRUCT)); + assertEquals("phase two names the id the server assigned", structId, updated.getDataTypeId()); + assertEquals(2, updated.getDefinition().getMembers().size()); + assertEquals("and its members point at ids from the same pass", + ids.get(new TypeKey("", "int", ServerDataType.Kind.BASE)), + updated.getDefinition().getMembers().get(0).getDataTypeId()); + } + + /// Base types complete in the create phase — they carry no definition, so there is nothing for + /// the update to say about them. + @Test + public void kindsWithoutADefinitionNeedNoSecondPhase() throws Exception { + var api = new WritingApi(List.of()); + var service = new AnalysisDataTypesService(api); + + service.ensure(new AnalysisID(1), List.of(new IntegerDataType())); + + assertEquals(List.of("list", "create"), api.calls); + } + + /// Every namespace the closure was written under, whichever create batch it went out in. + private static List createdNamespaces(WritingApi api, String name) { + return api.creates.stream() + .flatMap(request -> request.getDataTypes().stream()) + .map(entry -> entry.getActualInstance()) + .filter(instance -> name.equals(WritingApi.invoke(instance, "getName"))) + .map(instance -> WritingApi.invoke(instance, "getNamespace")) + .toList(); + } + + /// A Ghidra category is not only ever a server namespace. `/windows_vs12_32/DWORD` comes out of + /// one of Ghidra's own data-type archives, not out of this analysis, so reading that category as + /// a namespace would miss the `DWORD` the analysis holds at the root and create a second one — + /// on every edit, because the push is reactive. + @Test + public void aTypeInAGhidraArchiveCategoryResolvesToTheServerEntryAtTheRoot() throws Exception { + var dword = new TypedefDataType(new CategoryPath("/windows_vs12_32"), "DWORD", + new UnsignedIntegerDataType()); + var api = new WritingApi(List.of( + serverType(7L, "", "DWORD", ServerDataType.Kind.TYPEDEF), + serverType(8L, "", "uint", ServerDataType.Kind.BASE))); + var service = new AnalysisDataTypesService(api); + + var ids = service.ensure(new AnalysisID(1), List.of(dword)); + + assertFalse("the analysis already has this type, so nothing may be created", + api.calls.contains("create")); + assertEquals("and the Ghidra type resolves to it", Long.valueOf(7), + ids.get(new TypeKey("windows_vs12_32", "DWORD", ServerDataType.Kind.TYPEDEF))); + } + + /// The same type when the analysis does not have it yet: it is created once, at the root, and + /// the next push resolves what the first one wrote rather than creating it again. + @Test + public void aTypeInAGhidraArchiveCategoryIsCreatedOnceAtTheRoot() throws Exception { + var api = new WritingApi(List.of()); + var service = new AnalysisDataTypesService(api); + var analysis = new AnalysisID(1); + + var first = service.ensure(analysis, List.of( + new TypedefDataType(new CategoryPath("/windows_vs12_32"), "DWORD", + new UnsignedIntegerDataType()))); + + assertEquals("created at the root, not under the Ghidra archive's category", + List.of(""), createdNamespaces(api, "DWORD")); + + int createsAfterFirst = api.creates.size(); + var second = service.ensure(analysis, List.of( + new TypedefDataType(new CategoryPath("/windows_vs12_32"), "DWORD", + new UnsignedIntegerDataType()))); + + assertEquals("the second push resolves it instead", createsAfterFirst, api.creates.size()); + assertEquals("and lands on the same id", + first.get(new TypeKey("windows_vs12_32", "DWORD", ServerDataType.Kind.TYPEDEF)), + second.get(new TypeKey("windows_vs12_32", "DWORD", ServerDataType.Kind.TYPEDEF))); + } + + /// The root is only ever fallen back to for a namespace the analysis holds no such type in. Two + /// distinct types that share a name in different server namespaces are both in the catalogue + /// under their own namespace, so both resolve there and neither is flattened onto the other. + @Test + public void sameNameInDifferentServerNamespacesStillResolvesIndependently() throws Exception { + var fromLibA = new StructureDataType(new CategoryPath("/libA"), "Config", 0); + fromLibA.add(new IntegerDataType(), "a", null); + var fromLibB = new StructureDataType(new CategoryPath("/libB"), "Config", 0); + fromLibB.add(new CharDataType(), "b", null); + + var api = new WritingApi(List.of( + serverType(11L, "libA", "Config", ServerDataType.Kind.STRUCT), + serverType(22L, "libB", "Config", ServerDataType.Kind.STRUCT), + serverType(33L, "", "Config", ServerDataType.Kind.STRUCT), + serverType(8L, "", "int", ServerDataType.Kind.BASE), + serverType(9L, "", "char", ServerDataType.Kind.BASE))); + var service = new AnalysisDataTypesService(api); + + var ids = service.ensure(new AnalysisID(1), List.of(fromLibA, fromLibB)); + + assertFalse("both are already known", api.calls.contains("create")); + assertEquals(Long.valueOf(11), ids.get(new TypeKey("libA", "Config", ServerDataType.Kind.STRUCT))); + assertEquals(Long.valueOf(22), ids.get(new TypeKey("libB", "Config", ServerDataType.Kind.STRUCT))); + + // And each is written back to its own entry, still in its own namespace. + var byId = api.updates.get(0).getDataTypes().stream() + .map(entry -> (ai.reveng.model.UpdateStructDataType) entry.getActualInstance()) + .collect(java.util.stream.Collectors.toMap( + ai.reveng.model.UpdateStructDataType::getDataTypeId, + ai.reveng.model.UpdateStructDataType::getNamespace)); + assertEquals("libA", byId.get(11L)); + assertEquals("libB", byId.get(22L)); + } + + /// A namespace the analysis does have a type in is a server namespace, so it is kept — this is + /// what makes a type the plugin pulled from the server resolve back to the entry it came from. + @Test + public void aNamespaceTheAnalysisAlreadyUsesIsKept() throws Exception { + var file = new StructureDataType(new CategoryPath("/DWARF/stdio.h"), "FILE", 0); + file.add(new IntegerDataType(), "fd", null); + + var api = new WritingApi(List.of( + serverType(5L, "DWARF::stdio.h", "FILE", ServerDataType.Kind.STRUCT), + serverType(8L, "", "int", ServerDataType.Kind.BASE))); + var service = new AnalysisDataTypesService(api); + + var ids = service.ensure(new AnalysisID(1), List.of(file)); + + assertFalse(api.calls.contains("create")); + assertEquals(Long.valueOf(5), + ids.get(new TypeKey("DWARF::stdio.h", "FILE", ServerDataType.Kind.STRUCT))); + var updated = (ai.reveng.model.UpdateStructDataType) + api.updates.get(0).getDataTypes().get(0).getActualInstance(); + assertEquals("the update must not move the entry out of its namespace", + "DWARF::stdio.h", updated.getNamespace()); + } + + /// A local type the analysis has never seen is still created exactly once, and the second push + /// resolves it — the reactive case, where a miss would duplicate on every edit. + @Test + public void aGenuinelyNewLocalTypeIsCreatedOnceAndOnlyOnce() throws Exception { + var api = new WritingApi(List.of()); + var service = new AnalysisDataTypesService(api); + var analysis = new AnalysisID(1); + + service.ensure(analysis, List.of(packetHeader())); + service.ensure(analysis, List.of(packetHeader())); + + assertEquals("exactly one create of the struct, over both pushes", + List.of(""), createdNamespaces(api, "packet_header")); + } + + /// The write endpoints cap a request at 100 types, so a large closure has to be chunked. + @Test + public void chunksLargeCreateBatches() throws Exception { + var api = new WritingApi(List.of()); + var service = new AnalysisDataTypesService(api); + + var container = new StructureDataType("Big", 0); + for (int i = 0; i < 150; i++) { + container.add(new StructureDataType("Member" + i, 4), "m" + i, null); + } + + service.ensure(new AnalysisID(1), List.of(container)); + + assertTrue("more than one create request", api.creates.size() > 1); + api.creates.forEach(request -> + assertTrue("no batch exceeds the endpoint limit", request.getDataTypes().size() <= 100)); + } +} diff --git a/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/DataTypeEntryDeserialisationTest.java b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/DataTypeEntryDeserialisationTest.java new file mode 100644 index 00000000..2a3ab4b2 --- /dev/null +++ b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/DataTypeEntryDeserialisationTest.java @@ -0,0 +1,245 @@ +package ai.reveng.toolkit.ghidra.core.services.api; + +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.ServerDataType; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.ServerDataType.*; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.ServerDataTypeReader; +import com.google.gson.JsonParser; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.*; + +/** + * Covers {@link ServerDataTypeReader}, which reads the spec's {@code DataTypeEntry} straight into + * the flattened {@link ServerDataType} by switching on {@code kind}. + * + *

The generated {@code ai.reveng.model.DataTypeEntry} is deliberately not exercised: it cannot + * deserialise. Its adapter match-counts against all ten variants rather than using the + * discriminator, and every variant shares the same required fields so several always match. Nor can + * that be patched around from the plugin: the container models call the static + * {@code DataTypeEntry.validateJsonElement} before delegating, so an adapter registered for + * {@code DataTypeEntry} is never reached. The plugin reads the response body itself instead. + */ +public class DataTypeEntryDeserialisationTest { + + /// Every field the spec marks required on all ten variants, so payloads below stay realistic. + private static String entry(long id, String kind, String definition) { + return """ + { + "data_type_id": %d, + "kind": "%s", + "name": "T%d", + "namespace": "", + "source_type": "SYSTEM", + "has_definition": %b, + "created_at": "2026-08-13T10:00:00Z"%s + }""".formatted(id, kind, id, definition != null, definition == null ? "" : ",\n \"definition\": " + definition); + } + + private static ServerDataType read(String json) { + return ServerDataTypeReader.readEntry(JsonParser.parseString(json)); + } + + private static final String STRUCT_DEF = """ + {"members": [ + {"name": "sin_family", "offset": 0, "size": 2, "data_type_id": 18, "is_bitfield": false}, + {"name": "flag", "offset": 2, "size": 1, "data_type_id": 19, "is_bitfield": true, + "bit_offset": 3, "bit_size": 1} + ]}"""; + + private static final String ENUM_DEF = """ + {"values": [ + {"name": "AF_UNSPEC", "value": "0"}, + {"name": "NEG_ONE", "value": "-1"}, + {"name": "MAX_U64", "value": "18446744073709551615"} + ]}"""; + + private static final String FUNCTION_DEF = """ + {"return_data_type_id": 7, "parameters": [ + {"name": "argc", "ordinal": 0, "size": 4, "data_type_id": 18}, + {"name": "argv", "ordinal": 1, "size": 8, "data_type_id": 21} + ]}"""; + + @Test + public void allTenKindsFlattenToTheRightDefinition() { + assertEquals(Kind.STRUCT, read(entry(1, "STRUCT", STRUCT_DEF)).kind()); + assertTrue(read(entry(1, "STRUCT", STRUCT_DEF)).definition() instanceof StructDefinition); + assertTrue(read(entry(2, "UNION", """ + {"members": [{"name": "a", "offset": 0, "size": 4, "is_bitfield": false}]}""")) + .definition() instanceof UnionDefinition); + assertTrue(read(entry(3, "ENUM", ENUM_DEF)).definition() instanceof EnumDefinition); + assertTrue(read(entry(4, "TYPEDEF", """ + {"target_data_type_id": 18}""")).definition() instanceof TypedefDefinition); + assertTrue(read(entry(5, "POINTER", """ + {"pointee_data_type_id": 18}""")).definition() instanceof PointerDefinition); + assertTrue(read(entry(6, "ARRAY", """ + {"count": 8, "element_data_type_id": 18}""")).definition() instanceof ArrayDefinition); + assertTrue(read(entry(7, "FUNCTION_DEFINITION", FUNCTION_DEF)) + .definition() instanceof FunctionTypeDefinition); + + // The three kinds that never carry a definition. + for (String kind : List.of("BITFIELD", "BASE", "UNKNOWN")) { + ServerDataType type = read(entry(8, kind, null)); + assertEquals(Kind.valueOf(kind), type.kind()); + assertNull("kind " + kind + " must not carry a definition", type.definition()); + assertFalse(type.hasDefinition()); + } + } + + @Test + public void structMembersIncludingBitfieldsSurvive() { + var definition = (StructDefinition) read(entry(1, "STRUCT", STRUCT_DEF)).definition(); + assertEquals(2, definition.members().size()); + + Member first = definition.members().get(0); + assertEquals("sin_family", first.name()); + assertEquals(0L, first.offset()); + assertEquals(2L, first.size()); + assertEquals(Long.valueOf(18), first.dataTypeId()); + assertFalse(first.isBitfield()); + assertNull(first.bitOffset()); + + Member bitfield = definition.members().get(1); + assertTrue(bitfield.isBitfield()); + assertEquals(Long.valueOf(3), bitfield.bitOffset()); + assertEquals(Long.valueOf(1), bitfield.bitSize()); + } + + @Test + public void enumValuesStayDecimalStringsIncludingNegativeAndAboveSixtyFourBits() { + var definition = (EnumDefinition) read(entry(3, "ENUM", ENUM_DEF)).definition(); + assertEquals(3, definition.values().size()); + assertEquals("0", definition.values().get(0).value()); + assertEquals("NEG_ONE", definition.values().get(1).name()); + assertEquals("-1", definition.values().get(1).value()); + // Would not survive a long; the spec keeps it a string for exactly this reason. + assertEquals("18446744073709551615", definition.values().get(2).value()); + } + + @Test + public void functionParametersAndTargetIdsSurvive() { + var function = (FunctionTypeDefinition) read(entry(7, "FUNCTION_DEFINITION", FUNCTION_DEF)).definition(); + assertEquals(Long.valueOf(7), function.returnDataTypeId()); + assertEquals(2, function.parameters().size()); + assertEquals("argv", function.parameters().get(1).name()); + assertEquals(1L, function.parameters().get(1).ordinal()); + assertEquals(Long.valueOf(21), function.parameters().get(1).dataTypeId()); + + assertEquals(Long.valueOf(18), + ((TypedefDefinition) read(entry(4, "TYPEDEF", "{\"target_data_type_id\": 18}")).definition()) + .targetDataTypeId()); + assertEquals(Long.valueOf(18), + ((PointerDefinition) read(entry(5, "POINTER", "{\"pointee_data_type_id\": 18}")).definition()) + .pointeeDataTypeId()); + var array = (ArrayDefinition) read(entry(6, "ARRAY", "{\"count\": 8, \"element_data_type_id\": 18}")).definition(); + assertEquals(Long.valueOf(8), array.count()); + assertEquals(Long.valueOf(18), array.elementDataTypeId()); + } + + @Test + public void flatFieldsSurviveIncludingALargeSizeAndAZeroId() { + ServerDataType type = read(""" + { + "data_type_id": 0, + "kind": "BASE", + "name": "unsigned long long", + "namespace": "/DWARF/limits.h", + "source_type": "AI_DECOMP", + "has_definition": false, + "source_function_id": 987654321, + "size": 9223372036854775807, + "created_at": "2026-08-13T10:00:00Z" + }"""); + assertEquals(0L, type.id()); + assertEquals("unsigned long long", type.name()); + assertEquals("/DWARF/limits.h", type.namespace()); + assertEquals("AI_DECOMP", type.sourceType()); + assertEquals(Long.valueOf(Long.MAX_VALUE), type.size()); + assertEquals(Long.valueOf(987654321), type.sourceFunctionId()); + assertEquals("2026-08-13T10:00:00Z", type.createdAt()); + } + + @Test + public void absentOptionalsAndAKindDeclaredButNeverDefined() { + ServerDataType type = read(entry(9, "STRUCT", null)); + assertNull("size is absent when the server could not determine it", type.size()); + assertNull(type.sourceFunctionId()); + assertNull("a type referenced but never defined carries no definition", type.definition()); + assertFalse(type.hasDefinition()); + + // A definition present but with a null array still yields an empty list, never null. + var empty = (StructDefinition) read(entry(10, "STRUCT", "{\"members\": null}")).definition(); + assertNotNull(empty); + assertTrue(empty.members().isEmpty()); + } + + @Test + public void unrecognisedKindDegradesToUnknownRatherThanFailing() { + ServerDataType type = read(entry(11, "SOME_FUTURE_KIND", null)); + assertEquals(Kind.UNKNOWN, type.kind()); + } + + /// The real list read path: GET /v3/analyses/{analysis_id}/data-types. + @Test + public void listAnalysisDataTypesOutputBodyWithMixedKindsParses() { + String body = """ + { + "total_count": 4, + "items": [%s, %s, %s, %s] + }""".formatted( + entry(1, "STRUCT", STRUCT_DEF), + entry(3, "ENUM", ENUM_DEF), + entry(5, "POINTER", "{\"pointee_data_type_id\": 1}"), + entry(8, "BASE", null)); + + List types = ServerDataTypeReader.readEntries(JsonParser.parseString(body), "items"); + assertEquals(4, types.size()); + assertEquals(List.of(Kind.STRUCT, Kind.ENUM, Kind.POINTER, Kind.BASE), + types.stream().map(ServerDataType::kind).toList()); + assertEquals(2, ((StructDefinition) types.get(0).definition()).members().size()); + assertEquals("-1", ((EnumDefinition) types.get(1).definition()).values().get(1).value()); + assertEquals(Long.valueOf(1), ((PointerDefinition) types.get(2).definition()).pointeeDataTypeId()); + assertNull(types.get(3).definition()); + } + + /// The real signature read path: GET /v3/analyses/{analysis_id}/functions/{function_id}/signature. + @Test + public void functionSignatureBodyWithMixedKindsParses() { + String body = """ + { + "function_id": 4242, + "function_name": "main", + "has_signature": true, + "calling_convention": "__stdcall", + "created_at": "2026-08-13T10:00:00Z", + "return_data_type_id": 7, + "source_type": "USER", + "parameters": [ + {"ordinal": 0, "name": "argc", "data_type_id": 7}, + {"ordinal": 1, "name": "argv", "data_type_id": 5} + ], + "data_types": [%s, %s, %s] + }""".formatted( + entry(7, "BASE", null), + entry(5, "POINTER", "{\"pointee_data_type_id\": 7}"), + entry(2, "UNION", "{\"members\": [{\"name\": \"raw\", \"offset\": 0, \"size\": 8, \"is_bitfield\": false}]}")); + + List types = ServerDataTypeReader.readEntries(JsonParser.parseString(body), "data_types"); + assertEquals(3, types.size()); + assertEquals(List.of(Kind.BASE, Kind.POINTER, Kind.UNION), + types.stream().map(ServerDataType::kind).toList()); + assertEquals(Long.valueOf(7), ((PointerDefinition) types.get(1).definition()).pointeeDataTypeId()); + Member member = ((UnionDefinition) types.get(2).definition()).members().get(0); + assertEquals("raw", member.name()); + assertEquals(8L, member.size()); + } + + @Test + public void containerWithNoTypesParses() { + assertTrue(ServerDataTypeReader.readEntries( + JsonParser.parseString("{\"total_count\": 0, \"items\": null}"), "items").isEmpty()); + assertTrue(ServerDataTypeReader.readEntries( + JsonParser.parseString("{\"total_count\": 0}"), "items").isEmpty()); + } +} diff --git a/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/DisassemblyBlocksReaderTest.java b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/DisassemblyBlocksReaderTest.java new file mode 100644 index 00000000..4957ccb4 --- /dev/null +++ b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/DisassemblyBlocksReaderTest.java @@ -0,0 +1,115 @@ +package ai.reveng.toolkit.ghidra.core.services.api; + +import ai.reveng.invoker.JSON; +import ai.reveng.model.DisassemblyOutputBody; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * Covers {@link DisassemblyBlocksReader}, which pulls the assembly out of the untyped + * {@code basic_blocks} value of {@code GET /v3/functions/{function_id}/blocks}. + * + *

Each case is driven through the generated {@link DisassemblyOutputBody} rather than a + * hand-built {@code Object}, so the deserialisation the SDK actually performs — including the + * required {@code function_id} and {@code returns} fields, and Gson's untyped handling of + * {@code basic_blocks} — is exercised alongside the reader. + */ +public class DisassemblyBlocksReaderTest { + + private static List read(String json) { + DisassemblyOutputBody body = JSON.getGson().fromJson(json, DisassemblyOutputBody.class); + return DisassemblyBlocksReader.readAssembly(body.getBasicBlocks()); + } + + /// A block as v3 sends it: `{min_addr, max_addr, destinations, asm}`. Assembly lines carry the + /// address and the instruction separated by a tab, and a destination's `vaddr` is a number. + private static String block(long minAddr, long maxAddr, long destination, String... asm) { + return """ + { + "min_addr": %d, + "max_addr": %d, + "destinations": [{"vaddr": %d, "flowtype": "UNCONDITIONAL_JUMP"}], + "asm": [%s] + }""".formatted(minAddr, maxAddr, destination, + String.join(", ", List.of(asm).stream().map(l -> '"' + l + '"').toList())); + } + + /// The surrounding body, carrying the fields the spec marks required plus the sibling blobs v3 + /// added; passing null omits `basic_blocks` entirely, as v3 does for a function without one. + private static String body(String basicBlocks) { + return """ + { + "function_id": 1109480836, + "returns": true, + "return_type": "int", + "params": [], + "local_variables": [], + "global_variables": []%s + }""".formatted(basicBlocks == null ? "" + : ",\n \"basic_blocks\": [%s]".formatted(basicBlocks)); + } + + @Test + public void blocksAreConcatenatedInAddressOrder() { + // Deliberately out of order, and at load addresses past Integer.MAX_VALUE — the generated v2 + // block model narrows these to int, which is why the reader parses them itself. + String json = body(String.join(",\n", + block(0x140001010L, 0x140001018L, 0x140001020L, + "0x140001010\\tADD RSP,0x20", "0x140001014\\tRET"), + block(0x140001000L, 0x140001010L, 0x140001010L, + "0x140001000\\tPUSH RBP", "0x140001001\\tMOV RBP,RSP"))); + + assertEquals(List.of( + "0x140001000\tPUSH RBP", "0x140001001\tMOV RBP,RSP", + "0x140001010\tADD RSP,0x20", "0x140001014\tRET"), + read(json)); + } + + @Test + public void aBodyWithoutBlocksReadsAsNoAssembly() { + // How v3 reports a function that carries no stored disassembly: 200, block fields absent. + assertEquals(List.of(), read(body(null))); + } + + @Test + public void anEmptyBlockListReadsAsNoAssembly() { + assertEquals(List.of(), read(body(""))); + } + + @Test + public void blocksWithoutAssemblyContributeNothing() { + String json = body(String.join(",\n", + "{\"min_addr\": 4096, \"max_addr\": 4100, \"destinations\": []}", + block(0x1010L, 0x1018L, 0x1020L, "0x1010\\tRET"))); + + assertEquals(List.of("0x1010\tRET"), read(json)); + } + + @Test + public void blocksWithoutAStartAddressSortLast() { + String json = body(String.join(",\n", + "{\"asm\": [\"NOP\"]}", + block(0x1000L, 0x1004L, 0x1010L, "0x1000\\tPUSH RBP"))); + + assertEquals(List.of("0x1000\tPUSH RBP", "NOP"), read(json)); + } + + /// A block carrying fields the reader does not use must not derail the blocks around it. + @Test + public void legacyBlockFieldsAreIgnored() { + String json = body(""" + { + "id": 0, + "min_addr": 4096, + "max_addr": 4100, + "comment": null, + "destinations": [{"vaddr": "4128", "flowtype": "FALL_THROUGH", "destination_block_id": 1}], + "asm": ["0x1000\\tPUSH RBP"] + }"""); + + assertEquals(List.of("0x1000\tPUSH RBP"), read(json)); + } +} diff --git a/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/FunctionDetailsMappingTest.java b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/FunctionDetailsMappingTest.java new file mode 100644 index 00000000..b6bc8762 --- /dev/null +++ b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/FunctionDetailsMappingTest.java @@ -0,0 +1,74 @@ +package ai.reveng.toolkit.ghidra.core.services.api; + +import ai.reveng.invoker.JSON; +import ai.reveng.model.FunctionDetailsOutputBody; +import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionDetails; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * Covers {@link FunctionDetails#fromServerResponse}, which maps the body of + * {@code GET /v3/functions/{function_id}} onto the plugin's own record. Each case is driven + * through the generated {@link FunctionDetailsOutputBody} so the SDK's own deserialisation, + * including its required-field validation, is exercised alongside the mapping. + */ +public class FunctionDetailsMappingTest { + + private static FunctionDetails map(String json) { + return FunctionDetails.fromServerResponse( + JSON.getGson().fromJson(json, FunctionDetailsOutputBody.class)); + } + + private static final String FULL_BODY = """ + { + "analysis_id": 4321, + "binary_id": 99, + "creation": "2026-01-02T03:04:05Z", + "debug": true, + "function_id": 1234, + "function_name": "demangled_name", + "function_size": 256, + "function_vaddr": 16384, + "mangled_name": "_Z15mangled_namev", + "source_function_id": 7 + }"""; + + @Test + public void mapsEveryFieldThePluginReads() { + FunctionDetails details = map(FULL_BODY); + + assertEquals(1234L, details.functionId().value()); + assertEquals("_Z15mangled_namev", details.mangledFunctionName()); + assertEquals("demangled_name", details.demangledName()); + assertEquals(Long.valueOf(16384L), details.functionVaddr()); + assertEquals(Long.valueOf(256L), details.functionSize()); + assertEquals(4321, details.analysisId().id()); + } + + @Test + public void mangledNameIsOptional() { + FunctionDetails details = map(""" + { + "analysis_id": 4321, + "binary_id": 99, + "creation": "2026-01-02T03:04:05Z", + "debug": false, + "function_id": 1234, + "function_name": "demangled_name", + "function_size": 256, + "function_vaddr": 16384 + }"""); + + assertNull(details.mangledFunctionName()); + assertEquals("demangled_name", details.demangledName()); + } + + @Test + public void functionIdSurvivesValuesAboveTheIntRange() { + FunctionDetails details = map(FULL_BODY.replace("\"function_id\": 1234", "\"function_id\": 4294967296")); + + assertEquals(4294967296L, details.functionId().value()); + } +} diff --git a/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/FunctionSignatureServiceTest.java b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/FunctionSignatureServiceTest.java new file mode 100644 index 00000000..25d3db02 --- /dev/null +++ b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/FunctionSignatureServiceTest.java @@ -0,0 +1,185 @@ +package ai.reveng.toolkit.ghidra.core.services.api; + +import ai.reveng.invoker.ApiException; +import ai.reveng.model.BatchFunctionSignatureEntry; +import ai.reveng.model.UpdateFunctionSignatureInputBody; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface.AnalysisID; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface.FunctionID; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.FunctionSignatureBatch; +import ai.reveng.toolkit.ghidra.core.services.api.datatypes.ServerDataType; +import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.IntStream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/// Tests for {@link FunctionSignatureService}, which batches reads of `/v3/functions/signatures`. +public class FunctionSignatureServiceTest { + + /// Records what the service asked for and answers with one entry per requested id. + private static class RecordingApi extends UnimplementedAPI { + final List> requests = new ArrayList<>(); + final List includeDataTypesFlags = new ArrayList<>(); + + @Override + public FunctionSignatureBatch listFunctionSignatures(List functionIDs, boolean includeDataTypes) { + requests.add(List.copyOf(functionIDs)); + includeDataTypesFlags.add(includeDataTypes); + List items = functionIDs.stream().map(id -> { + var entry = new BatchFunctionSignatureEntry(); + entry.setAnalysisId(1L); + entry.setFunctionId(id.value()); + entry.setFunctionName("f" + id.value()); + entry.setHasSignature(id.value() % 2 == 0); + entry.setParameters(List.of()); + return entry; + }).toList(); + var type = new ServerDataType(id(functionIDs), "", "int", ServerDataType.Kind.BASE, + 4L, "AUTO", false, null, null, null); + return new FunctionSignatureBatch(items, Map.of(new AnalysisID(1), List.of(type))); + } + + private static long id(List ids) { + return ids.isEmpty() ? 0 : ids.get(0).value(); + } + } + + private static List ids(int count) { + return IntStream.rangeClosed(1, count).mapToObj(i -> new FunctionID(i)).toList(); + } + + /// The ids ride in the query string, so a whole-binary request has to be chunked or the request + /// URI overflows (HTTP 414). + @Test + public void chunksLargeIdListsIntoBatchesOfFifty() { + var api = new RecordingApi(); + var batch = new FunctionSignatureService(api).getMany(ids(120)); + + assertEquals(3, api.requests.size()); + assertEquals(50, api.requests.get(0).size()); + assertEquals(50, api.requests.get(1).size()); + assertEquals(20, api.requests.get(2).size()); + assertEquals("every requested id is answered for", 120, batch.items().size()); + } + + @Test + public void mergesDataTypesFromEveryChunk() { + var api = new RecordingApi(); + var batch = new FunctionSignatureService(api).getMany(ids(120)); + + assertEquals(1, batch.dataTypes().size()); + assertEquals("one type per chunk, all under the same analysis", + 3, batch.dataTypesFor(new AnalysisID(1)).size()); + } + + @Test + public void dedupesRepeatedIds() { + var api = new RecordingApi(); + new FunctionSignatureService(api).getMany(List.of( + new FunctionID(1), new FunctionID(1), new FunctionID(2))); + + assertEquals(1, api.requests.size()); + assertEquals(List.of(new FunctionID(1), new FunctionID(2)), api.requests.get(0)); + } + + @Test + public void emptyRequestDoesNotHitTheApi() { + var api = new RecordingApi(); + var batch = new FunctionSignatureService(api).getMany(List.of()); + + assertTrue(api.requests.isEmpty()); + assertTrue(batch.items().isEmpty()); + } + + /// A presence check does not need the type closure attached, which is far cheaper to fetch. + @Test + public void presenceOnlyReadsSkipDataTypes() { + var api = new RecordingApi(); + new FunctionSignatureService(api).getMany(ids(2), false); + + assertEquals(List.of(false), api.includeDataTypesFlags); + } + + @Test + public void getReturnsOnlyFunctionsTheServerHasASignatureFor() { + var service = new FunctionSignatureService(new RecordingApi()); + + // The stub reports a signature for even ids only. + var present = service.get(new FunctionID(2)); + assertTrue(present.isPresent()); + assertEquals("f2", present.get().entry().getFunctionName()); + assertFalse("the analysis' types come along with the signature", + present.get().dataTypes().isEmpty()); + + assertTrue(service.get(new FunctionID(3)).isEmpty()); + } + + /// Accepts a signature write for one function and answers 404 for every other, which is how the + /// endpoint reports a function it never extracted a signature for. + private static class WritingApi extends UnimplementedAPI { + final List written = new ArrayList<>(); + private final FunctionID extracted; + + WritingApi(FunctionID extracted) { + this.extracted = extracted; + } + + @Override + public void updateFunctionSignature(AnalysisID analysisID, FunctionID functionID, + UpdateFunctionSignatureInputBody signature) throws ApiException { + if (!extracted.equals(functionID)) { + throw new ApiException(404, "Not Found"); + } + written.add(functionID); + } + } + + @Test + public void writesTheSignatureOfAFunctionTheServerExtracted() throws Exception { + var api = new WritingApi(new FunctionID(1)); + var service = new FunctionSignatureService(api); + + assertTrue(service.put(new AnalysisID(1), new FunctionID(1), + new UpdateFunctionSignatureInputBody().parameters(List.of()))); + assertEquals(List.of(new FunctionID(1)), api.written); + } + + /// `has_signature` false is a normal state — a thunk, an external function, or an analysis where + /// type extraction never ran — and the endpoint reports it as a 404. The push is reactive on a + /// short debounce, so this must be a quiet skip rather than an exception or a warning. + @Test + public void skipsFunctionsTheServerHasNoExtractedSignatureFor() throws Exception { + var api = new WritingApi(new FunctionID(1)); + var service = new FunctionSignatureService(api); + + assertFalse(service.put(new AnalysisID(1), new FunctionID(2), + new UpdateFunctionSignatureInputBody().parameters(List.of()))); + assertTrue("nothing was written", api.written.isEmpty()); + } + + /// Anything that is not a missing signature is a real failure and has to reach the caller. + @Test + public void otherFailuresStillSurface() { + var service = new FunctionSignatureService(new UnimplementedAPI() { + @Override + public void updateFunctionSignature(AnalysisID analysisID, FunctionID functionID, + UpdateFunctionSignatureInputBody signature) throws ApiException { + throw new ApiException(403, "Forbidden"); + } + }); + + try { + service.put(new AnalysisID(1), new FunctionID(1), + new UpdateFunctionSignatureInputBody().parameters(List.of())); + org.junit.Assert.fail("a 403 must not be swallowed"); + } catch (ApiException e) { + assertEquals(403, e.getCode()); + } + } +} diff --git a/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/GetAnalysisBasicInfoTest.java b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/GetAnalysisBasicInfoTest.java new file mode 100644 index 00000000..c7e365dd --- /dev/null +++ b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/GetAnalysisBasicInfoTest.java @@ -0,0 +1,94 @@ +package ai.reveng.toolkit.ghidra.core.services.api; + +import com.sun.net.httpserver.HttpServer; +import org.junit.Test; + +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * {@link TypedApiImplementation#getAnalysisBasicInfo} memoises per analysis id and its ids are + * 64-bit on the wire, so this covers the cache and a binary id above 2^31. + */ +public class GetAnalysisBasicInfoTest extends AbstractStubServerTest { + + private static final int ANALYSIS_ID = 4321; + private static final long BINARY_ID = 5_000_000_000L; + + private final List requestPaths = new CopyOnWriteArrayList<>(); + + @Override + protected void configureStubs(HttpServer server) { + server.createContext("/v3/analyses", exchange -> { + requestPaths.add(exchange.getRequestURI().getPath()); + respondJson(exchange, body()); + }); + } + + @Test + public void getAnalysisBasicInfo_readsTheV3Endpoint() throws Exception { + var info = api().getAnalysisBasicInfo(new TypedApiInterface.AnalysisID(ANALYSIS_ID)); + + assertEquals(List.of("/v3/analyses/" + ANALYSIS_ID + "/basic"), requestPaths); + assertEquals("test_binary", info.getBinaryName()); + assertEquals("0".repeat(64), info.getSha256Hash()); + assertEquals("binnet-0.5", info.getModelName()); + } + + @Test + public void getAnalysisBasicInfo_widensIdsBeyondIntRange() throws Exception { + var info = api().getAnalysisBasicInfo(new TypedApiInterface.AnalysisID(ANALYSIS_ID)); + + assertEquals(Long.valueOf(BINARY_ID), info.getBinaryId()); + assertEquals(Long.valueOf(9_000_000_000L), info.getBinarySize()); + assertEquals(Long.valueOf(4_294_967_296L), info.getBaseAddress()); + } + + @Test + public void getAnalysisBasicInfo_secondReadOfTheSameIdIsServedFromCache() throws Exception { + var api = api(); + var first = api.getAnalysisBasicInfo(new TypedApiInterface.AnalysisID(ANALYSIS_ID)); + var second = api.getAnalysisBasicInfo(new TypedApiInterface.AnalysisID(ANALYSIS_ID)); + + assertEquals(1, requestPaths.size()); + assertTrue("a cache hit should return the memoised instance", first == second); + + api.getAnalysisBasicInfo(new TypedApiInterface.AnalysisID(ANALYSIS_ID + 1)); + assertEquals(2, requestPaths.size()); + assertEquals("/v3/analyses/" + (ANALYSIS_ID + 1) + "/basic", requestPaths.get(1)); + } + + private static String body() { + return """ + { + "analysis_scope": "PRIVATE", + "base_address": 4294967296, + "binary_id": %d, + "binary_name": "test_binary", + "binary_size": 9000000000, + "binary_uuid": "1a2b3c4d-0000-0000-0000-000000000000", + "creation": "2026-01-01T00:00:00Z", + "debug": false, + "detected_architecture": "x86_64", + "detected_binary_format": "ELF", + "detected_binary_type": "EXEC", + "function_count": 12, + "is_advanced": false, + "is_owner": true, + "is_system": false, + "model_id": 7, + "model_name": "binnet-0.5", + "owner_username": "tester", + "sequencer_version": null, + "sha_256_hash": "%s", + "supplied_architecture": "Auto", + "supplied_binary_format": "Auto", + "supplied_binary_type": "Auto", + "team_id": 3 + } + """.formatted(BINARY_ID, "0".repeat(64)); + } +} diff --git a/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/GetFunctionInfoPaginationTest.java b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/GetFunctionInfoPaginationTest.java index 21149969..126129da 100644 --- a/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/GetFunctionInfoPaginationTest.java +++ b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/GetFunctionInfoPaginationTest.java @@ -1,114 +1,136 @@ package ai.reveng.toolkit.ghidra.core.services.api; -import ai.reveng.invoker.ApiClient; -import ai.reveng.invoker.Configuration; import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionInfo; import com.sun.net.httpserver.HttpServer; -import ghidra.test.AbstractGhidraHeadlessIntegrationTest; -import org.junit.After; -import org.junit.Before; import org.junit.Test; -import java.io.OutputStream; -import java.net.InetSocketAddress; -import java.nio.charset.StandardCharsets; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; /** - * The v2 functions list endpoint caps page_size at 1000, so {@link TypedApiImplementation#getFunctionInfo} - * must page through every result. This stubs the endpoint with a two-page response and checks that all - * functions come back and that paging stops once the server reports no next page. + * The v3 functions list endpoint is paginated by offset and limit and reports the unpaginated + * population size as {@code total_count}, so {@link TypedApiImplementation#getFunctionInfo} must + * walk the offset forward until that many entries have arrived. These stubs serve a server that + * caps a page below the requested limit, which is also what forces the offset to advance by the + * number of entries actually returned. */ -public class GetFunctionInfoPaginationTest extends AbstractGhidraHeadlessIntegrationTest { +public class GetFunctionInfoPaginationTest extends AbstractStubServerTest { private static final int ANALYSIS_ID = 123; + private static final int SERVER_PAGE_CAP = 2; - private HttpServer server; - private ApiClient originalApiClient; private final List requestedQueries = new CopyOnWriteArrayList<>(); + private volatile List allFunctionIds = List.of(); - @Before - public void startStubServer() throws Exception { - // TypedApiImplementation mutates the shared default ApiClient (base path, stacked interceptors). - // Isolate this test from whatever ran before it in the same fork, and restore it afterwards. - originalApiClient = Configuration.getDefaultApiClient(); - Configuration.setDefaultApiClient(new ApiClient()); - - server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); - server.createContext("/v2/analyses/" + ANALYSIS_ID + "/functions/list", exchange -> { + @Override + protected void configureStubs(HttpServer server) { + server.createContext("/v3/analyses/" + ANALYSIS_ID + "/functions", exchange -> { String query = exchange.getRequestURI().getQuery(); requestedQueries.add(query); - int page = pageParam(query); - String body = page <= 1 ? pageResponse(List.of(10L, 11L), 1, true) - : pageResponse(List.of(12L), 2, false); - byte[] bytes = body.getBytes(StandardCharsets.UTF_8); - exchange.getResponseHeaders().add("Content-Type", "application/json"); - exchange.sendResponseHeaders(200, bytes.length); - try (OutputStream os = exchange.getResponseBody()) { - os.write(bytes); - } + int offset = Math.toIntExact(longParam(query, "offset", 0)); + int limit = Math.toIntExact(longParam(query, "limit", 100)); + + List ids = allFunctionIds; + int from = Math.min(offset, ids.size()); + int to = Math.min(from + Math.min(limit, SERVER_PAGE_CAP), ids.size()); + respondJson(exchange, pageResponse(ids.subList(from, to), ids.size())); }); - server.start(); } - @After - public void stopStubServer() { - if (server != null) { - server.stop(0); - } - if (originalApiClient != null) { - Configuration.setDefaultApiClient(originalApiClient); - } + /** A trailing partial page: the last request comes back short of the server's own cap. */ + @Test + public void getFunctionInfo_walksEveryPage() { + allFunctionIds = List.of(10L, 11L, 12L); + + List functions = fetch(); + + assertEquals("every page should be combined, in order", + List.of(10L, 11L, 12L), idsOf(functions)); + assertEquals("offset should advance by the entries actually returned", + List.of("offset=0&limit=500", "offset=2&limit=500"), requestedQueries); } + /** + * A final page that exactly reaches total_count. Paging has to stop on the count rather than + * on a short page, otherwise it issues one more request than it needs. + */ @Test - public void getFunctionInfo_walksEveryPage() { - var api = new TypedApiImplementation("http://127.0.0.1:" + server.getAddress().getPort(), "test-key"); + public void getFunctionInfo_stopsOnceTotalCountIsReached() { + allFunctionIds = List.of(10L, 11L, 12L, 13L); + + List functions = fetch(); + + assertEquals("every page should be combined, in order", + List.of(10L, 11L, 12L, 13L), idsOf(functions)); + assertEquals("a full final page should not trigger another request", + List.of("offset=0&limit=500", "offset=2&limit=500"), requestedQueries); + } + + /** An analysis with no functions still answers 200, with an empty list and a zero count. */ + @Test + public void getFunctionInfo_handlesAnEmptyAnalysis() { + allFunctionIds = List.of(); + + List functions = fetch(); + + assertEquals(List.of(), idsOf(functions)); + assertEquals("a single request is enough to learn the analysis is empty", + List.of("offset=0&limit=500"), requestedQueries); + } - List functions = api.getFunctionInfo(new TypedApiInterface.AnalysisID(ANALYSIS_ID)); + /** mangled_name is optional on the v3 entry; callers rely on the plugin type carrying one. */ + @Test + public void getFunctionInfo_fallsBackToTheFunctionNameWhenUnmangled() { + allFunctionIds = List.of(10L, UNMANGLED_ID); + + List functions = fetch(); + + assertEquals(List.of("mangled_10", "func_" + UNMANGLED_ID), + functions.stream().map(FunctionInfo::functionMangledName).collect(Collectors.toList())); + } - List ids = functions.stream().map(f -> f.functionID().value()).collect(Collectors.toList()); - assertEquals("both pages should be combined", List.of(10L, 11L, 12L), ids); + private List fetch() { + return api().getFunctionInfo(new TypedApiInterface.AnalysisID(ANALYSIS_ID)); + } - assertEquals("should stop after the page with has_next_page=false", 2, requestedQueries.size()); - assertTrue("first request should ask for page 1", requestedQueries.get(0).contains("page=1")); - assertTrue("second request should ask for page 2", requestedQueries.get(1).contains("page=2")); - assertTrue("should request the server's max page size", requestedQueries.get(0).contains("page_size=1000")); + private static List idsOf(List functions) { + return functions.stream().map(f -> f.functionID().value()).collect(Collectors.toList()); } - private static int pageParam(String query) { + private static long longParam(String query, String name, long fallback) { if (query == null) { - return 1; + return fallback; } for (String pair : query.split("&")) { int eq = pair.indexOf('='); - if (eq > 0 && pair.substring(0, eq).equals("page")) { - return Integer.parseInt(pair.substring(eq + 1)); + if (eq > 0 && pair.substring(0, eq).equals(name)) { + return Long.parseLong(pair.substring(eq + 1)); } } - return 1; + return fallback; } - private static String pageResponse(List functionIds, int pageNumber, boolean hasNextPage) { + private static String pageResponse(List functionIds, int totalCount) { String functions = functionIds.stream() .map(GetFunctionInfoPaginationTest::functionJson) .collect(Collectors.joining(",")); return """ - {"status":true,"message":"ok","errors":[],\ - "data":{"functions":[%s]},\ - "meta":{"pagination":{"page_size":1000,"page_number":%d,"has_next_page":%b}}}\ - """.formatted(functions, pageNumber, hasNextPage); + {"functions":[%s],"total_count":%d}\ + """.formatted(functions, totalCount); } + /** The id whose entry the stub serves without a mangled_name. */ + private static final long UNMANGLED_ID = 99L; + private static String functionJson(long id) { + String mangledName = id == UNMANGLED_ID ? "" : "\"mangled_name\":\"mangled_%d\",".formatted(id); return """ - {"function_id":%d,"function_name":"func_%d","function_mangled_name":"mangled_%d",\ - "function_vaddr":%d,"function_size":32,"debug":false}\ - """.formatted(id, id, id, 0x400000L + id); + {"function_id":%d,"function_name":"func_%d",%s\ + "function_vaddr":%d,"function_size":32,"binary_id":7,"debug":false,\ + "source_type":"analysis"}\ + """.formatted(id, id, mangledName, 0x400000L + id); } } diff --git a/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/ListAnalysesForHashTest.java b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/ListAnalysesForHashTest.java new file mode 100644 index 00000000..bc5aaec3 --- /dev/null +++ b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/ListAnalysesForHashTest.java @@ -0,0 +1,148 @@ +package ai.reveng.toolkit.ghidra.core.services.api; + +import com.sun.net.httpserver.HttpServer; +import org.junit.Test; + +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * {@link TypedApiImplementation#search} filters /v3/analyses by hash and follows next_page_token, + * so this covers the query it sends, the multi-page walk and the empty "no analyses for this hash" + * answer. + */ +public class ListAnalysesForHashTest extends AbstractStubServerTest { + + private static final String HASH = "b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6"; + + private final List requestQueries = new CopyOnWriteArrayList<>(); + private volatile String firstPage = emptyPage(); + private volatile String secondPage = emptyPage(); + + @Override + protected void configureStubs(HttpServer server) { + server.createContext("/v3/analyses", exchange -> { + String query = exchange.getRequestURI().getQuery(); + requestQueries.add(query); + respondJson(exchange, query != null && query.contains("next_page_token=") ? secondPage : firstPage); + }); + } + + @Test + public void search_filtersByHashOverEveryScope() { + firstPage = page(null, record(11, 22)); + + var results = api().search(new TypedApiInterface.BinaryHash(HASH)); + + assertEquals(1, requestQueries.size()); + String query = requestQueries.get(0); + assertTrue(query, query.contains("sha256_hash=" + HASH)); + assertTrue(query, query.contains("analysis_scope=PRIVATE")); + assertTrue(query, query.contains("analysis_scope=TEAM")); + assertTrue(query, query.contains("analysis_scope=PUBLIC")); + + assertEquals(1, results.size()); + assertEquals(Long.valueOf(11), results.get(0).getAnalysisId()); + assertEquals(Long.valueOf(22), results.get(0).getBinaryId()); + assertEquals("true", results.get(0).getBinaryName()); + assertEquals("Complete", results.get(0).getStatus()); + } + + @Test + public void search_followsNextPageToken() { + firstPage = page("cursor-1", record(11, 22)); + secondPage = page(null, record(12, 23)); + + var results = api().search(new TypedApiInterface.BinaryHash(HASH)); + + assertEquals(2, requestQueries.size()); + assertTrue(requestQueries.get(1), requestQueries.get(1).contains("next_page_token=cursor-1")); + assertEquals(List.of(11L, 12L), results.stream().map(r -> r.getAnalysisId()).toList()); + } + + @Test + public void search_returnsEmptyWhenNoAnalysisMatchesTheHash() { + var results = api().search(new TypedApiInterface.BinaryHash(HASH)); + + assertEquals(1, requestQueries.size()); + assertTrue(results.isEmpty()); + } + + /// Ids are 64-bit on the wire; base_address is what the Recent Analyses table matches against + /// the program's image base. + @Test + public void search_readsIdsAndBaseAddressBeyondIntRange() { + firstPage = page(null, """ + { + "analysis_id": 4321, + "analysis_scope": "PRIVATE", + "base_address": 4294967296, + "binary_id": 5000000000, + "binary_name": "true", + "binary_size": 9000000000, + "creation": "2024-04-19T08:57:18Z", + "detected_architecture": "x86_64", + "detected_binary_format": "ELF", + "detected_binary_type": "linux", + "function_boundaries_hash": "b48f61e8", + "is_owner": true, + "model_id": 1, + "model_name": "binnet-0.5", + "sha_256_hash": "%s", + "status": "Complete", + "supplied_architecture": "Auto", + "supplied_binary_format": "Auto", + "supplied_binary_type": "Auto", + "tags": [], + "username": "tester" + } + """.formatted(HASH)); + + var record = api().search(new TypedApiInterface.BinaryHash(HASH)).get(0); + + assertEquals(Long.valueOf(5_000_000_000L), record.getBinaryId()); + assertEquals(Long.valueOf(4_294_967_296L), record.getBaseAddress()); + assertEquals(Long.valueOf(9_000_000_000L), record.getBinarySize()); + assertEquals("2024-04-19T08:57:18Z", record.getCreation().toString()); + } + + private static String emptyPage() { + return page(null); + } + + private static String page(String nextPageToken, String... records) { + String token = nextPageToken == null ? "" : ", \"next_page_token\": \"" + nextPageToken + "\""; + return "{ \"page_size\": 50, \"results\": [" + String.join(",", records) + "]" + token + " }"; + } + + private static String record(long analysisId, long binaryId) { + return """ + { + "analysis_id": %d, + "analysis_scope": "PRIVATE", + "base_address": 4194304, + "binary_id": %d, + "binary_name": "true", + "binary_size": 1024, + "creation": "2024-04-19T08:57:18Z", + "detected_architecture": "x86_64", + "detected_binary_format": "ELF", + "detected_binary_type": "linux", + "function_boundaries_hash": "b48f61e8", + "is_owner": true, + "model_id": 1, + "model_name": "binnet-0.5", + "sha_256_hash": "%s", + "status": "Complete", + "supplied_architecture": "Auto", + "supplied_binary_format": "Auto", + "supplied_binary_type": "Auto", + "tags": [], + "username": "tester" + } + """.formatted(analysisId, binaryId, HASH); + } +} diff --git a/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/RenameFunctionBatchTest.java b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/RenameFunctionBatchTest.java new file mode 100644 index 00000000..a410a971 --- /dev/null +++ b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/RenameFunctionBatchTest.java @@ -0,0 +1,58 @@ +package ai.reveng.toolkit.ghidra.core.services.api; + +import com.sun.net.httpserver.HttpServer; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * {@link TypedApiImplementation#renameFunction} maps a single rename onto the v3 batch endpoint, + * which answers 200 with a renamed count instead of failing per item. This checks that the request + * carries the full 64-bit function id and that a count of zero reaches the caller as a failure. + */ +public class RenameFunctionBatchTest extends AbstractStubServerTest { + + private static final long FUNCTION_ID = 5_000_000_000L; + + private final List requestBodies = new CopyOnWriteArrayList<>(); + private final AtomicLong renamedCount = new AtomicLong(1); + + @Override + protected void configureStubs(HttpServer server) { + server.createContext("/v3/functions/rename", exchange -> { + requestBodies.add(new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8)); + respondJson(exchange, "{\"renamed_count\":%d}".formatted(renamedCount.get())); + }); + } + + @Test + public void renameFunction_postsOneItemBatchWithUnnarrowedId() { + api().renameFunction(new TypedApiInterface.FunctionID(FUNCTION_ID), "new_name", "new_mangled_name"); + + assertEquals(1, requestBodies.size()); + String body = requestBodies.get(0); + assertTrue("should send the full function id, not a narrowed int: " + body, + body.contains("\"function_id\":" + FUNCTION_ID)); + assertTrue(body.contains("\"new_name\":\"new_name\"")); + assertTrue(body.contains("\"new_mangled_name\":\"new_mangled_name\"")); + } + + @Test + public void renameFunction_failsWhenServerRenamedNothing() { + renamedCount.set(0); + + try { + api().renameFunction(new TypedApiInterface.FunctionID(FUNCTION_ID), "new_name", "new_mangled_name"); + fail("a renamed_count of zero should not be reported as a successful rename"); + } catch (RuntimeException e) { + assertTrue(e.getMessage(), e.getMessage().contains(String.valueOf(FUNCTION_ID))); + } + } +} diff --git a/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/SdkSchemaTest.java b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/SdkSchemaTest.java index c3b84f32..0c7a3f4c 100644 --- a/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/SdkSchemaTest.java +++ b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/SdkSchemaTest.java @@ -17,7 +17,7 @@ public class SdkSchemaTest { - private static final int[] PINNED = {3, 123, 0}; + private static final int[] PINNED = {4, 4, 0}; @Test public void installedSdkIsAtLeastPinned() { @@ -35,18 +35,26 @@ public void apiClientsExposeMethodsThePluginCalls() { apis.put("ai.reveng.api.SearchApi", new String[]{"searchBinaries"}); apis.put("ai.reveng.api.CollectionsApi", new String[]{"v3ListCollections"}); apis.put("ai.reveng.api.AnalysesCoreApi", new String[]{ - "uploadFile", "createAnalysis", "getAnalysisStatus", "getAnalysisBasicInfo", - "startAnalysisFunctionMatching", "getAnalysisFunctionMatchingStatus", "getAnalysisFunctionMatches"}); - apis.put("ai.reveng.api.AnalysesResultsMetadataApi", new String[]{"getFunctionsList"}); + "uploadFile", "createAnalysis", "getAnalysisStatus", "getAnalysisBasicInfo_0", + "startAnalysisFunctionMatching", "getAnalysisFunctionMatchingStatus", "getAnalysisFunctionMatches", + "v3GetAnalysisLogs", "v3ListAnalyses"}); apis.put("ai.reveng.api.FunctionsCoreApi", new String[]{ "startFunctionsMatching", "getFunctionsMatchingStatus", "getFunctionsMatches", - "getFunctionBlocks", "getFunctionDetails"}); - apis.put("ai.reveng.api.FunctionsRenamingHistoryApi", new String[]{ - "renameFunctionId", "batchRenameFunctions"}); - apis.put("ai.reveng.api.FunctionsDataTypesApi", new String[]{ - "listFunctionDataTypesForAnalysis", "listFunctionDataTypesForFunctions"}); + "listAnalysisFunctions", + // v3 endpoints; the generator suffixes _0 where the deprecated v2 name collides. + "getFunctionBlocks_0", "getFunctionDetails_0"}); + apis.put("ai.reveng.api.FunctionsRenamingHistoryApi", new String[]{"batchRenameFunctions"}); + apis.put("ai.reveng.api.DataTypesApi", new String[]{ + "v3ListFunctionSignaturesCall", "v3ListAnalysisDataTypesCall", + "v3GetFunctionSignatureHistory", + // The write path: the two batch data-type endpoints go through the call form + // because their responses embed DataTypeEntry, while the singular signature write + // uses the typed form so its status code survives on the ApiException. + "v3CreateAnalysisDataTypesCall", "v3UpdateAnalysisDataTypesCall", + "v3UpdateFunctionSignature"}); apis.put("ai.reveng.api.FunctionsAiDecompilationApi", new String[]{ - "createAiDecompilation", "getAiDecompilation", "getAiDecompilationTokenised", + "createAiDecompilation", "getAiDecompilation", "v3GetAiDecompilationTokens", + "v3UpsertAiDecompilationOverrides", "getAiDecompilationSummary", "getAiDecompilationSummaryStatus", "getAiDecompilationInlineComments", "getAiDecompilationInlineCommentsStatus", "regenerateAiDecompilationSummary", "regenerateAiDecompilationInlineComments", @@ -63,13 +71,47 @@ public void apiClientsExposeMethodsThePluginCalls() { public void modelTypesExposeAccessorsThePluginReliesOn() { List missing = new ArrayList<>(); - requireMethods(missing, "ai.reveng.model.FunctionInfo", "fromJson", "getFuncTypes", "getFuncDeps"); - requireMethods(missing, "ai.reveng.model.FunctionType", "getName", "getHeader", "getType"); - requireMethods(missing, "ai.reveng.model.FunctionHeader", "getName", "getArgs"); - requireMethods(missing, "ai.reveng.model.FunctionDataTypesList", "getItems"); - requireMethods(missing, "ai.reveng.model.FunctionDataTypesListItem", - "getDataTypes", "getCompleted", "getFunctionId"); - requireClass(missing, "ai.reveng.model.FuncDepsInner"); + // The data-type read path deserialises DataTypeEntry itself (see ServerDataTypeReader), so + // what has to stay stable is the signature surface around it: the entries the plugin reads + // out of the /v3/functions/signatures body, and the history models it reads whole. + requireMethods(missing, "ai.reveng.model.BatchFunctionSignatureEntry", + "getAnalysisId", "getFunctionId", "getFunctionName", "getHasSignature", + "getParameters", "getReturnDataTypeId"); + requireMethods(missing, "ai.reveng.model.SignatureParameterEntry", + "getName", "getOrdinal", "getDataTypeId", "getBitLength"); + requireMethods(missing, "ai.reveng.model.GetFunctionSignatureHistoryBody", "getVersions"); + + // The write path builds request bodies out of generated models — serialisation of the + // oneOf unions works even though deserialisation does not — so their setters are the + // surface that has to stay put. + requireMethods(missing, "ai.reveng.model.CreateAnalysisDataTypesInputBody", "setDataTypes"); + requireMethods(missing, "ai.reveng.model.UpdateAnalysisDataTypesInputBody", "setDataTypes"); + requireMethods(missing, "ai.reveng.model.CreateDataTypeEntry", "getActualInstance"); + requireMethods(missing, "ai.reveng.model.UpdateDataTypeEntry", "getActualInstance"); + requireMethods(missing, "ai.reveng.model.CreateStructDataType", + "kind", "name", "namespace", "size", "definition"); + requireMethods(missing, "ai.reveng.model.UpdateStructDataType", + "kind", "dataTypeId", "name", "namespace", "size", "definition"); + requireMethods(missing, "ai.reveng.model.DataTypeMemberEntry", + "name", "offset", "size", "dataTypeId", "isBitfield", "bitOffset", "bitSize"); + requireMethods(missing, "ai.reveng.model.DataTypeEnumValueEntry", "name", "value"); + requireMethods(missing, "ai.reveng.model.DataTypeFunctionParameterEntry", + "ordinal", "size", "name", "dataTypeId"); + requireMethods(missing, "ai.reveng.model.StructDefinition", "members"); + requireMethods(missing, "ai.reveng.model.UnionDefinition", "members"); + requireMethods(missing, "ai.reveng.model.EnumDefinition", "values"); + requireMethods(missing, "ai.reveng.model.TypedefDefinition", "targetDataTypeId"); + requireMethods(missing, "ai.reveng.model.PointerDefinition", "pointeeDataTypeId"); + requireMethods(missing, "ai.reveng.model.ArrayDefinition", "count", "elementDataTypeId"); + requireMethods(missing, "ai.reveng.model.FunctionTypeDefinition", + "parameters", "returnDataTypeId"); + requireMethods(missing, "ai.reveng.model.UpdateFunctionSignatureInputBody", + "setCallingConvention", "setParameters", "setReturnDataTypeId"); + requireMethods(missing, "ai.reveng.model.SignatureParameterInput", + "ordinal", "name", "dataTypeId", "bitLength", "storage"); + requireMethods(missing, "ai.reveng.model.SignatureStorageInput", "kind", "location"); + requireMethods(missing, "ai.reveng.model.FunctionSignatureVersion", + "getValue", "getUpdatedAt", "getUpdatedBy"); requireMethods(missing, "ai.reveng.model.AnalysisCreateRequest", "getFilename", "getSha256Hash", "getTags", "getAnalysisScope"); @@ -86,7 +128,51 @@ public void modelTypesExposeAccessorsThePluginReliesOn() { requireMethods(missing, "ai.reveng.model.BatchRenameInputBody", "setFunctions"); requireMethods(missing, "ai.reveng.model.BatchRenameItem", "setFunctionId", "setNewName", "setNewMangledName"); - requireMethods(missing, "ai.reveng.model.FunctionRename", "getNewName", "getNewMangledName"); + requireMethods(missing, "ai.reveng.model.BatchRenameOutputBody", "getRenamedCount"); + + // The v3 blocks body. Only the untyped basic_blocks value is read, by DisassemblyBlocksReader; + // the spec gives it no schema, so this pins the one accessor that carries the disassembly. + requireMethods(missing, "ai.reveng.model.DisassemblyOutputBody", "getBasicBlocks"); + + // The v3 function list. total_count drives paging termination, and the entry accessors are + // what FunctionInfo is built from. + requireMethods(missing, "ai.reveng.model.ListAnalysisFunctionsOutputBody", + "getFunctions", "getTotalCount"); + requireMethods(missing, "ai.reveng.model.AnalysisFunctionEntry", + "getFunctionId", "getFunctionName", "getMangledName", "getFunctionVaddr", + "getFunctionSize"); + + requireMethods(missing, "ai.reveng.model.AnalysisBasicInfoOutputBody", + "getBinaryName", "getSha256Hash", "getModelName"); + + // The v3 analysis log. Every entry field is rendered into the log view's single string. + requireMethods(missing, "ai.reveng.model.GetAnalysisLogsOutputBody", "getEntries"); + requireMethods(missing, "ai.reveng.model.AnalysisLogEntry", + "getTimestamp", "getLevel", "getSource", "getText"); + + // The v3 analysis list. next_page_token drives paging; AnalysisRecordBody is the row type + // the Recent Analyses table is built on. + requireMethods(missing, "ai.reveng.model.ListAnalysesOutputBody", + "getResults", "getNextPageToken"); + requireMethods(missing, "ai.reveng.model.AnalysisRecordBody", + "getAnalysisId", "getBinaryId", "getBinaryName", "getCreation", "getStatus", + "getBaseAddress"); + + requireMethods(missing, "ai.reveng.model.FunctionDetailsOutputBody", + "getFunctionId", "getMangledName", "getFunctionVaddr", "getFunctionSize", + "getAnalysisId", "getFunctionName"); + + // Resolving a double-clicked identifier back to the token to override reads the tokenised + // source and both name maps, which arrive unmerged. + requireMethods(missing, "ai.reveng.model.GetTokensResponse", + "getAiDecomp", "getPlaceholderToRenderedToken", "getPlaceholderToUserOverride"); + // Both maps hold different types. The rendered value is read out of either; the ids decide + // whether a token names something this decompilation owns, and so whether it can be renamed + // through the overrides endpoint at all. + requireMethods(missing, "ai.reveng.model.RenderedToken", + "getValue", "getDataTypeId", "getFunctionId", "getImportedFunctionId"); + requireMethods(missing, "ai.reveng.model.Token", "getValue"); + requireMethods(missing, "ai.reveng.model.UpsertOverridesInputBody", "getOverrides"); assertTrue("SDK model surface drifted: " + missing, missing.isEmpty()); } @@ -104,7 +190,10 @@ public void analysisScopeEnumHasPluginMembers() { } private static int[] installedSdkVersion() { - Class anchor = classOrNull("ai.reveng.model.FunctionInfo"); + // Anchored on the invoker rather than a model class: models come and go between SDK + // releases, and when the anchor disappears this assertion misreports the SDK as absent + // from the classpath entirely. + Class anchor = classOrNull("ai.reveng.invoker.ApiClient"); assertNotNull("ai.reveng:sdk is not on the test classpath", anchor); CodeSource codeSource = anchor.getProtectionDomain().getCodeSource(); assertNotNull("Could not locate the ai.reveng:sdk code source", codeSource); @@ -118,12 +207,6 @@ private static int[] installedSdkVersion() { }; } - private static void requireClass(List missing, String className) { - if (classOrNull(className) == null) { - missing.add(className + " (class)"); - } - } - private static void requireMethods(List missing, String className, String... methods) { Class cls = classOrNull(className); if (cls == null) { diff --git a/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisStatusTest.java b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisStatusTest.java new file mode 100644 index 00000000..b9c28b0f --- /dev/null +++ b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisStatusTest.java @@ -0,0 +1,57 @@ +package ai.reveng.toolkit.ghidra.core.services.api.types; + +import ai.reveng.model.StatusInput; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * Guards that every status the API can report resolves to an {@link AnalysisStatus} without throwing. + */ +public class AnalysisStatusTest { + + @Test + public void uploadedIsAModelledStatus() { + assertEquals(AnalysisStatus.Uploaded, AnalysisStatus.fromApiValue("Uploaded")); + } + + @Test + public void modelledStatusesResolveToThemselves() { + for (AnalysisStatus status : AnalysisStatus.values()) { + assertEquals(status, AnalysisStatus.fromApiValue(status.name())); + } + } + + @Test + public void unrecognisedValuesResolveToUnknown() { + assertEquals(AnalysisStatus.Unknown, AnalysisStatus.fromApiValue("Ludicrous")); + assertEquals(AnalysisStatus.Unknown, AnalysisStatus.fromApiValue("")); + assertEquals(AnalysisStatus.Unknown, AnalysisStatus.fromApiValue(null)); + } + + /** + * The SDK enum for the status endpoint is the contract the plugin has to survive, including the + * {@code All} filter sentinel and the generator's unknown placeholder. + */ + @Test + public void everyStatusTheSdkDeclaresResolves() { + for (StatusInput sdkStatus : StatusInput.values()) { + AnalysisStatus resolved = AnalysisStatus.fromApiValue(sdkStatus.getValue()); + assertNotNull("No status resolved for SDK value " + sdkStatus.getValue(), resolved); + } + } + + /** + * {@code RecentAnalysesTableModel} filters completed analyses by comparing this name against the + * raw status string from the API, so the constant name has to stay the wire value. + */ + @Test + public void constantNamesAreTheWireValues() { + assertEquals(StatusInput.COMPLETE.getValue(), AnalysisStatus.Complete.name()); + assertEquals(StatusInput.UPLOADED.getValue(), AnalysisStatus.Uploaded.name()); + assertEquals(StatusInput.QUEUED.getValue(), AnalysisStatus.Queued.name()); + assertEquals(StatusInput.PROCESSING.getValue(), AnalysisStatus.Processing.name()); + assertEquals(StatusInput.ERROR.getValue(), AnalysisStatus.Error.name()); + } +} diff --git a/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/types/TypePathAndNameTest.java b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/types/TypePathAndNameTest.java new file mode 100644 index 00000000..fd0da949 --- /dev/null +++ b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/types/TypePathAndNameTest.java @@ -0,0 +1,25 @@ +package ai.reveng.toolkit.ghidra.core.services.api.types; + +import org.junit.Test; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +public class TypePathAndNameTest { + + @Test + public void fromString_splitsTheNamespacePathFromTheName() { + var path = TypePathAndName.fromString("a::b::c"); + + assertEquals("the name is the last segment", "c", path.name()); + assertArrayEquals("the leading segments are the path", new String[]{"a", "b"}, path.path()); + } + + @Test + public void fromString_leavesAnUnqualifiedNameWithAnEmptyPath() { + var path = TypePathAndName.fromString("PlainName"); + + assertEquals("PlainName", path.name()); + assertArrayEquals("an unqualified name has no path segments", new String[]{}, path.path()); + } +} diff --git a/src/test/resources/FunctionDataTypeStatus.json b/src/test/resources/FunctionDataTypeStatus.json deleted file mode 100644 index bf07303a..00000000 --- a/src/test/resources/FunctionDataTypeStatus.json +++ /dev/null @@ -1,246 +0,0 @@ -{ - "status": true, - "data": { - "completed": true, - "status": "completed", - "data_types": { - "func_types": { - "last_change": null, - "addr": 8138, - "size": 113, - "header": { - "last_change": null, - "name": "FUN_00101fca", - "addr": 8138, - "type": "uint", - "args": { - "0x0": { - "last_change": null, - "offset": 0, - "name": "renamed_param_1", - "type": "char *", - "size": 8 - }, - "0x1": { - "last_change": null, - "offset": 1, - "name": "param_2", - "type": "char *", - "size": 8 - } - } - }, - "stack_vars": { - "-0x98": { - "last_change": null, - "offset": -152, - "name": "local_98", - "type": "stat64", - "size": 144, - "addr": 8138 - }, - "-0x128": { - "last_change": null, - "offset": -296, - "name": "sStack_128", - "type": "stat64", - "size": 144, - "addr": 8138 - } - }, - "name": "FUN_00101fca", - "type": "uint" - }, - "func_deps": [ - { - "last_change": null, - "name": "stat.h/stat64", - "size": 144, - "members": { - "0x0": { - "last_change": null, - "name": "st_dev", - "offset": 0, - "type": "__dev_t", - "size": 8 - }, - "0x8": { - "last_change": null, - "name": "st_ino", - "offset": 8, - "type": "__ino64_t", - "size": 8 - }, - "0x10": { - "last_change": null, - "name": "st_nlink", - "offset": 16, - "type": "__nlink_t", - "size": 8 - }, - "0x18": { - "last_change": null, - "name": "st_mode", - "offset": 24, - "type": "__mode_t", - "size": 4 - }, - "0x1c": { - "last_change": null, - "name": "st_uid", - "offset": 28, - "type": "__uid_t", - "size": 4 - }, - "0x20": { - "last_change": null, - "name": "st_gid", - "offset": 32, - "type": "__gid_t", - "size": 4 - }, - "0x24": { - "last_change": null, - "name": "__pad0", - "offset": 36, - "type": "int", - "size": 4 - }, - "0x28": { - "last_change": null, - "name": "st_rdev", - "offset": 40, - "type": "__dev_t", - "size": 8 - }, - "0x30": { - "last_change": null, - "name": "st_size", - "offset": 48, - "type": "__off_t", - "size": 8 - }, - "0x38": { - "last_change": null, - "name": "st_blksize", - "offset": 56, - "type": "__blksize_t", - "size": 8 - }, - "0x40": { - "last_change": null, - "name": "st_blocks", - "offset": 64, - "type": "__blkcnt64_t", - "size": 8 - }, - "0x48": { - "last_change": null, - "name": "st_atim", - "offset": 72, - "type": "timespec", - "size": 16 - }, - "0x58": { - "last_change": null, - "name": "st_mtim", - "offset": 88, - "type": "timespec", - "size": 16 - }, - "0x68": { - "last_change": null, - "name": "st_ctim", - "offset": 104, - "type": "timespec", - "size": 16 - }, - "0x78": { - "last_change": null, - "name": "__unused", - "offset": 120, - "type": "long[3]", - "size": 24 - } - } - }, - { - "last_change": null, - "name": "time.h/timespec", - "size": 16, - "members": { - "0x0": { - "last_change": null, - "name": "tv_sec", - "offset": 0, - "type": "__time_t", - "size": 8 - }, - "0x8": { - "last_change": null, - "name": "tv_nsec", - "offset": 8, - "type": "long", - "size": 8 - } - } - }, - { - "last_change": null, - "name": "types.h/__mode_t", - "type": "uint" - }, - { - "last_change": null, - "name": "types.h/__gid_t", - "type": "uint" - }, - { - "last_change": null, - "name": "types.h/__off_t", - "type": "long" - }, - { - "last_change": null, - "name": "types.h/__uid_t", - "type": "uint" - }, - { - "last_change": null, - "name": "types.h/__time_t", - "type": "long" - }, - { - "last_change": null, - "name": "types.h/__dev_t", - "type": "ulong" - }, - { - "last_change": null, - "name": "types.h/__blksize_t", - "type": "long" - }, - { - "last_change": null, - "name": "types.h/__nlink_t", - "type": "ulong" - }, - { - "last_change": null, - "name": "types.h/__blkcnt64_t", - "type": "long" - }, - { - "last_change": null, - "name": "types.h/__ino64_t", - "type": "ulong" - } - ] - } - }, - "message": null, - "errors": null, - "meta": { - "pagination": null - } -} \ No newline at end of file diff --git a/src/test/resources/ai_decomp_example.json b/src/test/resources/ai_decomp_example.json deleted file mode 100644 index e5994a41..00000000 --- a/src/test/resources/ai_decomp_example.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "status": true, - "data": { - "status": "success", - "decompilation": "\n(\n const char *\n)\n{\n struct ;\n if ((, &) == 0) {\n return .;\n }\n else {\n return -1;\n }\n}", - "raw_decompilation": "\n(\n const char *\n)\n{\n struct ;\n if ((, &) == 0) {\n return .;\n }\n else {\n return -1;\n }\n}", - "function_mapping": { - "": { - "name": "FUN_0000172e", - "addr": 5934, - "is_external": false - }, - "": { - "name": "__xstat64", - "addr": 4576, - "is_external": true - } - }, - "function_mapping_full": { - "inverse_string_map": {}, - "inverse_function_map": { - "": { - "name": "FUN_0000172e", - "addr": 5934, - "is_external": false - }, - "": { - "name": "__xstat64", - "addr": 4576, - "is_external": true - } - }, - "unmatched_functions": {}, - "unmatched_external_vars": {}, - "unmatched_custom_types": { - "": { - "value": "std::string" - }, - "": { - "value": "StringParser" - } - }, - "unmatched_strings": {}, - "unmatched_vars": { - "": { - "value": "inputString" - }, - "": { - "value": "parserResult" - } - }, - "unmatched_go_to_labels": {}, - "unmatched_custom_function_pointers": {}, - "unmatched_variadic_lists": {}, - "fields": { - "": { - "": { - "value": "resultCode" - } - } - } - }, - "summary": "The function takes a character pointer as input. It calls another function with the input and a pointer to a struct. If the call is successful, it returns a value from the struct. Otherwise, it returns -1.\n" - }, - "message": null, - "errors": null, - "meta": { - "pagination": null - } -} \ No newline at end of file diff --git a/src/test/resources/ai_decomp_type_field.json b/src/test/resources/ai_decomp_type_field.json deleted file mode 100644 index 26db4ab3..00000000 --- a/src/test/resources/ai_decomp_type_field.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - "status": true, - "data": { - "status": "success", - "decompilation": "std::string\nFUN_0000172e(\n const char *inputString\n)\n{\n struct StringParser parserResult;\n if (StringParser(inputString, &parserResult) == 0) {\n return parserResult.errorCode;\n }\n else {\n return -1;\n }\n}", - "raw_decompilation": "\n(\n const char *\n)\n{\n struct ;\n if ((, &) == 0) {\n return .;\n }\n else {\n return -1;\n }\n}", - "function_mapping": { - "": { - "name": "FUN_0000172e", - "addr": 5934, - "is_external": false - }, - "": { - "name": "__xstat64", - "addr": 4576, - "is_external": true - } - }, - "function_mapping_full": { - "inverse_string_map": {}, - "inverse_function_map": { - "": { - "name": "FUN_0000172e", - "addr": 5934, - "is_external": false - }, - "": { - "name": "__xstat64", - "addr": 4576, - "is_external": true - } - }, - "unmatched_functions": {}, - "unmatched_external_vars": {}, - "unmatched_custom_types": { - "": { - "value": "std::string" - }, - "": { - "value": "StringParser" - } - }, - "unmatched_strings": {}, - "unmatched_vars": { - "": { - "value": "inputString" - }, - "": { - "value": "parserResult" - } - }, - "unmatched_go_to_labels": {}, - "unmatched_custom_function_pointers": {}, - "unmatched_variadic_lists": {}, - "fields": { - "": { - "": { - "value": "errorCode" - } - } - } - }, - "summary": "The function takes a string as input. It attempts to populate a structure of type using the input string. If the population is successful, it returns the value of the member of the structure. Otherwise, it returns -1.\n", - "ai_summary": "The function FUN_0000172e takes a string as input. It attempts to populate a structure of type StringParser using the input string. If the population is successful, it returns the value of the member of the structure. Otherwise, it returns -1.\n", - "raw_ai_summary": "The function takes a string as input. It attempts to populate a structure of type using the input string. If the population is successful, it returns the value of the member of the structure. Otherwise, it returns -1.\n" - }, - "message": null, - "errors": null, - "meta": { - "pagination": null - } -} \ No newline at end of file diff --git a/src/test/resources/complex_pointer.json b/src/test/resources/complex_pointer.json deleted file mode 100644 index a529f9e3..00000000 --- a/src/test/resources/complex_pointer.json +++ /dev/null @@ -1,180 +0,0 @@ -{ - "status": true, - "data": { - "completed": true, - "status": "completed", - "data_types": { - "func_types": { - "last_change": null, - "addr": 11475, - "size": 130, - "header": { - "last_change": null, - "name": "registerpair", - "addr": 11475, - "type": "void", - "args": { - "0x0": { - "last_change": null, - "offset": 0, - "name": "matchlist", - "type": "file_t * *", - "size": 8 - }, - "0x1": { - "last_change": null, - "offset": 1, - "name": "newmatch", - "type": "file_t *", - "size": 8 - }, - "0x2": { - "last_change": null, - "offset": 2, - "name": "comparef", - "type": "_func_int_file_t_ptr_file_t_ptr *", - "size": 8 - } - } - }, - "stack_vars": null, - "name": "registerpair", - "type": "void" - }, - "func_deps": [ - { - "last_change": null, - "name": "DWARF/__ino64_t", - "type": "qword" - }, - { - "last_change": null, - "name": "DWARF/stat.h/dev_t", - "type": "__dev_t" - }, - { - "last_change": null, - "name": "DWARF/md5_byte_t", - "type": "uchar" - }, - { - "last_change": null, - "name": "DWARF/fdupes.c/_file", - "size": 80, - "members": { - "0x0": { - "last_change": null, - "name": "d_name", - "offset": 0, - "type": "char *", - "size": 8 - }, - "0x8": { - "last_change": null, - "name": "size", - "offset": 8, - "type": "off_t", - "size": 8 - }, - "0x10": { - "last_change": null, - "name": "crcpartial", - "offset": 16, - "type": "md5_byte_t *", - "size": 8 - }, - "0x18": { - "last_change": null, - "name": "crcsignature", - "offset": 24, - "type": "md5_byte_t *", - "size": 8 - }, - "0x20": { - "last_change": null, - "name": "device", - "offset": 32, - "type": "dev_t", - "size": 8 - }, - "0x28": { - "last_change": null, - "name": "inode", - "offset": 40, - "type": "ino_t", - "size": 8 - }, - "0x30": { - "last_change": null, - "name": "mtime", - "offset": 48, - "type": "time_t", - "size": 8 - }, - "0x38": { - "last_change": null, - "name": "hasdupes", - "offset": 56, - "type": "int", - "size": 4 - }, - "0x40": { - "last_change": null, - "name": "duplicates", - "offset": 64, - "type": "_file *", - "size": 8 - }, - "0x48": { - "last_change": null, - "name": "next", - "offset": 72, - "type": "_file *", - "size": 8 - } - } - }, - { - "last_change": null, - "name": "DWARF/stdio.h/off_t", - "type": "__off64_t" - }, - { - "last_change": null, - "name": "DWARF/stat.h/ino_t", - "type": "__ino64_t" - }, - { - "last_change": null, - "name": "DWARF/time.h/time_t", - "type": "__time_t" - }, - { - "last_change": null, - "name": "DWARF/fdupes.c/file_t", - "type": "_file" - }, - { - "last_change": null, - "name": "DWARF/__off64_t", - "type": "sqword" - }, - { - "last_change": null, - "name": "DWARF/__dev_t", - "type": "ulong" - }, - { - "last_change": null, - "name": "DWARF/__time_t", - "type": "long" - } - ] - } - }, - "message": null, - "errors": null, - "meta": { - "pagination": null - } -} \ No newline at end of file diff --git a/src/test/resources/confirmmatch_fdupes_77846700.json b/src/test/resources/confirmmatch_fdupes_77846700.json deleted file mode 100644 index 8a020dd6..00000000 --- a/src/test/resources/confirmmatch_fdupes_77846700.json +++ /dev/null @@ -1,326 +0,0 @@ -{ - "status": true, - "data": { - "completed": true, - "status": "completed", - "data_types": { - "func_types": { - "last_change": null, - "addr": 9098, - "size": 180, - "header": { - "last_change": null, - "name": "confirmmatch", - "addr": 9098, - "type": "int", - "args": { - "0x0": { - "last_change": null, - "offset": 0, - "name": "file1", - "type": "FILE *", - "size": 8 - }, - "0x1": { - "last_change": null, - "offset": 1, - "name": "file2", - "type": "FILE *", - "size": 8 - } - } - }, - "stack_vars": { - "-0x2038": { - "last_change": null, - "offset": -8248, - "name": "c1", - "type": "uchar[8192]", - "size": 8192, - "addr": 9098 - }, - "-0x4038": { - "last_change": null, - "offset": -16440, - "name": "c2", - "type": "uchar[8192]", - "size": 8192, - "addr": 9098 - } - }, - "name": "confirmmatch", - "type": "int" - }, - "func_deps": [ - { - "last_change": null, - "name": "DWARF/__off_t", - "type": "long" - }, - { - "last_change": null, - "name": "DWARF/libio.h/_IO_lock_t", - "type": "void" - }, - { - "last_change": null, - "name": "DWARF/__off64_t", - "type": "sqword" - }, - { - "last_change": null, - "name": "DWARF/size_t", - "type": "ulong" - }, - { - "last_change": null, - "name": "DWARF/stdio.h/FILE", - "type": "_IO_FILE" - }, - { - "last_change": null, - "name": "DWARF/libio.h/_IO_FILE", - "size": 216, - "members": { - "0x0": { - "last_change": null, - "name": "_flags", - "offset": 0, - "type": "int", - "size": 4 - }, - "0x8": { - "last_change": null, - "name": "_IO_read_ptr", - "offset": 8, - "type": "char *", - "size": 8 - }, - "0x10": { - "last_change": null, - "name": "_IO_read_end", - "offset": 16, - "type": "char *", - "size": 8 - }, - "0x18": { - "last_change": null, - "name": "_IO_read_base", - "offset": 24, - "type": "char *", - "size": 8 - }, - "0x20": { - "last_change": null, - "name": "_IO_write_base", - "offset": 32, - "type": "char *", - "size": 8 - }, - "0x28": { - "last_change": null, - "name": "_IO_write_ptr", - "offset": 40, - "type": "char *", - "size": 8 - }, - "0x30": { - "last_change": null, - "name": "_IO_write_end", - "offset": 48, - "type": "char *", - "size": 8 - }, - "0x38": { - "last_change": null, - "name": "_IO_buf_base", - "offset": 56, - "type": "char *", - "size": 8 - }, - "0x40": { - "last_change": null, - "name": "_IO_buf_end", - "offset": 64, - "type": "char *", - "size": 8 - }, - "0x48": { - "last_change": null, - "name": "_IO_save_base", - "offset": 72, - "type": "char *", - "size": 8 - }, - "0x50": { - "last_change": null, - "name": "_IO_backup_base", - "offset": 80, - "type": "char *", - "size": 8 - }, - "0x58": { - "last_change": null, - "name": "_IO_save_end", - "offset": 88, - "type": "char *", - "size": 8 - }, - "0x60": { - "last_change": null, - "name": "_markers", - "offset": 96, - "type": "_IO_marker *", - "size": 8 - }, - "0x68": { - "last_change": null, - "name": "_chain", - "offset": 104, - "type": "_IO_FILE *", - "size": 8 - }, - "0x70": { - "last_change": null, - "name": "_fileno", - "offset": 112, - "type": "int", - "size": 4 - }, - "0x74": { - "last_change": null, - "name": "_flags2", - "offset": 116, - "type": "int", - "size": 4 - }, - "0x78": { - "last_change": null, - "name": "_old_offset", - "offset": 120, - "type": "__off_t", - "size": 8 - }, - "0x80": { - "last_change": null, - "name": "_cur_column", - "offset": 128, - "type": "ushort", - "size": 2 - }, - "0x82": { - "last_change": null, - "name": "_vtable_offset", - "offset": 130, - "type": "char", - "size": 1 - }, - "0x83": { - "last_change": null, - "name": "_shortbuf", - "offset": 131, - "type": "char[1]", - "size": 1 - }, - "0x88": { - "last_change": null, - "name": "_lock", - "offset": 136, - "type": "_IO_lock_t *", - "size": 8 - }, - "0x90": { - "last_change": null, - "name": "_offset", - "offset": 144, - "type": "__off64_t", - "size": 8 - }, - "0x98": { - "last_change": null, - "name": "__pad1", - "offset": 152, - "type": "void *", - "size": 8 - }, - "0xa0": { - "last_change": null, - "name": "__pad2", - "offset": 160, - "type": "void *", - "size": 8 - }, - "0xa8": { - "last_change": null, - "name": "__pad3", - "offset": 168, - "type": "void *", - "size": 8 - }, - "0xb0": { - "last_change": null, - "name": "__pad4", - "offset": 176, - "type": "void *", - "size": 8 - }, - "0xb8": { - "last_change": null, - "name": "__pad5", - "offset": 184, - "type": "size_t", - "size": 8 - }, - "0xc0": { - "last_change": null, - "name": "_mode", - "offset": 192, - "type": "int", - "size": 4 - }, - "0xc4": { - "last_change": null, - "name": "_unused2", - "offset": 196, - "type": "char[20]", - "size": 20 - } - } - }, - { - "last_change": null, - "name": "DWARF/libio.h/_IO_marker", - "size": 24, - "members": { - "0x0": { - "last_change": null, - "name": "_next", - "offset": 0, - "type": "_IO_marker *", - "size": 8 - }, - "0x8": { - "last_change": null, - "name": "_sbuf", - "offset": 8, - "type": "_IO_FILE *", - "size": 8 - }, - "0x10": { - "last_change": null, - "name": "_pos", - "offset": 16, - "type": "int", - "size": 4 - } - } - } - ] - } - }, - "message": null, - "errors": null, - "meta": { - "pagination": null - } -} \ No newline at end of file diff --git a/src/test/resources/data_types_batch_response.json b/src/test/resources/data_types_batch_response.json deleted file mode 100644 index 52fa8ee4..00000000 --- a/src/test/resources/data_types_batch_response.json +++ /dev/null @@ -1,380 +0,0 @@ -{ - "status": true, - "data": { - "total_count": 2, - "total_data_types_count": 2, - "items": [ - { - "completed": true, - "status": "completed", - "data_types": { - "func_types": { - "last_change": null, - "addr": 5344, - "size": 72, - "header": { - "last_change": null, - "name": "sort_pairs_by_mtime", - "addr": 5344, - "type": "int", - "args": { - "0x0": { - "last_change": null, - "offset": 0, - "name": "f1", - "type": "file_t *", - "size": 8 - }, - "0x1": { - "last_change": null, - "offset": 1, - "name": "f2", - "type": "file_t *", - "size": 8 - } - } - }, - "stack_vars": null, - "name": "sort_pairs_by_mtime", - "type": "int", - "artifact_type": "Function" - }, - "func_deps": [ - { - "last_change": null, - "name": "DWARF/md5_byte_t", - "type": "uchar", - "artifact_type": "Typedef" - }, - { - "last_change": null, - "name": "DWARF/stat.h/dev_t", - "type": "__dev_t", - "artifact_type": "Typedef" - }, - { - "last_change": null, - "name": "types.h/__off64_t", - "type": "long", - "artifact_type": "Typedef" - }, - { - "last_change": null, - "name": "DWARF/fdupes.c/_file", - "size": 80, - "members": { - "0x0": { - "last_change": null, - "name": "d_name", - "offset": 0, - "type": "char *", - "size": 8 - }, - "0x8": { - "last_change": null, - "name": "size", - "offset": 8, - "type": "off_t", - "size": 8 - }, - "0x10": { - "last_change": null, - "name": "crcpartial", - "offset": 16, - "type": "md5_byte_t *", - "size": 8 - }, - "0x18": { - "last_change": null, - "name": "crcsignature", - "offset": 24, - "type": "md5_byte_t *", - "size": 8 - }, - "0x20": { - "last_change": null, - "name": "device", - "offset": 32, - "type": "dev_t", - "size": 8 - }, - "0x28": { - "last_change": null, - "name": "inode", - "offset": 40, - "type": "ino_t", - "size": 8 - }, - "0x30": { - "last_change": null, - "name": "mtime", - "offset": 48, - "type": "time_t", - "size": 8 - }, - "0x38": { - "last_change": null, - "name": "hasdupes", - "offset": 56, - "type": "int", - "size": 4 - }, - "0x40": { - "last_change": null, - "name": "duplicates", - "offset": 64, - "type": "_file *", - "size": 8 - }, - "0x48": { - "last_change": null, - "name": "next", - "offset": 72, - "type": "_file *", - "size": 8 - } - }, - "artifact_type": "Struct" - }, - { - "last_change": null, - "name": "DWARF/stdio.h/off_t", - "type": "__off64_t", - "artifact_type": "Typedef" - }, - { - "last_change": null, - "name": "DWARF/fdupes.c/file_t", - "type": "_file", - "artifact_type": "Typedef" - }, - { - "last_change": null, - "name": "types.h/__dev_t", - "type": "ulong", - "artifact_type": "Typedef" - }, - { - "last_change": null, - "name": "DWARF/stat.h/ino_t", - "type": "__ino64_t", - "artifact_type": "Typedef" - }, - { - "last_change": null, - "name": "DWARF/time.h/time_t", - "type": "__time_t", - "artifact_type": "Typedef" - }, - { - "last_change": null, - "name": "types.h/__time_t", - "type": "long", - "artifact_type": "Typedef" - }, - { - "last_change": null, - "name": "types.h/__ino64_t", - "type": "ulong", - "artifact_type": "Typedef" - }, - { - "last_change": null, - "addr": 2122944, - "name": "flags", - "type": "ulong", - "size": 8, - "artifact_type": "GlobalVariable" - } - ] - }, - "data_types_version": null, - "function_id": 266294328 - }, - { - "completed": true, - "status": "completed", - "data_types": { - "func_types": { - "last_change": null, - "addr": 5416, - "size": 20, - "header": { - "last_change": null, - "name": "sort_pairs_by_filename", - "addr": 5416, - "type": "int", - "args": { - "0x0": { - "last_change": null, - "offset": 0, - "name": "f1", - "type": "file_t *", - "size": 8 - }, - "0x1": { - "last_change": null, - "offset": 1, - "name": "f2", - "type": "file_t *", - "size": 8 - } - } - }, - "stack_vars": null, - "name": "sort_pairs_by_filename", - "type": "int", - "artifact_type": "Function" - }, - "func_deps": [ - { - "last_change": null, - "name": "DWARF/md5_byte_t", - "type": "uchar", - "artifact_type": "Typedef" - }, - { - "last_change": null, - "name": "DWARF/stat.h/dev_t", - "type": "__dev_t", - "artifact_type": "Typedef" - }, - { - "last_change": null, - "name": "types.h/__off64_t", - "type": "long", - "artifact_type": "Typedef" - }, - { - "last_change": null, - "name": "DWARF/fdupes.c/_file", - "size": 80, - "members": { - "0x0": { - "last_change": null, - "name": "d_name", - "offset": 0, - "type": "char *", - "size": 8 - }, - "0x8": { - "last_change": null, - "name": "size", - "offset": 8, - "type": "off_t", - "size": 8 - }, - "0x10": { - "last_change": null, - "name": "crcpartial", - "offset": 16, - "type": "md5_byte_t *", - "size": 8 - }, - "0x18": { - "last_change": null, - "name": "crcsignature", - "offset": 24, - "type": "md5_byte_t *", - "size": 8 - }, - "0x20": { - "last_change": null, - "name": "device", - "offset": 32, - "type": "dev_t", - "size": 8 - }, - "0x28": { - "last_change": null, - "name": "inode", - "offset": 40, - "type": "ino_t", - "size": 8 - }, - "0x30": { - "last_change": null, - "name": "mtime", - "offset": 48, - "type": "time_t", - "size": 8 - }, - "0x38": { - "last_change": null, - "name": "hasdupes", - "offset": 56, - "type": "int", - "size": 4 - }, - "0x40": { - "last_change": null, - "name": "duplicates", - "offset": 64, - "type": "_file *", - "size": 8 - }, - "0x48": { - "last_change": null, - "name": "next", - "offset": 72, - "type": "_file *", - "size": 8 - } - }, - "artifact_type": "Struct" - }, - { - "last_change": null, - "name": "DWARF/stdio.h/off_t", - "type": "__off64_t", - "artifact_type": "Typedef" - }, - { - "last_change": null, - "name": "DWARF/fdupes.c/file_t", - "type": "_file", - "artifact_type": "Typedef" - }, - { - "last_change": null, - "name": "types.h/__dev_t", - "type": "ulong", - "artifact_type": "Typedef" - }, - { - "last_change": null, - "name": "DWARF/stat.h/ino_t", - "type": "__ino64_t", - "artifact_type": "Typedef" - }, - { - "last_change": null, - "name": "DWARF/time.h/time_t", - "type": "__time_t", - "artifact_type": "Typedef" - }, - { - "last_change": null, - "name": "types.h/__time_t", - "type": "long", - "artifact_type": "Typedef" - }, - { - "last_change": null, - "name": "types.h/__ino64_t", - "type": "ulong", - "artifact_type": "Typedef" - } - ] - }, - "data_types_version": null, - "function_id": 266294329 - } - ] - }, - "message": null, - "errors": null, - "meta": { - "pagination": null - } -} \ No newline at end of file diff --git a/src/test/resources/errormsg.json b/src/test/resources/errormsg.json deleted file mode 100644 index 3dc17179..00000000 --- a/src/test/resources/errormsg.json +++ /dev/null @@ -1,196 +0,0 @@ -{ - "status": true, - "data": { - "completed": true, - "status": "completed", - "data_types": { - "func_types": { - "last_change": null, - "addr": 5436, - "size": 211, - "header": { - "last_change": null, - "name": "errormsg", - "addr": 5436, - "type": "void", - "args": { - "0x0": { - "last_change": null, - "offset": 0, - "name": "message", - "type": "char *", - "size": 8 - } - } - }, - "stack_vars": { - "-0xb8": { - "last_change": null, - "offset": -184, - "name": "local_b8", - "type": "char[8]", - "size": 8, - "addr": 5436 - }, - "-0xb0": { - "last_change": null, - "offset": -176, - "name": "local_b0", - "type": "char", - "size": 8, - "addr": 5436 - }, - "-0xd0": { - "last_change": null, - "offset": -208, - "name": "ap", - "type": "va_list", - "size": 24, - "addr": 5436 - }, - "-0xa8": { - "last_change": null, - "offset": -168, - "name": "local_a8", - "type": "char", - "size": 8, - "addr": 5436 - }, - "-0xa0": { - "last_change": null, - "offset": -160, - "name": "local_a0", - "type": "char", - "size": 8, - "addr": 5436 - }, - "-0x98": { - "last_change": null, - "offset": -152, - "name": "local_98", - "type": "char", - "size": 8, - "addr": 5436 - }, - "-0x90": { - "last_change": null, - "offset": -144, - "name": "local_90", - "type": "char", - "size": 8, - "addr": 5436 - }, - "-0x88": { - "last_change": null, - "offset": -136, - "name": "local_88", - "type": "char4", - "size": 4, - "addr": 5436 - }, - "-0x78": { - "last_change": null, - "offset": -120, - "name": "local_78", - "type": "char4", - "size": 4, - "addr": 5436 - }, - "-0x68": { - "last_change": null, - "offset": -104, - "name": "local_68", - "type": "char4", - "size": 4, - "addr": 5436 - }, - "-0x58": { - "last_change": null, - "offset": -88, - "name": "local_58", - "type": "char4", - "size": 4, - "addr": 5436 - }, - "-0x48": { - "last_change": null, - "offset": -72, - "name": "local_48", - "type": "char4", - "size": 4, - "addr": 5436 - }, - "-0x38": { - "last_change": null, - "offset": -56, - "name": "local_38", - "type": "char4", - "size": 4, - "addr": 5436 - }, - "-0x28": { - "last_change": null, - "offset": -40, - "name": "local_28", - "type": "char4", - "size": 4, - "addr": 5436 - }, - "-0x18": { - "last_change": null, - "offset": -24, - "name": "local_18", - "type": "char4", - "size": 4, - "addr": 5436 - } - }, - "name": "errormsg", - "type": "void" - }, - "func_deps": [ - { - "last_change": null, - "name": "DWARF/stdio.h/va_list", - "type": "__gnuc_va_list" - }, - { - "last_change": null, - "name": "DWARF/_UNCATEGORIZED_/__builtin_va_list", - "type": "__va_list_tag[1]" - }, - { - "last_change": null, - "name": "DWARF/stdarg.h/__gnuc_va_list", - "type": "__builtin_va_list" - }, - { - "last_change": null, - "addr": 16740, - "name": "DAT_00104164", - "type": "char", - "size": 1 - }, - { - "last_change": null, - "addr": 2131176, - "name": "program_name", - "type": "char *", - "size": 8 - }, - { - "last_change": null, - "addr": 2122912, - "name": "stderr", - "type": "char", - "size": 8 - } - ] - } - }, - "message": null, - "errors": null, - "meta": { - "pagination": null - } -} \ No newline at end of file diff --git a/src/test/resources/main_fdupes_77846709.json b/src/test/resources/main_fdupes_77846709.json deleted file mode 100644 index b562d408..00000000 --- a/src/test/resources/main_fdupes_77846709.json +++ /dev/null @@ -1,301 +0,0 @@ -{ - "status": true, - "data": { - "completed": true, - "status": "completed", - "data_types": { - "func_types": { - "last_change": null, - "addr": 12242, - "size": 1826, - "header": { - "last_change": null, - "name": "main", - "addr": 12242, - "type": "int", - "args": { - "0x0": { - "last_change": null, - "offset": 0, - "name": "argc", - "type": "int", - "size": 4 - }, - "0x1": { - "last_change": null, - "offset": 1, - "name": "argv", - "type": "char * *", - "size": 8 - } - } - }, - "stack_vars": { - "-0x40": { - "last_change": null, - "offset": -64, - "name": "files", - "type": "file_t *", - "size": 8, - "addr": 12242 - }, - "-0x48": { - "last_change": null, - "offset": -72, - "name": "checktree", - "type": "filetree_t *", - "size": 8, - "addr": 12242 - }, - "-0x58": { - "last_change": null, - "offset": -88, - "name": "local_58", - "type": "int", - "size": 4, - "addr": 12242 - } - }, - "name": "main", - "type": "int" - }, - "func_deps": [ - { - "last_change": null, - "name": "DWARF/stat.h/dev_t", - "type": "__dev_t" - }, - { - "last_change": null, - "name": "DWARF/time.h/time_t", - "type": "__time_t" - }, - { - "last_change": null, - "name": "DWARF/fdupes.c/_filetree", - "size": 24, - "members": { - "0x0": { - "last_change": null, - "name": "file", - "offset": 0, - "type": "file_t *", - "size": 8 - }, - "0x8": { - "last_change": null, - "name": "left", - "offset": 8, - "type": "_filetree *", - "size": 8 - }, - "0x10": { - "last_change": null, - "name": "right", - "offset": 16, - "type": "_filetree *", - "size": 8 - } - } - }, - { - "last_change": null, - "name": "DWARF/fdupes.c/file_t", - "type": "_file" - }, - { - "last_change": null, - "name": "DWARF/fdupes.c/_file", - "size": 80, - "members": { - "0x0": { - "last_change": null, - "name": "d_name", - "offset": 0, - "type": "char *", - "size": 8 - }, - "0x8": { - "last_change": null, - "name": "size", - "offset": 8, - "type": "off_t", - "size": 8 - }, - "0x10": { - "last_change": null, - "name": "crcpartial", - "offset": 16, - "type": "md5_byte_t *", - "size": 8 - }, - "0x18": { - "last_change": null, - "name": "crcsignature", - "offset": 24, - "type": "md5_byte_t *", - "size": 8 - }, - "0x20": { - "last_change": null, - "name": "device", - "offset": 32, - "type": "dev_t", - "size": 8 - }, - "0x28": { - "last_change": null, - "name": "inode", - "offset": 40, - "type": "ino_t", - "size": 8 - }, - "0x30": { - "last_change": null, - "name": "mtime", - "offset": 48, - "type": "time_t", - "size": 8 - }, - "0x38": { - "last_change": null, - "name": "hasdupes", - "offset": 56, - "type": "int", - "size": 4 - }, - "0x40": { - "last_change": null, - "name": "duplicates", - "offset": 64, - "type": "_file *", - "size": 8 - }, - "0x48": { - "last_change": null, - "name": "next", - "offset": 72, - "type": "_file *", - "size": 8 - } - } - }, - { - "last_change": null, - "name": "DWARF/__ino64_t", - "type": "qword" - }, - { - "last_change": null, - "name": "DWARF/__dev_t", - "type": "ulong" - }, - { - "last_change": null, - "name": "DWARF/md5_byte_t", - "type": "uchar" - }, - { - "last_change": null, - "name": "DWARF/__time_t", - "type": "long" - }, - { - "last_change": null, - "name": "DWARF/stat.h/ino_t", - "type": "__ino64_t" - }, - { - "last_change": null, - "name": "DWARF/__off64_t", - "type": "sqword" - }, - { - "last_change": null, - "name": "DWARF/stdio.h/off_t", - "type": "__off64_t" - }, - { - "last_change": null, - "name": "DWARF/fdupes.c/filetree_t", - "type": "_filetree" - }, - { - "last_change": null, - "addr": 20384, - "name": "DAT_00104fa0", - "type": "char", - "size": 1 - }, - { - "last_change": null, - "addr": 2131176, - "name": "program_name", - "type": "char *", - "size": 8 - }, - { - "last_change": null, - "addr": 2122944, - "name": "flags", - "type": "ulong", - "size": 8 - }, - { - "last_change": null, - "addr": 5416, - "name": "sort_pairs_by_filename", - "type": "char", - "size": 1 - }, - { - "last_change": null, - "addr": 2122864, - "name": "stdin", - "type": "FILE *", - "size": 8 - }, - { - "last_change": null, - "addr": 2122912, - "name": "stderr", - "type": "FILE *", - "size": 8 - }, - { - "last_change": null, - "addr": 2122872, - "name": "optind", - "type": "int", - "size": 4 - }, - { - "last_change": null, - "addr": 2122880, - "name": "optarg", - "type": "char *", - "size": 8 - }, - { - "last_change": null, - "addr": 5344, - "name": "sort_pairs_by_mtime", - "type": "char", - "size": 1 - }, - { - "last_change": null, - "addr": 2122080, - "name": "long_options", - "type": "option[24]", - "size": 768 - } - ] - } - }, - "message": null, - "errors": null, - "meta": { - "pagination": null - } -} \ No newline at end of file diff --git a/src/test/resources/md5_process_fdupes.json b/src/test/resources/md5_process_fdupes.json deleted file mode 100644 index 17771b92..00000000 --- a/src/test/resources/md5_process_fdupes.json +++ /dev/null @@ -1,93 +0,0 @@ -{ - "status": true, - "data": { - "completed": true, - "status": "completed", - "data_types": { - "func_types": { - "last_change": null, - "addr": 14068, - "size": 2067, - "header": { - "last_change": null, - "name": "md5_process", - "addr": 14068, - "type": "void", - "args": { - "0x0": { - "last_change": null, - "offset": 0, - "name": "pms", - "type": "md5_state_t *", - "size": 8 - }, - "0x1": { - "last_change": null, - "offset": 1, - "name": "data", - "type": "md5_byte_t *", - "size": 8 - } - } - }, - "stack_vars": { - "-0x70": { - "last_change": null, - "offset": -112, - "name": "xbuf", - "type": "md5_word_t[16]", - "size": 64, - "addr": 14068 - } - }, - "name": "md5_process", - "type": "void" - }, - "func_deps": [ - { - "last_change": null, - "name": "DWARF/md5.h/md5_state_t", - "type": "md5_state_s" - }, - { - "last_change": null, - "name": "DWARF/md5_byte_t", - "type": "uchar" - }, - { - "last_change": null, - "name": "DWARF/md5.h/md5_state_s", - "size": 88, - "members": { - "0x0": { - "last_change": null, - "name": "count", - "offset": 0, - "type": "md5_word_t[2]", - "size": 8 - }, - "0x8": { - "last_change": null, - "name": "abcd", - "offset": 8, - "type": "md5_word_t[4]", - "size": 16 - }, - "0x18": { - "last_change": null, - "name": "buf", - "offset": 24, - "type": "md5_byte_t[64]", - "size": 64 - } - } - } - ] - } - }, - "message": null, - "errors": null, - "meta": { - "pagination": null - } -} \ No newline at end of file diff --git a/src/test/resources/pending.json b/src/test/resources/pending.json deleted file mode 100644 index 5b89d838..00000000 --- a/src/test/resources/pending.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "status": true, - "data": { - "completed": false, - "status": "pending", - "data_types": null - }, - "message": null, - "errors": null, - "meta": { - "pagination": null - } -} \ No newline at end of file diff --git a/src/test/resources/simple_function_signature_response.json b/src/test/resources/simple_function_signature_response.json deleted file mode 100644 index c1d2dd7e..00000000 --- a/src/test/resources/simple_function_signature_response.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "status": true, - "data": { - "completed": true, - "status": "completed", - "data_types": { - "func_types": { - "last_change": null, - "addr": 12242, - "size": 1826, - "header": { - "last_change": null, - "name": "main", - "addr": 12242, - "type": "int", - "args": { - "0x0": { - "last_change": null, - "offset": 0, - "name": "argc", - "type": "int", - "size": 4 - }, - "0x1": { - "last_change": null, - "offset": 1, - "name": "argv", - "type": "char * *", - "size": 8 - } - } - }, - "stack_vars": {}, - "name": "main", - "type": "int" - }, - "func_deps": [] - } - }, - "message": null, - "errors": null, - "meta": { - "pagination": null - } -} \ No newline at end of file diff --git a/src/test/resources/summarizematches_fdupes.json b/src/test/resources/summarizematches_fdupes.json deleted file mode 100644 index 07d9a171..00000000 --- a/src/test/resources/summarizematches_fdupes.json +++ /dev/null @@ -1,187 +0,0 @@ -{ - "status": true, - "data": { - "completed": true, - "status": "completed", - "data_types": { - "func_types": { - "last_change": null, - "addr": 9278, - "size": 221, - "header": { - "last_change": null, - "name": "summarizematches", - "addr": 9278, - "type": "void", - "args": { - "0x0": { - "last_change": null, - "offset": 0, - "name": "files", - "type": "file_t *", - "size": 8 - } - } - }, - "stack_vars": null, - "name": "summarizematches", - "type": "void" - }, - "func_deps": [ - { - "last_change": null, - "name": "DWARF/time.h/time_t", - "type": "__time_t" - }, - { - "last_change": null, - "name": "DWARF/md5_byte_t", - "type": "uchar" - }, - { - "last_change": null, - "name": "DWARF/__time_t", - "type": "long" - }, - { - "last_change": null, - "name": "DWARF/stdio.h/off_t", - "type": "__off64_t" - }, - { - "last_change": null, - "name": "DWARF/fdupes.c/file_t", - "type": "_file" - }, - { - "last_change": null, - "name": "DWARF/stat.h/dev_t", - "type": "__dev_t" - }, - { - "last_change": null, - "name": "DWARF/stat.h/ino_t", - "type": "__ino64_t" - }, - { - "last_change": null, - "name": "DWARF/__dev_t", - "type": "ulong" - }, - { - "last_change": null, - "name": "DWARF/__ino64_t", - "type": "qword" - }, - { - "last_change": null, - "name": "DWARF/fdupes.c/_file", - "size": 80, - "members": { - "0x0": { - "last_change": null, - "name": "d_name", - "offset": 0, - "type": "char *", - "size": 8 - }, - "0x8": { - "last_change": null, - "name": "size", - "offset": 8, - "type": "off_t", - "size": 8 - }, - "0x10": { - "last_change": null, - "name": "crcpartial", - "offset": 16, - "type": "md5_byte_t *", - "size": 8 - }, - "0x18": { - "last_change": null, - "name": "crcsignature", - "offset": 24, - "type": "md5_byte_t *", - "size": 8 - }, - "0x20": { - "last_change": null, - "name": "device", - "offset": 32, - "type": "dev_t", - "size": 8 - }, - "0x28": { - "last_change": null, - "name": "inode", - "offset": 40, - "type": "ino_t", - "size": 8 - }, - "0x30": { - "last_change": null, - "name": "mtime", - "offset": 48, - "type": "time_t", - "size": 8 - }, - "0x38": { - "last_change": null, - "name": "hasdupes", - "offset": 56, - "type": "int", - "size": 4 - }, - "0x40": { - "last_change": null, - "name": "duplicates", - "offset": 64, - "type": "_file *", - "size": 8 - }, - "0x48": { - "last_change": null, - "name": "next", - "offset": 72, - "type": "_file *", - "size": 8 - } - } - }, - { - "last_change": null, - "name": "DWARF/__off64_t", - "type": "sqword" - }, - { - "last_change": null, - "addr": 20360, - "name": "DAT_00104f88", - "type": "double", - "size": 8 - }, - { - "last_change": null, - "addr": 20368, - "name": "DAT_00104f90", - "type": "double", - "size": 8 - }, - { - "last_change": null, - "addr": 20376, - "name": "DAT_00104f98", - "type": "char", - "size": 1 - } - ] - } - }, - "message": null, - "errors": null, - "meta": { - "pagination": null - } -} \ No newline at end of file