diff --git a/.gitattributes b/.gitattributes index 6313b56..10a98db 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,2 @@ * text=auto eol=lf +*.dll binary diff --git a/.gitignore b/.gitignore index 8790651..a1f37f3 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ __pycache__/ *.py[cod] *.egg-info/ *_decompiler/ +/dist +/build diff --git a/AGENTS.md b/AGENTS.md index e82d0ee..c72139e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,6 @@ # AGENTS -This repository contains ToCode, a Python-only binary exporter. ToCode takes one binary, IDA database, or Android APK path and writes one source-like project directory for reverse-engineering agents. +This repository contains ToCode, a Python-only binary exporter. ToCode takes one binary, IDA database, Android APK, or .NET assembly/bundle/package path and writes one source-like project directory for reverse-engineering agents. ## Scope @@ -16,6 +16,8 @@ This repository contains ToCode, a Python-only binary exporter. ToCode takes one - `src/tocode/__init__.py`: package version and `export_from_binaryview()`, the in-UI Binary Ninja library entry. - `src/tocode/analysis.py`: backend-neutral binary inventory and call graph normalization. - `src/tocode/backends/`: IDA Domain, radare2, angr, and Binary Ninja (`binja.py`) session adapters, plus the ASC/droidasc APK backend (`asc.py`: APK set discovery, DEX inventory, worker-side class decompilation). +- `src/tocode/backends/dotnet.py` / `dotnet_pe.py`: .NET backend (input discovery for assemblies, apphosts, single-file bundles and NuGet packages; dnlib inventory; pure-Python PE/CLR/bundle parsing and IL token scan; worker-side C#/IL decompilation). `backends/dotnet_lib/` holds the bundled dnlib and ICSharpCode.Decompiler DLLs (MIT) with their provenance. +- `src/tocode/dotnet.py`: .NET export pipeline; `dotnet_metadata.py`: .NET JSON documents and the generated `AGENTS.md`. - `src/tocode/apk.py`: APK export pipeline (extraction, decompile pool, resource decoding, metadata); `apk_metadata.py`: manifest parsing and Android JSON documents; `apk_native.py`: extraction and background native export of every `.so`. - `src/tocode/exporter.py`: project writer, function rendering, worker-session rendering, generated export `AGENTS.md`. - `src/tocode/metadata.py`: JSON metadata and triage documents. @@ -71,6 +73,17 @@ APK input (`.apk`, `.apks`, `.xapk`) uses the ASC backend and writes instead: `base.apk` merges sibling `split_*.apk` files unless `--no-splits`; bundles are unpacked and merged. `--backend` selects the native backend for the `.so` exports (`binja` is rejected). +.NET input (a managed PE, an apphost launcher with its `.dll`, a single-file bundle, or a `.nupkg`; detected by content, `--as-native` opts out) uses the .NET backend and writes instead: + +- `src/raw///**/.cs` and `.il` (C# and IL with raw bytecode per top-level type; nested types inside), `src/raw//Properties/AssemblyInfo.cs` and `Manifest.il`, `src/raw//.csproj` +- `assemblies.json`, `types.json`, `namespace-graph.json`, `resources.json`, `container.json`, `native-libs.json` +- `functions.json`, `function-index.json` (`c` = C#, `asm` = IL), `strings.json`, `imports.json`, `exports.json`, `sections.json`, `reachable.json`, `triage.json`, `project.json`, `export-manifest.json` +- `data/resources//*` (embedded resources, `.resources` decoded to JSON), `data/assemblies/*` and `data//*` (extracted container entries) +- `lib//*` and `native///` for mixed-mode code, bundled native libraries, and P/Invoke targets next to the input (unless `--no-native`) +- `tocode.log`, generated `AGENTS.md` and `CLAUDE.md` + +Framework/runtime assemblies and native runtime libraries inside bundles and packages are inventoried but only decompiled with `--include-framework`. + ## Development - Prefer `uv` for local commands. @@ -87,7 +100,8 @@ APK input (`.apk`, `.apks`, `.xapk`) uses the ASC backend and writes instead: - radare2/r2pipe is a fallback backend. - angr is the optional pure-Python fallback backend (`[angr]` extra). - ASC (`droidasc`, PyPI) is the APK/DEX backend and a core runtime dependency. It pulls androguard. `backends/asc.py` is the only module that imports it; it pre-imports the modules ASC would otherwise stub in `sys.modules` and silences androguard's loguru logging. Class decompilation is not thread-safe, so it runs in spawned worker processes (recycled in rounds; do not use `max_tasks_per_child`, it deadlocks spawn pools on some CPython builds). The big per-method/per-class/per-string JSON documents are streamed row by row (`apk_metadata.write_json_rows`), never built as one object. -- APK native libraries are exported by `apk_native.py` on a background thread, each `export_binary` in its own spawned process so a backend OOM-kill or crash only fails that library (`native-libs.json` status). The thread waits for `TOCODE_APK_NATIVE_MIN_FREE_MB` (default 1024) of free memory before starting each library so it does not starve the DEX pool. `TOCODE_WORKER_TMP_DIR` also places the unpacked `.apks` bundle. +- APK native libraries are exported by `apk_native.py` on a background thread, each `export_binary` in its own spawned process so a backend OOM-kill or crash only fails that library (`native-libs.json` status). The thread waits for `TOCODE_NATIVE_MIN_FREE_MB` (default 1024) of free memory before starting each library so it does not starve the DEX pool. `TOCODE_WORKER_TMP_DIR` also places the unpacked `.apks` bundle. +- The .NET backend needs `pythonnet` (core dependency, pinned per Python version) and a .NET 9+ runtime on the host. dnlib 4.5.0 and ICSharpCode.Decompiler 11.1 are vendored under `src/tocode/backends/dotnet_lib/` (package data; hashes in its README and checked by `tests/test_dotnet_backend.py`); ICSharpCode.Decompiler 11 needs System.Reflection.Metadata 9, hence .NET 9+. Decompilation runs in spawned worker processes (recycled per round; a batch that crashes or hangs a worker is retried type by type so only the culprit fails). Tests must not require .NET: use `tests/dotnet_fixtures.py` (synthetic PE/bundles) and fake sessions; runtime-backed tests skip without it. - Binary Ninja is an opt-in backend (`--backend binja`, never auto-selected). The `binaryninja` module is supplied by the Binary Ninja install (in-UI) or the remote VM, so it is not a pip dependency. `rpyc`, the client used for the diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c67b338 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 buzzer-re + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 4e70536..4f6c8c8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ToCode -ToCode exports a binary, IDA database, or Android APK into a source-like project tree: raw recovered C (or Java for APKs), matching assembly, function summaries, section data, optional IDA database, and metadata that coding agents can read directly. +ToCode exports a binary, IDA database, Android APK, or .NET assembly into a source-like project tree: raw recovered C (Java for APKs, C# and IL for .NET), matching assembly, function summaries, section data, optional IDA database, and metadata that coding agents can read directly. ## Why @@ -24,6 +24,10 @@ sample_decompiler/ src/raw/**/*.asm src/raw/**/*.summary src/raw//**/*.java # Only for APK files (replaces the .c/.asm/.summary tree) + src/raw///**/*.cs # Only for .NET files (replaces the .c/.asm/.summary tree) + src/raw///**/*.il # Only for .NET files: IL with RVA + raw bytecode per instruction + src/raw//Properties/* # Only for .NET files: AssemblyInfo.cs and the Manifest.il + src/raw//*.csproj # Only for .NET files: recovered project file include/*.h include/*.types.h data/*.bin @@ -32,13 +36,19 @@ sample_decompiler/ data/apk//** # Only for APK files: every non-code APK entry, verbatim data/res//**/*.xml # Only for APK files: binary XML resources decoded to text data/resources.json # Only for APK files: decoded resources.arsc - lib//*.so # Only for APK files: extracted native libraries - native/// # Only for APK files: full nested ToCode export per .so + data/resources//* # Only for .NET files: embedded resources (.resources decoded to JSON) + data/assemblies/* # Only for .NET files: assemblies extracted from bundles/packages + lib//*.so # Only for APK and .NET files: extracted native libraries + native/// # Only for APK and .NET files: full nested ToCode export per native library AndroidManifest.xml # Only for APK files manifest.json # Only for APK files classes.json # Only for APK files package-graph.json # Only for APK files (replaces cluster-graph.json) - native-libs.json # Only for APK files + native-libs.json # Only for APK and .NET files + assemblies.json # Only for .NET files + namespace-graph.json # Only for .NET files (replaces cluster-graph.json) + resources.json # Only for .NET files + container.json # Only for .NET files: bundle / NuGet package layout function-index.json functions.json types.json @@ -62,14 +72,19 @@ sample_decompiler/ | `include` | Generated headers, including `*.types.h` with the structs/enums/typedefs recovered from the binary. Not written for APK files. | | `data` | Raw section dumps and variable metadata. | | `data/apk`, `data/res`, `data/resources.json` | **Only for APK files.** Every non-code entry of each APK in the set verbatim, `res/**/*.xml` decoded from binary XML, and the decoded `resources.arsc` tables. | -| `lib//*.so` | **Only for APK files.** Every native library found in the APK set (all ABIs), always extracted. | -| `native///` | **Only for APK files.** A complete nested ToCode export for each native library (own `AGENTS.md` with an Origin section naming the APK, `src/raw/*.c`, `functions.json`, `exports.json`, ...). Skipped with `--no-native`. | +| `src/raw///**/*.cs` and `*.il` | **Only for .NET files.** Decompiled C# (ICSharpCode.Decompiler) and IL disassembly (RVA, raw bytecode, metadata tokens) per top-level type, nested types inside, folders follow assemblies and namespaces like a dnSpy/ILSpy project export. `Properties/AssemblyInfo.cs`, `Properties/Manifest.il`, and a `.csproj` per assembly. | +| `data/resources/`, `data/assemblies` | **Only for .NET files.** Embedded resources (`.resources` decoded to JSON) and the assemblies extracted from single-file bundles or NuGet packages. | +| `lib//*.so` | **Only for APK and .NET files.** Every native library found (APK: all ABIs; .NET: mixed-mode code, bundled libraries, P/Invoke targets next to the input), always extracted. | +| `native///` | **Only for APK and .NET files.** A complete nested ToCode export for each native library (own `AGENTS.md` with an Origin section naming the APK or .NET program, `src/raw/*.c`, `functions.json`, `exports.json`, ...). Skipped with `--no-native`. | | `AndroidManifest.xml` / `manifest.json` | **Only for APK files.** Decoded manifest and its parsed form: package, versions, SDKs, permissions, components with intent filters and exported state, application attributes, split manifests. | | `classes.json` | **Only for APK files.** Every class with superclass, interfaces, access flags, fields, methods, source file, and Java file/line ranges. | | `package-graph.json` | **Only for APK files.** Inter-package call graph (the APK counterpart of `cluster-graph.json`). | -| `native-libs.json` | **Only for APK files.** ABI, hash, source APK, export directory, and decompilation status of every native library. | +| `native-libs.json` | **Only for APK and .NET files.** ABI/arch, hash, source, export directory, and decompilation status of every native library. | +| `assemblies.json` | **Only for .NET files.** Every assembly found: version, target framework, entry point, CLR flags, ReadyToRun/mixed-mode, references, resources, obfuscation hints, and why skipped ones were not decompiled. | +| `namespace-graph.json` | **Only for .NET files.** Inter-namespace and inter-assembly call graph (the .NET counterpart of `cluster-graph.json`). | +| `resources.json` / `container.json` | **Only for .NET files.** Embedded/linked resources index; layout of the bundle, package, or assembly and where each entry was extracted. | | `types.json` | Catalog of types recovered from the binary's debug info or type library, with C declarations. | -| `*.json` | Functions (with recovered types and original source decl file/line), sections, strings, imports, exports, relocations, reachability, clusters, triage, project metadata, and export manifest. For APK files the same documents describe DEX methods, strings, framework imports, exported components/JNI methods, and reachability from manifest components; `relocations.json`, `cluster-graph.json`, and `types.json` are not written. | +| `*.json` | Functions (with recovered types and original source decl file/line), sections, strings, imports, exports, relocations, reachability, clusters, triage, project metadata, and export manifest. For APK files the same documents describe DEX methods, strings, framework imports, exported components/JNI methods, and reachability from manifest components; `relocations.json`, `cluster-graph.json`, and `types.json` are not written. For .NET files they describe methods (address `!0x`, C# and IL line ranges), the user-string heap, cross-assembly and P/Invoke imports, entry points/public API, and `types.json` lists .NET types with members; `relocations.json` and `cluster-graph.json` are not written. | | `tocode.log` | Export log with checkpoint, resume, and per-function render history. For APK files it also carries the native library export status. | | `AGENTS.md` / `CLAUDE.md` | Instructions for agents analyzing the exported binary. | | `src/tree` | Optional scanner-friendly C output when `--tree` is used. | @@ -95,6 +110,23 @@ Three backends are supported, selected with `--backend` (default `auto`, which p Other disassemblers may be added in the future. +### .NET assemblies + +`tocode App.dll` (also `.exe`, an apphost launcher next to its `.dll`, single-file bundles, and `.nupkg` packages) uses a built-in .NET backend: [dnlib](https://github.com/0xd4d/dnlib) for metadata and [ICSharpCode.Decompiler](https://github.com/icsharpcode/ILSpy) (ILSpy's engine) for C#, both bundled with ToCode and loaded through [pythonnet](https://pypi.org/project/pythonnet/). No dnSpy/ILSpy install is needed; only a **.NET 9+ runtime** (e.g. `sudo apt install dotnet-runtime-10.0`, or the SDK; `DOTNET_ROOT` is honoured). + +- `src/raw///.../.cs`: C# per top-level type, folders follow assemblies and namespaces like a dnSpy/ILSpy project export; `.il` next to it keeps the bytecode (IL with RVA, raw bytes, and metadata tokens per instruction). +- Per-method C# and IL line ranges in `functions.json`/`function-index.json` (address `!0x`), with lambdas and async/iterator state machines linked to the method that owns them. +- `assemblies.json`, `types.json`, `strings.json` (user strings with `ldstr` xrefs), `imports.json` (cross-assembly members and P/Invoke), `exports.json`, `reachable.json`, `namespace-graph.json`, `resources.json`, `container.json`, `triage.json` (P/Invoke, suspicious API families, obfuscation hints, strings of interest). +- Native code goes through the regular native backends on a background thread, like APK `.so` files: mixed-mode (C++/CLI) assemblies, native libraries inside bundles/packages, and P/Invoke targets shipped next to the input. `--no-native` skips it. +- Single-file bundles and packages carry the .NET runtime/framework; those assemblies and runtime native libraries are listed but only decompiled with `--include-framework`. NuGet packages decompile each assembly once, from its newest target framework. +- `--as-native` exports a managed PE with the native backend instead (e.g. to look at a ReadyToRun or mixed-mode image in IDA). + +```bash +tocode App.dll # assembly (+ P/Invoke libraries next to it) +tocode publish/App # single-file bundle or apphost launcher +tocode Vendor.Lib.1.0.0.nupkg # NuGet package +``` + ### Android APKs `tocode app.apk` (also `.apks`/`.xapk` bundles) uses [ASC](https://github.com/MG1937/ASC) (`droidasc`, a core dependency) for the DEX side and the regular native backends for every shared object in the package: @@ -102,7 +134,7 @@ Other disassemblers may be added in the future. - `src/raw//.java`: one decompiled Java file per class, folders follow Java packages (no clustering, no summaries). - `AndroidManifest.xml` + `manifest.json`: decoded and parsed manifest (permissions, components with intent filters and exported state, application attributes, split manifests). - `classes.json`, `functions.json`, `function-index.json`, `strings.json`, `imports.json`, `exports.json` (exported components + JNI methods), `reachable.json` (from manifest components), `package-graph.json`, `sections.json`, `triage.json`. -- `lib//*.so`: every native library extracted; `native///`: a complete nested ToCode export per library (all ABIs), produced on a background thread while the DEX side decompiles. `native-libs.json` records the status of each (each library runs in its own process; one failing or being OOM-killed never fails the APK export, and the native thread waits for `TOCODE_APK_NATIVE_MIN_FREE_MB`, default 1024 MB, of free memory before each library). Pass `--no-native` to skip the native decompilation (libraries are still extracted). `--backend` picks the native backend (`auto`/`ida`/`r2`/`angr`; `binja` is not supported for APKs). +- `lib//*.so`: every native library extracted; `native///`: a complete nested ToCode export per library (all ABIs), produced on a background thread while the DEX side decompiles. `native-libs.json` records the status of each (each library runs in its own process; one failing or being OOM-killed never fails the APK export, and the native thread waits for `TOCODE_NATIVE_MIN_FREE_MB`, default 1024 MB, of free memory before each library). Pass `--no-native` to skip the native decompilation (libraries are still extracted). `--backend` picks the native backend (`auto`/`ida`/`r2`/`angr`; `binja` is not supported for APKs). - `data/apk/**`: every other APK entry verbatim; `data/res/**/*.xml` and `data/resources.json`: decoded binary XML and `resources.arsc`. `base.apk` automatically merges sibling `split_*.apk` files (config and ABI splits) into the same project; `--no-splits` exports it alone. The default output directory is `_decompiler`. diff --git a/pyproject.toml b/pyproject.toml index becc2c6..7397b14 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,22 +4,50 @@ build-backend = "setuptools.build_meta" [project] name = "tocode-cli" -version = "0.1.0" -description = "Export compiled binaries as source-like projects for reverse engineering agents." +dynamic = ["version"] +description = "Export compiled binaries, IDA databases, Android APKs, and .NET assemblies as source-like projects for reverse engineering agents." readme = "README.md" requires-python = ">=3.10" +authors = [{ name = "buzzer-re" }] +keywords = [ + "reverse-engineering", + "decompiler", + "ida", + "radare2", + "angr", + "binary-ninja", + "android", + "apk", + "dex", + "dotnet", + "csharp", + "il", + "agents", +] dependencies = [ "droidasc==0.1.1.post1", "ida-domain==0.5.0", "idapro==0.0.9", + "pythonnet==3.0.5 ; python_full_version < '3.11'", + "pythonnet==3.2.0 ; python_full_version >= '3.11'", "r2pipe==1.9.8", "rpyc==6.0.2", "tqdm==4.67.3", ] license = "MIT" +license-files = ["LICENSE"] classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX :: Linux", "Operating System :: MacOS", @@ -27,6 +55,11 @@ classifiers = [ "Topic :: Software Development :: Disassemblers", ] +[project.urls] +Homepage = "https://github.com/buzzer-re/ToCode" +Repository = "https://github.com/buzzer-re/ToCode" +Issues = "https://github.com/buzzer-re/ToCode/issues" + [project.scripts] tocode = "tocode.cli:main" @@ -65,8 +98,23 @@ ignore_missing_imports = true module = ["droidasc.*", "androguard.*"] ignore_missing_imports = true +# The .NET backend imports CLR namespaces through pythonnet at runtime; they +# have no Python stubs. +[[tool.mypy.overrides]] +module = ["clr", "clr_loader.*", "pythonnet.*", "System", "System.*", "dnlib.*", "ICSharpCode.*"] +ignore_missing_imports = true + [tool.setuptools] package-dir = { "" = "src" } +# Single source of truth for the version: src/tocode/__init__.py. +[tool.setuptools.dynamic] +version = { attr = "tocode.__version__" } + [tool.setuptools.packages.find] where = ["src"] + +# dnlib and ICSharpCode.Decompiler (MIT) ship inside the package so .NET input +# needs only a .NET runtime; see src/tocode/backends/dotnet_lib/README.md. +[tool.setuptools.package-data] +"tocode.backends" = ["dotnet_lib/*.dll", "dotnet_lib/*.txt", "dotnet_lib/*.md"] diff --git a/src/tocode/apk.py b/src/tocode/apk.py index 0526dbf..ee9a0c6 100644 --- a/src/tocode/apk.py +++ b/src/tocode/apk.py @@ -24,7 +24,6 @@ from . import __version__ from . import apk_metadata as meta from .apk_native import ( - DEFAULT_NATIVE_MIN_FREE_MB, NATIVE_HEARTBEAT_SECONDS, NativeExporter, NativeLib, @@ -34,6 +33,7 @@ default_native_exporter, extract_native_libs, native_libs_json, + native_min_free_mb, ) from .backends.asc import ( ApkSet, @@ -344,7 +344,7 @@ def _start_native(context: ApkContext, exporter: NativeExporter | None) -> None: root=root, progress=context.progress, exporter=exporter or default_native_exporter(context.options.native_options), - min_free_mb=_native_min_free_mb(), + min_free_mb=native_min_free_mb(), available_memory=available_memory_mb, ) context.progress.log( @@ -355,16 +355,6 @@ def _start_native(context: ApkContext, exporter: NativeExporter | None) -> None: context.native_runner = runner -def _native_min_free_mb() -> int: - raw = os.environ.get("TOCODE_APK_NATIVE_MIN_FREE_MB", "").strip() - if raw: - try: - return max(0, int(raw)) - except ValueError: - pass - return DEFAULT_NATIVE_MIN_FREE_MB - - def _join_native(context: ApkContext) -> None: runner = context.native_runner if runner is None: diff --git a/src/tocode/apk_native.py b/src/tocode/apk_native.py index bbb3b3a..a3af729 100644 --- a/src/tocode/apk_native.py +++ b/src/tocode/apk_native.py @@ -68,6 +68,9 @@ class NativeLib: seconds: float | None = None # Package of the APK the library was extracted from (for the nested AGENTS.md). package: str = "" + # Full provenance paragraph for the nested AGENTS.md; overrides the APK text + # (used when the library comes from a .NET bundle, package, or assembly). + origin: str | None = None @property def stem(self) -> str: @@ -140,8 +143,22 @@ def native_export_dir(root: Path, lib: NativeLib) -> Path: NativeExporter = Callable[[NativeLib, Path, Progress], tuple[str, str, int, int]] +def native_min_free_mb() -> int: + """Free-memory floor before each native export (0 disables the wait).""" + for name in ("TOCODE_NATIVE_MIN_FREE_MB", "TOCODE_APK_NATIVE_MIN_FREE_MB"): + raw = os.environ.get(name, "").strip() + if raw: + try: + return max(0, int(raw)) + except ValueError: + continue + return DEFAULT_NATIVE_MIN_FREE_MB + + def describe_origin(lib: NativeLib) -> str: """Provenance paragraph for the nested export's AGENTS.md.""" + if lib.origin: + return lib.origin package = f"`{lib.package}`" if lib.package else "an Android app" return ( f"This is a decompiled native library from the {package} APK: " @@ -451,11 +468,20 @@ class NativeRunner: max_wait_seconds: float = NATIVE_MEMORY_MAX_WAIT_SECONDS mirror_logs: bool = True current: NativeLib | None = None + waiting: tuple[NativeLib, int] | None = None current_started: float = 0.0 current_tail: NestedLogTail | None = None def describe_current(self) -> str | None: """One line about what the native thread is doing right now.""" + waiting = self.waiting + if waiting is not None: + pending, available = waiting + return ( + f"native: waiting for memory before {pending.name} ({pending.abi}): " + f"{available} MB free, need {self.min_free_mb} MB " + "(set TOCODE_NATIVE_MIN_FREE_MB to change)" + ) lib = self.current if lib is None: return None @@ -502,6 +528,7 @@ def _wait_for_memory(self, lib: NativeLib) -> None: available = self.available_memory() if available is None or available >= self.min_free_mb: break + self.waiting = (lib, available) if not logged: self.progress.log( f"native: waiting for memory before {lib.name} ({lib.abi}): " @@ -510,6 +537,7 @@ def _wait_for_memory(self, lib: NativeLib) -> None: logged = True self.stop_event.wait(self.poll_seconds) waited += self.poll_seconds + self.waiting = None if logged and not self.stop_event.is_set(): self.progress.file_log( f"native: waited {waited:.0f}s for memory before {lib.name}" diff --git a/src/tocode/backends/dotnet.py b/src/tocode/backends/dotnet.py new file mode 100644 index 0000000..0d945b7 --- /dev/null +++ b/src/tocode/backends/dotnet.py @@ -0,0 +1,1624 @@ +""".NET backend: dnlib inventory plus ICSharpCode.Decompiler C#/IL output. + +Both libraries ship with ToCode (``dotnet_lib/``) and are loaded in-process +through pythonnet, so exporting a .NET binary needs a .NET 9+ runtime and no +other tool. The parent process inventories every assembly with dnlib (types, +members, P/Invoke, resources, references) and scans IL bytes in pure Python for +call/string/field cross-references; spawned workers decompile types to C# and +disassemble them to IL (with raw bytecode), returning per-method line ranges. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +import importlib.util +import json +import os +from pathlib import Path +import re +import shutil +import sys +from typing import Any +import zipfile + +from ..errors import ToCodeError +from .dotnet_pe import ( + PE_MACHINES, + BundleManifest, + PeInfo, + extract_bundle_entry, + is_apphost_launcher, + iter_user_strings, + method_body, + read_bundle, + read_pe_info, + scan_il_tokens, +) + +DOTNET_LIB_DIR = Path(__file__).with_name("dotnet_lib") +DOTNET_LIBRARIES = ("dnlib.dll", "ICSharpCode.Decompiler.dll") +MIN_RUNTIME_MAJOR = 9 +NUPKG_SUFFIXES = frozenset({".nupkg", ".snupkg"}) +MAX_STRING_XREFS = 64 +MAX_FIELD_REFS = 32 + +# Assemblies and native libraries of the .NET runtime itself. A self-contained +# app or bundle carries a couple of hundred of them; they are inventoried and +# extracted but only decompiled with --include-framework. +_FRAMEWORK_ASSEMBLY_RX = re.compile( + r"^(System(\..+)?|mscorlib|netstandard|WindowsBase|PresentationCore|" + r"PresentationFramework|Microsoft\.(CSharp|VisualBasic(\..+)?|Win32\..+|" + r"NETCore\..+|Extensions\.DependencyModel))\.dll$", + re.IGNORECASE, +) +_FRAMEWORK_NATIVE_RX = re.compile( + r"^(lib)?(coreclr|clrjit|clrgc(exp)?|hostfxr|hostpolicy|mscordaccore|mscordbi|" + r"mscorrc|createdump|coreclrtraceptprovider|System(\.[A-Za-z]+)*\.Native|" + r"Microsoft\.DiaSymReader\.Native\.[a-z0-9]+|D3DCompiler_47_cor3|" + r"PenImc_cor3|PresentationNative_cor3|vcruntime140_cor3|wpfgfx_cor3)" + r"(\.so|\.dylib|\.dll)?$", + re.IGNORECASE, +) +_ELF_MACHINES = { + 0x03: "x86", + 0x28: "arm", + 0x3E: "x86_64", + 0xB7: "arm64", + 0xF3: "riscv64", +} + + +# -------------------------------------------------------------------------- +# Input classification +# -------------------------------------------------------------------------- + + +def is_nupkg(path: Path) -> bool: + return path.suffix.lower() in NUPKG_SUFFIXES + + +def is_dotnet_input(path: Path) -> bool: + """Managed PE, single-file bundle, apphost launcher, or NuGet package.""" + if is_nupkg(path): + return zipfile.is_zipfile(path) + info = read_pe_info(path) + if info is not None and info.is_managed: + return True + if read_bundle(path) is not None: + return True + return _apphost_assembly(path) is not None + + +def _apphost_assembly(path: Path) -> Path | None: + if not is_apphost_launcher(path): + return None + stem = path.name[:-4] if path.name.lower().endswith(".exe") else path.name + candidate = path.with_name(f"{stem}.dll") + info = read_pe_info(candidate) if candidate.is_file() else None + return candidate if info is not None and info.is_managed else None + + +def native_arch(path: Path) -> str: + try: + head = path.read_bytes()[:0x400] if path.stat().st_size < 0x400 else None + if head is None: + with path.open("rb") as handle: + head = handle.read(0x400) + except OSError: + return "unknown" + if head[:4] == b"\x7fELF" and len(head) >= 20: + return _ELF_MACHINES.get(int.from_bytes(head[18:20], "little"), "unknown") + info = read_pe_info(path) + if info is not None: + return info.machine + if head[:4] in (b"\xcf\xfa\xed\xfe", b"\xce\xfa\xed\xfe"): + cpu = int.from_bytes(head[4:8], "little") + return {0x01000007: "x86_64", 0x0100000C: "arm64", 7: "x86", 12: "arm"}.get( + cpu, "unknown" + ) + return "unknown" + + +def is_native_binary(path: Path) -> bool: + try: + with path.open("rb") as handle: + head = handle.read(4) + except OSError: + return False + if head[:4] == b"\x7fELF" or head[:4] in ( + b"\xcf\xfa\xed\xfe", + b"\xce\xfa\xed\xfe", + b"\xca\xfe\xba\xbe", + ): + return True + if head[:2] == b"MZ": + info = read_pe_info(path) + return info is not None and not info.is_managed + return False + + +def is_framework_assembly(name: str) -> bool: + return bool(_FRAMEWORK_ASSEMBLY_RX.match(Path(name).name)) + + +def is_framework_native(name: str) -> bool: + return bool(_FRAMEWORK_NATIVE_RX.match(Path(name).name)) + + +# -------------------------------------------------------------------------- +# Input discovery +# -------------------------------------------------------------------------- + + +@dataclass(slots=True) +class AssemblyFile: + path: Path + entry: str # path inside the container (or the file name) + source: str # container the assembly came from + framework: bool = False + include: bool = True + reason: str | None = None + tfm: str | None = None + + +@dataclass(slots=True) +class NativeFile: + name: str + path: Path + entry: str + source: str + arch: str + framework: bool = False + reason: str | None = None + + +@dataclass(slots=True) +class ExtraFile: + entry: str + path: Path + kind: str + + +@dataclass(slots=True) +class DotnetInput: + kind: str # assembly | apphost | bundle | nupkg + input: Path + assemblies: list[AssemblyFile] + natives: list[NativeFile] = field(default_factory=list) + extras: list[ExtraFile] = field(default_factory=list) + search_dirs: list[Path] = field(default_factory=list) + bundle: BundleManifest | None = None + workdir: Path | None = None + + @property + def included(self) -> list[AssemblyFile]: + return [item for item in self.assemblies if item.include] + + +def discover_dotnet_input( + path: Path, *, workdir: Path, include_framework: bool = False +) -> DotnetInput: + path = Path(path).resolve() + if is_nupkg(path): + return _discover_nupkg(path, workdir, include_framework) + info = read_pe_info(path) + if info is not None and info.is_managed: + assembly = AssemblyFile(path=path, entry=path.name, source=path.name) + found = DotnetInput("assembly", path, [assembly], search_dirs=[path.parent]) + if info.mixed_mode: + found.natives.append( + NativeFile( + name=path.name, + path=path, + entry="(native code of the mixed-mode assembly)", + source=path.name, + arch=info.machine, + ) + ) + return found + bundle = read_bundle(path) + if bundle is not None: + return _discover_bundle(path, bundle, workdir, include_framework) + sibling = _apphost_assembly(path) + if sibling is not None: + assembly = AssemblyFile( + path=sibling, entry=sibling.name, source=f"{path.name} (apphost launcher)" + ) + return DotnetInput("apphost", path, [assembly], search_dirs=[path.parent]) + raise ToCodeError(f"not a .NET assembly, bundle, or NuGet package: {path}") + + +def _framework_names_from_deps(deps_path: Path) -> set[str]: + try: + deps = json.loads(deps_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return set() + names: set[str] = set() + libraries = deps.get("libraries", {}) + for target in deps.get("targets", {}).values(): + for key, value in target.items(): + library = libraries.get(key, {}) + if library.get("type") != "runtimepack" and not key.lower().startswith( + "runtimepack." + ): + continue + for group in ("runtime", "native"): + for asset in value.get(group, {}): + names.add(Path(asset).name.lower()) + return names + + +def _discover_bundle( + path: Path, bundle: BundleManifest, workdir: Path, include_framework: bool +) -> DotnetInput: + root = workdir / "bundle" + found = DotnetInput("bundle", path, [], bundle=bundle, workdir=workdir) + found.search_dirs.append(root) + extracted: list[tuple[Any, Path]] = [] + for entry in bundle.entries: + target = _safe_join(root, entry.path) + if target is None: + continue + extract_bundle_entry(path, entry, target) + extracted.append((entry, target)) + deps = next( + (target for entry, target in extracted if entry.kind == "deps-json"), None + ) + framework_names = _framework_names_from_deps(deps) if deps else set() + for entry, target in extracted: + lower = Path(entry.path).name.lower() + info = read_pe_info(target) if entry.kind in {"assembly", "unknown"} else None + if info is not None and info.is_managed: + framework = lower in framework_names or is_framework_assembly(lower) + found.assemblies.append( + AssemblyFile( + path=target, + entry=entry.path, + source=path.name, + framework=framework, + include=include_framework or not framework, + reason=None + if include_framework or not framework + else "framework assembly (use --include-framework)", + ) + ) + elif entry.kind == "native" or is_native_binary(target): + framework = lower in framework_names or is_framework_native(lower) + found.natives.append( + NativeFile( + name=Path(entry.path).name, + path=target, + entry=entry.path, + source=path.name, + arch=native_arch(target), + framework=framework, + reason="framework native library (use --include-framework)" + if framework and not include_framework + else None, + ) + ) + else: + found.extras.append(ExtraFile(entry.path, target, entry.kind)) + if not found.assemblies: + raise ToCodeError(f"single-file bundle contains no managed assemblies: {path}") + return found + + +def _tfm_rank(tfm: str) -> tuple[int, int, int]: + """Order target frameworks: net5.0+ > netcoreapp > netstandard > .NET Framework. + + Modern TFMs carry a dot (``net6.0``, ``net10.0-windows``); .NET Framework + ones do not (``net35``, ``net472``). + """ + text = tfm.lower() + for family, rank in (("net", 3), ("netcoreapp", 2), ("netstandard", 1)): + match = re.match(rf"^{family}(\d+)\.(\d+)", text) + if match: + return (rank, int(match.group(1)), int(match.group(2))) + match = re.match(r"^net(\d)(\d)(\d)?$", text) + if match: + return ( + 0, + int(match.group(1)), + int(match.group(2)) * 10 + int(match.group(3) or 0), + ) + return (-1, 0, 0) + + +def _discover_nupkg(path: Path, workdir: Path, include_framework: bool) -> DotnetInput: + root = workdir / "nupkg" + found = DotnetInput("nupkg", path, [], workdir=workdir) + with zipfile.ZipFile(path) as archive: + for info in archive.infolist(): + if info.is_dir(): + continue + target = _safe_join(root, info.filename) + if target is None: + continue + target.parent.mkdir(parents=True, exist_ok=True) + with archive.open(info) as source, target.open("wb") as sink: + shutil.copyfileobj(source, sink, 1 << 20) + parts = info.filename.split("/") + pe = read_pe_info(target) + if pe is not None and pe.is_managed: + tfm = _nupkg_tfm(parts) + found.assemblies.append( + AssemblyFile( + path=target, + entry=info.filename, + source=path.name, + framework=is_framework_assembly(parts[-1]), + tfm=tfm, + ) + ) + elif is_native_binary(target): + found.natives.append( + NativeFile( + name=parts[-1], + path=target, + entry=info.filename, + source=path.name, + arch=_nupkg_rid(parts) or native_arch(target), + ) + ) + else: + found.extras.append(ExtraFile(info.filename, target, "package-file")) + if not found.assemblies: + raise ToCodeError(f"NuGet package contains no managed assemblies: {path}") + # One copy per assembly name: the newest target framework wins; reference + # assemblies (ref/) only when nothing else exists. + best: dict[str, AssemblyFile] = {} + for item in found.assemblies: + key = Path(item.entry).name.lower() + current = best.get(key) + if current is None or _nupkg_preference(item) > _nupkg_preference(current): + best[key] = item + for item in found.assemblies: + chosen = best[Path(item.entry).name.lower()] + if item is not chosen: + item.include = False + item.reason = f"duplicate of {chosen.entry} (newer target framework)" + elif item.framework and not include_framework: + item.include = False + item.reason = "framework assembly (use --include-framework)" + found.search_dirs.extend(sorted({item.path.parent for item in found.included})) + return found + + +def _nupkg_tfm(parts: list[str]) -> str | None: + for marker in ("lib", "ref", "tools"): + if marker in parts: + index = parts.index(marker) + if index + 1 < len(parts) - 1: + return parts[index + 1] + return None + + +def _nupkg_rid(parts: list[str]) -> str | None: + if len(parts) > 2 and parts[0] == "runtimes": + return parts[1] + return None + + +def _nupkg_preference(item: AssemblyFile) -> tuple[int, tuple[int, int, int]]: + top = item.entry.split("/", 1)[0] + kind = {"lib": 3, "runtimes": 2, "tools": 2, "ref": 0}.get(top, 1) + return (kind, _tfm_rank(item.tfm or "")) + + +def _safe_join(root: Path, name: str) -> Path | None: + parts = [ + part + for part in name.replace("\\", "/").split("/") + if part not in {"", ".", ".."} + ] + if not parts: + return None + return root.joinpath(*parts) + + +def pinvoke_neighbours( + input_dir: Path, module_names: set[str] +) -> list[tuple[str, Path]]: + """Native libraries next to the input that P/Invoke declarations name.""" + found: list[tuple[str, Path]] = [] + seen: set[Path] = set() + for module in sorted(module_names): + base = Path(module).name + stem = base + for suffix in (".so", ".dll", ".dylib"): + if stem.lower().endswith(suffix): + stem = stem[: -len(suffix)] + candidates = { + base, + f"{stem}.so", + f"lib{stem}.so", + f"{stem}.dll", + f"{stem}.dylib", + f"lib{stem}.dylib", + } + for name in sorted(candidates): + path = (input_dir / name).resolve() + if path in seen or not path.is_file(): + continue + if is_native_binary(path): + seen.add(path) + found.append((module, path)) + return found + + +# -------------------------------------------------------------------------- +# .NET runtime (pythonnet) +# -------------------------------------------------------------------------- + +_runtime: dict[str, Any] | None = None + + +def _version_tuple(text: str) -> tuple[int, ...]: + numbers = re.findall(r"\d+", text.split("-", 1)[0]) + return tuple(int(item) for item in numbers[:3]) + + +def dotnet_root_candidates() -> list[Path]: + candidates: list[Path] = [] + for name in ("DOTNET_ROOT", "DOTNET_ROOT_ARM64", "DOTNET_ROOT_X64"): + value = os.environ.get(name, "").strip() + if value: + candidates.append(Path(value).expanduser()) + exe = shutil.which("dotnet") + if exe: + candidates.append(Path(exe).resolve().parent) + candidates.extend( + Path(item) + for item in ( + "/usr/lib/dotnet", + "/usr/share/dotnet", + "/usr/local/share/dotnet", + "/opt/dotnet", + str(Path.home() / ".dotnet"), + "C:/Program Files/dotnet", + ) + ) + return candidates + + +def find_runtime() -> tuple[Path, str] | None: + """Newest installed Microsoft.NETCore.App >= 9 as ``(dotnet root, version)``.""" + best: tuple[tuple[int, ...], Path, str] | None = None + for root in dotnet_root_candidates(): + shared = root / "shared" / "Microsoft.NETCore.App" + if not shared.is_dir(): + continue + for item in shared.iterdir(): + version = _version_tuple(item.name) + if not version or version[0] < MIN_RUNTIME_MAJOR: + continue + if not (item / "System.Private.CoreLib.dll").is_file(): + continue + if best is None or version > best[0]: + best = (version, root, item.name) + if best is None: + return None + return best[1], best[2] + + +def probe_dotnet() -> tuple[bool, str]: + if importlib.util.find_spec("pythonnet") is None: + return False, "pythonnet is not installed (pip install pythonnet)" + missing = [ + name for name in DOTNET_LIBRARIES if not (DOTNET_LIB_DIR / name).is_file() + ] + if missing: + return False, f"bundled .NET libraries missing: {', '.join(missing)}" + found = find_runtime() + if found is None: + return ( + False, + f"no .NET runtime >= {MIN_RUNTIME_MAJOR} found (install e.g. " + "dotnet-runtime-10.0, or set DOTNET_ROOT)", + ) + return True, f".NET {found[1]} at {found[0]}" + + +def load_runtime() -> dict[str, Any]: + """Load CoreCLR and the bundled assemblies once per process.""" + global _runtime + if _runtime is not None: + return _runtime + ok, reason = probe_dotnet() + if not ok: + raise ToCodeError(f".NET input requires the .NET backend: {reason}") + found = find_runtime() + assert found is not None + root, version = found + os.environ.setdefault("DOTNET_ROOT", str(root)) + # Keep the runtime quiet and lean inside a Python host. + os.environ.setdefault("DOTNET_CLI_TELEMETRY_OPTOUT", "1") + os.environ.setdefault("DOTNET_gcServer", "0") + import pythonnet + + try: + pythonnet.load("coreclr", dotnet_root=str(root)) + except RuntimeError as exc: + if "already" not in str(exc).lower(): + raise ToCodeError(f"cannot load the .NET runtime: {exc}") from exc + import clr # type: ignore[import-not-found] + + for name in DOTNET_LIBRARIES: + clr.AddReference(str(DOTNET_LIB_DIR / name)) + import System # type: ignore[import-not-found] + + actual = str(System.Environment.Version) + if _version_tuple(actual)[:1] < (MIN_RUNTIME_MAJOR,): + raise ToCodeError( + f".NET runtime {actual} was loaded; ToCode needs {MIN_RUNTIME_MAJOR}+" + ) + _runtime = { + "clr": clr, + "System": System, + "root": root, + "version": actual, + "framework_dir": root / "shared" / "Microsoft.NETCore.App" / version, + } + return _runtime + + +# -------------------------------------------------------------------------- +# Inventory records +# -------------------------------------------------------------------------- + + +@dataclass(slots=True) +class FieldInfo: + token: int + name: str + type: str + flags: list[str] + constant: str | None = None + + +@dataclass(slots=True) +class PropertyInfo: + token: int + name: str + type: str + getter: int | None + setter: int | None + + +@dataclass(slots=True) +class EventInfo: + token: int + name: str + type: str + add: int | None + remove: int | None + + +@dataclass(slots=True) +class MethodInfo: + id: int + assembly: int + token: int + name: str + type_token: int + type_name: str + full_name: str + return_type: str + params: list[tuple[str, str]] + flags: list[str] + impl_flags: list[str] + rva: int + il_size: int + is_static: bool + is_virtual: bool + is_abstract: bool + is_constructor: bool + compiler_generated: bool + pinvoke_module: str | None = None + pinvoke_entry: str | None = None + unmanaged_entry: str | None = None + state_machine_type: int | None = None + owner_method: int | None = None # state machine / closure owner (method id) + attributes: list[str] = field(default_factory=list) + callees: list[int] = field(default_factory=list) + external_calls: list[str] = field(default_factory=list) + callers: list[int] = field(default_factory=list) + string_ref_count: int = 0 + field_refs: list[str] = field(default_factory=list) + cs_start: int | None = None + cs_end: int | None = None + il_start: int | None = None + il_end: int | None = None + + @property + def address(self) -> str: + return f"{self.assembly_name}!0x{self.token:08x}" + + # Set by the session after inventory (kept out of __init__ for brevity). + assembly_name: str = "" + + @property + def prototype(self) -> str: + params = ", ".join(f"{kind} {name}" for name, kind in self.params) + return f"{self.return_type} {self.name}({params})" + + +@dataclass(slots=True) +class TypeInfo: + assembly: int + token: int + full_name: str # dnlib style, nested with '/' + reflection_name: str # nested with '+' + namespace: str + name: str + kind: str + flags: list[str] + visibility: str + base_type: str | None + interfaces: list[str] + generic_params: list[str] + declaring_type: int | None + top_level: int # token of the top-level type that owns the file + attributes: list[str] + fields: list[FieldInfo] + properties: list[PropertyInfo] + events: list[EventInfo] + methods: list[int] # method ids + compiler_generated: bool = False + cs_path: Path | None = None + il_path: Path | None = None + cs_lines: int = 0 + il_lines: int = 0 + error: str | None = None + il_error: str | None = None + + +@dataclass(slots=True) +class ResourceInfo: + name: str + kind: str # embedded | linked | assembly-linked + size: int | None + public: bool + file: str | None = None + path: Path | None = None + entries: list[dict[str, Any]] | None = None + error: str | None = None + + +@dataclass(slots=True) +class AssemblyInfo: + index: int + file: AssemblyFile + pe: PeInfo + name: str + full_name: str + version: str + culture: str + public_key_token: str | None + module_name: str + mvid: str + kind: str + target_framework: str | None + entry_point: int | None + assembly_refs: list[dict[str, Any]] + module_refs: list[str] + attributes: list[str] + resources: list[ResourceInfo] + folder: str = "" + types: list[int] = field(default_factory=list) # tokens + user_strings: list[tuple[int, str]] = field(default_factory=list) + obfuscation_hints: list[str] = field(default_factory=list) + error: str | None = None + + +def _str(value: Any) -> str: + return "" if value is None else str(value) + + +def _flag_list(value: Any) -> list[str]: + text = _str(value) + return [ + item.strip() for item in text.split(",") if item.strip() and item.strip() != "0" + ] + + +def _attribute_text(attribute: Any) -> str: + try: + name = _str(attribute.TypeFullName) + args = [] + for arg in attribute.ConstructorArguments: + args.append(_short_value(arg.Value)) + for named in attribute.NamedArguments: + args.append(f"{_str(named.Name)}={_short_value(named.Argument.Value)}") + return f"{name}({', '.join(args)})" if args else name + except Exception: # pragma: no cover - malformed blobs + return "" + + +def _short_value(value: Any) -> str: + text = _str(value) + return text if len(text) <= 200 else text[:197] + "..." + + +def _type_sig_token(signature: Any) -> int | None: + """TypeDef token behind a ``typeof(X)`` attribute argument (a TypeSig).""" + target = getattr(signature, "TypeDefOrRef", None) + if target is None: + return None + token = int(target.MDToken.Raw) + if token >> 24 == 0x02: + return token + try: + resolved = target.ResolveTypeDef() + except Exception: + return None + return int(resolved.MDToken.Raw) if resolved is not None else None + + +def type_kind(type_def: Any) -> str: + if type_def.IsInterface: + return "interface" + if type_def.IsEnum: + return "enum" + if type_def.IsDelegate: + return "delegate" + if type_def.IsValueType: + return "struct" + if type_def.IsAbstract and type_def.IsSealed: + return "static class" + return "class" + + +_COMPILER_GENERATED = "System.Runtime.CompilerServices.CompilerGeneratedAttribute" +_GENERATED_MEMBER_RX = re.compile(r"^<([^>]+)>(?:[bg]__|$)") +_STATE_MACHINE_ATTRIBUTES = { + "System.Runtime.CompilerServices.AsyncStateMachineAttribute", + "System.Runtime.CompilerServices.IteratorStateMachineAttribute", + "System.Runtime.CompilerServices.AsyncIteratorStateMachineAttribute", +} + + +class DotnetSession: + """dnlib-backed inventory over every included assembly.""" + + backend_name = "dotnet" + backend_label = ".NET" + decompiler_label = "ICSharpCode.Decompiler + dnlib" + + def __init__(self, found: DotnetInput) -> None: + self.input = found + self.assemblies: list[AssemblyInfo] = [] + self.types: dict[tuple[int, int], TypeInfo] = {} + self.methods: list[MethodInfo] = [] + self._method_by_token: dict[tuple[int, int], MethodInfo] = {} + self._method_by_name: dict[str, MethodInfo] = {} + # (assembly index, #US offset) -> ids of the methods that load it + self.string_xrefs: dict[tuple[int, int], list[int]] = {} + self.external_refs: dict[str, dict[str, list[int]]] = {} + self.pinvoke_modules: set[str] = set() + + # -- loading ----------------------------------------------------------- + + def load(self, progress: Any = None) -> None: + runtime = load_runtime() + from dnlib.DotNet import ModuleDefMD # type: ignore[import-not-found] + + modules: list[Any] = [] + for index, item in enumerate(self.input.included): + pe = read_pe_info(item.path) + if pe is None: + continue + try: + module = ModuleDefMD.Load(str(item.path)) + except Exception as exc: + raise ToCodeError(f"dnlib cannot load {item.entry}: {exc}") from exc + info = self._assembly_info(len(self.assemblies), item, pe, module) + self.assemblies.append(info) + modules.append(module) + self._types_and_methods(info, module) + del runtime + for info, module in zip(self.assemblies, modules): + self._link_owners(info) + self._scan_references(info, module) + self._modules = modules + + def _assembly_info( + self, index: int, item: AssemblyFile, pe: PeInfo, module: Any + ) -> AssemblyInfo: + assembly = module.Assembly + target_framework = None + attributes: list[str] = [] + if assembly is not None: + for attribute in assembly.CustomAttributes: + text = _attribute_text(attribute) + attributes.append(text) + if _str(attribute.TypeFullName).endswith("TargetFrameworkAttribute"): + args = list(attribute.ConstructorArguments) + if args: + target_framework = _str(args[0].Value) + for attribute in module.CustomAttributes: + attributes.append(f"module: {_attribute_text(attribute)}") + refs = [] + for ref in module.GetAssemblyRefs(): + refs.append( + { + "name": _str(ref.Name), + "version": _str(ref.Version), + "culture": _str(ref.Culture) or None, + "public_key_token": _str(ref.PublicKeyOrToken.Token) + if ref.PublicKeyOrToken is not None + else None, + "full_name": _str(ref.FullName), + } + ) + module_refs = sorted({_str(ref.Name) for ref in module.GetModuleRefs()}) + entry = module.ManagedEntryPoint + entry_token = int(entry.MDToken.Raw) if entry is not None else None + name = _str(assembly.Name) if assembly is not None else _str(module.Name) + info = AssemblyInfo( + index=index, + file=item, + pe=pe, + name=name or item.path.stem, + full_name=_str(assembly.FullName) + if assembly is not None + else _str(module.Name), + version=_str(assembly.Version) if assembly is not None else "", + culture=_str(assembly.Culture) if assembly is not None else "", + public_key_token=_str(assembly.PublicKeyToken) + if assembly is not None and assembly.PublicKeyToken is not None + else None, + module_name=_str(module.Name), + mvid=_str(module.Mvid), + kind=_str(module.Kind), + target_framework=target_framework, + entry_point=entry_token, + assembly_refs=refs, + module_refs=module_refs, + attributes=attributes, + resources=[], + ) + for resource in module.Resources: + kind = _str(resource.ResourceType).lower() + size = None + file_name = None + if "embedded" in kind: + try: + size = int(resource.Length) + except Exception: + size = None + elif "linked" in kind and "assembly" not in kind: + file_name = _str(getattr(resource, "FileName", None)) or None + info.resources.append( + ResourceInfo( + name=_str(resource.Name), + kind="embedded" + if "embedded" in kind + else ("assembly-linked" if "assembly" in kind else "linked"), + size=size, + public=bool(resource.IsPublic), + file=file_name, + ) + ) + heap = pe.stream("#US") + if heap is not None: + data = item.path.read_bytes()[heap.offset : heap.offset + heap.size] + info.user_strings = iter_user_strings(data) + return info + + def _types_and_methods(self, info: AssemblyInfo, module: Any) -> None: + odd_names = 0 + total = 0 + for type_def in module.GetTypes(): + token = int(type_def.MDToken.Raw) + top = type_def + while top.DeclaringType is not None: + top = top.DeclaringType + name = _str(type_def.Name) + total += 1 + if name and not re.match(r"^[A-Za-z_<][\w`<>.$-]*$", name): + odd_names += 1 + attributes = [_attribute_text(item) for item in type_def.CustomAttributes] + compiler_generated = name.startswith("<") or any( + item.startswith(_COMPILER_GENERATED) for item in attributes + ) + record = TypeInfo( + assembly=info.index, + token=token, + full_name=_str(type_def.FullName), + reflection_name=_str(type_def.ReflectionFullName), + namespace=_str(top.Namespace), + name=name, + kind=type_kind(type_def), + flags=_flag_list(type_def.Attributes), + visibility=_str(type_def.Visibility), + base_type=_str(type_def.BaseType.FullName) + if type_def.BaseType is not None + else None, + interfaces=[ + _str(item.Interface.FullName) + for item in type_def.Interfaces + if item.Interface is not None + ], + generic_params=[_str(item.Name) for item in type_def.GenericParameters], + declaring_type=int(type_def.DeclaringType.MDToken.Raw) + if type_def.DeclaringType is not None + else None, + top_level=int(top.MDToken.Raw), + attributes=attributes, + fields=[self._field(item) for item in type_def.Fields], + properties=[self._property(item) for item in type_def.Properties], + events=[self._event(item) for item in type_def.Events], + methods=[], + compiler_generated=compiler_generated, + ) + self.types[(info.index, token)] = record + info.types.append(token) + for method in type_def.Methods: + self._method(info, record, method) + if total and odd_names / total > 0.2: + info.obfuscation_hints.append( + f"{odd_names}/{total} type names are not valid C# identifiers" + ) + for attribute in info.attributes: + lowered = attribute.lower() + if any( + marker in lowered + for marker in ( + "confusedby", + "dotfuscator", + "obfuscat", + "suppressildasm", + ) + ): + info.obfuscation_hints.append(f"attribute: {attribute}") + + def _field(self, item: Any) -> FieldInfo: + constant = None + if item.HasConstant and item.Constant is not None: + constant = _short_value(item.Constant.Value) + return FieldInfo( + token=int(item.MDToken.Raw), + name=_str(item.Name), + type=_str(item.FieldType.FullName) if item.FieldType is not None else "?", + flags=_flag_list(item.Attributes), + constant=constant, + ) + + def _property(self, item: Any) -> PropertyInfo: + getter = item.GetMethod + setter = item.SetMethod + signature = item.PropertySig + return PropertyInfo( + token=int(item.MDToken.Raw), + name=_str(item.Name), + type=_str(signature.RetType.FullName) + if signature is not None and signature.RetType is not None + else "?", + getter=int(getter.MDToken.Raw) if getter is not None else None, + setter=int(setter.MDToken.Raw) if setter is not None else None, + ) + + def _event(self, item: Any) -> EventInfo: + add = item.AddMethod + remove = item.RemoveMethod + return EventInfo( + token=int(item.MDToken.Raw), + name=_str(item.Name), + type=_str(item.EventType.FullName) if item.EventType is not None else "?", + add=int(add.MDToken.Raw) if add is not None else None, + remove=int(remove.MDToken.Raw) if remove is not None else None, + ) + + def _method(self, info: AssemblyInfo, owner: TypeInfo, method: Any) -> None: + signature = method.MethodSig + params: list[tuple[str, str]] = [] + if signature is not None: + names = [ + _str(item.Name) + for item in method.Parameters + if not item.IsHiddenThisParameter + ] + for position, kind in enumerate(signature.Params): + label = ( + names[position] + if position < len(names) and names[position] + else f"p{position}" + ) + params.append((label, _str(kind.FullName))) + attributes = [_attribute_text(item) for item in method.CustomAttributes] + state_machine = None + unmanaged_entry = None + for attribute in method.CustomAttributes: + type_name = _str(attribute.TypeFullName) + if type_name in _STATE_MACHINE_ATTRIBUTES: + args = list(attribute.ConstructorArguments) + if args and args[0].Value is not None: + state_machine = _type_sig_token(args[0].Value) + elif type_name.endswith("UnmanagedCallersOnlyAttribute"): + for named in attribute.NamedArguments: + if _str(named.Name) == "EntryPoint": + unmanaged_entry = _str(named.Argument.Value) or None + impl_map = method.ImplMap + pinvoke_module = None + pinvoke_entry = None + if impl_map is not None: + pinvoke_module = ( + _str(impl_map.Module.Name) if impl_map.Module is not None else None + ) + pinvoke_entry = _str(impl_map.Name) or _str(method.Name) + if pinvoke_module: + self.pinvoke_modules.add(pinvoke_module) + body_size = 0 + rva = int(method.RVA) + name = _str(method.Name) + record = MethodInfo( + id=len(self.methods), + assembly=info.index, + token=int(method.MDToken.Raw), + name=name, + type_token=owner.token, + type_name=owner.full_name, + full_name=_str(method.FullName), + return_type=_str(signature.RetType.FullName) + if signature is not None and signature.RetType is not None + else "System.Void", + params=params, + flags=_flag_list(method.Attributes), + impl_flags=_flag_list(method.ImplAttributes), + rva=rva, + il_size=body_size, + is_static=bool(method.IsStatic), + is_virtual=bool(method.IsVirtual), + is_abstract=bool(method.IsAbstract), + is_constructor=bool(method.IsConstructor), + compiler_generated=owner.compiler_generated + or name.startswith("<") + or any(item.startswith(_COMPILER_GENERATED) for item in attributes), + pinvoke_module=pinvoke_module, + pinvoke_entry=pinvoke_entry, + unmanaged_entry=unmanaged_entry, + state_machine_type=state_machine, + attributes=attributes, + ) + record.assembly_name = info.name + self.methods.append(record) + owner.methods.append(record.id) + self._method_by_token[(info.index, record.token)] = record + self._method_by_name.setdefault(f"{info.name}|{record.full_name}", record) + + def _link_owners(self, info: AssemblyInfo) -> None: + """Point compiler-generated methods back to the method that owns them. + + State machines come from [AsyncStateMachine]/[IteratorStateMachine]; + lambdas (``b__N``) and local functions (``g__Name|N``) + are matched by the C# compiler's naming scheme against the enclosing + type and, for closure classes (``<>c``, ``<>c__DisplayClass``), its + declaring type. + """ + for method in self.methods: + if method.assembly != info.index or method.state_machine_type is None: + continue + machine = self.types.get((info.index, method.state_machine_type)) + if machine is None: + continue + for member in machine.methods: + self.methods[member].owner_method = method.id + for method in self.methods: + if method.assembly != info.index or method.owner_method is not None: + continue + match = _GENERATED_MEMBER_RX.match(method.name) + if match is None: + continue + owner_name = match.group(1) + record = self.types.get((info.index, method.type_token)) + while record is not None: + candidates = [ + self.methods[item] + for item in record.methods + if self.methods[item].name == owner_name and item != method.id + ] + if candidates: + method.owner_method = candidates[0].id + break + parent = record.declaring_type + record = self.types.get((info.index, parent)) if parent else None + + # -- references --------------------------------------------------------- + + def _scan_references(self, info: AssemblyInfo, module: Any) -> None: + data = info.file.path.read_bytes() + member_refs = self._member_refs(info, module) + method_specs = self._method_specs(module) + field_names = { + field_info.token: f"{record.full_name}::{field_info.name}" + for (assembly, _), record in self.types.items() + if assembly == info.index + for field_info in record.fields + } + for method in self.methods: + if method.assembly != info.index or method.rva == 0: + continue + offset = info.pe.rva_to_offset(method.rva) + if offset is None: + continue + code = method_body(data, offset) + if code is None: + continue + method.il_size = len(code) + callees: list[int] = [] + external: list[str] = [] + fields: list[str] = [] + seen: set[int] = set() + strings = 0 + for kind, token in scan_il_tokens(code): + if kind == "string": + strings += 1 + bucket = self.string_xrefs.setdefault( + (info.index, token & 0xFFFFFF), [] + ) + if len(bucket) < MAX_STRING_XREFS and method.id not in bucket: + bucket.append(method.id) + continue + if token in seen: + continue + seen.add(token) + if kind == "method": + table = token >> 24 + if table == 0x2B: + token = method_specs.get(token & 0xFFFFFF, token) + table = token >> 24 + if table == 0x06: + target = self._method_by_token.get((info.index, token)) + if target is not None: + callees.append(target.id) + target.callers.append(method.id) + continue + if table == 0x0A: + ref = member_refs.get(token) + if ref is None: + continue + full_name, scope, internal = ref + target = internal + if target is None: + target = self._method_by_name.get(f"{scope}|{full_name}") + if target is not None: + callees.append(target.id) + target.callers.append(method.id) + else: + external.append(f"[{scope}]{full_name}") + refs = self.external_refs.setdefault(scope, {}).setdefault( + full_name, [] + ) + if len(refs) < 32: + refs.append(method.id) + elif kind == "field" and len(fields) < MAX_FIELD_REFS: + table = token >> 24 + if table == 0x04: + name = field_names.get(token) + if name: + fields.append(name) + elif table == 0x0A: + ref = member_refs.get(token) + if ref is not None: + fields.append(f"[{ref[1]}]{ref[0]}") + method.callees = callees + method.external_calls = external + method.string_ref_count = strings + method.field_refs = fields + + def _member_refs( + self, info: AssemblyInfo, module: Any + ) -> dict[int, tuple[str, str, MethodInfo | None]]: + result: dict[int, tuple[str, str, MethodInfo | None]] = {} + rows = int(module.Metadata.TablesStream.MemberRefTable.Rows) + for rid in range(1, rows + 1): + try: + ref = module.ResolveMemberRef(rid) + except Exception: + continue + if ref is None: + continue + scope = info.name + internal: MethodInfo | None = None + declaring = ref.DeclaringType + try: + if declaring is not None and declaring.DefinitionAssembly is not None: + scope = _str(declaring.DefinitionAssembly.Name) + elif ( + ref.Class is not None + and _str(ref.Class.GetType().Name) == "ModuleRefUser" + ): + scope = _str(ref.Class.Name) + except Exception: + pass + if scope == info.name and ref.IsMethodRef: + try: + resolved = ref.ResolveMethod() + except Exception: + resolved = None + if ( + resolved is not None + and resolved.Module is not None + and _str(resolved.Module.Name) == info.module_name + ): + internal = self._method_by_token.get( + (info.index, int(resolved.MDToken.Raw)) + ) + result[0x0A000000 | rid] = (_str(ref.FullName), scope, internal) + return result + + def _method_specs(self, module: Any) -> dict[int, int]: + result: dict[int, int] = {} + rows = int(module.Metadata.TablesStream.MethodSpecTable.Rows) + for rid in range(1, rows + 1): + try: + spec = module.ResolveMethodSpec(rid) + except Exception: + continue + if spec is not None and spec.Method is not None: + result[rid] = int(spec.Method.MDToken.Raw) + return result + + # -- resources ---------------------------------------------------------- + + def save_resource(self, info: AssemblyInfo, name: str, target: Path) -> int | None: + """Write an embedded resource's bytes to ``target``; its size or None.""" + import System # type: ignore[import-not-found] + + module = self._modules[info.index] + for resource in module.Resources: + if _str(resource.Name) != name: + continue + try: + raw = resource.CreateReader().ToArray() + except Exception: + return None + target.parent.mkdir(parents=True, exist_ok=True) + System.IO.File.WriteAllBytes(str(target), raw) + return int(raw.Length) + return None + + def decode_resources_file( + self, info: AssemblyInfo, name: str + ) -> list[dict[str, Any]]: + """Entries of a ``.resources`` blob via dnlib's ResourceReader.""" + from dnlib.DotNet.Resources import ResourceReader # type: ignore[import-not-found] + + module = self._modules[info.index] + resource = next(item for item in module.Resources if _str(item.Name) == name) + elements = ResourceReader.Read(module, resource.CreateReader()) + rows: list[dict[str, Any]] = [] + for element in elements.ResourceElements: + data = element.ResourceData + code = _str(data.Code) + value: Any + built_in = getattr(data, "Data", None) + if built_in is not None and not hasattr(built_in, "Length"): + value = _short_value(built_in) + else: + raw = getattr(data, "Data", None) + try: + value = ( + f"<{len(bytes(raw))} bytes>" if raw is not None else _str(data) + ) + except Exception: + value = _str(data) + rows.append({"name": _str(element.Name), "type": code, "value": value}) + return rows + + def close(self) -> None: + for module in getattr(self, "_modules", []): + try: + module.Dispose() + except Exception: + pass + self._modules = [] + + # -- lookups ------------------------------------------------------------ + + def method(self, assembly: int, token: int) -> MethodInfo | None: + return self._method_by_token.get((assembly, token)) + + def _type_index( + self, + ) -> tuple[dict[int, list[TypeInfo]], dict[tuple[int, int], list[TypeInfo]]]: + cached = getattr(self, "_index_cache", None) + if cached is not None and cached[0] == len(self.types): + return cached[1], cached[2] + tops: dict[int, list[TypeInfo]] = {} + groups: dict[tuple[int, int], list[TypeInfo]] = {} + for (index, _), record in self.types.items(): + if record.declaring_type is None: + tops.setdefault(index, []).append(record) + groups.setdefault((index, record.top_level), []).append(record) + self._index_cache = (len(self.types), tops, groups) + return tops, groups + + def top_level_types(self, assembly: int) -> list[TypeInfo]: + return self._type_index()[0].get(assembly, []) + + def nested_types(self, assembly: int, top: int) -> list[TypeInfo]: + """The top-level type ``top`` and every type nested in it.""" + return self._type_index()[1].get((assembly, top), []) + + +# -------------------------------------------------------------------------- +# Worker-side decompilation +# -------------------------------------------------------------------------- + +_METHOD_IL_RX = re.compile(r"^\s*\.method /\* (06[0-9A-Fa-f]{6}) \*/") +_END_METHOD_RX = re.compile(r"^\s*\} // end of method ") +_DECLARATION_NODES = { + "MethodDeclaration", + "ConstructorDeclaration", + "DestructorDeclaration", + "OperatorDeclaration", + "PropertyDeclaration", + "IndexerDeclaration", + "EventDeclaration", + "CustomEventDeclaration", + "Accessor", + "LambdaExpression", + "AnonymousMethodExpression", + "LocalFunctionDeclarationStatement", +} + + +@dataclass(slots=True) +class DecompiledType: + assembly: int + token: int + cs: str | None + cs_error: str | None + il: str | None + il_error: str | None + # metadata token -> (first line, last line) in the respective text + cs_ranges: dict[int, tuple[int, int]] + il_ranges: dict[int, tuple[int, int]] + + +@dataclass(slots=True) +class WorkItem: + assembly: int + path: str + token: int # 0 = assembly/module attributes + manifest + + +_worker: dict[str, Any] = {} + + +def init_dotnet_worker(search_dirs: list[str]) -> None: + load_runtime() + _worker["search_dirs"] = search_dirs + _worker["decompilers"] = {} + _worker["files"] = {} + + +def _decompiler_for(path: str) -> tuple[Any, Any, Any]: + cached = _worker["decompilers"].get(path) + if cached is not None: + return cached + from ICSharpCode.Decompiler import DecompilerSettings # type: ignore[import-not-found] + from ICSharpCode.Decompiler.CSharp import CSharpDecompiler # type: ignore[import-not-found] + from ICSharpCode.Decompiler.Metadata import ( # type: ignore[import-not-found] + DotNetCorePathFinderExtensions, + PEFile, + UniversalAssemblyResolver, + ) + + settings = DecompilerSettings() + settings.ThrowOnAssemblyResolveErrors = False + settings.ShowXmlDocumentation = False + pe = PEFile(path) + try: + target = DotNetCorePathFinderExtensions.DetectTargetFrameworkId(pe) + except Exception: + target = "" + resolver = UniversalAssemblyResolver(path, False, target) + for directory in _worker.get("search_dirs", []): + resolver.AddSearchDirectory(directory) + framework_dir = (_runtime or {}).get("framework_dir") + if framework_dir is not None: + resolver.AddSearchDirectory(str(framework_dir)) + decompiler = CSharpDecompiler(pe, resolver, settings) + cached = (decompiler, pe, settings) + _worker["decompilers"][path] = cached + return cached + + +def decompile_batch_in_worker(items: list[WorkItem]) -> list[DecompiledType]: + return [decompile_item(item) for item in items] + + +def decompile_item(item: WorkItem) -> DecompiledType: + result = DecompiledType(item.assembly, item.token, None, None, None, None, {}, {}) + try: + decompiler, pe, settings = _decompiler_for(item.path) + except Exception as exc: + result.cs_error = result.il_error = f"cannot open assembly: {_exc_text(exc)}" + return result + if item.token == 0: + try: + result.cs = str(decompiler.DecompileModuleAndAssemblyAttributesToString()) + except Exception as exc: + result.cs_error = _exc_text(exc) + try: + result.il = _manifest_il(pe) + except Exception as exc: + result.il_error = _exc_text(exc) + return result + try: + result.cs, result.cs_ranges = _decompile_type(decompiler, settings, item.token) + except Exception as exc: + result.cs_error = _exc_text(exc) + try: + result.il, result.il_ranges = _disassemble_type(pe, item.token) + except Exception as exc: + result.il_error = _exc_text(exc) + return result + + +def _exc_text(exc: BaseException) -> str: + text = str(exc).strip().splitlines() + first = text[0] if text else type(exc).__name__ + return first[:500] + + +def _decompile_type( + decompiler: Any, settings: Any, token: int +) -> tuple[str, dict[int, tuple[int, int]]]: + from System.IO import StringWriter # type: ignore[import-not-found] + from System.Reflection.Metadata.Ecma335 import MetadataTokens # type: ignore[import-not-found] + from ICSharpCode.Decompiler.CSharp import AnnotationExtensions # type: ignore[import-not-found] + from ICSharpCode.Decompiler.CSharp.OutputVisitor import ( # type: ignore[import-not-found] + CSharpOutputVisitor, + TokenWriter, + ) + from ICSharpCode.Decompiler.TypeSystem import IEntity, IEvent, IProperty # type: ignore[import-not-found] + + # DecompileTypes (unlike Decompile(EntityHandle)) wraps the type in its + # namespace declaration, as a project export would. + from System.Collections.Generic import List # type: ignore[import-not-found] + from System.Reflection.Metadata import TypeDefinitionHandle # type: ignore[import-not-found] + + handles = List[TypeDefinitionHandle]() + handles.Add(MetadataTokens.TypeDefinitionHandle(token & 0xFFFFFF)) + tree = decompiler.DecompileTypes(handles) + writer = StringWriter() + tree.AcceptVisitor( + CSharpOutputVisitor( + TokenWriter.CreateWriterThatSetsLocationsInAST(writer, " "), + settings.CSharpFormattingOptions, + ) + ) + ranges: dict[int, tuple[int, int]] = {} + + def record(entity: Any, start: int, end: int) -> None: + try: + value = int(MetadataTokens.GetToken(IEntity(entity).MetadataToken)) + except Exception: + return + if value and value not in ranges: + ranges[value] = (start, end) + + for node in tree.Descendants: + kind = type(node).__name__ + if kind not in _DECLARATION_NODES: + continue + start = int(node.StartLocation.Line) + end = int(node.EndLocation.Line) + if start <= 0: + continue + symbol = AnnotationExtensions.GetSymbol(node) + if symbol is None: + for annotation in node.Annotations: + if type(annotation).__name__ == "ILFunction": + method = getattr(annotation, "Method", None) + if method is not None: + record(method, start, end) + continue + record(symbol, start, end) + # Properties/events: map their accessor methods to the same span when + # the accessors have no separate Accessor node (expression bodies). + try: + prop = IProperty(symbol) + for accessor in (prop.Getter, prop.Setter): + if accessor is not None: + record(accessor, start, end) + except Exception: + pass + try: + event = IEvent(symbol) + for accessor in ( + event.AddAccessor, + event.RemoveAccessor, + event.InvokeAccessor, + ): + if accessor is not None: + record(accessor, start, end) + except Exception: + pass + text = str(writer.ToString()).replace("\r\n", "\n") + return text, ranges + + +def _new_disassembler(output: Any) -> Any: + from System.Threading import CancellationToken # type: ignore[import-not-found] + from ICSharpCode.Decompiler.Disassembler import ReflectionDisassembler # type: ignore[import-not-found] + + disassembler = ReflectionDisassembler(output, getattr(CancellationToken, "None")) + disassembler.ShowMetadataTokens = True + disassembler.ShowRawRVAOffsetAndBytes = True + disassembler.DetectControlStructure = True + disassembler.ExpandMemberDefinitions = True + return disassembler + + +def _disassemble_type(pe: Any, token: int) -> tuple[str, dict[int, tuple[int, int]]]: + from System.Reflection.Metadata.Ecma335 import MetadataTokens # type: ignore[import-not-found] + from ICSharpCode.Decompiler import PlainTextOutput # type: ignore[import-not-found] + + output = PlainTextOutput() + output.IndentationString = " " + _new_disassembler(output).DisassembleType( + pe, MetadataTokens.TypeDefinitionHandle(token & 0xFFFFFF) + ) + text = str(output.ToString()).replace("\r\n", "\n") + return text, il_method_ranges(text) + + +def _manifest_il(pe: Any) -> str: + from ICSharpCode.Decompiler import PlainTextOutput # type: ignore[import-not-found] + + output = PlainTextOutput() + output.IndentationString = " " + disassembler = _new_disassembler(output) + disassembler.WriteAssemblyReferences(pe.Metadata) + if pe.Metadata.IsAssembly: + disassembler.WriteAssemblyHeader(pe) + output.WriteLine() + disassembler.WriteModuleHeader(pe) + return str(output.ToString()).replace("\r\n", "\n") + + +def il_method_ranges(text: str) -> dict[int, tuple[int, int]]: + """Line ranges of every ``.method`` block in ReflectionDisassembler output.""" + ranges: dict[int, tuple[int, int]] = {} + current: tuple[int, int] | None = None + for number, line in enumerate(text.split("\n"), start=1): + match = _METHOD_IL_RX.match(line) + if match: + current = (int(match.group(1), 16), number) + continue + if current is not None and _END_METHOD_RX.match(line): + ranges[current[0]] = (current[1], number) + current = None + return ranges + + +def runtime_description() -> str: + runtime = _runtime or {} + return f".NET {runtime.get('version', '?')} ({sys.platform})" + + +__all__ = [ + "PE_MACHINES", + "AssemblyFile", + "AssemblyInfo", + "DecompiledType", + "DotnetInput", + "DotnetSession", + "MethodInfo", + "NativeFile", + "TypeInfo", + "WorkItem", + "decompile_batch_in_worker", + "discover_dotnet_input", + "init_dotnet_worker", + "is_dotnet_input", + "load_runtime", + "pinvoke_neighbours", + "probe_dotnet", +] diff --git a/src/tocode/backends/dotnet_lib/ICSharpCode.Decompiler.dll b/src/tocode/backends/dotnet_lib/ICSharpCode.Decompiler.dll new file mode 100644 index 0000000..f954514 Binary files /dev/null and b/src/tocode/backends/dotnet_lib/ICSharpCode.Decompiler.dll differ diff --git a/src/tocode/backends/dotnet_lib/LICENSE-ICSharpCode.Decompiler.txt b/src/tocode/backends/dotnet_lib/LICENSE-ICSharpCode.Decompiler.txt new file mode 100644 index 0000000..8ae196d --- /dev/null +++ b/src/tocode/backends/dotnet_lib/LICENSE-ICSharpCode.Decompiler.txt @@ -0,0 +1,19 @@ +MIT License + +Copyright 2011-2026 AlphaSierraPapa for the SharpDevelop Team (ILSpy Contributors) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this +software and associated documentation files (the "Software"), to deal in the Software +without restriction, including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons +to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or +substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE +FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/src/tocode/backends/dotnet_lib/LICENSE-dnlib.txt b/src/tocode/backends/dotnet_lib/LICENSE-dnlib.txt new file mode 100644 index 0000000..74858b8 --- /dev/null +++ b/src/tocode/backends/dotnet_lib/LICENSE-dnlib.txt @@ -0,0 +1,20 @@ +Copyright (C) 2012-2019 de4dot@gmail.com + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/src/tocode/backends/dotnet_lib/README.md b/src/tocode/backends/dotnet_lib/README.md new file mode 100644 index 0000000..bdc8ee3 --- /dev/null +++ b/src/tocode/backends/dotnet_lib/README.md @@ -0,0 +1,19 @@ +# Bundled .NET libraries + +ToCode's .NET backend loads these assemblies in-process through pythonnet, so +exporting a .NET binary needs only a .NET 9+ runtime -- no dnSpy/ILSpy install. +They are the unmodified `lib/netstandard2.0/` assemblies of the NuGet packages +below (both MIT licensed; license texts are next to them). + +| File | NuGet package | Version | nupkg sha256 | dll sha256 | +| --- | --- | --- | --- | --- | +| `dnlib.dll` | [dnlib](https://www.nuget.org/packages/dnlib) | 4.5.0 | `63bc2f9579568204cc8b30fa9f6700a231bcf868a8032e09118887af3eafee58` | `566fdab59c91a3c2eab14a22b67f01cb3c0cdb48fa9b9b6677e2b32998636efb` | +| `ICSharpCode.Decompiler.dll` | [ICSharpCode.Decompiler](https://www.nuget.org/packages/ICSharpCode.Decompiler) | 11.1.0.9782 | `1f76df4d35193ba4eeb50ffcd84eae381a2c48c67f95c1104dc406759448a811` | `38e6abf7497845d79d9d01d56351a3203991356de1d3496b8c51142f495e9f9c` | + +ICSharpCode.Decompiler 11.x references System.Reflection.Metadata and +System.Collections.Immutable 9.0, which ship inbox with the .NET 9+ runtime; +that is why the backend requires .NET 9 or newer. + +To update: download the `.nupkg` from nuget.org, extract +`lib/netstandard2.0/.dll`, and update the hashes above and in +`tests/test_dotnet_backend.py`. diff --git a/src/tocode/backends/dotnet_lib/dnlib.dll b/src/tocode/backends/dotnet_lib/dnlib.dll new file mode 100644 index 0000000..81e682a Binary files /dev/null and b/src/tocode/backends/dotnet_lib/dnlib.dll differ diff --git a/src/tocode/backends/dotnet_pe.py b/src/tocode/backends/dotnet_pe.py new file mode 100644 index 0000000..0cfe765 --- /dev/null +++ b/src/tocode/backends/dotnet_pe.py @@ -0,0 +1,589 @@ +"""Pure-Python readers for .NET containers: PE/CLR headers, single-file bundles. + +Nothing here loads the .NET runtime, so input routing (``is_dotnet_input``) +stays cheap for every binary ToCode is pointed at, and the readers are +testable without a .NET install. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +import mmap +from pathlib import Path +import struct +import zlib + +# IMAGE_DIRECTORY_ENTRY_COM_DESCRIPTOR +_CLR_DIRECTORY = 14 +COMIMAGE_FLAGS_ILONLY = 0x1 +COMIMAGE_FLAGS_32BITREQUIRED = 0x2 +COMIMAGE_FLAGS_STRONGNAMESIGNED = 0x8 +COMIMAGE_FLAGS_NATIVE_ENTRYPOINT = 0x10 +COMIMAGE_FLAGS_32BITPREFERRED = 0x20000 + +PE_MACHINES = { + 0x14C: "x86", + 0x8664: "x86_64", + 0x1C0: "arm", + 0x1C4: "arm", + 0xAA64: "arm64", + 0x200: "ia64", +} +IMAGE_SCN_MEM_EXECUTE = 0x20000000 +IMAGE_SCN_MEM_READ = 0x40000000 +IMAGE_SCN_MEM_WRITE = 0x80000000 + +# Marker the .NET host (apphost / singlefilehost) uses to find the bundle +# manifest; the 8 bytes before it hold the manifest offset (0 = not a bundle). +BUNDLE_SIGNATURE = bytes( + [ + 0x8B, 0x12, 0x02, 0xB9, 0x6A, 0x61, 0x20, 0x38, + 0x72, 0x7B, 0x93, 0x02, 0x14, 0xD7, 0xA0, 0x32, + 0x13, 0xF5, 0xB9, 0xE6, 0xEF, 0xAE, 0x33, 0x18, + 0xEE, 0x3B, 0x2D, 0xCE, 0x24, 0xB3, 0x6A, 0xAE, + ] +) # fmt: skip +BUNDLE_ENTRY_KINDS = { + 0: "unknown", + 1: "assembly", + 2: "native", + 3: "deps-json", + 4: "runtime-config", + 5: "symbols", +} +_EXECUTABLE_MAGICS = ( + b"MZ", + b"\x7fELF", + b"\xcf\xfa\xed\xfe", # Mach-O 64 + b"\xce\xfa\xed\xfe", # Mach-O 32 + b"\xca\xfe\xba\xbe", # Mach-O fat +) + + +@dataclass(slots=True) +class PeSection: + name: str + virtual_address: int + virtual_size: int + raw_offset: int + raw_size: int + characteristics: int + + @property + def perms(self) -> str: + flags = self.characteristics + return ( + ("r" if flags & IMAGE_SCN_MEM_READ else "-") + + ("w" if flags & IMAGE_SCN_MEM_WRITE else "-") + + ("x" if flags & IMAGE_SCN_MEM_EXECUTE else "-") + ) + + +@dataclass(slots=True) +class MetadataStream: + name: str + offset: int # file offset + size: int + + +@dataclass(slots=True) +class PeInfo: + machine: str + pe32_plus: bool + image_base: int + sections: list[PeSection] + clr_rva: int = 0 + clr_size: int = 0 + runtime_version: str | None = None + cor_flags: int = 0 + entry_point_token: int = 0 + metadata_rva: int = 0 + metadata_size: int = 0 + resources_rva: int = 0 + resources_size: int = 0 + strong_name_rva: int = 0 + managed_native_header_rva: int = 0 + streams: list[MetadataStream] = field(default_factory=list) + + @property + def is_managed(self) -> bool: + return self.clr_rva != 0 and self.clr_size != 0 + + @property + def il_only(self) -> bool: + return bool(self.cor_flags & COMIMAGE_FLAGS_ILONLY) + + @property + def mixed_mode(self) -> bool: + # C++/CLI images clear ILONLY because they carry native code. So do + # ReadyToRun images, but their native code is precompiled IL that the + # IL decompiler already covers, so they are not "mixed" for our purpose. + return self.is_managed and not self.il_only and not self.ready_to_run + + @property + def ready_to_run(self) -> bool: + return self.managed_native_header_rva != 0 + + @property + def cor_flag_names(self) -> list[str]: + names = [] + for bit, name in ( + (COMIMAGE_FLAGS_ILONLY, "ILOnly"), + (COMIMAGE_FLAGS_32BITREQUIRED, "32BitRequired"), + (COMIMAGE_FLAGS_STRONGNAMESIGNED, "StrongNameSigned"), + (COMIMAGE_FLAGS_NATIVE_ENTRYPOINT, "NativeEntryPoint"), + (COMIMAGE_FLAGS_32BITPREFERRED, "32BitPreferred"), + ): + if self.cor_flags & bit: + names.append(name) + return names + + def rva_to_offset(self, rva: int) -> int | None: + for section in self.sections: + span = max(section.virtual_size, section.raw_size) + if section.virtual_address <= rva < section.virtual_address + span: + delta = rva - section.virtual_address + if delta >= section.raw_size: + return None + return section.raw_offset + delta + return None + + def stream(self, name: str) -> MetadataStream | None: + return next((item for item in self.streams if item.name == name), None) + + +def parse_pe(data: bytes | mmap.mmap) -> PeInfo | None: + """Parse the PE headers we need; ``None`` for anything that is not a PE.""" + try: + if data[:2] != b"MZ": + return None + pe_offset = struct.unpack_from(" _CLR_DIRECTORY: + clr_rva, clr_size = struct.unpack_from( + " None: + offset = info.rva_to_offset(info.clr_rva) + if offset is None: + info.clr_rva = info.clr_size = 0 + return + ( + _cb, + _major, + _minor, + info.metadata_rva, + info.metadata_size, + info.cor_flags, + info.entry_point_token, + info.resources_rva, + info.resources_size, + info.strong_name_rva, + ) = struct.unpack_from(" PeInfo | None: + try: + with path.open("rb") as handle: + head = handle.read(2) + if head != b"MZ": + return None + handle.seek(0) + with mmap.mmap(handle.fileno(), 0, access=mmap.ACCESS_READ) as view: + return parse_pe(view) + except (OSError, ValueError): + return None + + +def is_managed_pe(path: Path) -> bool: + info = read_pe_info(path) + return info is not None and info.is_managed + + +# -------------------------------------------------------------------------- +# User string heap (#US) +# -------------------------------------------------------------------------- + + +def _compressed_uint(data: bytes, offset: int) -> tuple[int, int]: + first = data[offset] + if first & 0x80 == 0: + return first, 1 + if first & 0xC0 == 0x80: + return ((first & 0x3F) << 8) | data[offset + 1], 2 + return ( + ((first & 0x1F) << 24) + | (data[offset + 1] << 16) + | (data[offset + 2] << 8) + | data[offset + 3], + 4, + ) + + +def iter_user_strings(heap: bytes) -> list[tuple[int, str]]: + """``(heap offset, value)`` for every string in a #US heap.""" + result: list[tuple[int, str]] = [] + offset = 1 # offset 0 is the empty blob + length = len(heap) + while offset < length: + try: + size, used = _compressed_uint(heap, offset) + except IndexError: + break + start = offset + used + if size == 0: + offset = start + continue + chars = (size - 1) & ~1 # drop the trailing flag byte + if start + chars > length: + break + value = heap[start : start + chars].decode("utf-16-le", errors="replace") + result.append((offset, value)) + offset = start + size + return result + + +# -------------------------------------------------------------------------- +# Method bodies and IL token scan +# -------------------------------------------------------------------------- + +# Operand kinds; the scan only needs sizes plus the token-bearing kinds. +_N, _B1, _B2, _B4, _B8 = "none", "i1", "i2", "i4", "i8" +TOK_METHOD, TOK_FIELD, TOK_TYPE, TOK_STRING, TOK_SIG, TOK_TOKEN = ( + "method", + "field", + "type", + "string", + "sig", + "token", +) +_SWITCH = "switch" + + +def _one_byte_table() -> dict[int, str]: + table: dict[int, str] = {} + for code in range(0x00, 0x0E): # nop .. stloc.3 + table[code] = _N + for code in range(0x0E, 0x14): # ldarg.s .. stloc.s + table[code] = _B1 + for code in range(0x14, 0x1F): # ldnull, ldc.i4.m1 .. ldc.i4.8 + table[code] = _N + table[0x1F] = _B1 # ldc.i4.s + table[0x20] = _B4 # ldc.i4 + table[0x21] = _B8 # ldc.i8 + table[0x22] = _B4 # ldc.r4 + table[0x23] = _B8 # ldc.r8 + table[0x25] = _N # dup + table[0x26] = _N # pop + table[0x27] = TOK_METHOD # jmp + table[0x28] = TOK_METHOD # call + table[0x29] = TOK_SIG # calli + table[0x2A] = _N # ret + for code in range(0x2B, 0x38): # short branches + table[code] = _B1 + for code in range(0x38, 0x45): # long branches + table[code] = _B4 + table[0x45] = _SWITCH + for code in range(0x46, 0x6F): # ldind/stind/arith/conv + table[code] = _N + table[0x6F] = TOK_METHOD # callvirt + table[0x70] = TOK_TYPE # cpobj + table[0x71] = TOK_TYPE # ldobj + table[0x72] = TOK_STRING # ldstr + table[0x73] = TOK_METHOD # newobj + table[0x74] = TOK_TYPE # castclass + table[0x75] = TOK_TYPE # isinst + table[0x76] = _N # conv.r.un + table[0x79] = TOK_TYPE # unbox + table[0x7A] = _N # throw + for code in range(0x7B, 0x81): # ldfld .. stsfld + table[code] = TOK_FIELD + table[0x81] = TOK_TYPE # stobj + for code in range(0x82, 0x8C): # conv.ovf.*.un + table[code] = _N + table[0x8C] = TOK_TYPE # box + table[0x8D] = TOK_TYPE # newarr + table[0x8E] = _N # ldlen + table[0x8F] = TOK_TYPE # ldelema + for code in range(0x90, 0xA3): # ldelem.* / stelem.* + table[code] = _N + table[0xA3] = TOK_TYPE # ldelem + table[0xA4] = TOK_TYPE # stelem + table[0xA5] = TOK_TYPE # unbox.any + for code in range(0xB3, 0xBB): # conv.ovf.* + table[code] = _N + table[0xC2] = TOK_TYPE # refanyval + table[0xC3] = _N # ckfinite + table[0xC6] = TOK_TYPE # mkrefany + table[0xD0] = TOK_TOKEN # ldtoken + for code in range(0xD1, 0xDD): # conv.u2 .. endfinally + table[code] = _N + table[0xDD] = _B4 # leave + table[0xDE] = _B1 # leave.s + table[0xDF] = _N # stind.i + table[0xE0] = _N # conv.u + return table + + +def _two_byte_table() -> dict[int, str]: + table: dict[int, str] = {code: _N for code in range(0x00, 0x06)} + table[0x06] = TOK_METHOD # ldftn + table[0x07] = TOK_METHOD # ldvirtftn + for code in range(0x09, 0x0F): # ldarg .. stloc + table[code] = _B2 + table[0x0F] = _N # localloc + table[0x11] = _N # endfilter + table[0x12] = _B1 # unaligned. + table[0x13] = _N # volatile. + table[0x14] = _N # tail. + table[0x15] = TOK_TYPE # initobj + table[0x16] = TOK_TYPE # constrained. + table[0x17] = _N # cpblk + table[0x18] = _N # initblk + table[0x19] = _B1 # no. + table[0x1A] = _N # rethrow + table[0x1C] = TOK_TYPE # sizeof + table[0x1D] = _N # refanytype + table[0x1E] = _N # readonly. + return table + + +ONE_BYTE_OPCODES = _one_byte_table() +TWO_BYTE_OPCODES = _two_byte_table() +_OPERAND_SIZES = {_N: 0, _B1: 1, _B2: 2, _B4: 4, _B8: 8} +_TOKEN_KINDS = {TOK_METHOD, TOK_FIELD, TOK_TYPE, TOK_STRING, TOK_SIG, TOK_TOKEN} + + +def method_body(data: bytes | mmap.mmap, offset: int) -> bytes | None: + """IL code bytes of the method body at ``offset`` (tiny or fat header).""" + try: + first = data[offset] + except IndexError: + return None + if first & 0x3 == 0x2: # tiny + size = first >> 2 + return bytes(data[offset + 1 : offset + 1 + size]) + if first & 0x3 == 0x3: # fat + flags_size = struct.unpack_from("> 12) * 4 + code_size = struct.unpack_from(" list[tuple[str, int]]: + """Every ``(kind, metadata token)`` an IL stream references, in order.""" + refs: list[tuple[str, int]] = [] + pc = 0 + length = len(code) + while pc < length: + opcode = code[pc] + pc += 1 + if opcode == 0xFE: + if pc >= length: + break + kind = TWO_BYTE_OPCODES.get(code[pc]) + pc += 1 + else: + kind = ONE_BYTE_OPCODES.get(opcode) + if kind is None: + break # invalid/encrypted body: stop rather than misread + if kind == _SWITCH: + if pc + 4 > length: + break + count = struct.unpack_from(" length: + break + refs.append((kind, struct.unpack_from(" tuple[str, int]: + length = 0 + shift = 0 + while True: + byte = data[offset] + offset += 1 + length |= (byte & 0x7F) << shift + shift += 7 + if byte < 0x80: + break + raw = bytes(data[offset : offset + length]) + return raw.decode("utf-8", errors="replace"), offset + length + + +def find_bundle_header(data: bytes | mmap.mmap) -> tuple[int, int] | None: + """``(signature offset, manifest offset)``; manifest 0 = plain apphost.""" + position = data.find(BUNDLE_SIGNATURE) + if position < 8: + return None + header = struct.unpack_from(" BundleManifest | None: + found = find_bundle_header(data) + if found is None: + return None + _signature, header = found + if header <= 0 or header >= len(data): + return None + try: + major, minor, count = struct.unpack_from("= 2: + cursor += 40 # deps.json / runtimeconfig.json locations + flags + entries: list[BundleEntry] = [] + for _ in range(max(0, count)): + offset, size = struct.unpack_from("= 6: + compressed = struct.unpack_from(" BundleManifest | None: + try: + with path.open("rb") as handle: + with mmap.mmap(handle.fileno(), 0, access=mmap.ACCESS_READ) as view: + if not bytes(view[:4]).startswith(_EXECUTABLE_MAGICS): + return None + return parse_bundle(view) + except (OSError, ValueError): + return None + + +def is_apphost_launcher(path: Path) -> bool: + """A non-bundled .NET apphost: the real code is ``.dll`` beside it.""" + try: + with path.open("rb") as handle: + with mmap.mmap(handle.fileno(), 0, access=mmap.ACCESS_READ) as view: + if not bytes(view[:4]).startswith(_EXECUTABLE_MAGICS): + return False + found = find_bundle_header(view) + return found is not None and found[1] == 0 + except (OSError, ValueError): + return False + + +def extract_bundle_entry(source: Path, entry: BundleEntry, target: Path) -> None: + target.parent.mkdir(parents=True, exist_ok=True) + with source.open("rb") as handle: + handle.seek(entry.offset) + if entry.compressed_size: + payload = handle.read(entry.compressed_size) + data = zlib.decompress(payload, -15) + else: + data = handle.read(entry.size) + target.write_bytes(data) diff --git a/src/tocode/cli.py b/src/tocode/cli.py index 8f8b546..be06b9f 100644 --- a/src/tocode/cli.py +++ b/src/tocode/cli.py @@ -10,6 +10,7 @@ from .apk import ApkExportOptions, export_apk from .apk_native import NativeOptions from .backends.asc import is_apk_input +from .backends.dotnet import is_dotnet_input from .errors import ToCodeError from .exporter import export_binary from .naming import default_output_name @@ -78,6 +79,16 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="APK input: only export the DEX/Android side; native .so libraries are extracted but not decompiled.", ) + parser.add_argument( + "--as-native", + action="store_true", + help=".NET input: skip the .NET backend and export the file with the native backend (IDA/r2/angr) like any PE/ELF.", + ) + parser.add_argument( + "--include-framework", + action="store_true", + help=".NET input: also decompile .NET framework/runtime assemblies and native runtime libraries found in bundles and packages.", + ) parser.add_argument( "--no-splits", action="store_true", @@ -187,6 +198,16 @@ def main(argv: list[str] | None = None) -> int: "--backend binja is not supported for APK input; native libraries " "use ida, r2, or angr" ) + if ( + args.binary is not None + and not args.as_native + and args.binary.is_file() + and is_dotnet_input(args.binary) + ): + parser.error( + "--backend binja is not supported for .NET input; native code " + "uses ida, r2, or angr (or pass --as-native)" + ) return _run_binja(args, progress, parser, argv) if args.binary is None: @@ -199,6 +220,8 @@ def main(argv: list[str] | None = None) -> int: ) if is_apk_input(binary): return _run_apk(binary, args=args, progress=progress, parser=parser) + if not args.as_native and binary.is_file() and is_dotnet_input(binary): + return _run_dotnet(binary, args=args, progress=progress) log_root = ( args.out_dir if args.out_dir is not None @@ -242,16 +265,7 @@ def _run_apk( jobs=args.jobs, native=not args.no_native, splits=not args.no_splits, - native_options=NativeOptions( - backend=args.backend, - analysis_command=args.analysis, - idadir=args.idadir, - ida_domain_path=args.ida_domain_path, - purge_cache=args.purge_cache, - jobs=args.jobs, - tree=args.tree, - entropy=args.entropy, - ), + native_options=_native_options(args), ) started = time.monotonic() try: @@ -277,6 +291,54 @@ def _run_apk( return 0 +def _native_options(args: argparse.Namespace) -> NativeOptions: + return NativeOptions( + backend=args.backend, + analysis_command=args.analysis, + idadir=args.idadir, + ida_domain_path=args.ida_domain_path, + purge_cache=args.purge_cache, + jobs=args.jobs, + tree=args.tree, + entropy=args.entropy, + ) + + +def _run_dotnet(binary: Path, *, args: argparse.Namespace, progress: Progress) -> int: + from .dotnet import DotnetExportOptions, export_dotnet + + options = DotnetExportOptions( + out_dir=args.out_dir, + jobs=args.jobs, + native=not args.no_native, + include_framework=args.include_framework, + native_options=_native_options(args), + ) + started = time.monotonic() + try: + summary = export_dotnet(binary, options=options, progress=progress) + except KeyboardInterrupt: + progress.log("tocode: interrupted") + print("tocode: interrupted", file=sys.stderr) + return 130 + except ToCodeError as exc: + progress.log(f"tocode: {exc}") + print(f"tocode: {exc}", file=sys.stderr) + return 1 + if not args.quiet: + print(f"Project: {summary.root_dir}") + print( + f"Summary: kind={summary.kind} assemblies={len(summary.assemblies)} " + f"types={summary.type_count} methods={summary.method_count} " + f"failures={len(summary.failed_types)} " + f"natives={summary.native_done}/{summary.native_total}" + ) + for line in summary.native_errors: + print(f"native: {line}", file=sys.stderr) + print(f"Exported in {_format_duration(time.monotonic() - started)}") + return 0 + + def _format_duration(seconds: float) -> str: if seconds >= 60: minutes, secs = divmod(int(round(seconds)), 60) diff --git a/src/tocode/dotnet.py b/src/tocode/dotnet.py new file mode 100644 index 0000000..bbe17b1 --- /dev/null +++ b/src/tocode/dotnet.py @@ -0,0 +1,1078 @@ +""".NET export pipeline (dnlib + ICSharpCode.Decompiler backend). + +Writes one project tree for a managed assembly, an apphost launcher, a +single-file bundle, or a NuGet package: C# and IL per top-level type under +``src/raw///``, assembly attributes, a project file, +resources, JSON metadata, and (unless disabled) a nested native ToCode export +for every native library found (mixed-mode code, bundled libraries, and +P/Invoke targets shipped next to the input). +""" + +from __future__ import annotations + +from collections import deque +from concurrent.futures import FIRST_COMPLETED, Future, ProcessPoolExecutor, wait +from concurrent.futures.process import BrokenProcessPool +from dataclasses import dataclass, field +import multiprocessing +import os +from pathlib import Path +import re +import shutil +import tempfile +import time +from typing import Any, Callable + +from . import __version__ +from . import dotnet_metadata as meta +from .apk_metadata import write_json_rows +from .apk_native import ( + NATIVE_HEARTBEAT_SECONDS, + NativeExporter, + NativeLib, + NativeOptions, + NativeRunner, + default_native_exporter, + native_libs_json, + native_min_free_mb, +) +from .backends.asc import sha256_file +from .backends.dotnet import ( + AssemblyInfo, + DecompiledType, + DotnetInput, + DotnetSession, + NativeFile, + TypeInfo, + WorkItem, + decompile_batch_in_worker, + discover_dotnet_input, + init_dotnet_worker, + is_framework_native, + native_arch, + pinvoke_neighbours, +) +from .errors import ToCodeError +from .exporter import _terminate_executor +from .metadata import write_json, write_text_atomic +from .naming import clean_path_component +from .parallel import available_memory_mb, choose_jobs +from .progress import Progress +from .schema import DotnetExportSummary + +DECOMPILE_BATCH_SIZE = 16 +# A worker holds the CLR, the decompiler's type system for the assemblies it +# touches, and the current syntax tree; large assemblies reach ~600 MB. +DOTNET_WORKER_MEMORY_MB = 1024 +DOTNET_BATCHES_PER_ROUND = 64 +DEFAULT_TYPE_TIMEOUT_SECONDS = 300 +_INVALID_FILE_CHARS = re.compile(r"[^0-9A-Za-z_.\-$@+]") +_TFM_ATTRIBUTE_RX = re.compile( + r"^\.(NETCoreApp|NETStandard|NETFramework),Version=v(\d+)\.(\d+)(?:\.(\d+))?" +) + + +@dataclass(slots=True) +class DotnetExportOptions: + out_dir: Path | None = None + jobs: int | None = None + native: bool = True + include_framework: bool = False + native_options: NativeOptions = field(default_factory=NativeOptions) + + +@dataclass +class DotnetContext: + input_path: Path + options: DotnetExportOptions + progress: Progress + found: DotnetInput | None = None + session: DotnetSession | None = None + root: Path | None = None + raw_dir: Path | None = None + data_dir: Path | None = None + workdir: Path | None = None + title: str = "" + libs: list[NativeLib] = field(default_factory=list) + native_runner: NativeRunner | None = None + source_files: list[Path] = field(default_factory=list) + il_files: list[Path] = field(default_factory=list) + project_files: list[Path] = field(default_factory=list) + failures: list[tuple[str, str]] = field(default_factory=list) + extracted: dict[str, str | None] = field(default_factory=dict) + reachable: dict[str, Any] = field(default_factory=dict) + roots: set[int] = field(default_factory=set) + worker_count: int = 1 + render_mode: str = "single" + started: float = 0.0 + + +def export_dotnet( + input_path: Path, + *, + options: DotnetExportOptions | None = None, + progress: Progress | None = None, + native_exporter: NativeExporter | None = None, + decompiler_factory: Callable[[DotnetSession], Any] | None = None, + session_factory: Callable[[DotnetInput], DotnetSession] | None = None, +) -> DotnetExportSummary: + options = options or DotnetExportOptions() + progress = progress or Progress() + context = DotnetContext( + input_path=Path(input_path).resolve(), + options=options, + progress=progress, + started=time.monotonic(), + ) + try: + _discover(context) + _prepare_root(context) + _inventory(context, session_factory) + _collect_natives(context) + _start_native(context, native_exporter) + _assign_paths(context) + _decompile(context, decompiler_factory) + _write_projects(context) + _write_resources(context) + _write_metadata(context) + _join_native(context) + _write_final_documents(context) + except BaseException: + if context.native_runner is not None: + context.native_runner.stop() + raise + finally: + if context.session is not None: + context.session.close() + if context.workdir is not None: + shutil.rmtree(context.workdir, ignore_errors=True) + return _summary(context) + + +# -------------------------------------------------------------------------- +# Steps +# -------------------------------------------------------------------------- + + +def _work_root() -> str | None: + explicit = os.environ.get("TOCODE_WORKER_TMP_DIR", "").strip() + if not explicit: + return None + candidate = Path(explicit).expanduser() + try: + candidate.mkdir(parents=True, exist_ok=True) + except OSError: + return None + return str(candidate) + + +def _discover(context: DotnetContext) -> None: + path = context.input_path + if not path.is_file(): + raise ToCodeError(f".NET input not found: {path}") + context.workdir = Path(tempfile.mkdtemp(prefix="tocode-dotnet-", dir=_work_root())) + context.found = discover_dotnet_input( + path, + workdir=context.workdir, + include_framework=context.options.include_framework, + ) + if not context.found.included: + raise ToCodeError( + f"no assemblies to decompile in {path.name} (all are framework " + "assemblies; pass --include-framework)" + ) + stem = path.name + for suffix in (".dll", ".exe", ".nupkg", ".snupkg", ".winmd"): + if stem.lower().endswith(suffix): + stem = stem[: -len(suffix)] + context.title = stem or path.stem + + +def _root_dir(context: DotnetContext) -> Path: + out_dir = context.options.out_dir + source = context.input_path + if out_dir is not None: + out = Path(out_dir).expanduser() + return out.resolve() if out.is_absolute() else (source.parent / out).resolve() + name = f"{clean_path_component(context.title)}_decompiler" + default_root = os.environ.get("TOCODE_DEFAULT_OUT_ROOT", "").strip() + if default_root: + return (Path(default_root).expanduser().resolve() / name).resolve() + return (source.parent / name).resolve() + + +def _prepare_root(context: DotnetContext) -> None: + found = _need(context.found) + root = _root_dir(context) + context.root = root + context.raw_dir = root / "src" / "raw" + context.data_dir = root / "data" + if context.progress.log_path is None: + context.progress.set_log_path(root / "tocode.log") + context.raw_dir.mkdir(parents=True, exist_ok=True) + context.data_dir.mkdir(parents=True, exist_ok=True) + included = found.included + skipped = len(found.assemblies) - len(included) + context.progress.log("Export run started") + context.progress.log( + f"Using .NET (dnlib + ICSharpCode.Decompiler) as backend for {context.title} " + f"({found.kind})" + ) + context.progress.log( + f"Assemblies: {len(included)} to decompile" + + (f", {skipped} skipped (framework/duplicates)" if skipped else "") + + (f", {len(found.natives)} native libraries" if found.natives else "") + ) + for item in found.assemblies: + if not item.include: + context.progress.file_log(f"skip {item.entry}: {item.reason}") + + +def _inventory( + context: DotnetContext, + session_factory: Callable[[DotnetInput], DotnetSession] | None, +) -> None: + found = _need(context.found) + session = session_factory(found) if session_factory else DotnetSession(found) + with context.progress.bar(total=1, desc="inventory", unit="step") as bar: + session.load() + bar.update(1) + context.session = session + context.progress.log( + f"Inventory: {len(session.assemblies)} assemblies, {len(session.types)} types, " + f"{len(session.methods)} methods, " + f"{sum(len(item.user_strings) for item in session.assemblies)} strings" + ) + for info in session.assemblies: + for hint in info.obfuscation_hints: + context.progress.log(f"warning: {info.name}: possible obfuscation ({hint})") + + +def _native_origin(context: DotnetContext, native: NativeFile) -> str: + found = _need(context.found) + container = { + "bundle": "single-file bundle", + "nupkg": "NuGet package", + "apphost": "app", + "assembly": "assembly", + }.get(found.kind, found.kind) + if native.path == found.input or native.entry.startswith("(native code"): + how = "the native (C++/CLI) half of the mixed-mode assembly" + elif native.source.startswith("beside "): + how = f"a native library next to the input that P/Invoke declarations name (`{native.entry}`)" + else: + how = f"`{native.entry}` inside `{native.source}`" + return ( + f"This is a decompiled native library from the `{context.title}` .NET " + f"{container}: `{native.name}` ({native.arch}), {how}. " + "The .NET export that owns it is three directories up (`../../..`): its " + "`imports.json` lists the P/Invoke declarations (module and entry point) " + "that call into native code, `functions.json` marks those managed methods " + "with `pinvoke`, `native-libs.json` lists the other native libraries, and " + "`src/raw//` holds the decompiled C# that uses them." + ) + + +def _collect_natives(context: DotnetContext) -> None: + found = _need(context.found) + session = _need(context.session) + root = _need(context.root) + natives = list(found.natives) + if found.kind in {"assembly", "apphost"}: + known = {native.path.resolve() for native in natives} + for module, path in pinvoke_neighbours( + found.input.parent, session.pinvoke_modules + ): + if path.resolve() in known: + continue + known.add(path.resolve()) + natives.append( + NativeFile( + name=path.name, + path=path, + entry=module, + source=f"beside {found.input.name}", + arch=native_arch(path), + framework=is_framework_native(path.name), + ) + ) + libs: list[NativeLib] = [] + for native in natives: + arch = clean_path_component(native.arch or "unknown") + target = root / "lib" / arch / native.name + if native.path != found.input: + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(native.path, target) + path = target + else: + path = native.path # mixed-mode: decompile the assembly file itself + lib = NativeLib( + abi=native.arch or "unknown", + name=native.name, + entry=native.entry, + source_apk=native.source, + size=path.stat().st_size, + sha256=sha256_file(path), + path=path, + package=context.title, + origin=_native_origin(context, native), + ) + if native.framework and not context.options.include_framework: + lib.status = "skipped: .NET runtime library (use --include-framework)" + libs.append(lib) + context.extracted[native.entry] = ( + target.relative_to(root).as_posix() if native.path != found.input else None + ) + context.libs = libs + if libs: + context.progress.log( + f"Native libraries: {len(libs)} found, " + f"{sum(1 for lib in libs if lib.status == 'pending')} to decompile" + ) + _write_native_libs(context) + + +def _start_native(context: DotnetContext, exporter: NativeExporter | None) -> None: + pending = [lib for lib in context.libs if lib.status == "pending"] + if not pending: + return + if not context.options.native: + for lib in pending: + lib.status = "skipped: --no-native" + context.progress.log( + f"Native decompilation skipped (--no-native) for {len(pending)} libraries" + ) + _write_native_libs(context) + return + runner = NativeRunner( + libs=pending, + root=_need(context.root), + progress=context.progress, + exporter=exporter or default_native_exporter(context.options.native_options), + min_free_mb=native_min_free_mb(), + available_memory=available_memory_mb, + ) + context.progress.log( + f"Native decompilation started in background for {len(pending)} libraries " + f"(backend: {context.options.native_options.backend})" + ) + runner.start() + context.native_runner = runner + + +def _join_native(context: DotnetContext) -> None: + runner = context.native_runner + if runner is None: + return + thread = runner.thread + if thread is not None and thread.is_alive(): + context.progress.log( + "Managed export done; waiting for native library export(s) " + "(progress below is mirrored from native///tocode.log)" + ) + while thread.is_alive(): + thread.join(timeout=NATIVE_HEARTBEAT_SECONDS) + if thread.is_alive(): + status = runner.describe_current() + if status: + context.progress.log(status) + runner.join() + done = sum(1 for lib in context.libs if lib.status == "done") + context.progress.log( + f"Native decompilation finished: {done}/{len(context.libs)} libraries exported" + ) + _write_native_libs(context) + + +def _write_native_libs(context: DotnetContext) -> None: + root = _need(context.root) + write_json(root / "native-libs.json", native_libs_json(context.libs, root)) + + +# -- paths --------------------------------------------------------------------- + + +def safe_file_name(name: str) -> str: + cleaned = _INVALID_FILE_CHARS.sub("_", name).strip(".") + return cleaned[:120] or "_" + + +def type_file_stem(record: TypeInfo) -> str: + """``List`1`` -> ``List-1``; compiler/obfuscator names made filesystem-safe.""" + name = record.name + if "`" in name: + base, _, arity = name.partition("`") + name = f"{base}-{arity}" if arity.isdigit() else name.replace("`", "-") + return safe_file_name(name) + + +def namespace_parts(namespace: str) -> list[str]: + return [safe_file_name(part) for part in namespace.split(".") if part] + + +def _assembly_folders(session: DotnetSession) -> None: + used: dict[str, int] = {} + for info in session.assemblies: + base = safe_file_name(info.name) + count = used.get(base.lower(), 0) + used[base.lower()] = count + 1 + info.folder = base if count == 0 else f"{base}~{count}" + + +def _assign_paths(context: DotnetContext) -> None: + session = _need(context.session) + raw_dir = _need(context.raw_dir) + _assembly_folders(session) + seen: dict[str, int] = {} + for info in session.assemblies: + base = raw_dir / info.folder + for record in session.top_level_types(info.index): + folder = base.joinpath(*namespace_parts(record.namespace)) + stem = type_file_stem(record) + key = (folder / stem).as_posix().lower() + count = seen.get(key, 0) + seen[key] = count + 1 + if count: + stem = f"{stem}~{count}" + record.cs_path = folder / f"{stem}.cs" + record.il_path = folder / f"{stem}.il" + for nested in session.nested_types(info.index, record.token): + nested.cs_path = record.cs_path + nested.il_path = record.il_path + + +# -- decompilation --------------------------------------------------------------- + + +class _PoolBroken(RuntimeError): + pass + + +def _work_items(context: DotnetContext) -> list[WorkItem]: + session = _need(context.session) + items: list[WorkItem] = [] + for info in session.assemblies: + items.append(WorkItem(info.index, str(info.file.path), 0)) + for record in session.top_level_types(info.index): + if record.name == "" and not record.methods and not record.fields: + continue + items.append(WorkItem(info.index, str(info.file.path), record.token)) + return items + + +def _select_workers(context: DotnetContext, count: int) -> int: + chosen = choose_jobs( + function_count=count, + analysis_seconds=0.0, + requested=context.options.jobs, + backend="dotnet", + ) + memory = available_memory_mb() + if memory is not None and chosen > 1: + ceiling = max(1, memory // DOTNET_WORKER_MEMORY_MB) + if ceiling < chosen: + context.progress.log( + f"Limiting decompile workers to {ceiling} ({memory} MB available, " + f"{DOTNET_WORKER_MEMORY_MB} MB per worker)" + ) + chosen = ceiling + context.worker_count = max(1, chosen) + context.render_mode = "process" if context.worker_count > 1 else "single" + return context.worker_count + + +def _type_timeout() -> int: + raw = os.environ.get("TOCODE_DOTNET_TYPE_TIMEOUT_SECONDS", "").strip() + try: + return max(10, int(raw)) if raw else DEFAULT_TYPE_TIMEOUT_SECONDS + except ValueError: + return DEFAULT_TYPE_TIMEOUT_SECONDS + + +def _decompile( + context: DotnetContext, decompiler_factory: Callable[[DotnetSession], Any] | None +) -> None: + session = _need(context.session) + items = _work_items(context) + type_items = sum(1 for item in items if item.token) + _remove_stale_sources(context) + if decompiler_factory is not None: + context.progress.log(f"Decompiling {type_items} types in-process") + decompiler = decompiler_factory(session) + with context.progress.bar( + total=len(items), desc="decompile", unit="type" + ) as bar: + for item in items: + _store_result(context, decompiler.decompile(item)) + bar.update(1) + else: + workers = _select_workers(context, type_items) + context.progress.log( + f"Decompiling {type_items} types to C# and IL with {workers} worker(s)" + ) + with context.progress.bar( + total=len(items), desc="decompile", unit="type" + ) as bar: + _decompile_pool(context, items, workers, bar) + _inherit_owner_ranges(session) + context.progress.log( + f"Decompiled {type_items - len(context.failures)} types, " + f"{len(context.failures)} failed" + ) + + +def _decompile_pool( + context: DotnetContext, items: list[WorkItem], workers: int, bar: Any +) -> None: + found = _need(context.found) + search_dirs = [str(path) for path in found.search_dirs] + queue: deque[tuple[list[WorkItem], bool]] = deque( + (items[start : start + DECOMPILE_BATCH_SIZE], False) + for start in range(0, len(items), DECOMPILE_BATCH_SIZE) + ) + timeout = _type_timeout() + while queue: + # A fresh pool per round recycles worker memory (cached type systems). + round_batches = [ + queue.popleft() + for _ in range(min(len(queue), workers * DOTNET_BATCHES_PER_ROUND)) + ] + failed = _run_round(context, round_batches, workers, search_dirs, timeout, bar) + for batch, isolated, reason in failed: + if len(batch) > 1: + # Re-run each type alone to find the one that crashed or hung. + context.progress.file_log( + f"decompile: batch of {len(batch)} failed ({reason}); isolating" + ) + queue.extendleft((([item], True) for item in reversed(batch))) + continue + item = batch[0] + if not isolated: + queue.appendleft(([item], True)) + continue + _store_result( + context, + DecompiledType( + item.assembly, item.token, None, reason, None, reason, {}, {} + ), + ) + bar.update(1) + + +def _run_round( + context: DotnetContext, + batches: list[tuple[list[WorkItem], bool]], + workers: int, + search_dirs: list[str], + timeout: int, + bar: Any, +) -> list[tuple[list[WorkItem], bool, str]]: + executor = ProcessPoolExecutor( + max_workers=max(1, min(workers, len(batches))), + mp_context=multiprocessing.get_context("spawn"), + initializer=init_dotnet_worker, + initargs=(search_dirs,), + ) + futures: dict[Future[list[DecompiledType]], tuple[list[WorkItem], bool, float]] = {} + failed: list[tuple[list[WorkItem], bool, str]] = [] + pending = deque(batches) + broken = False + + def submit() -> None: + while pending and len(futures) < max(1, workers) * 2: + batch, isolated = pending.popleft() + futures[executor.submit(decompile_batch_in_worker, batch)] = ( + batch, + isolated, + time.monotonic(), + ) + + try: + submit() + while futures: + done, _ = wait(list(futures), timeout=5, return_when=FIRST_COMPLETED) + for future in done: + batch, isolated, _started = futures.pop(future) + try: + results = future.result() + except BrokenProcessPool: + failed.append((batch, isolated, "decompiler process crashed")) + broken = True + continue + except Exception as exc: + failed.append((batch, isolated, f"{type(exc).__name__}: {exc}")) + continue + for result in results: + _store_result(context, result) + bar.update(1) + now = time.monotonic() + overdue = [ + future + for future, (batch, _isolated, started) in futures.items() + if future.running() and now - started > timeout + 30 * len(batch) + ] + if overdue or broken: + for future in list(futures): + batch, isolated, _started = futures.pop(future) + reason = ( + f"decompilation timed out after {timeout}s" + if future in overdue + else "decompiler process crashed" + ) + if future not in overdue and not broken: + reason = "worker pool restarted" + failed.append((batch, isolated, reason)) + failed.extend( + (batch, isolated, "worker pool restarted") + for batch, isolated in pending + ) + pending.clear() + break + submit() + finally: + if broken or failed: + _terminate_executor(executor) + else: + executor.shutdown(wait=True, cancel_futures=True) + return failed + + +def _store_result(context: DotnetContext, result: DecompiledType) -> None: + session = _need(context.session) + info = session.assemblies[result.assembly] + if result.token == 0: + _store_assembly_attributes(context, info, result) + return + record = session.types.get((result.assembly, result.token)) + if record is None or record.cs_path is None or record.il_path is None: + return + header = f"// ToCode: {record.full_name.replace('/', '+')} (0x{record.token:08x}) from {info.name} {info.version}\n" + if result.cs is not None: + cs_text = header + result.cs.rstrip("\n") + "\n" + cs_offset = 1 + else: + record.error = result.cs_error or "unknown decompilation error" + context.failures.append((record.full_name, record.error)) + cs_text = header + _failure_stub(session, record) + cs_offset = 0 + context.progress.file_log( + f"export {info.name}!{record.full_name} failed: {record.error}" + ) + if result.il is not None: + il_text = header + result.il.rstrip("\n") + "\n" + else: + record.il_error = result.il_error or "unknown disassembly error" + il_text = header + f"// IL disassembly failed: {record.il_error}\n" + write_text_atomic(record.cs_path, cs_text) + write_text_atomic(record.il_path, il_text) + record.cs_lines = cs_text.count("\n") + record.il_lines = il_text.count("\n") + context.source_files.append(record.cs_path) + context.il_files.append(record.il_path) + for nested in session.nested_types(record.assembly, record.token): + nested.error = record.error + nested.il_error = record.il_error + nested.cs_lines = record.cs_lines + nested.il_lines = record.il_lines + for method_id in nested.methods: + method = session.methods[method_id] + span = result.cs_ranges.get(method.token) + if span is not None and result.cs is not None: + method.cs_start, method.cs_end = ( + span[0] + cs_offset, + span[1] + cs_offset, + ) + elif result.cs is None: + method.cs_start, method.cs_end = 1, record.cs_lines + il_span = result.il_ranges.get(method.token) + if il_span is not None: + method.il_start, method.il_end = il_span[0] + 1, il_span[1] + 1 + if record.error is None: + context.progress.file_log( + f"export {info.name}!{record.full_name} - {record.cs_lines} C# / " + f"{record.il_lines} IL lines done" + ) + + +def _store_assembly_attributes( + context: DotnetContext, info: AssemblyInfo, result: DecompiledType +) -> None: + folder = _need(context.raw_dir) / info.folder / "Properties" + header = f"// ToCode: assembly and module attributes of {info.full_name}\n" + cs = header + (result.cs or f"// decompilation failed: {result.cs_error}\n") + il = header + (result.il or f"// IL manifest failed: {result.il_error}\n") + write_text_atomic( + folder / "AssemblyInfo.cs", cs if cs.endswith("\n") else cs + "\n" + ) + write_text_atomic(folder / "Manifest.il", il if il.endswith("\n") else il + "\n") + context.source_files.append(folder / "AssemblyInfo.cs") + context.il_files.append(folder / "Manifest.il") + + +def _failure_stub(session: DotnetSession, record: TypeInfo) -> str: + lines = [ + f"// decompilation failed: {record.error}", + "// The IL next to this file (.il) is complete; read it instead.", + ] + if record.namespace: + lines.append(f"namespace {record.namespace};") + lines.append(f"// {record.visibility} {record.kind} {record.name}") + for member in session.nested_types(record.assembly, record.token): + for method_id in member.methods: + method = session.methods[method_id] + lines.append(f"// {member.name}::{method.prototype}") + return "\n".join(lines) + "\n" + + +def _inherit_owner_ranges(session: DotnetSession) -> None: + """State machine / closure methods without their own C# span point at the + method that the decompiler folded them into.""" + for method in session.methods: + if method.cs_start is not None or method.owner_method is None: + continue + owner = session.methods[method.owner_method] + if owner.cs_start is not None: + method.cs_start, method.cs_end = owner.cs_start, owner.cs_end + + +def _remove_stale_sources(context: DotnetContext) -> None: + raw_dir = _need(context.raw_dir) + session = _need(context.session) + expected = { + path + for record in session.types.values() + for path in (record.cs_path, record.il_path) + if path is not None + } + for info in session.assemblies: + expected.add(raw_dir / info.folder / "Properties" / "AssemblyInfo.cs") + expected.add(raw_dir / info.folder / "Properties" / "Manifest.il") + expected.add(raw_dir / info.folder / f"{info.name}.csproj") + removed = 0 + for pattern in ("*.cs", "*.il", "*.csproj"): + for path in raw_dir.rglob(pattern): + if path not in expected: + path.unlink(missing_ok=True) + removed += 1 + if removed: + context.progress.file_log(f"removed {removed} stale source files") + + +# -- project files / resources ----------------------------------------------------- + + +def target_framework_moniker(attribute: str | None) -> str | None: + if not attribute: + return None + match = _TFM_ATTRIBUTE_RX.match(attribute) + if not match: + return None + family, major, minor = match.group(1), int(match.group(2)), int(match.group(3)) + patch = match.group(4) + if family == "NETCoreApp": + return f"net{major}.{minor}" if major >= 5 else f"netcoreapp{major}.{minor}" + if family == "NETStandard": + return f"netstandard{major}.{minor}" + return f"net{major}{minor}{patch or ''}" + + +def build_csproj(info: AssemblyInfo, *, internal_refs: set[str]) -> str: + framework = target_framework_moniker(info.target_framework) or "net10.0" + output = "Exe" if info.entry_point is not None else "Library" + if output == "Exe" and "Windows" in info.kind: + output = "WinExe" + references = [ + ref["name"] + for ref in info.assembly_refs + if ref["name"] in internal_refs + or not re.match( + r"^(System|Microsoft|mscorlib|netstandard|WindowsBase)\b", ref["name"] + ) + ] + lines = [ + "", + '', + " ", + f" {info.name}", + f" {output}", + f" {framework}", + " latest", + " true", + " annotations", + " false", + " true", + " ", + ] + if references: + lines.append(" ") + for name in sorted(set(references)): + if name in internal_refs: + lines.append( + f' ' + ) + else: + lines.append(f' ') + lines.append(" ") + lines.append("") + return "\n".join(lines) + "\n" + + +def _write_projects(context: DotnetContext) -> None: + session = _need(context.session) + raw_dir = _need(context.raw_dir) + internal = {info.name for info in session.assemblies} + for info in session.assemblies: + path = raw_dir / info.folder / f"{info.name}.csproj" + write_text_atomic( + path, build_csproj(info, internal_refs=internal - {info.name}) + ) + context.project_files.append(path) + + +def _write_resources(context: DotnetContext) -> None: + session = _need(context.session) + found = _need(context.found) + root = _need(context.root) + data_dir = _need(context.data_dir) + count = 0 + for info in session.assemblies: + folder = data_dir / "resources" / info.folder + for resource in info.resources: + if resource.kind != "embedded": + continue + target = folder / safe_file_name(resource.name) + try: + size = session.save_resource(info, resource.name, target) + except Exception as exc: # pragma: no cover - backend specific + resource.error = f"{type(exc).__name__}: {exc}" + continue + if size is None: + resource.error = "cannot read resource data" + continue + resource.path = target + resource.size = size + count += 1 + if resource.name.endswith(".resources"): + try: + resource.entries = session.decode_resources_file( + info, resource.name + ) + write_json( + target.with_name(target.name + ".json"), + {"resource": resource.name, "entries": resource.entries}, + ) + except Exception as exc: + resource.error = f"cannot decode .resources: {exc}" + for extra in found.extras: + target = data_dir / found.kind / safe_file_name(extra.entry.replace("/", "__")) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(extra.path, target) + context.extracted[extra.entry] = target.relative_to(root).as_posix() + for item in found.assemblies: + if item.path.is_relative_to(context.workdir or Path("/nonexistent")): + target = ( + data_dir / "assemblies" / safe_file_name(item.entry.replace("/", "__")) + ) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(item.path, target) + context.extracted[item.entry] = target.relative_to(root).as_posix() + else: + context.extracted.setdefault(item.entry, None) + if count: + context.progress.log(f"Extracted {count} embedded resources") + + +# -- metadata --------------------------------------------------------------------- + + +def _write_metadata(context: DotnetContext) -> None: + root = _need(context.root) + session = _need(context.session) + found = _need(context.found) + with context.progress.bar(total=11, desc="metadata", unit="doc") as bar: + context.reachable, context.roots = meta.reachable_json(session) + write_json(root / "reachable.json", context.reachable) + bar.update(1) + write_json(root / "assemblies.json", meta.assemblies_json(session, found, root)) + bar.update(1) + write_json_rows( + root / "types.json", + "types", + meta.iter_type_rows(session, root), + header={"count": len(session.types)}, + ) + bar.update(1) + write_json_rows( + root / "functions.json", + "functions", + meta.iter_function_rows(session, root, context.roots), + ) + bar.update(1) + write_json_rows( + root / "function-index.json", + "functions", + meta.iter_function_index_rows(session, root), + header={"schema_version": 2, "language": "csharp", "asm_language": "il"}, + ) + bar.update(1) + write_json_rows( + root / "strings.json", "strings", meta.iter_string_rows(session) + ) + bar.update(1) + write_json(root / "imports.json", meta.imports_json(session)) + bar.update(1) + write_json(root / "exports.json", meta.exports_json(session)) + bar.update(1) + write_json(root / "sections.json", meta.sections_json(session)) + bar.update(1) + write_json(root / "namespace-graph.json", meta.namespace_graph_json(session)) + bar.update(1) + write_json(root / "resources.json", meta.resources_json(session, root)) + bar.update(1) + + +def _write_final_documents(context: DotnetContext) -> None: + root = _need(context.root) + session = _need(context.session) + found = _need(context.found) + libs_doc = native_libs_json(context.libs, root) + write_json(root / "container.json", meta.container_json(found, context.extracted)) + write_json( + root / "triage.json", + meta.triage_json(session, found, context.reachable, libs_doc["libs"]), + ) + native_exports = [ + { + "abi": lib.abi, + "name": lib.name, + "export_dir": str(lib.export_dir) if lib.export_dir else None, + "status": lib.status, + "backend": lib.backend, + } + for lib in context.libs + ] + common: dict[str, Any] = { + "binary": str(context.input_path), + "kind": found.kind, + "assemblies": [info.name for info in session.assemblies], + "backend": "dotnet", + "decompiler": DotnetSession.decompiler_label, + "root_dir": str(root), + "src_dir": str(_need(context.raw_dir)), + "raw_src_dir": str(_need(context.raw_dir)), + "tree_src_dir": None, + "include_dir": None, + "data_dir": str(_need(context.data_dir)), + "header": None, + "agents": str(root / "AGENTS.md"), + "claude": str(root / "CLAUDE.md"), + "log": str(root / "tocode.log"), + "ida_database": None, + "source_files": [str(item) for item in context.source_files], + "il_files": [str(item) for item in context.il_files], + "project_files": [str(item) for item in context.project_files], + "tree_source_files": [], + "summary_files": [], + "asm_files": [str(item) for item in context.il_files], + "function_index": str(root / "function-index.json"), + "tree_function_index": None, + "function_count": len(session.methods), + "type_count": len(session.types), + "cluster_count": 0, + "failure_count": len(context.failures), + "requested_worker_count": context.options.jobs + if context.options.jobs is not None + else "auto", + "worker_count": context.worker_count, + "parallel_mode": context.render_mode, + "native_enabled": context.options.native, + "include_framework": context.options.include_framework, + "native_exports": native_exports, + } + write_json(root / "project.json", common) + manifest: dict[str, Any] = dict(common) + manifest.update( + { + "schema_version": 2, + "tocode_version": __version__, + "format": f"dotnet-{found.kind}", + "arch": "cil", + "bits": 64, + "entrypoints": [ + method.address + for kind, method in meta.export_roots(session) + if kind == "entrypoint" + ], + "raw_source_files": [str(item) for item in context.source_files], + "data_variable_count": 0, + "triage": str(root / "triage.json"), + "imports": str(root / "imports.json"), + "exports": str(root / "exports.json"), + "reachable": str(root / "reachable.json"), + "cluster_graph": None, + "namespace_graph": str(root / "namespace-graph.json"), + "assemblies_index": str(root / "assemblies.json"), + "types": str(root / "types.json"), + "type_count": len(session.types), + "strings": str(root / "strings.json"), + "sections": str(root / "sections.json"), + "resources": str(root / "resources.json"), + "container": str(root / "container.json"), + "native_libs": str(root / "native-libs.json"), + "variables_interesting": None, + "failures": [ + {"address": None, "name": name, "error": error} + for name, error in context.failures + ], + "native_errors": context.native_runner.errors + if context.native_runner + else [], + } + ) + write_json(root / "export-manifest.json", manifest) + write_text_atomic( + root / "AGENTS.md", + meta.build_dotnet_agents( + title=context.title, + kind=found.kind, + assemblies=[info.name for info in session.assemblies], + native=context.options.native + and any( + lib.status not in {"pending"} and not lib.status.startswith("skipped") + for lib in context.libs + ), + native_count=sum( + 1 for lib in context.libs if not lib.status.startswith("skipped") + ), + ) + + "\n", + ) + write_text_atomic(root / "CLAUDE.md", "@./AGENTS.md\n") + context.progress.log("Wrote project.json, export-manifest.json, AGENTS.md") + + +def _summary(context: DotnetContext) -> DotnetExportSummary: + session = context.session + found = _need(context.found) + return DotnetExportSummary( + root_dir=_need(context.root), + kind=found.kind, + assemblies=[info.name for info in session.assemblies] if session else [], + type_count=len(session.types) if session else 0, + method_count=len(session.methods) if session else 0, + source_files=list(context.source_files), + failed_types=list(context.failures), + native_total=len(context.libs), + native_done=sum(1 for lib in context.libs if lib.status == "done"), + native_errors=list(context.native_runner.errors) + if context.native_runner + else [], + seconds=time.monotonic() - context.started, + ) + + +def _need(value: Any) -> Any: + if value is None: + raise RuntimeError("export context not initialized") + return value diff --git a/src/tocode/dotnet_metadata.py b/src/tocode/dotnet_metadata.py new file mode 100644 index 0000000..20d08a3 --- /dev/null +++ b/src/tocode/dotnet_metadata.py @@ -0,0 +1,877 @@ +"""JSON metadata documents for .NET exports. + +Shapes follow ``metadata.py``/``apk_metadata.py`` wherever a native or Android +concept has a .NET equivalent (functions, function-index, strings, imports, +exports, sections, reachable, triage), so tooling that reads other ToCode +exports keeps working. .NET-only facts live in ``assemblies.json``, +``types.json``, ``namespace-graph.json``, ``resources.json`` and +``container.json``. Big per-row documents are generators for +``apk_metadata.write_json_rows``. +""" + +from __future__ import annotations + +from collections import deque +from pathlib import Path +import re +from typing import Any, Iterable, Iterator + +from .apk_metadata import INTERESTING_STRING_TIERS +from .backends.dotnet import ( + AssemblyInfo, + DotnetInput, + DotnetSession, + MethodInfo, + TypeInfo, +) + +# External API families worth surfacing first when triaging a .NET binary. +SUSPICIOUS_API_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = ( + ( + "dynamic code loading / reflection", + re.compile( + r"System\.Reflection\.Assembly::(Load|LoadFrom|LoadFile|UnsafeLoadFrom)\b|" + # CreateInstance() is what the compiler emits for `new T()`. + r"System\.AppDomain::(Load|ExecuteAssembly)|System\.Activator::CreateInstance(?!<)|" + r"System\.Reflection\.MethodBase::Invoke|System\.Type::InvokeMember|" + r"System\.Reflection\.Emit\.|AssemblyLoadContext::Load" + ), + ), + ("process execution", re.compile(r"System\.Diagnostics\.Process::Start")), + ( + "network", + re.compile( + r"System\.Net\.(Http\.HttpClient|WebClient|WebRequest|Sockets\.|Dns::|" + r"HttpWebRequest|WebSockets\.)" + ), + ), + ("cryptography", re.compile(r"System\.Security\.Cryptography\.")), + ("registry", re.compile(r"Microsoft\.Win32\.Registry")), + ( + "native interop", + re.compile( + r"System\.Runtime\.InteropServices\.(Marshal::(GetDelegateForFunctionPointer|" + r"Copy|AllocHGlobal|PtrToStructure)|NativeLibrary::)" + ), + ), + ("file system", re.compile(r"System\.IO\.(File|Directory|FileStream)::")), + ( + "unsafe deserialization", + re.compile( + r"BinaryFormatter|NetDataContractSerializer|SoapFormatter|LosFormatter|" + r"ObjectStateFormatter" + ), + ), + ( + "environment / host info", + re.compile( + r"System\.Environment::(get_MachineName|get_UserName|get_UserDomainName|" + r"get_OSVersion|GetEnvironmentVariable|SetEnvironmentVariable|GetFolderPath|" + r"get_CommandLine|GetCommandLineArgs|Exit|FailFast)" + ), + ), +) + + +def _rel(path: Path | None, root: Path) -> str | None: + if path is None: + return None + try: + return path.relative_to(root).as_posix() + except ValueError: + return str(path) + + +def _method_label(method: MethodInfo) -> str: + return f"{method.type_name.replace('/', '.')}.{method.name}" + + +# -------------------------------------------------------------------------- +# assemblies.json / container.json +# -------------------------------------------------------------------------- + + +def assemblies_json( + session: DotnetSession, found: DotnetInput, root: Path +) -> dict[str, Any]: + rows: list[dict[str, Any]] = [] + loaded = {info.file.path: info for info in session.assemblies} + for item in found.assemblies: + info = loaded.get(item.path) + base: dict[str, Any] = { + "file": item.entry, + "source": item.source, + "framework": item.framework, + "decompiled": info is not None, + "skip_reason": item.reason, + "target_framework_folder": item.tfm, + } + if info is None: + base["name"] = Path(item.entry).stem + rows.append(base) + continue + method_count = sum( + len(session.types[(info.index, token)].methods) for token in info.types + ) + base.update( + { + "name": info.name, + "full_name": info.full_name, + "version": info.version, + "culture": info.culture or None, + "public_key_token": info.public_key_token, + "module": info.module_name, + "mvid": info.mvid, + "kind": info.kind, + "target_framework": info.target_framework, + "runtime_version": info.pe.runtime_version, + "machine": info.pe.machine, + "pe32_plus": info.pe.pe32_plus, + "cor_flags": info.pe.cor_flag_names, + "ready_to_run": info.pe.ready_to_run, + "mixed_mode": info.pe.mixed_mode, + "strong_name_signed": "StrongNameSigned" in info.pe.cor_flag_names, + "entry_point": _entry_label(session, info), + "folder": info.folder, + "project": f"{info.folder}/{info.name}.csproj" if info.folder else None, + "type_count": len(info.types), + "method_count": method_count, + "user_string_count": len(info.user_strings), + "assembly_refs": info.assembly_refs, + "module_refs": info.module_refs, + "attributes": info.attributes, + "resources": [ + { + "name": resource.name, + "kind": resource.kind, + "size": resource.size, + "public": resource.public, + "path": _rel(resource.path, root), + } + for resource in info.resources + ], + "obfuscation_hints": info.obfuscation_hints, + } + ) + rows.append(base) + return {"count": len(rows), "assemblies": rows} + + +def _entry_label(session: DotnetSession, info: AssemblyInfo) -> dict[str, Any] | None: + if info.entry_point is None: + return None + method = session.method(info.index, info.entry_point) + if method is None: + return {"token": f"0x{info.entry_point:08x}", "name": None, "address": None} + return { + "token": f"0x{method.token:08x}", + "name": method.full_name, + "address": method.address, + } + + +def container_json( + found: DotnetInput, extracted: dict[str, str | None] +) -> dict[str, Any]: + entries: list[dict[str, Any]] = [] + if found.bundle is not None: + for entry in found.bundle.entries: + entries.append( + { + "path": entry.path, + "kind": entry.kind, + "offset": entry.offset, + "size": entry.size, + "compressed_size": entry.compressed_size or None, + "exported_path": extracted.get(entry.path), + } + ) + else: + for item in found.assemblies: + entries.append( + { + "path": item.entry, + "kind": "assembly", + "exported_path": extracted.get(item.entry), + } + ) + for native in found.natives: + entries.append( + { + "path": native.entry, + "kind": "native", + "exported_path": extracted.get(native.entry), + } + ) + for extra in found.extras: + entries.append( + { + "path": extra.entry, + "kind": extra.kind, + "exported_path": extracted.get(extra.entry), + } + ) + return { + "kind": found.kind, + "input": str(found.input), + "bundle": { + "version": f"{found.bundle.major}.{found.bundle.minor}", + "bundle_id": found.bundle.bundle_id, + "manifest_offset": found.bundle.header_offset, + } + if found.bundle is not None + else None, + "entries": entries, + } + + +# -------------------------------------------------------------------------- +# types.json / functions.json / function-index.json +# -------------------------------------------------------------------------- + + +def iter_type_rows(session: DotnetSession, root: Path) -> Iterator[dict[str, Any]]: + for record in session.types.values(): + assembly = session.assemblies[record.assembly] + yield { + "assembly": assembly.name, + "token": f"0x{record.token:08x}", + "name": record.full_name.replace("/", "+"), + "namespace": record.namespace, + "simple_name": record.name, + "kind": record.kind, + "size": None, + "visibility": record.visibility, + "flags": record.flags, + "base_type": record.base_type, + "interfaces": record.interfaces, + "generic_parameters": record.generic_params, + "declaring_type": f"0x{record.declaring_type:08x}" + if record.declaring_type + else None, + "compiler_generated": record.compiler_generated, + "attributes": record.attributes, + "cs_file": _rel(record.cs_path, root), + "il_file": _rel(record.il_path, root), + "decompile_error": record.error, + "disassemble_error": record.il_error, + "members": [ + { + "name": item.name, + "token": f"0x{item.token:08x}", + "type": item.type, + "flags": item.flags, + "constant": item.constant, + } + for item in record.fields + ], + "properties": [ + { + "name": item.name, + "token": f"0x{item.token:08x}", + "type": item.type, + "getter": f"0x{item.getter:08x}" if item.getter else None, + "setter": f"0x{item.setter:08x}" if item.setter else None, + } + for item in record.properties + ], + "events": [ + { + "name": item.name, + "token": f"0x{item.token:08x}", + "type": item.type, + } + for item in record.events + ], + "methods": [ + _method_summary(session.methods[method_id]) + for method_id in record.methods + ], + "c_decl": None, + } + + +def _method_summary(method: MethodInfo) -> dict[str, Any]: + return { + "name": method.name, + "token": f"0x{method.token:08x}", + "address": method.address, + "prototype": method.prototype, + "flags": method.flags, + "rva": f"0x{method.rva:x}" if method.rva else None, + "il_size": method.il_size, + "cs_lines": [method.cs_start, method.cs_end] if method.cs_start else None, + "il_lines": [method.il_start, method.il_end] if method.il_start else None, + } + + +def _source_paths( + session: DotnetSession, method: MethodInfo, root: Path +) -> tuple[str | None, str | None]: + record = session.types.get((method.assembly, method.type_token)) + if record is None: + return None, None + return _rel(record.cs_path, root), _rel(record.il_path, root) + + +def is_dead(session: DotnetSession, method: MethodInfo, roots: set[int]) -> bool: + if method.callers or method.id in roots or method.owner_method is not None: + return False + if method.is_virtual or method.is_constructor or method.unmanaged_entry: + return False + return method.name not in {".cctor", "Finalize", "Dispose"} + + +def iter_function_rows( + session: DotnetSession, root: Path, roots: set[int] +) -> Iterator[dict[str, Any]]: + by_id = session.methods + for method in session.methods: + cs_file, il_file = _source_paths(session, method, root) + yield { + "address": method.address, + "name": method.full_name, + "c_name": _method_label(method), + "prototype": method.prototype, + "size": method.il_size, + "calltype": "static" + if method.is_static + else ("virtual" if method.is_virtual else "instance"), + "return_type": method.return_type, + "params": [{"name": name, "type": kind} for name, kind in method.params], + "locals": [], + "decl_file": None, + "decl_dir": None, + "decl_line": None, + "nargs": len(method.params), + "nlocals": None, + "stackframe": None, + "callees": [by_id[item].address for item in method.callees], + "callees_imports": method.external_calls, + "callee_count": len(method.callees) + len(method.external_calls), + "callers": [by_id[item].address for item in method.callers], + "caller_count": len(method.callers), + "dead_code": is_dead(session, method, roots), + "source_file": cs_file, + "source_line_start": method.cs_start, + "source_line_end": method.cs_end, + "tree_source_file": None, + "tree_source_line_start": None, + "tree_source_line_end": None, + "asm_file": il_file, + "asm_line_start": method.il_start, + "asm_line_end": method.il_end, + "assembly": method.assembly_name, + "token": f"0x{method.token:08x}", + "rva": f"0x{method.rva:x}" if method.rva else None, + "flags": method.flags, + "impl_flags": method.impl_flags, + "attributes": method.attributes, + "compiler_generated": method.compiler_generated, + "owner": by_id[method.owner_method].address + if method.owner_method is not None + else None, + "pinvoke": {"module": method.pinvoke_module, "entry": method.pinvoke_entry} + if method.pinvoke_module + else None, + "unmanaged_export": method.unmanaged_entry, + "string_ref_count": method.string_ref_count, + "field_refs": method.field_refs, + } + + +def iter_function_index_rows( + session: DotnetSession, root: Path +) -> Iterator[dict[str, Any]]: + for method in session.methods: + cs_file, il_file = _source_paths(session, method, root) + yield { + "address": method.address, + "name": method.full_name, + "c": { + "path": cs_file, + "line_start": method.cs_start, + "line_end": method.cs_end, + } + if cs_file and method.cs_start + else None, + "asm": { + "path": il_file, + "line_start": method.il_start, + "line_end": method.il_end, + } + if il_file and method.il_start + else None, + "origin": None, + } + + +# -------------------------------------------------------------------------- +# strings / imports / exports / sections +# -------------------------------------------------------------------------- + + +def iter_string_rows(session: DotnetSession) -> Iterator[dict[str, Any]]: + by_id = session.methods + for info in session.assemblies: + heap = info.pe.stream("#US") + base = heap.offset if heap is not None else 0 + for offset, value in info.user_strings: + xrefs = [ + { + "function": by_id[method_id].full_name, + "address": by_id[method_id].address, + "access": "read", + } + for method_id in session.string_xrefs.get((info.index, offset), ()) + ] + yield { + "vaddr": f"{info.name}!0x{0x70000000 | offset:08x}", + "paddr": f"0x{base + offset:x}", + "size": len(value.encode("utf-16-le")), + "length": len(value), + "section": f"{info.name}#US", + "type": "utf16", + "value": value, + "xrefs": xrefs, + } + + +def imports_json(session: DotnetSession) -> dict[str, Any]: + groups: list[dict[str, Any]] = [] + for scope, members in sorted(session.external_refs.items()): + groups.append( + { + "dll": scope, + "kind": "assembly", + "functions": [ + { + "name": name, + "address": None, + "delay": False, + "callers": [session.methods[item].address for item in callers], + } + for name, callers in sorted(members.items()) + ], + } + ) + pinvoke: dict[str, list[MethodInfo]] = {} + for method in session.methods: + if method.pinvoke_module: + pinvoke.setdefault(method.pinvoke_module, []).append(method) + for module, methods in sorted(pinvoke.items()): + groups.append( + { + "dll": module, + "kind": "pinvoke", + "functions": [ + { + "name": method.pinvoke_entry or method.name, + "address": method.address, + "delay": True, + "managed": method.full_name, + "callers": [ + session.methods[item].address for item in method.callers + ], + } + for method in methods + ], + } + ) + return {"imports": groups} + + +def _type_visible(session: DotnetSession, record: TypeInfo) -> bool: + current: TypeInfo | None = record + while current is not None: + if current.visibility not in { + "Public", + "NestedPublic", + "NestedFamily", + "NestedFamORAssem", + }: + return False + parent = current.declaring_type + current = session.types.get((current.assembly, parent)) if parent else None + return True + + +def export_roots(session: DotnetSession) -> list[tuple[str, MethodInfo]]: + """(kind, method) for entry points, native exports, and public API.""" + rows: list[tuple[str, MethodInfo]] = [] + for info in session.assemblies: + if info.entry_point is not None: + method = session.method(info.index, info.entry_point) + if method is not None: + rows.append(("entrypoint", method)) + for method in session.methods: + if method.unmanaged_entry: + rows.append(("unmanaged-export", method)) + elif method.type_name == "" and method.name == ".cctor": + rows.append(("module-initializer", method)) + for method in session.methods: + info = session.assemblies[method.assembly] + if info.entry_point is not None: + continue # executables: the entry point is the API + if "Public" not in method.flags or method.compiler_generated: + continue + record = session.types.get((method.assembly, method.type_token)) + if record is not None and _type_visible(session, record): + rows.append(("public", method)) + return rows + + +def exports_json(session: DotnetSession) -> dict[str, Any]: + rows = [] + for ordinal, (kind, method) in enumerate(export_roots(session)): + rows.append( + { + "ordinal": ordinal, + "name": method.full_name, + "address": method.address, + "is_forwarder": False, + "forwarder_target": method.unmanaged_entry, + "kind": kind, + "assembly": method.assembly_name, + } + ) + return {"exports": rows} + + +def sections_json(session: DotnetSession) -> dict[str, Any]: + rows: list[dict[str, Any]] = [] + for info in session.assemblies: + for section in info.pe.sections: + rows.append( + { + "assembly": info.name, + "name": section.name, + "vaddr": f"0x{info.pe.image_base + section.virtual_address:x}", + "paddr": f"0x{section.raw_offset:x}", + "size": section.raw_size, + "vsize": section.virtual_size, + "type": "pe-section", + "permissions": section.perms, + "rwx": section.perms == "rwx", + "entropy": None, + } + ) + for stream in info.pe.streams: + rows.append( + { + "assembly": info.name, + "name": stream.name, + "vaddr": None, + "paddr": f"0x{stream.offset:x}", + "size": stream.size, + "vsize": stream.size, + "type": "metadata-stream", + "permissions": "r--", + "rwx": False, + "entropy": None, + } + ) + return {"sections": rows} + + +# -------------------------------------------------------------------------- +# reachable / namespace graph / triage +# -------------------------------------------------------------------------- + + +def reachable_json(session: DotnetSession) -> tuple[dict[str, Any], set[int]]: + roots = [method.id for _, method in export_roots(session)] + owned: dict[int, list[int]] = {} + for method in session.methods: + if method.owner_method is not None: + owned.setdefault(method.owner_method, []).append(method.id) + depths: dict[int, int] = {} + queue: deque[int] = deque() + for root in roots: + if root not in depths: + depths[root] = 0 + queue.append(root) + while queue: + current = queue.popleft() + depth = depths[current] + method = session.methods[current] + for target in [*method.callees, *owned.get(current, ())]: + if target not in depths: + depths[target] = depth + 1 + queue.append(target) + rows = [ + { + "address": session.methods[item].address, + "name": session.methods[item].full_name, + "depth": depth, + } + for item, depth in sorted(depths.items(), key=lambda pair: (pair[1], pair[0])) + ] + return ( + { + "reachable": rows, + "unreachable_count": len(session.methods) - len(depths), + "seed_count": len(roots), + "note": "virtual/interface dispatch and reflection are not followed", + }, + set(roots), + ) + + +def namespace_graph_json(session: DotnetSession) -> dict[str, Any]: + def key(method: MethodInfo) -> tuple[str, str]: + record = session.types.get((method.assembly, method.type_token)) + return method.assembly_name, record.namespace if record else "" + + counts: dict[tuple[str, str], int] = {} + folders: dict[tuple[str, str], str] = {} + for record in session.types.values(): + name = (session.assemblies[record.assembly].name, record.namespace) + counts[name] = counts.get(name, 0) + 1 + if record.cs_path is not None and name not in folders: + folders[name] = record.cs_path.parent.as_posix() + calls: dict[tuple[str, str], dict[str, int]] = {} + external: dict[tuple[str, str], dict[str, int]] = {} + for method in session.methods: + source = key(method) + for callee in method.callees: + target = key(session.methods[callee]) + if target != source: + label = f"{target[0]}:{target[1] or ''}" + bucket = calls.setdefault(source, {}) + bucket[label] = bucket.get(label, 0) + 1 + for call in method.external_calls: + scope = call[1:].split("]", 1)[0] if call.startswith("[") else "?" + bucket = external.setdefault(source, {}) + bucket[scope] = bucket.get(scope, 0) + 1 + rows = [] + for name in sorted(counts): + rows.append( + { + "assembly": name[0], + "namespace": name[1] or "", + "types": counts[name], + "folder": folders.get(name), + "calls_namespaces": [ + {"namespace": target, "calls": count} + for target, count in sorted( + calls.get(name, {}).items(), key=lambda pair: -pair[1] + ) + ], + "calls_assemblies": [ + {"assembly": target, "calls": count} + for target, count in sorted( + external.get(name, {}).items(), key=lambda pair: -pair[1] + ) + ], + } + ) + return {"namespaces": rows} + + +def interesting_strings( + values: Iterable[tuple[str, str]], *, limit: int = 100 +) -> list[dict[str, Any]]: + buckets: dict[int, list[dict[str, Any]]] = { + tier: [] for tier, _ in INTERESTING_STRING_TIERS + } + seen: set[str] = set() + for address, value in values: + if len(value) < 6 or len(value) > 512 or value in seen: + continue + for tier, pattern in INTERESTING_STRING_TIERS: + if pattern.search(value): + seen.add(value) + buckets[tier].append({"value": value, "address": address, "tier": tier}) + break + rows: list[dict[str, Any]] = [] + for tier in sorted(buckets): + rows.extend(buckets[tier]) + return rows[:limit] + + +def suspicious_apis(session: DotnetSession) -> list[dict[str, Any]]: + found: dict[str, dict[str, set[str]]] = {} + for method in session.methods: + for name in method.external_calls: + for label, pattern in SUSPICIOUS_API_PATTERNS: + if pattern.search(name): + found.setdefault(label, {}).setdefault(name, set()).add( + method.address + ) + break + return [ + { + "category": label, + "apis": [ + { + "name": name, + "caller_count": len(callers), + "callers": sorted(callers)[:10], + } + for name, callers in sorted(apis.items()) + ], + } + for label, apis in found.items() + ] + + +def triage_json( + session: DotnetSession, + found: DotnetInput, + reachable: dict[str, Any], + native_libs: list[dict[str, Any]], +) -> dict[str, Any]: + strings = ( + (f"{info.name}!0x{0x70000000 | offset:08x}", value) + for info in session.assemblies + for offset, value in info.user_strings + ) + pinvoke: dict[str, list[str]] = {} + for method in session.methods: + if method.pinvoke_module: + pinvoke.setdefault(method.pinvoke_module, []).append( + method.pinvoke_entry or method.name + ) + main = session.assemblies[0] if session.assemblies else None + return { + "binary_type": f"dotnet-{found.kind}", + "arch": "cil", + "bits": 64 if main is not None and main.pe.pe32_plus else 32, + "assemblies": [ + { + "name": info.name, + "version": info.version, + "kind": info.kind, + "target_framework": info.target_framework, + "entry_point": _entry_label(session, info), + "ready_to_run": info.pe.ready_to_run, + "mixed_mode": info.pe.mixed_mode, + "strong_name_signed": "StrongNameSigned" in info.pe.cor_flag_names, + "types": len(info.types), + "obfuscation_hints": info.obfuscation_hints, + } + for info in session.assemblies + ], + "skipped_assemblies": [ + {"file": item.entry, "reason": item.reason} + for item in found.assemblies + if not item.include + ][:200], + "type_count": len(session.types), + "method_count": len(session.methods), + "pinvoke": [ + {"module": module, "functions": sorted(set(names))[:50]} + for module, names in sorted(pinvoke.items()) + ], + "suspicious_apis": suspicious_apis(session), + "resources": [ + {"assembly": info.name, "name": resource.name, "size": resource.size} + for info in session.assemblies + for resource in info.resources + ][:100], + "native_libs": [ + { + "arch": item["abi"], + "name": item["name"], + "status": item["status"], + "export_dir": item.get("export_dir"), + } + for item in native_libs + ], + "strings_of_interest": interesting_strings(strings), + "reachable_count": len(reachable.get("reachable", [])), + "unreachable_count": reachable.get("unreachable_count", 0), + } + + +def resources_json(session: DotnetSession, root: Path) -> dict[str, Any]: + return { + "resources": [ + { + "assembly": info.name, + "name": resource.name, + "kind": resource.kind, + "size": resource.size, + "public": resource.public, + "linked_file": resource.file, + "path": _rel(resource.path, root), + "entries": resource.entries, + "error": resource.error, + } + for info in session.assemblies + for resource in info.resources + ] + } + + +# -------------------------------------------------------------------------- +# Generated AGENTS.md +# -------------------------------------------------------------------------- + + +def build_dotnet_agents( + *, title: str, kind: str, assemblies: list[str], native: bool, native_count: int +) -> str: + shown = ", ".join(f"`{name}`" for name in assemblies[:8]) + if len(assemblies) > 8: + shown += ", ..." + lines = [ + "# AGENTS", + "", + f"You are working inside a ToCode .NET export of `{title}` ({kind}; assemblies: {shown or 'none'}).", + "Treat the recovered C#, the IL, the metadata, and the extracted resources as evidence for reverse engineering.", + "", + "## Mission", + "", + "Reverse the program, answer user questions, or serve as an oracle/helper to the user regarding this recovered source code.", + "Write a report only when the user asks for one; otherwise keep findings focused on the question or task at hand.", + "Do not refactor or modify the generated export unless the user explicitly asks for edits.", + "Use subagents only for narrow evidence-gathering tasks such as strings, entry paths, or one namespace.", + "", + "## Files", + "", + "- `src/raw///.../.cs`: decompiled C# (ICSharpCode.Decompiler), one file per top-level type with its nested types, folders follow namespaces like a dnSpy/ILSpy project export.", + "- `src/raw///.../.il`: IL disassembly of the same type with RVA, raw bytecode, and metadata tokens per instruction. Read it when the C# is ambiguous, obfuscated, or failed to decompile.", + "- `src/raw//Properties/AssemblyInfo.cs` and `Manifest.il`: assembly/module attributes, references, and the IL manifest; `.csproj`: a project file for the recovered sources.", + "- `assemblies.json`: every assembly found (version, target framework, entry point, flags, ReadyToRun/mixed-mode, references, resources, obfuscation hints, and why skipped assemblies were not decompiled).", + "- `types.json`: every type with base type, interfaces, attributes, fields, properties, events, and methods with their C#/IL line ranges.", + "- `functions.json`: per-method callers/callees, external (framework/library) calls, P/Invoke targets, field references, and C#/IL line ranges. `address` is `!0x`.", + "- `function-index.json`: exact C# (`c`) and IL (`asm`) file and line range for each method.", + "- `strings.json`: the user-string heap (`ldstr` literals) with the methods that load each string.", + "- `imports.json`: members referenced from other assemblies (grouped by assembly) and P/Invoke imports (grouped by native library).", + "- `exports.json`: entry points, module initializers, `UnmanagedCallersOnly` native exports, and the public API of libraries.", + "- `reachable.json`: methods reachable from those roots with call depth (virtual dispatch and reflection are not followed).", + "- `namespace-graph.json`: inter-namespace and inter-assembly call graph.", + "- `sections.json`: PE sections and metadata streams of every assembly.", + "- `resources.json` and `data/resources//`: embedded resources, with `.resources` files decoded to JSON.", + "- `container.json`: the single-file bundle / NuGet package / assembly layout and where each entry was extracted.", + "- `triage.json`: first-read summary: assemblies, entry points, P/Invoke, suspicious API families, resources, native libraries, strings of interest, reachability counts.", + ] + if native: + lines.append( + f"- `native///`: a complete nested ToCode export for each of the {native_count} native libraries (mixed-mode code, bundled or P/Invoke'd libraries). `native-libs.json` lists their status; `lib//` holds the extracted files.", + ) + else: + lines.append( + "- `native/`: not generated (no native code found, or `--no-native`); see `native-libs.json`.", + ) + lines.extend( + [ + "- `project.json` and `export-manifest.json`: top-level export paths and artifact inventory.", + "- `tocode.log`: export history including per-type decompilation and native export status.", + "- `CLAUDE.md`: Claude entrypoint that references `AGENTS.md`.", + "", + "## Working Style", + "", + "- Start with `triage.json`, `assemblies.json`, `strings.json`, and `reachable.json`.", + "- Use `functions.json` and `function-index.json` to open only the method line ranges you need, in C# or IL.", + "- Compiler-generated types (`<>c`, `d__N`) are folded back into their owners in the C#; `owner` in `functions.json` links state machines and lambdas to the method that created them.", + "- Follow P/Invoke methods into `native/` exports when the logic lives in native code.", + "- Treat obfuscated names as opaque; rely on strings, framework calls, and resources to recover intent.", + "- Cite file paths, type and method names, metadata tokens, and line numbers for every major claim.", + "- Be explicit about uncertainty, decompilation failures (`types.json` `decompile_error`), and unresolved references.", + "", + ] + ) + return "\n".join(lines) diff --git a/src/tocode/exporter.py b/src/tocode/exporter.py index 646dbbb..68cfcb5 100644 --- a/src/tocode/exporter.py +++ b/src/tocode/exporter.py @@ -43,6 +43,7 @@ SHARED_CLUSTER_ID, NameBook, asm_file_name, + bounded_path_component, build_name_book, c_file_name, clean_c_identifier, @@ -265,7 +266,7 @@ def covers_all(self, addresses: list[int]) -> bool: def cluster_record_path(self, cluster: Cluster) -> Path: return ( self.clusters_dir - / f"{cluster.root:016x}_{clean_path_component(cluster.label)}.json" + / f"{cluster.root:016x}_{bounded_path_component(cluster.label)}.json" ) def load_cluster_record(self, cluster: Cluster) -> dict[str, Any] | None: diff --git a/src/tocode/naming.py b/src/tocode/naming.py index ad50377..b56577a 100644 --- a/src/tocode/naming.py +++ b/src/tocode/naming.py @@ -1,6 +1,7 @@ from __future__ import annotations from dataclasses import dataclass, field +import hashlib from pathlib import Path import re @@ -24,6 +25,24 @@ def clean_path_component(value: str) -> str: return "".join(parts).strip("_") or "unnamed" +# Filesystems cap a name at 255 bytes; leave room for prefixes and suffixes +# such as "
_" and ".json..tmp". +MAX_NAME_COMPONENT = 120 + + +def bounded_path_component(value: str, limit: int = MAX_NAME_COMPONENT) -> str: + """``clean_path_component`` capped at ``limit`` chars, kept unique by a hash. + + Mangled names (NativeAOT generics, C++ templates) can run to hundreds of + characters, longer than a single path component may be. + """ + cleaned = clean_path_component(value) + if len(cleaned) <= limit: + return cleaned + digest = hashlib.sha1(value.encode("utf-8", "surrogatepass")).hexdigest()[:10] + return f"{cleaned[: limit - 11]}_{digest}" + + def clean_c_identifier(value: str) -> str: text = _IDENT_BAD.sub("_", value) if not re.search(r"[0-9A-Za-z_]", text): @@ -37,7 +56,7 @@ def default_output_name(binary: Path) -> str: def c_file_name(cluster: Cluster) -> str: if cluster.file_base: - return f"{clean_path_component(cluster.file_base)}.c" + return f"{bounded_path_component(cluster.file_base)}.c" if cluster.root == SHARED_CLUSTER_ID: return "utils.c" return f"cluster_{cluster.root:016x}.c" diff --git a/src/tocode/schema.py b/src/tocode/schema.py index 62a8c73..4a08054 100644 --- a/src/tocode/schema.py +++ b/src/tocode/schema.py @@ -279,3 +279,18 @@ class ApkExportSummary: native_errors: list[str] = field(default_factory=list) manifest_path: Path | None = None seconds: float = 0.0 + + +@dataclass(slots=True) +class DotnetExportSummary: + root_dir: Path + kind: str + assemblies: list[str] + type_count: int + method_count: int + source_files: list[Path] + failed_types: list[tuple[str, str]] + native_total: int + native_done: int + native_errors: list[str] = field(default_factory=list) + seconds: float = 0.0 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..e014a93 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +import pytest + + +@pytest.fixture(autouse=True) +def _no_native_memory_wait(monkeypatch: pytest.MonkeyPatch) -> None: + """Exports in tests must not wait for free RAM on the host running them.""" + monkeypatch.setenv("TOCODE_NATIVE_MIN_FREE_MB", "0") diff --git a/tests/dotnet_fixtures.py b/tests/dotnet_fixtures.py new file mode 100644 index 0000000..3e538d2 --- /dev/null +++ b/tests/dotnet_fixtures.py @@ -0,0 +1,153 @@ +"""Synthetic .NET containers for tests (no .NET SDK or runtime needed).""" + +from __future__ import annotations + +from pathlib import Path +import struct +import zlib + +from tocode.backends.dotnet_pe import BUNDLE_SIGNATURE + +TEXT_RVA = 0x2000 +TEXT_OFFSET = 0x200 +COR20_RVA = TEXT_RVA + 0x8 +METADATA_RVA = TEXT_RVA + 0x100 + + +def _compressed(value: int) -> bytes: + if value < 0x80: + return bytes([value]) + if value < 0x4000: + return struct.pack(">H", value | 0x8000) + return struct.pack(">I", value | 0xC0000000) + + +def user_string_heap(values: list[str]) -> bytes: + heap = bytearray(b"\x00") + for value in values: + encoded = value.encode("utf-16-le") + heap += _compressed(len(encoded) + 1) + encoded + b"\x00" + while len(heap) % 4: + heap += b"\x00" + return bytes(heap) + + +def managed_pe( + *, + strings: list[str] | None = None, + cor_flags: int = 0x1, + ready_to_run: bool = False, + machine: int = 0x8664, +) -> bytes: + """A minimal PE32+ image with a CLR header, metadata root, and #US heap.""" + heap = user_string_heap(strings or ["hello", "https://example.invalid/api"]) + table_stream = b"\x00" * 8 + version = b"v4.0.30319\x00\x00" + streams = [("#~", table_stream), ("#US", heap)] + header = bytearray(b"BSJB" + struct.pack(" bytes: + data = bytearray(b"\x7fELF\x02\x01\x01" + b"\x00" * 57) + struct.pack_into(" bytes: + raw = value.encode() + length = len(raw) + prefix = bytearray() + while True: + byte = length & 0x7F + length >>= 7 + prefix.append(byte | (0x80 if length else 0)) + if not length: + break + return bytes(prefix) + raw + + +def single_file_bundle( + entries: list[tuple[str, int, bytes, bool]], *, launcher: bool = False +) -> bytes: + """An ELF "apphost" with a v6 bundle manifest. + + ``entries`` are ``(path, kind, data, compress)``; ``launcher=True`` writes + a plain apphost (signature with a zero manifest offset). + """ + host = bytearray(native_elf()) + host += b"\x00" * 64 + marker_at = len(host) + host += b"\x00" * 8 + BUNDLE_SIGNATURE + b"\x00" * 32 + if launcher: + return bytes(host) + payload = bytearray(host) + records: list[tuple[int, int, int, int, str]] = [] + for path, kind, data, compress in entries: + stored = data + compressed = 0 + if compress: + deflater = zlib.compressobj(9, zlib.DEFLATED, -15) + stored = deflater.compress(data) + deflater.flush() + compressed = len(stored) + records.append((len(payload), len(data), compressed, kind, path)) + payload += stored + manifest_at = len(payload) + manifest = bytearray(struct.pack(" Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return path diff --git a/tests/test_algorithms.py b/tests/test_algorithms.py index 3a24e1a..5fd722a 100644 --- a/tests/test_algorithms.py +++ b/tests/test_algorithms.py @@ -275,3 +275,21 @@ def cmd(self, command: str) -> str: with pytest.raises(BackendError, match="r2ghidra SLEIGH languages"): session.ensure_decompiler() + + +def test_bounded_path_component_caps_long_mangled_names() -> None: + from tocode.naming import MAX_NAME_COMPONENT, bounded_path_component + + short = bounded_path_component("sub_1000") + long_name = ( + "S_P_CoreLib_System_Collections_Generic_Dictionary_2<" + + "Int32__" * 60 + + ">__TryInsert" + ) + first = bounded_path_component(long_name) + second = bounded_path_component(long_name + "_other") + + assert short == "sub_1000" + assert len(first) == MAX_NAME_COMPONENT and len(second) == MAX_NAME_COMPONENT + assert first != second # the hash keeps distinct long names distinct + assert first.startswith("S_P_CoreLib_System_Collections_Generic_Dictionary_2_") diff --git a/tests/test_apk_export.py b/tests/test_apk_export.py index 1541dfc..658f5f5 100644 --- a/tests/test_apk_export.py +++ b/tests/test_apk_export.py @@ -987,3 +987,31 @@ def exporter( ) assert seen == ["com.example.app"] * 3 + + +def test_native_runner_heartbeat_reports_memory_wait(tmp_path: Path) -> None: + lib = NativeLib( + "arm64", + "libfoo.so", + "lib/arm64/libfoo.so", + "base.apk", + 1, + "h", + tmp_path / "a.so", + ) + runner = NativeRunner( + libs=[lib], + root=tmp_path, + progress=Progress(enabled=False), + exporter=_fake_native_exporter(), + min_free_mb=2048, + ) + runner.waiting = (lib, 900) + + status = runner.describe_current() + + assert status is not None + assert ( + "waiting for memory before libfoo.so (arm64): 900 MB free, need 2048 MB" + in status + ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 76e5d48..dfe9b6b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -95,3 +95,99 @@ def fake_export(binary, *, options, progress): assert seen["options"].jobs == 3 assert seen["options"].native_options.backend == "r2" assert seen["options"].native_options.jobs == 3 + + +def test_package_installs_a_tocode_console_script() -> None: + """The PyPI distribution is `tocode-cli`, but the command stays `tocode`.""" + import importlib.metadata as metadata + + try: + distribution = metadata.distribution("tocode-cli") + except metadata.PackageNotFoundError: # pragma: no cover - not installed + pytest.skip("tocode-cli is not installed in this environment") + + scripts = { + entry.name: entry.value + for entry in distribution.entry_points + if entry.group == "console_scripts" + } + + assert scripts == {"tocode": "tocode.cli:main"} + + +def test_parser_has_dotnet_flags() -> None: + args = build_parser().parse_args(["--as-native", "--include-framework", "app.dll"]) + + assert args.as_native is True and args.include_framework is True + assert build_parser().parse_args(["app.dll"]).as_native is False + + +def test_main_routes_dotnet_input_to_dotnet_export(tmp_path, monkeypatch) -> None: + from types import SimpleNamespace + + from dotnet_fixtures import managed_pe + + from tocode import cli, dotnet + + binary = tmp_path / "App.dll" + binary.write_bytes(managed_pe()) + seen = {} + + def fake_export(path, *, options, progress): + seen["path"] = path + seen["options"] = options + return SimpleNamespace( + root_dir=tmp_path / "out", + kind="assembly", + assemblies=["App"], + type_count=1, + method_count=1, + failed_types=[], + native_total=0, + native_done=0, + native_errors=[], + ) + + monkeypatch.setattr(dotnet, "export_dotnet", fake_export) + + assert ( + cli.main(["--include-framework", "--no-native", "-j", "2", "-q", str(binary)]) + == 0 + ) + assert seen["path"] == binary.resolve() + assert seen["options"].include_framework is True + assert seen["options"].native is False + assert seen["options"].jobs == 2 + + +def test_main_as_native_skips_the_dotnet_backend(tmp_path, monkeypatch) -> None: + from dotnet_fixtures import managed_pe + + from tocode import cli + + binary = tmp_path / "App.dll" + binary.write_bytes(managed_pe()) + called = {} + + def fake_run_one(path, *, args, progress, out_dir): + called["path"] = path + raise cli.ToCodeError("stop here") + + monkeypatch.setattr(cli, "_run_one", fake_run_one) + + assert cli.main(["--as-native", "-q", str(binary)]) == 1 + assert called["path"] == binary.resolve() + + +def test_main_rejects_binja_backend_for_dotnet_input(tmp_path) -> None: + from dotnet_fixtures import managed_pe + + from tocode.cli import main + + binary = tmp_path / "App.dll" + binary.write_bytes(managed_pe()) + + with pytest.raises(SystemExit) as info: + main(["--backend", "binja", str(binary)]) + + assert info.value.code == 2 diff --git a/tests/test_dotnet_backend.py b/tests/test_dotnet_backend.py new file mode 100644 index 0000000..ee04a13 --- /dev/null +++ b/tests/test_dotnet_backend.py @@ -0,0 +1,516 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +import struct +import zipfile + +import pytest + +from dotnet_fixtures import managed_pe, native_elf, single_file_bundle, write +from tocode.backends import dotnet as backend +from tocode.backends.dotnet import ( + DOTNET_LIB_DIR, + discover_dotnet_input, + il_method_ranges, + is_dotnet_input, + is_framework_assembly, + is_framework_native, + native_arch, + pinvoke_neighbours, +) +from tocode.backends.dotnet_pe import ( + COMIMAGE_FLAGS_32BITREQUIRED, + COMIMAGE_FLAGS_ILONLY, + COMIMAGE_FLAGS_STRONGNAMESIGNED, + ONE_BYTE_OPCODES, + TWO_BYTE_OPCODES, + extract_bundle_entry, + is_apphost_launcher, + iter_user_strings, + method_body, + parse_bundle, + parse_pe, + read_bundle, + read_pe_info, + scan_il_tokens, +) +from tocode.errors import ToCodeError + + +# -------------------------------------------------------------------------- +# PE / CLR headers +# -------------------------------------------------------------------------- + + +def test_parse_pe_reads_clr_header_and_metadata_streams() -> None: + data = managed_pe( + strings=["hello", "https://example.invalid/api"], + cor_flags=COMIMAGE_FLAGS_ILONLY | COMIMAGE_FLAGS_STRONGNAMESIGNED, + ) + + info = parse_pe(data) + + assert info is not None + assert info.is_managed and info.il_only and not info.mixed_mode + assert info.machine == "x86_64" and info.pe32_plus + assert info.image_base == 0x180000000 + assert info.runtime_version == "v4.0.30319" + assert info.entry_point_token == 0x06000001 + assert info.cor_flag_names == ["ILOnly", "StrongNameSigned"] + assert [stream.name for stream in info.streams] == ["#~", "#US"] + assert info.sections[0].name == ".text" and info.sections[0].perms == "r-x" + heap = info.stream("#US") + assert heap is not None + assert [ + value + for _, value in iter_user_strings(data[heap.offset : heap.offset + heap.size]) + ] == [ + "hello", + "https://example.invalid/api", + ] + + +def test_mixed_mode_versus_ready_to_run() -> None: + mixed = parse_pe(managed_pe(cor_flags=COMIMAGE_FLAGS_32BITREQUIRED)) + r2r = parse_pe(managed_pe(cor_flags=0, ready_to_run=True)) + + assert mixed is not None and mixed.mixed_mode and not mixed.ready_to_run + # ReadyToRun clears ILONLY too, but its native code is precompiled IL. + assert r2r is not None and r2r.ready_to_run and not r2r.mixed_mode + + +def test_parse_pe_rejects_non_pe_input() -> None: + assert parse_pe(b"\x7fELF" + b"\x00" * 64) is None + assert parse_pe(b"MZ" + b"\x00" * 10) is None + assert parse_pe(b"") is None + + +def test_rva_to_offset_maps_into_sections() -> None: + info = parse_pe(managed_pe()) + assert info is not None + assert info.rva_to_offset(0x2010) == 0x210 + assert info.rva_to_offset(0x9000) is None + + +def test_user_strings_handle_multibyte_lengths() -> None: + from dotnet_fixtures import user_string_heap + + long_value = "x" * 300 # 601 bytes -> 2-byte compressed length + heap = user_string_heap(["a", long_value, "ñandú"]) + values = iter_user_strings(heap) + + assert [value for _, value in values] == ["a", long_value, "ñandú"] + assert values[0][0] == 1 + + +# -------------------------------------------------------------------------- +# IL bodies and token scan +# -------------------------------------------------------------------------- + + +def test_method_body_reads_tiny_and_fat_headers() -> None: + tiny = bytes([(3 << 2) | 0x2, 0x02, 0x2A, 0x00]) + fat = struct.pack(" None: + code = bytes.fromhex( + "72" + "01000070" # ldstr string 1 + "28" + "0a00000a" # call memberref + "45" + "02000000" + "00000000" + "00000000" # switch with 2 targets + "fe06" + "0300002b" # ldftn methodspec + "7b" + "05000004" # ldfld field + "20" + "ffffffff" # ldc.i4 (no token) + "fe0c" + "0100" # ldloc 1 (2-byte operand) + "8d" + "0100001b" # newarr typespec + "d0" + "02000002" # ldtoken typedef + "2a" # ret + ) + + assert scan_il_tokens(code) == [ + ("string", 0x70000001), + ("method", 0x0A00000A), + ("method", 0x2B000003), + ("field", 0x04000005), + ("type", 0x1B000001), + ("token", 0x02000002), + ] + + +def test_scan_il_tokens_stops_at_invalid_opcode() -> None: + # 0x24 is unassigned: an encrypted/garbled body must not be misread. + assert scan_il_tokens(bytes.fromhex("72010000702428020000062a")) == [ + ("string", 0x70000001) + ] + + +def test_opcode_tables_leave_only_the_unassigned_ecma_codes_out() -> None: + unassigned = ( + {0x24, 0x77, 0x78, 0xC4, 0xC5} + | set(range(0xA6, 0xB3)) + | set(range(0xBB, 0xC2)) + | set(range(0xC7, 0xD0)) + ) + assert {code for code in range(0xE1) if code not in ONE_BYTE_OPCODES} == unassigned + assert {code for code in range(0x1F) if code not in TWO_BYTE_OPCODES} == { + 0x08, + 0x10, + 0x1B, + } + assert TWO_BYTE_OPCODES[0x06] == "method" and TWO_BYTE_OPCODES[0x16] == "type" + + +# -------------------------------------------------------------------------- +# Single-file bundles +# -------------------------------------------------------------------------- + + +def _bundle(tmp_path: Path) -> Path: + deps = { + "targets": { + ".NETCoreApp,Version=v10.0/linux-arm64": { + "runtimepack.Microsoft.NETCore.App.Runtime.linux-arm64/10.0.0": { + "runtime": {"System.Private.CoreLib.dll": {}}, + "native": {"libcoreclr.so": {}}, + } + } + }, + "libraries": { + "runtimepack.Microsoft.NETCore.App.Runtime.linux-arm64/10.0.0": { + "type": "runtimepack" + } + }, + } + return write( + tmp_path / "App", + single_file_bundle( + [ + ("App.dll", 1, managed_pe(strings=["app"]), True), + ( + "System.Private.CoreLib.dll", + 1, + managed_pe(strings=["corelib"]), + False, + ), + ("Vendor.Lib.dll", 1, managed_pe(strings=["vendor"]), True), + ("libcoreclr.so", 2, native_elf(), False), + ("libcustom.so", 2, native_elf(0x3E), True), + ("App.deps.json", 3, json.dumps(deps).encode(), False), + ("App.runtimeconfig.json", 4, b"{}", False), + ] + ), + ) + + +def test_parse_bundle_lists_entries_and_extracts_compressed_data( + tmp_path: Path, +) -> None: + path = _bundle(tmp_path) + + manifest = read_bundle(path) + + assert manifest is not None + assert (manifest.major, manifest.minor, manifest.bundle_id) == ( + 6, + 0, + "testbundle01", + ) + assert [(entry.path, entry.kind) for entry in manifest.entries][:4] == [ + ("App.dll", "assembly"), + ("System.Private.CoreLib.dll", "assembly"), + ("Vendor.Lib.dll", "assembly"), + ("libcoreclr.so", "native"), + ] + first = manifest.entries[0] + assert first.compressed_size and first.compressed_size < first.size + extract_bundle_entry(path, first, tmp_path / "out" / "App.dll") + assert (tmp_path / "out" / "App.dll").read_bytes() == managed_pe(strings=["app"]) + assert not is_apphost_launcher(path) + + +def test_plain_apphost_is_a_launcher_not_a_bundle(tmp_path: Path) -> None: + host = write(tmp_path / "Tool", single_file_bundle([], launcher=True)) + + assert parse_bundle(host.read_bytes()) is None + assert is_apphost_launcher(host) + assert not is_dotnet_input(host) # no Tool.dll next to it + write(tmp_path / "Tool.dll", managed_pe()) + assert is_dotnet_input(host) + + found = discover_dotnet_input(host, workdir=tmp_path / "work") + assert found.kind == "apphost" + assert [item.path.name for item in found.assemblies] == ["Tool.dll"] + + +def test_discover_bundle_splits_app_framework_and_natives(tmp_path: Path) -> None: + path = _bundle(tmp_path) + + found = discover_dotnet_input(path, workdir=tmp_path / "work") + + assert found.kind == "bundle" + assert {item.entry: item.include for item in found.assemblies} == { + "App.dll": True, + "System.Private.CoreLib.dll": False, + "Vendor.Lib.dll": True, + } + assert {item.name: item.framework for item in found.natives} == { + "libcoreclr.so": True, + "libcustom.so": False, + } + assert {item.name: item.arch for item in found.natives}["libcustom.so"] == "x86_64" + assert {extra.kind for extra in found.extras} == {"deps-json", "runtime-config"} + assert found.search_dirs == [tmp_path / "work" / "bundle"] + + everything = discover_dotnet_input( + path, workdir=tmp_path / "work2", include_framework=True + ) + assert all(item.include for item in everything.assemblies) + + +# -------------------------------------------------------------------------- +# NuGet packages and plain assemblies +# -------------------------------------------------------------------------- + + +def test_discover_nupkg_prefers_the_newest_target_framework(tmp_path: Path) -> None: + package = tmp_path / "Vendor.Lib.1.0.0.nupkg" + with zipfile.ZipFile(package, "w") as archive: + archive.writestr("Vendor.Lib.nuspec", "") + archive.writestr( + "lib/netstandard2.0/Vendor.Lib.dll", managed_pe(strings=["ns20"]) + ) + archive.writestr("lib/net8.0/Vendor.Lib.dll", managed_pe(strings=["net8"])) + archive.writestr("lib/net48/Vendor.Lib.dll", managed_pe(strings=["net48"])) + archive.writestr("ref/net8.0/Vendor.Lib.dll", managed_pe(strings=["ref"])) + archive.writestr("runtimes/linux-x64/native/libvendor.so", native_elf(0x3E)) + + found = discover_dotnet_input(package, workdir=tmp_path / "work") + + assert found.kind == "nupkg" + chosen = [item.entry for item in found.included] + assert chosen == ["lib/net8.0/Vendor.Lib.dll"] + skipped = {item.entry: item.reason for item in found.assemblies if not item.include} + assert set(skipped) == { + "lib/netstandard2.0/Vendor.Lib.dll", + "lib/net48/Vendor.Lib.dll", + "ref/net8.0/Vendor.Lib.dll", + } + assert [(item.name, item.arch) for item in found.natives] == [ + ("libvendor.so", "linux-x64") + ] + assert [extra.entry for extra in found.extras] == ["Vendor.Lib.nuspec"] + + +def test_tfm_ranking_prefers_modern_targets() -> None: + from tocode.backends.dotnet import _tfm_rank + + newtonsoft = [ + "net20", + "net35", + "net40", + "net45", + "net6.0", + "netstandard1.0", + "netstandard1.3", + "netstandard2.0", + ] + assert max(newtonsoft, key=_tfm_rank) == "net6.0" + assert sorted( + ["net48", "net472", "netcoreapp3.1", "net10.0-windows", "netstandard2.1"], + key=_tfm_rank, + ) == [ + "net472", + "net48", + "netstandard2.1", + "netcoreapp3.1", + "net10.0-windows", + ] + + +def test_discover_plain_assembly_and_mixed_mode(tmp_path: Path) -> None: + plain = write(tmp_path / "Plain.dll", managed_pe()) + mixed = write( + tmp_path / "Mixed.dll", managed_pe(cor_flags=COMIMAGE_FLAGS_32BITREQUIRED) + ) + + assert discover_dotnet_input(plain, workdir=tmp_path / "w1").natives == [] + found = discover_dotnet_input(mixed, workdir=tmp_path / "w2") + assert [(item.name, item.path) for item in found.natives] == [("Mixed.dll", mixed)] + + +def test_is_dotnet_input_rejects_native_and_garbage(tmp_path: Path) -> None: + assert not is_dotnet_input(write(tmp_path / "libfoo.so", native_elf())) + assert not is_dotnet_input(write(tmp_path / "junk.bin", b"hello")) + assert is_dotnet_input(write(tmp_path / "App.exe", managed_pe())) + with pytest.raises(ToCodeError): + discover_dotnet_input(tmp_path / "libfoo.so", workdir=tmp_path / "w") + + +def test_pinvoke_neighbours_finds_platform_named_libraries(tmp_path: Path) -> None: + write(tmp_path / "libnative_helper.so", native_elf()) + write(tmp_path / "crypto.dll", managed_pe()) # managed: not a native target + write(tmp_path / "libzstd.so", native_elf()) + + found = pinvoke_neighbours( + tmp_path, {"native_helper", "crypto", "libzstd.so", "libc"} + ) + + assert sorted((module, path.name) for module, path in found) == [ + ("libzstd.so", "libzstd.so"), + ("native_helper", "libnative_helper.so"), + ] + + +def test_framework_name_heuristics() -> None: + assert is_framework_assembly("System.Private.CoreLib.dll") + assert is_framework_assembly("mscorlib.dll") + assert is_framework_assembly("Microsoft.CSharp.dll") + assert not is_framework_assembly("Microsoft.Build.dll") + assert not is_framework_assembly("Newtonsoft.Json.dll") + assert is_framework_native("libcoreclr.so") + assert is_framework_native("libSystem.Native.so") + assert is_framework_native("clrjit.dll") + assert not is_framework_native("libsqlite3.so") + + +def test_native_arch_reads_elf_and_pe_machines(tmp_path: Path) -> None: + assert native_arch(write(tmp_path / "a.so", native_elf(0xB7))) == "arm64" + assert native_arch(write(tmp_path / "b.so", native_elf(0x3E))) == "x86_64" + assert native_arch(write(tmp_path / "c.dll", managed_pe(machine=0x14C))) == "x86" + + +def test_read_pe_info_on_missing_file(tmp_path: Path) -> None: + assert read_pe_info(tmp_path / "missing.dll") is None + + +# -------------------------------------------------------------------------- +# Decompiler output parsing and runtime discovery +# -------------------------------------------------------------------------- + + +def test_il_method_ranges_follow_method_blocks() -> None: + text = "\n".join( + [ + ".class /* 02000002 */ public C", + "{", + " .method /* 06000001 */ public hidebysig", + " instance void M () cil managed", + " {", + " IL_0000: ret", + " } // end of method C::M", + "", + " .method /* 0600000A */ private static", + " void N () cil managed", + " {", + " } // end of method C::N", + "} // end of class C", + ] + ) + + assert il_method_ranges(text) == {0x06000001: (3, 7), 0x0600000A: (9, 12)} + + +def test_find_runtime_requires_net9_or_newer( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = tmp_path / "dotnet" + for version in ("8.0.10", "9.0.4", "10.0.12"): + folder = root / "shared" / "Microsoft.NETCore.App" / version + folder.mkdir(parents=True) + (folder / "System.Private.CoreLib.dll").write_bytes(b"MZ") + monkeypatch.setattr(backend, "dotnet_root_candidates", lambda: [root]) + assert backend.find_runtime() == (root, "10.0.12") + + for version in ("9.0.4", "10.0.12"): + ( + root + / "shared" + / "Microsoft.NETCore.App" + / version + / "System.Private.CoreLib.dll" + ).unlink() + assert backend.find_runtime() is None + + +def test_vendored_libraries_match_the_documented_hashes() -> None: + expected = { + "dnlib.dll": "566fdab59c91a3c2eab14a22b67f01cb3c0cdb48fa9b9b6677e2b32998636efb", + "ICSharpCode.Decompiler.dll": "38e6abf7497845d79d9d01d56351a3203991356de1d3496b8c51142f495e9f9c", + } + for name, digest in expected.items(): + path = DOTNET_LIB_DIR / name + assert hashlib.sha256(path.read_bytes()).hexdigest() == digest, name + assert digest in (DOTNET_LIB_DIR / "README.md").read_text(encoding="utf-8") + assert (DOTNET_LIB_DIR / "LICENSE-dnlib.txt").is_file() + assert (DOTNET_LIB_DIR / "LICENSE-ICSharpCode.Decompiler.txt").is_file() + + +def _runtime_available() -> bool: + try: + return backend.probe_dotnet()[0] + except Exception: # pragma: no cover + return False + + +@pytest.mark.skipif( + not _runtime_available(), reason="needs a .NET 9+ runtime and pythonnet" +) +def test_opcode_tables_agree_with_dnlib() -> None: + backend.load_runtime() + from dnlib.DotNet.Emit import OpCodes, OperandType # type: ignore[import-not-found] + + token_kinds = { + int(OperandType.InlineMethod): "method", + int(OperandType.InlineField): "field", + int(OperandType.InlineType): "type", + int(OperandType.InlineString): "string", + int(OperandType.InlineSig): "sig", + int(OperandType.InlineTok): "token", + int(OperandType.InlineSwitch): "switch", + } + sizes = { + int(OperandType.InlineNone): "none", + int(OperandType.ShortInlineBrTarget): "i1", + int(OperandType.ShortInlineI): "i1", + int(OperandType.ShortInlineVar): "i1", + int(OperandType.InlineVar): "i2", + int(OperandType.InlineBrTarget): "i4", + int(OperandType.InlineI): "i4", + int(OperandType.ShortInlineR): "i4", + int(OperandType.InlineI8): "i8", + int(OperandType.InlineR): "i8", + } + for table, ours in ( + (OpCodes.OneByteOpCodes, ONE_BYTE_OPCODES), + (OpCodes.TwoByteOpCodes, TWO_BYTE_OPCODES), + ): + for index, opcode in enumerate(table): + if str(opcode.Name).startswith("prefix"): + continue # reserved prefix bytes, not instructions + if str(opcode.Name).startswith("UNKNOWN"): + assert index not in ours, hex(index) + continue + if index == 0xFE and table is OpCodes.OneByteOpCodes: + continue # prefix byte + kind = int(opcode.OperandType) + assert ours.get(index) == token_kinds.get(kind, sizes.get(kind)), ( + str(opcode.Name), + hex(index), + ) diff --git a/tests/test_dotnet_export.py b/tests/test_dotnet_export.py new file mode 100644 index 0000000..b25ae57 --- /dev/null +++ b/tests/test_dotnet_export.py @@ -0,0 +1,751 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from dotnet_fixtures import managed_pe, native_elf, single_file_bundle, write +from tocode import dotnet as pipeline +from tocode import dotnet_metadata as meta +from tocode.apk_native import NativeLib +from tocode.backends.dotnet import ( + AssemblyInfo, + DecompiledType, + DotnetSession, + MethodInfo, + ResourceInfo, + TypeInfo, + WorkItem, +) +from tocode.backends.dotnet_pe import parse_pe +from tocode.dotnet import ( + DotnetExportOptions, + build_csproj, + export_dotnet, + namespace_parts, + safe_file_name, + target_framework_moniker, + type_file_stem, +) +from tocode.errors import ToCodeError +from tocode.progress import Progress + +PROGRAM = 0x02000002 +CLOSURE = 0x02000003 +CLIENT = 0x02000004 +BROKEN = 0x02000005 +BOX = 0x02000006 +MODULE = 0x02000001 + +PROGRAM_CS = """using System; + +namespace Contoso.App; + +public static class Program +{ + public static void Main(string[] args) + { + var client = new Client(); + Action a = () => client.Send(); + a(); + } +} +""" + + +def _type( + token: int, + name: str, + namespace: str, + *, + declaring: int | None = None, + top: int | None = None, + kind: str = "class", + visibility: str = "Public", + parent_name: str | None = None, +) -> TypeInfo: + full = f"{namespace}.{name}" if namespace else name + if parent_name is not None: + full = f"{parent_name}/{name}" # dnlib's nested-type naming + return TypeInfo( + assembly=0, + token=token, + full_name=full, + reflection_name=full, + namespace=namespace, + name=name, + kind=kind, + flags=["Public"], + visibility=visibility, + base_type="System.Object", + interfaces=[], + generic_params=["T"] if "`" in name else [], + declaring_type=declaring, + top_level=top if top is not None else token, + attributes=[], + fields=[], + properties=[], + events=[], + methods=[], + compiler_generated=name.startswith("<"), + ) + + +class FakeSession(DotnetSession): + """Inventory built by hand: no .NET runtime involved.""" + + def load(self, progress: Any = None) -> None: + item = self.input.included[0] + pe = parse_pe(item.path.read_bytes()) + assert pe is not None + info = AssemblyInfo( + index=0, + file=item, + pe=pe, + name="Contoso.App", + full_name="Contoso.App, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null", + version="1.0.0.0", + culture="", + public_key_token=None, + module_name="Contoso.App.dll", + mvid="00000000-0000-0000-0000-000000000001", + kind="Console", + target_framework=".NETCoreApp,Version=v10.0", + entry_point=0x06000001, + assembly_refs=[ + {"name": "System.Runtime", "version": "10.0.0.0"}, + {"name": "Vendor.Lib", "version": "2.0.0.0"}, + ], + module_refs=["native_helper"], + attributes=[ + "System.Runtime.Versioning.TargetFrameworkAttribute(.NETCoreApp,Version=v10.0)" + ], + resources=[ + ResourceInfo("Contoso.App.Strings.resources", "embedded", 10, True), + ResourceInfo("config.json", "embedded", 2, False), + ResourceInfo("Other.dll", "linked", None, True, file="Other.dll"), + ], + user_strings=[(1, "https://api.example.invalid/v1"), (64, "hello world")], + ) + self.assemblies = [info] + types = [ + _type(MODULE, "", ""), + _type(PROGRAM, "Program", "Contoso.App", kind="static class"), + _type( + CLOSURE, + "<>c", + "Contoso.App", + declaring=PROGRAM, + top=PROGRAM, + visibility="NestedPrivate", + parent_name="Contoso.App.Program", + ), + _type(CLIENT, "Client", "Contoso.App.Net"), + _type(BROKEN, "Broken", "Contoso.App"), + _type(BOX, "Box`1", "Contoso.App"), + ] + for record in types: + self.types[(0, record.token)] = record + info.types.append(record.token) + + def method(token: int, owner: int, name: str, **extra: Any) -> MethodInfo: + record = self.types[(0, owner)] + item = MethodInfo( + id=len(self.methods), + assembly=0, + token=token, + name=name, + type_token=owner, + type_name=record.full_name, + full_name=f"System.Void {record.full_name}::{name}()", + return_type="System.Void", + params=[], + flags=["Public"] if not name.startswith("<") else ["Private"], + impl_flags=[], + rva=0x2050 + token % 0x100, + il_size=12, + is_static=True, + is_virtual=False, + is_abstract=False, + is_constructor=name in {".ctor", ".cctor"}, + compiler_generated=name.startswith("<") or record.compiler_generated, + **extra, + ) + item.assembly_name = "Contoso.App" + self.methods.append(item) + record.methods.append(item.id) + self._method_by_token[(0, token)] = item + return item + + main = method(0x06000001, PROGRAM, "Main") + lambda_ = method(0x06000002, CLOSURE, "
b__0_0") + send = method(0x06000003, CLIENT, "Send") + native = method( + 0x06000004, + CLIENT, + "compute", + pinvoke_module="native_helper", + pinvoke_entry="helper_compute", + ) + method(0x06000005, BROKEN, "Explode") + cctor = method(0x06000006, MODULE, ".cctor") + method(0x06000007, BOX, "Get") + lambda_.owner_method = main.id + main.callees = [send.id] + send.callers = [main.id] + lambda_.callees = [send.id] + send.callers.append(lambda_.id) + send.callees = [native.id] + native.callers = [send.id] + send.external_calls = [ + "[System.Net.Http]System.Net.Http.HttpResponseMessage System.Net.Http.HttpClient::Send(System.Net.Http.HttpRequestMessage)" + ] + self.external_refs = { + "System.Net.Http": { + send.external_calls[0][len("[System.Net.Http]") :]: [send.id] + } + } + main.string_ref_count = 1 + self.string_xrefs = {(0, 1): [main.id]} + self.pinvoke_modules = {"native_helper"} + del cctor + + def save_resource(self, info: AssemblyInfo, name: str, target: Path) -> int | None: + payload = b"{}" if name == "config.json" else b"\x00resources" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(payload) + return len(payload) + + def decode_resources_file( + self, info: AssemblyInfo, name: str + ) -> list[dict[str, Any]]: + return [{"name": "Greeting", "type": "String", "value": "hi"}] + + def close(self) -> None: + return + + +class FakeDecompiler: + def __init__(self, session: DotnetSession) -> None: + self.session = session + self.items: list[WorkItem] = [] + + def decompile(self, item: WorkItem) -> DecompiledType: + self.items.append(item) + if item.token == 0: + return DecompiledType( + 0, + 0, + "[assembly: Fake]\n", + None, + ".assembly Contoso.App\n", + None, + {}, + {}, + ) + if item.token == BROKEN: + return DecompiledType( + 0, BROKEN, None, "boom", ".class Broken\n", None, {}, {} + ) + if item.token == PROGRAM: + il = "\n".join( + [ + ".class Program", + "{", + " .method /* 06000001 */ public static void Main", + " {", + " } // end of method Program::Main", + "}", + ] + ) + return DecompiledType( + 0, + PROGRAM, + PROGRAM_CS, + None, + il, + None, + {0x06000001: (7, 12), 0x06000002: (10, 10)}, + {0x06000001: (3, 5)}, + ) + return DecompiledType( + 0, + item.token, + f"// type {item.token:#x}\n", + None, + f"// il {item.token:#x}\n", + None, + {}, + {}, + ) + + +def _fake_native_exporter(calls: list[str]) -> Any: + def run( + lib: NativeLib, out_dir: Path, progress: Progress + ) -> tuple[str, str, int, int]: + calls.append(lib.name) + out_dir.mkdir(parents=True, exist_ok=True) + (out_dir / "AGENTS.md").write_text(lib.origin or "", encoding="utf-8") + return ("ida", "Hex-Rays", 3, 0) + + return run + + +def _export(tmp_path: Path, **options: Any) -> Any: + calls: list[str] = [] + summary = export_dotnet( + tmp_path / "Contoso.App.dll", + options=DotnetExportOptions(out_dir=tmp_path / "export", **options), + progress=Progress(enabled=False), + native_exporter=_fake_native_exporter(calls), + decompiler_factory=FakeDecompiler, + session_factory=FakeSession, + ) + return summary, calls + + +@pytest.fixture +def assembly(tmp_path: Path) -> Path: + write(tmp_path / "libnative_helper.so", native_elf()) + return write(tmp_path / "Contoso.App.dll", managed_pe(strings=["x"])) + + +# -------------------------------------------------------------------------- +# helpers +# -------------------------------------------------------------------------- + + +def test_file_naming_helpers() -> None: + assert ( + safe_file_name("") + == "_PrivateImplementationDetails_" + ) + assert safe_file_name("..hidden") == "hidden" + assert namespace_parts("Contoso.App.Net") == ["Contoso", "App", "Net"] + assert namespace_parts("") == [] + assert type_file_stem(_type(1, "Dictionary`2", "System")) == "Dictionary-2" + assert type_file_stem(_type(1, "\u0001\u0002", "")) == "__" + + +def test_target_framework_monikers() -> None: + assert target_framework_moniker(".NETCoreApp,Version=v10.0") == "net10.0" + assert target_framework_moniker(".NETCoreApp,Version=v3.1") == "netcoreapp3.1" + assert target_framework_moniker(".NETStandard,Version=v2.0") == "netstandard2.0" + assert target_framework_moniker(".NETFramework,Version=v4.7.2") == "net472" + assert target_framework_moniker(".NETFramework,Version=v4.8") == "net48" + assert target_framework_moniker(None) is None + assert target_framework_moniker("Unity") is None + + +# -------------------------------------------------------------------------- +# end-to-end with fakes +# -------------------------------------------------------------------------- + + +def test_export_dotnet_writes_a_project_tree(tmp_path: Path, assembly: Path) -> None: + summary, calls = _export(tmp_path, jobs=1) + + root = summary.root_dir + assert root == tmp_path / "export" + assert summary.kind == "assembly" and summary.assemblies == ["Contoso.App"] + assert summary.failed_types == [("Contoso.App.Broken", "boom")] + assert summary.native_total == 1 and summary.native_done == 1 + assert calls == ["libnative_helper.so"] + + raw = root / "src" / "raw" / "Contoso.App" + for name in ( + "Contoso/App/Program.cs", + "Contoso/App/Program.il", + "Contoso/App/Net/Client.cs", + "Contoso/App/Net/Client.il", + "Contoso/App/Broken.cs", + "Contoso/App/Box-1.cs", + "_Module_.cs", + "Properties/AssemblyInfo.cs", + "Properties/Manifest.il", + "Contoso.App.csproj", + ): + assert (raw / name).is_file(), name + program = (raw / "Contoso/App/Program.cs").read_text(encoding="utf-8") + assert program.startswith( + "// ToCode: Contoso.App.Program (0x02000002) from Contoso.App 1.0.0.0\n" + ) + assert "namespace Contoso.App;" in program + broken = (raw / "Contoso/App/Broken.cs").read_text(encoding="utf-8") + assert "decompilation failed: boom" in broken and "Broken::System.Void" in broken + csproj = (raw / "Contoso.App.csproj").read_text(encoding="utf-8") + assert "Exe" in csproj + assert "net10.0" in csproj + assert '' in csproj + assert "System.Runtime" not in csproj + + for name in ( + "AGENTS.md", + "CLAUDE.md", + "assemblies.json", + "types.json", + "functions.json", + "function-index.json", + "strings.json", + "imports.json", + "exports.json", + "sections.json", + "reachable.json", + "namespace-graph.json", + "resources.json", + "container.json", + "triage.json", + "native-libs.json", + "project.json", + "export-manifest.json", + "tocode.log", + ): + assert (root / name).is_file(), name + + functions = { + row["c_name"]: row + for row in json.loads((root / "functions.json").read_text())["functions"] + } + main = functions["Contoso.App.Program.Main"] + # +1 for the ToCode header line written above the decompiler output. + assert (main["source_line_start"], main["source_line_end"]) == (8, 13) + assert (main["asm_line_start"], main["asm_line_end"]) == (4, 6) + assert main["source_file"] == "src/raw/Contoso.App/Contoso/App/Program.cs" + assert main["asm_file"] == "src/raw/Contoso.App/Contoso/App/Program.il" + assert main["callees"] == ["Contoso.App!0x06000003"] + assert main["dead_code"] is False + lambda_ = functions["Contoso.App.Program.<>c.
b__0_0"] + assert lambda_["source_line_start"] == 11 + assert lambda_["owner"] == "Contoso.App!0x06000001" + native = functions["Contoso.App.Net.Client.compute"] + assert native["pinvoke"] == {"module": "native_helper", "entry": "helper_compute"} + explode = functions["Contoso.App.Broken.Explode"] + assert explode["source_line_start"] == 1 # failure stub spans the file + + index = json.loads((root / "function-index.json").read_text()) + assert index["language"] == "csharp" and index["asm_language"] == "il" + entry = next( + row for row in index["functions"] if row["address"] == "Contoso.App!0x06000001" + ) + assert entry["c"]["line_start"] == 8 and entry["asm"]["line_start"] == 4 + + imports = json.loads((root / "imports.json").read_text())["imports"] + assert [(group["dll"], group["kind"]) for group in imports] == [ + ("System.Net.Http", "assembly"), + ("native_helper", "pinvoke"), + ] + assert imports[1]["functions"][0]["name"] == "helper_compute" + + exports = json.loads((root / "exports.json").read_text())["exports"] + assert [(row["kind"], row["address"]) for row in exports] == [ + ("entrypoint", "Contoso.App!0x06000001"), + ("module-initializer", "Contoso.App!0x06000006"), + ] + + reachable = json.loads((root / "reachable.json").read_text()) + depths = {row["address"]: row["depth"] for row in reachable["reachable"]} + assert depths["Contoso.App!0x06000003"] == 1 + assert depths["Contoso.App!0x06000002"] == 1 # lambda via its owner + assert depths["Contoso.App!0x06000004"] == 2 + + strings = json.loads((root / "strings.json").read_text())["strings"] + assert strings[0]["value"] == "https://api.example.invalid/v1" + assert strings[0]["vaddr"] == "Contoso.App!0x70000001" + assert strings[0]["xrefs"][0]["address"] == "Contoso.App!0x06000001" + + types = { + row["name"]: row + for row in json.loads((root / "types.json").read_text())["types"] + } + assert ( + types["Contoso.App.Program+<>c"]["cs_file"] + == "src/raw/Contoso.App/Contoso/App/Program.cs" + ) + assert types["Contoso.App.Broken"]["decompile_error"] == "boom" + + assemblies = json.loads((root / "assemblies.json").read_text())["assemblies"] + assert assemblies[0]["entry_point"]["address"] == "Contoso.App!0x06000001" + assert assemblies[0]["project"] == "Contoso.App/Contoso.App.csproj" + + resources = json.loads((root / "resources.json").read_text())["resources"] + by_name = {row["name"]: row for row in resources} + assert by_name["config.json"]["path"] == "data/resources/Contoso.App/config.json" + assert by_name["Contoso.App.Strings.resources"]["entries"][0]["value"] == "hi" + assert by_name["Other.dll"]["kind"] == "linked" + assert ( + root / "data/resources/Contoso.App/Contoso.App.Strings.resources.json" + ).is_file() + + triage = json.loads((root / "triage.json").read_text()) + assert triage["binary_type"] == "dotnet-assembly" + assert triage["pinvoke"] == [ + {"module": "native_helper", "functions": ["helper_compute"]} + ] + assert triage["suspicious_apis"][0]["category"] == "network" + assert triage["strings_of_interest"][0]["value"] == "https://api.example.invalid/v1" + + libs = json.loads((root / "native-libs.json").read_text())["libs"] + assert libs[0]["status"] == "done" and libs[0]["path"].startswith("lib/arm64/") + origin = (root / libs[0]["export_dir"] / "AGENTS.md").read_text(encoding="utf-8") + assert "`Contoso.App` .NET assembly" in origin + assert "P/Invoke declarations name (`native_helper`)" in origin + + graph = json.loads((root / "namespace-graph.json").read_text())["namespaces"] + app = next(row for row in graph if row["namespace"] == "Contoso.App") + assert app["calls_namespaces"] == [ + {"namespace": "Contoso.App:Contoso.App.Net", "calls": 2} + ] + + agents = (root / "AGENTS.md").read_text(encoding="utf-8") + assert "Contoso.App" in agents and "native///" in agents + assert (root / "CLAUDE.md").read_text(encoding="utf-8") == "@./AGENTS.md\n" + manifest = json.loads((root / "export-manifest.json").read_text()) + assert manifest["format"] == "dotnet-assembly" + assert manifest["failures"] == [ + {"address": None, "name": "Contoso.App.Broken", "error": "boom"} + ] + log = (root / "tocode.log").read_text(encoding="utf-8") + assert "export Contoso.App!Contoso.App.Broken failed: boom" in log + + +def test_export_dotnet_no_native_and_stale_cleanup( + tmp_path: Path, assembly: Path +) -> None: + stale = tmp_path / "export" / "src" / "raw" / "Old" / "Gone.cs" + stale.parent.mkdir(parents=True) + stale.write_text("stale", encoding="utf-8") + + summary, calls = _export(tmp_path, native=False) + + assert calls == [] + assert not stale.exists() + libs = json.loads((summary.root_dir / "native-libs.json").read_text())["libs"] + assert libs[0]["status"] == "skipped: --no-native" + assert (summary.root_dir / "lib" / "arm64" / "libnative_helper.so").is_file() + assert not (summary.root_dir / "native").exists() + + +def test_export_dotnet_default_out_dir( + tmp_path: Path, assembly: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("TOCODE_DEFAULT_OUT_ROOT", raising=False) + summary = export_dotnet( + assembly, + options=DotnetExportOptions(native=False), + progress=Progress(enabled=False), + decompiler_factory=FakeDecompiler, + session_factory=FakeSession, + ) + assert summary.root_dir == tmp_path / "Contoso_App_decompiler" + + +def test_export_dotnet_bundle_skips_runtime_natives(tmp_path: Path) -> None: + bundle = write( + tmp_path / "App", + single_file_bundle( + [ + ("Contoso.App.dll", 1, managed_pe(strings=["x"]), True), + ("libcoreclr.so", 2, native_elf(), False), + ("libcustom.so", 2, native_elf(), False), + ] + ), + ) + calls: list[str] = [] + summary = export_dotnet( + bundle, + options=DotnetExportOptions(out_dir=tmp_path / "export"), + progress=Progress(enabled=False), + native_exporter=_fake_native_exporter(calls), + decompiler_factory=FakeDecompiler, + session_factory=FakeSession, + ) + + assert summary.kind == "bundle" + assert calls == ["libcustom.so"] + libs = { + row["name"]: row["status"] + for row in json.loads((summary.root_dir / "native-libs.json").read_text())[ + "libs" + ] + } + assert libs == { + "libcoreclr.so": "skipped: .NET runtime library (use --include-framework)", + "libcustom.so": "done", + } + container = json.loads((summary.root_dir / "container.json").read_text()) + assert ( + container["kind"] == "bundle" + and container["bundle"]["bundle_id"] == "testbundle01" + ) + assert [entry["path"] for entry in container["entries"]] == [ + "Contoso.App.dll", + "libcoreclr.so", + "libcustom.so", + ] + assert (summary.root_dir / "data" / "assemblies" / "Contoso.App.dll").is_file() + + +def test_export_dotnet_rejects_missing_input(tmp_path: Path) -> None: + with pytest.raises(ToCodeError): + export_dotnet(tmp_path / "missing.dll", progress=Progress(enabled=False)) + + +def test_decompile_pool_isolates_crashing_types( + tmp_path: Path, assembly: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A batch that kills its worker is re-run type by type; only the culprit fails.""" + rounds: list[list[list[int]]] = [] + + def fake_round( + context: Any, + batches: list[tuple[list[WorkItem], bool]], + workers: int, + search_dirs: list[str], + timeout: int, + bar: Any, + ) -> list[tuple[list[WorkItem], bool, str]]: + rounds.append([[item.token for item in batch] for batch, _ in batches]) + failed = [] + fake = FakeDecompiler(context.session) + for batch, isolated in batches: + if any(item.token == CLIENT for item in batch): + failed.append((batch, isolated, "decompiler process crashed")) + continue + for item in batch: + pipeline._store_result(context, fake.decompile(item)) + return failed + + monkeypatch.setattr(pipeline, "_run_round", fake_round) + summary = export_dotnet( + assembly, + options=DotnetExportOptions(out_dir=tmp_path / "export", native=False), + progress=Progress(enabled=False), + session_factory=FakeSession, + ) + + assert ( + "Contoso.App.Net.Client", + "decompiler process crashed", + ) in summary.failed_types + assert ("Contoso.App.Program", "boom") not in summary.failed_types + # first round: one batch; then each item alone; then the culprit once more alone. + assert len(rounds[0]) == 1 and all(len(batch) == 1 for batch in rounds[1]) + client = ( + summary.root_dir / "src/raw/Contoso.App/Contoso/App/Net/Client.il" + ).read_text() + assert "IL disassembly failed: decompiler process crashed" in client + + +# -------------------------------------------------------------------------- +# metadata helpers +# -------------------------------------------------------------------------- + + +def test_build_csproj_for_library_with_internal_refs(tmp_path: Path) -> None: + pe = parse_pe(managed_pe()) + assert pe is not None + from tocode.backends.dotnet import AssemblyFile + + info = AssemblyInfo( + index=0, + file=AssemblyFile(tmp_path / "A.dll", "A.dll", "A.dll"), + pe=pe, + name="A", + full_name="A", + version="1.0.0.0", + culture="", + public_key_token=None, + module_name="A.dll", + mvid="", + kind="Dll", + target_framework=".NETStandard,Version=v2.0", + entry_point=None, + assembly_refs=[ + {"name": "B"}, + {"name": "netstandard"}, + {"name": "Newtonsoft.Json"}, + ], + module_refs=[], + attributes=[], + resources=[], + ) + text = build_csproj(info, internal_refs={"B"}) + assert "Library" in text + assert "netstandard2.0" in text + assert '' in text + assert '' in text + assert 'netstandard"' not in text + + +def test_interesting_strings_rank_secrets_first() -> None: + rows = meta.interesting_strings( + [ + ("a", "AES/CBC/PKCS7Padding"), + ("b", "connect to https://c2.example.invalid/x"), + ("c", "short"), + ("d", "/proc/self/maps"), + ] + ) + assert [row["address"] for row in rows] == ["b", "d", "a"] + + +def _real_runtime() -> bool: + from tocode.backends.dotnet import probe_dotnet + + return probe_dotnet()[0] + + +@pytest.mark.skipif( + not __import__("os").environ.get("TOCODE_TEST_DOTNET") or not _real_runtime(), + reason="set TOCODE_TEST_DOTNET to a managed assembly (needs .NET 9+ and pythonnet)", +) +def test_export_real_assembly(tmp_path: Path) -> None: + import os + + source = Path(os.environ["TOCODE_TEST_DOTNET"]) + summary = export_dotnet( + source, + options=DotnetExportOptions(out_dir=tmp_path / "export", native=False, jobs=1), + progress=Progress(enabled=False), + ) + + assert summary.type_count > 0 and not summary.failed_types + index = json.loads((summary.root_dir / "function-index.json").read_text()) + with_body = [row for row in index["functions"] if row["asm"]] + assert with_body and all(row["asm"]["line_start"] > 1 for row in with_body) + cs_files = list((summary.root_dir / "src" / "raw").rglob("*.cs")) + il_files = list((summary.root_dir / "src" / "raw").rglob("*.il")) + assert cs_files and len(il_files) == len(cs_files) + + +def test_suspicious_apis_ignore_compiler_generated_calls( + tmp_path: Path, assembly: Path +) -> None: + from tocode.backends.dotnet import discover_dotnet_input + + session = FakeSession(discover_dotnet_input(assembly, workdir=tmp_path / "work")) + session.load() + session.methods[0].external_calls = [ + "[System.Runtime]!!0 System.Activator::CreateInstance()", + "[System.Runtime]System.Int32 System.Environment::get_CurrentManagedThreadId()", + "[System.Runtime]System.Object System.Activator::CreateInstance(System.Type)", + "[System.Runtime]System.String System.Environment::GetEnvironmentVariable(System.String)", + ] + session.methods[2].external_calls = [] + + found = { + row["category"]: [api["name"] for api in row["apis"]] + for row in meta.suspicious_apis(session) + } + + assert found == { + "dynamic code loading / reflection": [ + "[System.Runtime]System.Object System.Activator::CreateInstance(System.Type)" + ], + "environment / host info": [ + "[System.Runtime]System.String System.Environment::GetEnvironmentVariable(System.String)" + ], + } diff --git a/uv.lock b/uv.lock index c9e8fcb..e69bc57 100644 --- a/uv.lock +++ b/uv.lock @@ -554,6 +554,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl", hash = "sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360", size = 125251, upload-time = "2026-08-26T13:33:12.928Z" }, ] +[[package]] +name = "clr-loader" +version = "0.2.10" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11'", +] +dependencies = [ + { name = "cffi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/24/c12faf3f61614b3131b5c98d3bf0d376b49c7feaa73edca559aeb2aee080/clr_loader-0.2.10.tar.gz", hash = "sha256:81f114afbc5005bafc5efe5af1341d400e22137e275b042a8979f3feb9fc9446", size = 83605, upload-time = "2026-01-03T23:13:06.984Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/61/cf819f8e8bb4d4c74661acf2498ba8d4a296714be3478d21eaabf64f5b9b/clr_loader-0.2.10-py3-none-any.whl", hash = "sha256:ebbbf9d511a7fe95fa28a95a4e04cd195b097881dfe66158dc2c281d3536f282", size = 56483, upload-time = "2026-01-03T23:13:05.439Z" }, +] + +[[package]] +name = "clr-loader" +version = "0.3.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", +] +dependencies = [ + { name = "cffi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/46/7eea92b6aa2d68af78e049cbecec5f757f1aad44ecdecdc16bbad7eead51/clr_loader-0.3.1.tar.gz", hash = "sha256:2e073e9aaf49d1ae2f56ecba27987ad5fb68be4bcd9dd34a5bed8f0e4e128366", size = 86805, upload-time = "2026-04-18T17:49:44.287Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/da/ec1a6e36624000b6df0dd61183c42342ee5814c073315e802cadaad04d2f/clr_loader-0.3.1-py3-none-any.whl", hash = "sha256:cbad189de20d202a7d621956b0fc38049e13c9bf7ca2923441eff725cd121aa1", size = 55730, upload-time = "2026-04-18T17:49:42.99Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -1736,6 +1767,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" }, ] +[[package]] +name = "pythonnet" +version = "3.0.5" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11'", +] +dependencies = [ + { name = "clr-loader", version = "0.2.10", source = { registry = "https://pypi.org/simple" } }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9a/d6/1afd75edd932306ae9bd2c2d961d603dc2b52fcec51b04afea464f1f6646/pythonnet-3.0.5.tar.gz", hash = "sha256:48e43ca463941b3608b32b4e236db92d8d40db4c58a75ace902985f76dac21cf", size = 239212, upload-time = "2024-12-13T08:30:44.393Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/f1/bfb6811df4745f92f14c47a29e50e89a36b1533130fcc56452d4660bd2d6/pythonnet-3.0.5-py3-none-any.whl", hash = "sha256:f6702d694d5d5b163c9f3f5cc34e0bed8d6857150237fae411fefb883a656d20", size = 297506, upload-time = "2024-12-13T08:30:40.661Z" }, +] + +[[package]] +name = "pythonnet" +version = "3.2.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", +] +dependencies = [ + { name = "clr-loader", version = "0.3.1", source = { registry = "https://pypi.org/simple" } }, +] +sdist = { url = "https://files.pythonhosted.org/packages/17/e7/ee7af9cebbb4992fdd32ffc363f8ab7d22609665745a46b1d0b4d9d4ccc9/pythonnet-3.2.0.tar.gz", hash = "sha256:880bed9452646624f2cca1627d463424d15abefa16933c46ba00dc10da78d184", size = 257801, upload-time = "2026-09-29T10:51:37.932Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/42/75f9429ae418f92e0366a334fd07d8313a13c8e8c0ce6ba650f6ebfb0cf6/pythonnet-3.2.0-cp311.cp312.cp313.cp314.cp315-none-any.whl", hash = "sha256:c8c7946d291156ffc1d5f22c001a63c6776947ce3c64da800ccea8181ddb1abe", size = 222406, upload-time = "2026-09-29T10:51:58.583Z" }, +] + [[package]] name = "pyvex" version = "9.2.213" @@ -2079,12 +2141,13 @@ wheels = [ [[package]] name = "tocode-cli" -version = "0.1.0" source = { editable = "." } dependencies = [ { name = "droidasc" }, { name = "ida-domain" }, { name = "idapro" }, + { name = "pythonnet", version = "3.0.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "pythonnet", version = "3.2.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "r2pipe" }, { name = "rpyc" }, { name = "tqdm" }, @@ -2106,6 +2169,8 @@ requires-dist = [ { name = "ida-domain", specifier = "==0.5.0" }, { name = "idapro", specifier = "==0.0.9" }, { name = "pytest", marker = "extra == 'dev'", specifier = "==8.4.2" }, + { name = "pythonnet", marker = "python_full_version < '3.11'", specifier = "==3.0.5" }, + { name = "pythonnet", marker = "python_full_version >= '3.11'", specifier = "==3.2.0" }, { name = "r2pipe", specifier = "==1.9.8" }, { name = "rpyc", specifier = "==6.0.2" }, { name = "tqdm", specifier = "==4.67.3" },