From 6a8af30f1a4022db6275a2cb26522c16330fbc91 Mon Sep 17 00:00:00 2001 From: buzzer-re <22428720+buzzer-re@users.noreply.github.com> Date: Sun, 20 Sep 2026 20:19:38 +0900 Subject: [PATCH 1/6] Make the package PyPI-ready Add the MIT LICENSE file and ship it in the wheel, fill in authors, URLs, keywords, and classifiers, and read the version from tocode.__version__ so it is defined once. `uv build` produces an sdist and a wheel that pass `twine check`; the wheel contains only the tocode package. Co-Authored-By: Claude Opus 5 --- .gitignore | 2 ++ LICENSE | 21 +++++++++++++++++++++ pyproject.toml | 36 ++++++++++++++++++++++++++++++++++-- uv.lock | 1 - 4 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 LICENSE diff --git a/.gitignore b/.gitignore index 8790651..a1f37f3 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ __pycache__/ *.py[cod] *.egg-info/ *_decompiler/ +/dist +/build diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c67b338 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 buzzer-re + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pyproject.toml b/pyproject.toml index becc2c6..63fcbbe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,10 +4,23 @@ build-backend = "setuptools.build_meta" [project] name = "tocode-cli" -version = "0.1.0" -description = "Export compiled binaries as source-like projects for reverse engineering agents." +dynamic = ["version"] +description = "Export compiled binaries, IDA databases, and Android APKs as source-like projects for reverse engineering agents." readme = "README.md" requires-python = ">=3.10" +authors = [{ name = "buzzer-re" }] +keywords = [ + "reverse-engineering", + "decompiler", + "ida", + "radare2", + "angr", + "binary-ninja", + "android", + "apk", + "dex", + "agents", +] dependencies = [ "droidasc==0.1.1.post1", "ida-domain==0.5.0", @@ -17,9 +30,19 @@ dependencies = [ "tqdm==4.67.3", ] license = "MIT" +license-files = ["LICENSE"] classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX :: Linux", "Operating System :: MacOS", @@ -27,6 +50,11 @@ classifiers = [ "Topic :: Software Development :: Disassemblers", ] +[project.urls] +Homepage = "https://github.com/buzzer-re/ToCode" +Repository = "https://github.com/buzzer-re/ToCode" +Issues = "https://github.com/buzzer-re/ToCode/issues" + [project.scripts] tocode = "tocode.cli:main" @@ -68,5 +96,9 @@ ignore_missing_imports = true [tool.setuptools] package-dir = { "" = "src" } +# Single source of truth for the version: src/tocode/__init__.py. +[tool.setuptools.dynamic] +version = { attr = "tocode.__version__" } + [tool.setuptools.packages.find] where = ["src"] diff --git a/uv.lock b/uv.lock index c9e8fcb..5aaaffb 100644 --- a/uv.lock +++ b/uv.lock @@ -2079,7 +2079,6 @@ wheels = [ [[package]] name = "tocode-cli" -version = "0.1.0" source = { editable = "." } dependencies = [ { name = "droidasc" }, From accdb9cf020e6220e79219528c887f101d708fc4 Mon Sep 17 00:00:00 2001 From: buzzer-re <22428720+buzzer-re@users.noreply.github.com> Date: Mon, 21 Sep 2026 23:24:33 +0900 Subject: [PATCH 2/6] Pin the console script name to tocode The PyPI distribution is tocode-cli; assert the installed command stays `tocode` so a rename cannot slip through. Co-Authored-By: Claude Opus 5 --- tests/test_cli.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index 76e5d48..d8167e3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -95,3 +95,21 @@ def fake_export(binary, *, options, progress): assert seen["options"].jobs == 3 assert seen["options"].native_options.backend == "r2" assert seen["options"].native_options.jobs == 3 + + +def test_package_installs_a_tocode_console_script() -> None: + """The PyPI distribution is `tocode-cli`, but the command stays `tocode`.""" + import importlib.metadata as metadata + + try: + distribution = metadata.distribution("tocode-cli") + except metadata.PackageNotFoundError: # pragma: no cover - not installed + pytest.skip("tocode-cli is not installed in this environment") + + scripts = { + entry.name: entry.value + for entry in distribution.entry_points + if entry.group == "console_scripts" + } + + assert scripts == {"tocode": "tocode.cli:main"} From 31f69994944aec11d557ba86d12a57334d42c495 Mon Sep 17 00:00:00 2001 From: buzzer-re <22428720+buzzer-re@users.noreply.github.com> Date: Mon, 5 Oct 2026 11:00:32 +0900 Subject: [PATCH 3/6] Add .NET support (dnlib + ICSharpCode.Decompiler backend) `tocode App.dll` (also .exe, apphost launchers, single-file bundles, and .nupkg packages, detected by content) now exports managed code without dnSpy/ILSpy installed: dnlib and ICSharpCode.Decompiler (MIT) are loaded through pythonnet, so only a .NET 9+ runtime is needed. (The libraries are downloaded on demand; see the follow-up commit.) - src/raw///.../.cs per top-level type, with a matching .il that keeps the bytecode (RVA, raw bytes, metadata tokens), plus Properties/AssemblyInfo.cs, Manifest.il, and a .csproj per assembly. - Per-method C# and IL line ranges; lambdas, local functions, and async/iterator state machines are linked to the method that owns them. - assemblies.json, types.json, functions.json, function-index.json, strings.json (user strings with ldstr xrefs), imports.json (cross-assembly members and P/Invoke), exports.json, reachable.json, namespace-graph.json, resources.json (+ decoded .resources), container.json, and triage.json. - Native code (mixed-mode images, bundled native libraries, P/Invoke targets next to the input) goes through the native runner like APK .so files; framework/runtime parts of bundles and packages are listed but only decompiled with --include-framework. --as-native keeps the old path. - PE/CLR/bundle parsing and the IL token scan are pure Python (validated against dnlib on every opcode); decompilation runs in recycled worker processes and a crashing or hanging type fails alone. The native memory floor is now TOCODE_NATIVE_MIN_FREE_MB (the APK name still works) and the waiting state shows in the heartbeat. Co-Authored-By: Claude Opus 5.5 --- .gitattributes | 1 + AGENTS.md | 18 +- README.md | 50 +- pyproject.toml | 18 +- src/tocode/apk.py | 14 +- src/tocode/apk_native.py | 28 + src/tocode/backends/dotnet.py | 1624 ++++++++++++++++++++++++++++++ src/tocode/backends/dotnet_pe.py | 589 +++++++++++ src/tocode/cli.py | 82 +- src/tocode/dotnet.py | 1078 ++++++++++++++++++++ src/tocode/dotnet_metadata.py | 877 ++++++++++++++++ src/tocode/schema.py | 15 + tests/conftest.py | 9 + tests/dotnet_fixtures.py | 153 +++ tests/test_apk_export.py | 28 + tests/test_cli.py | 78 ++ tests/test_dotnet_backend.py | 516 ++++++++++ tests/test_dotnet_export.py | 751 ++++++++++++++ uv.lock | 66 ++ 19 files changed, 5961 insertions(+), 34 deletions(-) create mode 100644 src/tocode/backends/dotnet.py create mode 100644 src/tocode/backends/dotnet_pe.py create mode 100644 src/tocode/dotnet.py create mode 100644 src/tocode/dotnet_metadata.py create mode 100644 tests/conftest.py create mode 100644 tests/dotnet_fixtures.py create mode 100644 tests/test_dotnet_backend.py create mode 100644 tests/test_dotnet_export.py diff --git a/.gitattributes b/.gitattributes index 6313b56..10a98db 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,2 @@ * text=auto eol=lf +*.dll binary diff --git a/AGENTS.md b/AGENTS.md index e82d0ee..c72139e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,6 @@ # AGENTS -This repository contains ToCode, a Python-only binary exporter. ToCode takes one binary, IDA database, or Android APK path and writes one source-like project directory for reverse-engineering agents. +This repository contains ToCode, a Python-only binary exporter. ToCode takes one binary, IDA database, Android APK, or .NET assembly/bundle/package path and writes one source-like project directory for reverse-engineering agents. ## Scope @@ -16,6 +16,8 @@ This repository contains ToCode, a Python-only binary exporter. ToCode takes one - `src/tocode/__init__.py`: package version and `export_from_binaryview()`, the in-UI Binary Ninja library entry. - `src/tocode/analysis.py`: backend-neutral binary inventory and call graph normalization. - `src/tocode/backends/`: IDA Domain, radare2, angr, and Binary Ninja (`binja.py`) session adapters, plus the ASC/droidasc APK backend (`asc.py`: APK set discovery, DEX inventory, worker-side class decompilation). +- `src/tocode/backends/dotnet.py` / `dotnet_pe.py`: .NET backend (input discovery for assemblies, apphosts, single-file bundles and NuGet packages; dnlib inventory; pure-Python PE/CLR/bundle parsing and IL token scan; worker-side C#/IL decompilation). `backends/dotnet_lib/` holds the bundled dnlib and ICSharpCode.Decompiler DLLs (MIT) with their provenance. +- `src/tocode/dotnet.py`: .NET export pipeline; `dotnet_metadata.py`: .NET JSON documents and the generated `AGENTS.md`. - `src/tocode/apk.py`: APK export pipeline (extraction, decompile pool, resource decoding, metadata); `apk_metadata.py`: manifest parsing and Android JSON documents; `apk_native.py`: extraction and background native export of every `.so`. - `src/tocode/exporter.py`: project writer, function rendering, worker-session rendering, generated export `AGENTS.md`. - `src/tocode/metadata.py`: JSON metadata and triage documents. @@ -71,6 +73,17 @@ APK input (`.apk`, `.apks`, `.xapk`) uses the ASC backend and writes instead: `base.apk` merges sibling `split_*.apk` files unless `--no-splits`; bundles are unpacked and merged. `--backend` selects the native backend for the `.so` exports (`binja` is rejected). +.NET input (a managed PE, an apphost launcher with its `.dll`, a single-file bundle, or a `.nupkg`; detected by content, `--as-native` opts out) uses the .NET backend and writes instead: + +- `src/raw///**/.cs` and `.il` (C# and IL with raw bytecode per top-level type; nested types inside), `src/raw//Properties/AssemblyInfo.cs` and `Manifest.il`, `src/raw//.csproj` +- `assemblies.json`, `types.json`, `namespace-graph.json`, `resources.json`, `container.json`, `native-libs.json` +- `functions.json`, `function-index.json` (`c` = C#, `asm` = IL), `strings.json`, `imports.json`, `exports.json`, `sections.json`, `reachable.json`, `triage.json`, `project.json`, `export-manifest.json` +- `data/resources//*` (embedded resources, `.resources` decoded to JSON), `data/assemblies/*` and `data//*` (extracted container entries) +- `lib//*` and `native///` for mixed-mode code, bundled native libraries, and P/Invoke targets next to the input (unless `--no-native`) +- `tocode.log`, generated `AGENTS.md` and `CLAUDE.md` + +Framework/runtime assemblies and native runtime libraries inside bundles and packages are inventoried but only decompiled with `--include-framework`. + ## Development - Prefer `uv` for local commands. @@ -87,7 +100,8 @@ APK input (`.apk`, `.apks`, `.xapk`) uses the ASC backend and writes instead: - radare2/r2pipe is a fallback backend. - angr is the optional pure-Python fallback backend (`[angr]` extra). - ASC (`droidasc`, PyPI) is the APK/DEX backend and a core runtime dependency. It pulls androguard. `backends/asc.py` is the only module that imports it; it pre-imports the modules ASC would otherwise stub in `sys.modules` and silences androguard's loguru logging. Class decompilation is not thread-safe, so it runs in spawned worker processes (recycled in rounds; do not use `max_tasks_per_child`, it deadlocks spawn pools on some CPython builds). The big per-method/per-class/per-string JSON documents are streamed row by row (`apk_metadata.write_json_rows`), never built as one object. -- APK native libraries are exported by `apk_native.py` on a background thread, each `export_binary` in its own spawned process so a backend OOM-kill or crash only fails that library (`native-libs.json` status). The thread waits for `TOCODE_APK_NATIVE_MIN_FREE_MB` (default 1024) of free memory before starting each library so it does not starve the DEX pool. `TOCODE_WORKER_TMP_DIR` also places the unpacked `.apks` bundle. +- APK native libraries are exported by `apk_native.py` on a background thread, each `export_binary` in its own spawned process so a backend OOM-kill or crash only fails that library (`native-libs.json` status). The thread waits for `TOCODE_NATIVE_MIN_FREE_MB` (default 1024) of free memory before starting each library so it does not starve the DEX pool. `TOCODE_WORKER_TMP_DIR` also places the unpacked `.apks` bundle. +- The .NET backend needs `pythonnet` (core dependency, pinned per Python version) and a .NET 9+ runtime on the host. dnlib 4.5.0 and ICSharpCode.Decompiler 11.1 are vendored under `src/tocode/backends/dotnet_lib/` (package data; hashes in its README and checked by `tests/test_dotnet_backend.py`); ICSharpCode.Decompiler 11 needs System.Reflection.Metadata 9, hence .NET 9+. Decompilation runs in spawned worker processes (recycled per round; a batch that crashes or hangs a worker is retried type by type so only the culprit fails). Tests must not require .NET: use `tests/dotnet_fixtures.py` (synthetic PE/bundles) and fake sessions; runtime-backed tests skip without it. - Binary Ninja is an opt-in backend (`--backend binja`, never auto-selected). The `binaryninja` module is supplied by the Binary Ninja install (in-UI) or the remote VM, so it is not a pip dependency. `rpyc`, the client used for the diff --git a/README.md b/README.md index 4e70536..4f6c8c8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ToCode -ToCode exports a binary, IDA database, or Android APK into a source-like project tree: raw recovered C (or Java for APKs), matching assembly, function summaries, section data, optional IDA database, and metadata that coding agents can read directly. +ToCode exports a binary, IDA database, Android APK, or .NET assembly into a source-like project tree: raw recovered C (Java for APKs, C# and IL for .NET), matching assembly, function summaries, section data, optional IDA database, and metadata that coding agents can read directly. ## Why @@ -24,6 +24,10 @@ sample_decompiler/ src/raw/**/*.asm src/raw/**/*.summary src/raw//**/*.java # Only for APK files (replaces the .c/.asm/.summary tree) + src/raw///**/*.cs # Only for .NET files (replaces the .c/.asm/.summary tree) + src/raw///**/*.il # Only for .NET files: IL with RVA + raw bytecode per instruction + src/raw//Properties/* # Only for .NET files: AssemblyInfo.cs and the Manifest.il + src/raw//*.csproj # Only for .NET files: recovered project file include/*.h include/*.types.h data/*.bin @@ -32,13 +36,19 @@ sample_decompiler/ data/apk//** # Only for APK files: every non-code APK entry, verbatim data/res//**/*.xml # Only for APK files: binary XML resources decoded to text data/resources.json # Only for APK files: decoded resources.arsc - lib//*.so # Only for APK files: extracted native libraries - native/// # Only for APK files: full nested ToCode export per .so + data/resources//* # Only for .NET files: embedded resources (.resources decoded to JSON) + data/assemblies/* # Only for .NET files: assemblies extracted from bundles/packages + lib//*.so # Only for APK and .NET files: extracted native libraries + native/// # Only for APK and .NET files: full nested ToCode export per native library AndroidManifest.xml # Only for APK files manifest.json # Only for APK files classes.json # Only for APK files package-graph.json # Only for APK files (replaces cluster-graph.json) - native-libs.json # Only for APK files + native-libs.json # Only for APK and .NET files + assemblies.json # Only for .NET files + namespace-graph.json # Only for .NET files (replaces cluster-graph.json) + resources.json # Only for .NET files + container.json # Only for .NET files: bundle / NuGet package layout function-index.json functions.json types.json @@ -62,14 +72,19 @@ sample_decompiler/ | `include` | Generated headers, including `*.types.h` with the structs/enums/typedefs recovered from the binary. Not written for APK files. | | `data` | Raw section dumps and variable metadata. | | `data/apk`, `data/res`, `data/resources.json` | **Only for APK files.** Every non-code entry of each APK in the set verbatim, `res/**/*.xml` decoded from binary XML, and the decoded `resources.arsc` tables. | -| `lib//*.so` | **Only for APK files.** Every native library found in the APK set (all ABIs), always extracted. | -| `native///` | **Only for APK files.** A complete nested ToCode export for each native library (own `AGENTS.md` with an Origin section naming the APK, `src/raw/*.c`, `functions.json`, `exports.json`, ...). Skipped with `--no-native`. | +| `src/raw///**/*.cs` and `*.il` | **Only for .NET files.** Decompiled C# (ICSharpCode.Decompiler) and IL disassembly (RVA, raw bytecode, metadata tokens) per top-level type, nested types inside, folders follow assemblies and namespaces like a dnSpy/ILSpy project export. `Properties/AssemblyInfo.cs`, `Properties/Manifest.il`, and a `.csproj` per assembly. | +| `data/resources/`, `data/assemblies` | **Only for .NET files.** Embedded resources (`.resources` decoded to JSON) and the assemblies extracted from single-file bundles or NuGet packages. | +| `lib//*.so` | **Only for APK and .NET files.** Every native library found (APK: all ABIs; .NET: mixed-mode code, bundled libraries, P/Invoke targets next to the input), always extracted. | +| `native///` | **Only for APK and .NET files.** A complete nested ToCode export for each native library (own `AGENTS.md` with an Origin section naming the APK or .NET program, `src/raw/*.c`, `functions.json`, `exports.json`, ...). Skipped with `--no-native`. | | `AndroidManifest.xml` / `manifest.json` | **Only for APK files.** Decoded manifest and its parsed form: package, versions, SDKs, permissions, components with intent filters and exported state, application attributes, split manifests. | | `classes.json` | **Only for APK files.** Every class with superclass, interfaces, access flags, fields, methods, source file, and Java file/line ranges. | | `package-graph.json` | **Only for APK files.** Inter-package call graph (the APK counterpart of `cluster-graph.json`). | -| `native-libs.json` | **Only for APK files.** ABI, hash, source APK, export directory, and decompilation status of every native library. | +| `native-libs.json` | **Only for APK and .NET files.** ABI/arch, hash, source, export directory, and decompilation status of every native library. | +| `assemblies.json` | **Only for .NET files.** Every assembly found: version, target framework, entry point, CLR flags, ReadyToRun/mixed-mode, references, resources, obfuscation hints, and why skipped ones were not decompiled. | +| `namespace-graph.json` | **Only for .NET files.** Inter-namespace and inter-assembly call graph (the .NET counterpart of `cluster-graph.json`). | +| `resources.json` / `container.json` | **Only for .NET files.** Embedded/linked resources index; layout of the bundle, package, or assembly and where each entry was extracted. | | `types.json` | Catalog of types recovered from the binary's debug info or type library, with C declarations. | -| `*.json` | Functions (with recovered types and original source decl file/line), sections, strings, imports, exports, relocations, reachability, clusters, triage, project metadata, and export manifest. For APK files the same documents describe DEX methods, strings, framework imports, exported components/JNI methods, and reachability from manifest components; `relocations.json`, `cluster-graph.json`, and `types.json` are not written. | +| `*.json` | Functions (with recovered types and original source decl file/line), sections, strings, imports, exports, relocations, reachability, clusters, triage, project metadata, and export manifest. For APK files the same documents describe DEX methods, strings, framework imports, exported components/JNI methods, and reachability from manifest components; `relocations.json`, `cluster-graph.json`, and `types.json` are not written. For .NET files they describe methods (address `!0x`, C# and IL line ranges), the user-string heap, cross-assembly and P/Invoke imports, entry points/public API, and `types.json` lists .NET types with members; `relocations.json` and `cluster-graph.json` are not written. | | `tocode.log` | Export log with checkpoint, resume, and per-function render history. For APK files it also carries the native library export status. | | `AGENTS.md` / `CLAUDE.md` | Instructions for agents analyzing the exported binary. | | `src/tree` | Optional scanner-friendly C output when `--tree` is used. | @@ -95,6 +110,23 @@ Three backends are supported, selected with `--backend` (default `auto`, which p Other disassemblers may be added in the future. +### .NET assemblies + +`tocode App.dll` (also `.exe`, an apphost launcher next to its `.dll`, single-file bundles, and `.nupkg` packages) uses a built-in .NET backend: [dnlib](https://github.com/0xd4d/dnlib) for metadata and [ICSharpCode.Decompiler](https://github.com/icsharpcode/ILSpy) (ILSpy's engine) for C#, both bundled with ToCode and loaded through [pythonnet](https://pypi.org/project/pythonnet/). No dnSpy/ILSpy install is needed; only a **.NET 9+ runtime** (e.g. `sudo apt install dotnet-runtime-10.0`, or the SDK; `DOTNET_ROOT` is honoured). + +- `src/raw///.../.cs`: C# per top-level type, folders follow assemblies and namespaces like a dnSpy/ILSpy project export; `.il` next to it keeps the bytecode (IL with RVA, raw bytes, and metadata tokens per instruction). +- Per-method C# and IL line ranges in `functions.json`/`function-index.json` (address `!0x`), with lambdas and async/iterator state machines linked to the method that owns them. +- `assemblies.json`, `types.json`, `strings.json` (user strings with `ldstr` xrefs), `imports.json` (cross-assembly members and P/Invoke), `exports.json`, `reachable.json`, `namespace-graph.json`, `resources.json`, `container.json`, `triage.json` (P/Invoke, suspicious API families, obfuscation hints, strings of interest). +- Native code goes through the regular native backends on a background thread, like APK `.so` files: mixed-mode (C++/CLI) assemblies, native libraries inside bundles/packages, and P/Invoke targets shipped next to the input. `--no-native` skips it. +- Single-file bundles and packages carry the .NET runtime/framework; those assemblies and runtime native libraries are listed but only decompiled with `--include-framework`. NuGet packages decompile each assembly once, from its newest target framework. +- `--as-native` exports a managed PE with the native backend instead (e.g. to look at a ReadyToRun or mixed-mode image in IDA). + +```bash +tocode App.dll # assembly (+ P/Invoke libraries next to it) +tocode publish/App # single-file bundle or apphost launcher +tocode Vendor.Lib.1.0.0.nupkg # NuGet package +``` + ### Android APKs `tocode app.apk` (also `.apks`/`.xapk` bundles) uses [ASC](https://github.com/MG1937/ASC) (`droidasc`, a core dependency) for the DEX side and the regular native backends for every shared object in the package: @@ -102,7 +134,7 @@ Other disassemblers may be added in the future. - `src/raw//.java`: one decompiled Java file per class, folders follow Java packages (no clustering, no summaries). - `AndroidManifest.xml` + `manifest.json`: decoded and parsed manifest (permissions, components with intent filters and exported state, application attributes, split manifests). - `classes.json`, `functions.json`, `function-index.json`, `strings.json`, `imports.json`, `exports.json` (exported components + JNI methods), `reachable.json` (from manifest components), `package-graph.json`, `sections.json`, `triage.json`. -- `lib//*.so`: every native library extracted; `native///`: a complete nested ToCode export per library (all ABIs), produced on a background thread while the DEX side decompiles. `native-libs.json` records the status of each (each library runs in its own process; one failing or being OOM-killed never fails the APK export, and the native thread waits for `TOCODE_APK_NATIVE_MIN_FREE_MB`, default 1024 MB, of free memory before each library). Pass `--no-native` to skip the native decompilation (libraries are still extracted). `--backend` picks the native backend (`auto`/`ida`/`r2`/`angr`; `binja` is not supported for APKs). +- `lib//*.so`: every native library extracted; `native///`: a complete nested ToCode export per library (all ABIs), produced on a background thread while the DEX side decompiles. `native-libs.json` records the status of each (each library runs in its own process; one failing or being OOM-killed never fails the APK export, and the native thread waits for `TOCODE_NATIVE_MIN_FREE_MB`, default 1024 MB, of free memory before each library). Pass `--no-native` to skip the native decompilation (libraries are still extracted). `--backend` picks the native backend (`auto`/`ida`/`r2`/`angr`; `binja` is not supported for APKs). - `data/apk/**`: every other APK entry verbatim; `data/res/**/*.xml` and `data/resources.json`: decoded binary XML and `resources.arsc`. `base.apk` automatically merges sibling `split_*.apk` files (config and ABI splits) into the same project; `--no-splits` exports it alone. The default output directory is `_decompiler`. diff --git a/pyproject.toml b/pyproject.toml index 63fcbbe..7397b14 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "tocode-cli" dynamic = ["version"] -description = "Export compiled binaries, IDA databases, and Android APKs as source-like projects for reverse engineering agents." +description = "Export compiled binaries, IDA databases, Android APKs, and .NET assemblies as source-like projects for reverse engineering agents." readme = "README.md" requires-python = ">=3.10" authors = [{ name = "buzzer-re" }] @@ -19,12 +19,17 @@ keywords = [ "android", "apk", "dex", + "dotnet", + "csharp", + "il", "agents", ] dependencies = [ "droidasc==0.1.1.post1", "ida-domain==0.5.0", "idapro==0.0.9", + "pythonnet==3.0.5 ; python_full_version < '3.11'", + "pythonnet==3.2.0 ; python_full_version >= '3.11'", "r2pipe==1.9.8", "rpyc==6.0.2", "tqdm==4.67.3", @@ -93,6 +98,12 @@ ignore_missing_imports = true module = ["droidasc.*", "androguard.*"] ignore_missing_imports = true +# The .NET backend imports CLR namespaces through pythonnet at runtime; they +# have no Python stubs. +[[tool.mypy.overrides]] +module = ["clr", "clr_loader.*", "pythonnet.*", "System", "System.*", "dnlib.*", "ICSharpCode.*"] +ignore_missing_imports = true + [tool.setuptools] package-dir = { "" = "src" } @@ -102,3 +113,8 @@ version = { attr = "tocode.__version__" } [tool.setuptools.packages.find] where = ["src"] + +# dnlib and ICSharpCode.Decompiler (MIT) ship inside the package so .NET input +# needs only a .NET runtime; see src/tocode/backends/dotnet_lib/README.md. +[tool.setuptools.package-data] +"tocode.backends" = ["dotnet_lib/*.dll", "dotnet_lib/*.txt", "dotnet_lib/*.md"] diff --git a/src/tocode/apk.py b/src/tocode/apk.py index 0526dbf..ee9a0c6 100644 --- a/src/tocode/apk.py +++ b/src/tocode/apk.py @@ -24,7 +24,6 @@ from . import __version__ from . import apk_metadata as meta from .apk_native import ( - DEFAULT_NATIVE_MIN_FREE_MB, NATIVE_HEARTBEAT_SECONDS, NativeExporter, NativeLib, @@ -34,6 +33,7 @@ default_native_exporter, extract_native_libs, native_libs_json, + native_min_free_mb, ) from .backends.asc import ( ApkSet, @@ -344,7 +344,7 @@ def _start_native(context: ApkContext, exporter: NativeExporter | None) -> None: root=root, progress=context.progress, exporter=exporter or default_native_exporter(context.options.native_options), - min_free_mb=_native_min_free_mb(), + min_free_mb=native_min_free_mb(), available_memory=available_memory_mb, ) context.progress.log( @@ -355,16 +355,6 @@ def _start_native(context: ApkContext, exporter: NativeExporter | None) -> None: context.native_runner = runner -def _native_min_free_mb() -> int: - raw = os.environ.get("TOCODE_APK_NATIVE_MIN_FREE_MB", "").strip() - if raw: - try: - return max(0, int(raw)) - except ValueError: - pass - return DEFAULT_NATIVE_MIN_FREE_MB - - def _join_native(context: ApkContext) -> None: runner = context.native_runner if runner is None: diff --git a/src/tocode/apk_native.py b/src/tocode/apk_native.py index bbb3b3a..a3af729 100644 --- a/src/tocode/apk_native.py +++ b/src/tocode/apk_native.py @@ -68,6 +68,9 @@ class NativeLib: seconds: float | None = None # Package of the APK the library was extracted from (for the nested AGENTS.md). package: str = "" + # Full provenance paragraph for the nested AGENTS.md; overrides the APK text + # (used when the library comes from a .NET bundle, package, or assembly). + origin: str | None = None @property def stem(self) -> str: @@ -140,8 +143,22 @@ def native_export_dir(root: Path, lib: NativeLib) -> Path: NativeExporter = Callable[[NativeLib, Path, Progress], tuple[str, str, int, int]] +def native_min_free_mb() -> int: + """Free-memory floor before each native export (0 disables the wait).""" + for name in ("TOCODE_NATIVE_MIN_FREE_MB", "TOCODE_APK_NATIVE_MIN_FREE_MB"): + raw = os.environ.get(name, "").strip() + if raw: + try: + return max(0, int(raw)) + except ValueError: + continue + return DEFAULT_NATIVE_MIN_FREE_MB + + def describe_origin(lib: NativeLib) -> str: """Provenance paragraph for the nested export's AGENTS.md.""" + if lib.origin: + return lib.origin package = f"`{lib.package}`" if lib.package else "an Android app" return ( f"This is a decompiled native library from the {package} APK: " @@ -451,11 +468,20 @@ class NativeRunner: max_wait_seconds: float = NATIVE_MEMORY_MAX_WAIT_SECONDS mirror_logs: bool = True current: NativeLib | None = None + waiting: tuple[NativeLib, int] | None = None current_started: float = 0.0 current_tail: NestedLogTail | None = None def describe_current(self) -> str | None: """One line about what the native thread is doing right now.""" + waiting = self.waiting + if waiting is not None: + pending, available = waiting + return ( + f"native: waiting for memory before {pending.name} ({pending.abi}): " + f"{available} MB free, need {self.min_free_mb} MB " + "(set TOCODE_NATIVE_MIN_FREE_MB to change)" + ) lib = self.current if lib is None: return None @@ -502,6 +528,7 @@ def _wait_for_memory(self, lib: NativeLib) -> None: available = self.available_memory() if available is None or available >= self.min_free_mb: break + self.waiting = (lib, available) if not logged: self.progress.log( f"native: waiting for memory before {lib.name} ({lib.abi}): " @@ -510,6 +537,7 @@ def _wait_for_memory(self, lib: NativeLib) -> None: logged = True self.stop_event.wait(self.poll_seconds) waited += self.poll_seconds + self.waiting = None if logged and not self.stop_event.is_set(): self.progress.file_log( f"native: waited {waited:.0f}s for memory before {lib.name}" diff --git a/src/tocode/backends/dotnet.py b/src/tocode/backends/dotnet.py new file mode 100644 index 0000000..0d945b7 --- /dev/null +++ b/src/tocode/backends/dotnet.py @@ -0,0 +1,1624 @@ +""".NET backend: dnlib inventory plus ICSharpCode.Decompiler C#/IL output. + +Both libraries ship with ToCode (``dotnet_lib/``) and are loaded in-process +through pythonnet, so exporting a .NET binary needs a .NET 9+ runtime and no +other tool. The parent process inventories every assembly with dnlib (types, +members, P/Invoke, resources, references) and scans IL bytes in pure Python for +call/string/field cross-references; spawned workers decompile types to C# and +disassemble them to IL (with raw bytecode), returning per-method line ranges. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +import importlib.util +import json +import os +from pathlib import Path +import re +import shutil +import sys +from typing import Any +import zipfile + +from ..errors import ToCodeError +from .dotnet_pe import ( + PE_MACHINES, + BundleManifest, + PeInfo, + extract_bundle_entry, + is_apphost_launcher, + iter_user_strings, + method_body, + read_bundle, + read_pe_info, + scan_il_tokens, +) + +DOTNET_LIB_DIR = Path(__file__).with_name("dotnet_lib") +DOTNET_LIBRARIES = ("dnlib.dll", "ICSharpCode.Decompiler.dll") +MIN_RUNTIME_MAJOR = 9 +NUPKG_SUFFIXES = frozenset({".nupkg", ".snupkg"}) +MAX_STRING_XREFS = 64 +MAX_FIELD_REFS = 32 + +# Assemblies and native libraries of the .NET runtime itself. A self-contained +# app or bundle carries a couple of hundred of them; they are inventoried and +# extracted but only decompiled with --include-framework. +_FRAMEWORK_ASSEMBLY_RX = re.compile( + r"^(System(\..+)?|mscorlib|netstandard|WindowsBase|PresentationCore|" + r"PresentationFramework|Microsoft\.(CSharp|VisualBasic(\..+)?|Win32\..+|" + r"NETCore\..+|Extensions\.DependencyModel))\.dll$", + re.IGNORECASE, +) +_FRAMEWORK_NATIVE_RX = re.compile( + r"^(lib)?(coreclr|clrjit|clrgc(exp)?|hostfxr|hostpolicy|mscordaccore|mscordbi|" + r"mscorrc|createdump|coreclrtraceptprovider|System(\.[A-Za-z]+)*\.Native|" + r"Microsoft\.DiaSymReader\.Native\.[a-z0-9]+|D3DCompiler_47_cor3|" + r"PenImc_cor3|PresentationNative_cor3|vcruntime140_cor3|wpfgfx_cor3)" + r"(\.so|\.dylib|\.dll)?$", + re.IGNORECASE, +) +_ELF_MACHINES = { + 0x03: "x86", + 0x28: "arm", + 0x3E: "x86_64", + 0xB7: "arm64", + 0xF3: "riscv64", +} + + +# -------------------------------------------------------------------------- +# Input classification +# -------------------------------------------------------------------------- + + +def is_nupkg(path: Path) -> bool: + return path.suffix.lower() in NUPKG_SUFFIXES + + +def is_dotnet_input(path: Path) -> bool: + """Managed PE, single-file bundle, apphost launcher, or NuGet package.""" + if is_nupkg(path): + return zipfile.is_zipfile(path) + info = read_pe_info(path) + if info is not None and info.is_managed: + return True + if read_bundle(path) is not None: + return True + return _apphost_assembly(path) is not None + + +def _apphost_assembly(path: Path) -> Path | None: + if not is_apphost_launcher(path): + return None + stem = path.name[:-4] if path.name.lower().endswith(".exe") else path.name + candidate = path.with_name(f"{stem}.dll") + info = read_pe_info(candidate) if candidate.is_file() else None + return candidate if info is not None and info.is_managed else None + + +def native_arch(path: Path) -> str: + try: + head = path.read_bytes()[:0x400] if path.stat().st_size < 0x400 else None + if head is None: + with path.open("rb") as handle: + head = handle.read(0x400) + except OSError: + return "unknown" + if head[:4] == b"\x7fELF" and len(head) >= 20: + return _ELF_MACHINES.get(int.from_bytes(head[18:20], "little"), "unknown") + info = read_pe_info(path) + if info is not None: + return info.machine + if head[:4] in (b"\xcf\xfa\xed\xfe", b"\xce\xfa\xed\xfe"): + cpu = int.from_bytes(head[4:8], "little") + return {0x01000007: "x86_64", 0x0100000C: "arm64", 7: "x86", 12: "arm"}.get( + cpu, "unknown" + ) + return "unknown" + + +def is_native_binary(path: Path) -> bool: + try: + with path.open("rb") as handle: + head = handle.read(4) + except OSError: + return False + if head[:4] == b"\x7fELF" or head[:4] in ( + b"\xcf\xfa\xed\xfe", + b"\xce\xfa\xed\xfe", + b"\xca\xfe\xba\xbe", + ): + return True + if head[:2] == b"MZ": + info = read_pe_info(path) + return info is not None and not info.is_managed + return False + + +def is_framework_assembly(name: str) -> bool: + return bool(_FRAMEWORK_ASSEMBLY_RX.match(Path(name).name)) + + +def is_framework_native(name: str) -> bool: + return bool(_FRAMEWORK_NATIVE_RX.match(Path(name).name)) + + +# -------------------------------------------------------------------------- +# Input discovery +# -------------------------------------------------------------------------- + + +@dataclass(slots=True) +class AssemblyFile: + path: Path + entry: str # path inside the container (or the file name) + source: str # container the assembly came from + framework: bool = False + include: bool = True + reason: str | None = None + tfm: str | None = None + + +@dataclass(slots=True) +class NativeFile: + name: str + path: Path + entry: str + source: str + arch: str + framework: bool = False + reason: str | None = None + + +@dataclass(slots=True) +class ExtraFile: + entry: str + path: Path + kind: str + + +@dataclass(slots=True) +class DotnetInput: + kind: str # assembly | apphost | bundle | nupkg + input: Path + assemblies: list[AssemblyFile] + natives: list[NativeFile] = field(default_factory=list) + extras: list[ExtraFile] = field(default_factory=list) + search_dirs: list[Path] = field(default_factory=list) + bundle: BundleManifest | None = None + workdir: Path | None = None + + @property + def included(self) -> list[AssemblyFile]: + return [item for item in self.assemblies if item.include] + + +def discover_dotnet_input( + path: Path, *, workdir: Path, include_framework: bool = False +) -> DotnetInput: + path = Path(path).resolve() + if is_nupkg(path): + return _discover_nupkg(path, workdir, include_framework) + info = read_pe_info(path) + if info is not None and info.is_managed: + assembly = AssemblyFile(path=path, entry=path.name, source=path.name) + found = DotnetInput("assembly", path, [assembly], search_dirs=[path.parent]) + if info.mixed_mode: + found.natives.append( + NativeFile( + name=path.name, + path=path, + entry="(native code of the mixed-mode assembly)", + source=path.name, + arch=info.machine, + ) + ) + return found + bundle = read_bundle(path) + if bundle is not None: + return _discover_bundle(path, bundle, workdir, include_framework) + sibling = _apphost_assembly(path) + if sibling is not None: + assembly = AssemblyFile( + path=sibling, entry=sibling.name, source=f"{path.name} (apphost launcher)" + ) + return DotnetInput("apphost", path, [assembly], search_dirs=[path.parent]) + raise ToCodeError(f"not a .NET assembly, bundle, or NuGet package: {path}") + + +def _framework_names_from_deps(deps_path: Path) -> set[str]: + try: + deps = json.loads(deps_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return set() + names: set[str] = set() + libraries = deps.get("libraries", {}) + for target in deps.get("targets", {}).values(): + for key, value in target.items(): + library = libraries.get(key, {}) + if library.get("type") != "runtimepack" and not key.lower().startswith( + "runtimepack." + ): + continue + for group in ("runtime", "native"): + for asset in value.get(group, {}): + names.add(Path(asset).name.lower()) + return names + + +def _discover_bundle( + path: Path, bundle: BundleManifest, workdir: Path, include_framework: bool +) -> DotnetInput: + root = workdir / "bundle" + found = DotnetInput("bundle", path, [], bundle=bundle, workdir=workdir) + found.search_dirs.append(root) + extracted: list[tuple[Any, Path]] = [] + for entry in bundle.entries: + target = _safe_join(root, entry.path) + if target is None: + continue + extract_bundle_entry(path, entry, target) + extracted.append((entry, target)) + deps = next( + (target for entry, target in extracted if entry.kind == "deps-json"), None + ) + framework_names = _framework_names_from_deps(deps) if deps else set() + for entry, target in extracted: + lower = Path(entry.path).name.lower() + info = read_pe_info(target) if entry.kind in {"assembly", "unknown"} else None + if info is not None and info.is_managed: + framework = lower in framework_names or is_framework_assembly(lower) + found.assemblies.append( + AssemblyFile( + path=target, + entry=entry.path, + source=path.name, + framework=framework, + include=include_framework or not framework, + reason=None + if include_framework or not framework + else "framework assembly (use --include-framework)", + ) + ) + elif entry.kind == "native" or is_native_binary(target): + framework = lower in framework_names or is_framework_native(lower) + found.natives.append( + NativeFile( + name=Path(entry.path).name, + path=target, + entry=entry.path, + source=path.name, + arch=native_arch(target), + framework=framework, + reason="framework native library (use --include-framework)" + if framework and not include_framework + else None, + ) + ) + else: + found.extras.append(ExtraFile(entry.path, target, entry.kind)) + if not found.assemblies: + raise ToCodeError(f"single-file bundle contains no managed assemblies: {path}") + return found + + +def _tfm_rank(tfm: str) -> tuple[int, int, int]: + """Order target frameworks: net5.0+ > netcoreapp > netstandard > .NET Framework. + + Modern TFMs carry a dot (``net6.0``, ``net10.0-windows``); .NET Framework + ones do not (``net35``, ``net472``). + """ + text = tfm.lower() + for family, rank in (("net", 3), ("netcoreapp", 2), ("netstandard", 1)): + match = re.match(rf"^{family}(\d+)\.(\d+)", text) + if match: + return (rank, int(match.group(1)), int(match.group(2))) + match = re.match(r"^net(\d)(\d)(\d)?$", text) + if match: + return ( + 0, + int(match.group(1)), + int(match.group(2)) * 10 + int(match.group(3) or 0), + ) + return (-1, 0, 0) + + +def _discover_nupkg(path: Path, workdir: Path, include_framework: bool) -> DotnetInput: + root = workdir / "nupkg" + found = DotnetInput("nupkg", path, [], workdir=workdir) + with zipfile.ZipFile(path) as archive: + for info in archive.infolist(): + if info.is_dir(): + continue + target = _safe_join(root, info.filename) + if target is None: + continue + target.parent.mkdir(parents=True, exist_ok=True) + with archive.open(info) as source, target.open("wb") as sink: + shutil.copyfileobj(source, sink, 1 << 20) + parts = info.filename.split("/") + pe = read_pe_info(target) + if pe is not None and pe.is_managed: + tfm = _nupkg_tfm(parts) + found.assemblies.append( + AssemblyFile( + path=target, + entry=info.filename, + source=path.name, + framework=is_framework_assembly(parts[-1]), + tfm=tfm, + ) + ) + elif is_native_binary(target): + found.natives.append( + NativeFile( + name=parts[-1], + path=target, + entry=info.filename, + source=path.name, + arch=_nupkg_rid(parts) or native_arch(target), + ) + ) + else: + found.extras.append(ExtraFile(info.filename, target, "package-file")) + if not found.assemblies: + raise ToCodeError(f"NuGet package contains no managed assemblies: {path}") + # One copy per assembly name: the newest target framework wins; reference + # assemblies (ref/) only when nothing else exists. + best: dict[str, AssemblyFile] = {} + for item in found.assemblies: + key = Path(item.entry).name.lower() + current = best.get(key) + if current is None or _nupkg_preference(item) > _nupkg_preference(current): + best[key] = item + for item in found.assemblies: + chosen = best[Path(item.entry).name.lower()] + if item is not chosen: + item.include = False + item.reason = f"duplicate of {chosen.entry} (newer target framework)" + elif item.framework and not include_framework: + item.include = False + item.reason = "framework assembly (use --include-framework)" + found.search_dirs.extend(sorted({item.path.parent for item in found.included})) + return found + + +def _nupkg_tfm(parts: list[str]) -> str | None: + for marker in ("lib", "ref", "tools"): + if marker in parts: + index = parts.index(marker) + if index + 1 < len(parts) - 1: + return parts[index + 1] + return None + + +def _nupkg_rid(parts: list[str]) -> str | None: + if len(parts) > 2 and parts[0] == "runtimes": + return parts[1] + return None + + +def _nupkg_preference(item: AssemblyFile) -> tuple[int, tuple[int, int, int]]: + top = item.entry.split("/", 1)[0] + kind = {"lib": 3, "runtimes": 2, "tools": 2, "ref": 0}.get(top, 1) + return (kind, _tfm_rank(item.tfm or "")) + + +def _safe_join(root: Path, name: str) -> Path | None: + parts = [ + part + for part in name.replace("\\", "/").split("/") + if part not in {"", ".", ".."} + ] + if not parts: + return None + return root.joinpath(*parts) + + +def pinvoke_neighbours( + input_dir: Path, module_names: set[str] +) -> list[tuple[str, Path]]: + """Native libraries next to the input that P/Invoke declarations name.""" + found: list[tuple[str, Path]] = [] + seen: set[Path] = set() + for module in sorted(module_names): + base = Path(module).name + stem = base + for suffix in (".so", ".dll", ".dylib"): + if stem.lower().endswith(suffix): + stem = stem[: -len(suffix)] + candidates = { + base, + f"{stem}.so", + f"lib{stem}.so", + f"{stem}.dll", + f"{stem}.dylib", + f"lib{stem}.dylib", + } + for name in sorted(candidates): + path = (input_dir / name).resolve() + if path in seen or not path.is_file(): + continue + if is_native_binary(path): + seen.add(path) + found.append((module, path)) + return found + + +# -------------------------------------------------------------------------- +# .NET runtime (pythonnet) +# -------------------------------------------------------------------------- + +_runtime: dict[str, Any] | None = None + + +def _version_tuple(text: str) -> tuple[int, ...]: + numbers = re.findall(r"\d+", text.split("-", 1)[0]) + return tuple(int(item) for item in numbers[:3]) + + +def dotnet_root_candidates() -> list[Path]: + candidates: list[Path] = [] + for name in ("DOTNET_ROOT", "DOTNET_ROOT_ARM64", "DOTNET_ROOT_X64"): + value = os.environ.get(name, "").strip() + if value: + candidates.append(Path(value).expanduser()) + exe = shutil.which("dotnet") + if exe: + candidates.append(Path(exe).resolve().parent) + candidates.extend( + Path(item) + for item in ( + "/usr/lib/dotnet", + "/usr/share/dotnet", + "/usr/local/share/dotnet", + "/opt/dotnet", + str(Path.home() / ".dotnet"), + "C:/Program Files/dotnet", + ) + ) + return candidates + + +def find_runtime() -> tuple[Path, str] | None: + """Newest installed Microsoft.NETCore.App >= 9 as ``(dotnet root, version)``.""" + best: tuple[tuple[int, ...], Path, str] | None = None + for root in dotnet_root_candidates(): + shared = root / "shared" / "Microsoft.NETCore.App" + if not shared.is_dir(): + continue + for item in shared.iterdir(): + version = _version_tuple(item.name) + if not version or version[0] < MIN_RUNTIME_MAJOR: + continue + if not (item / "System.Private.CoreLib.dll").is_file(): + continue + if best is None or version > best[0]: + best = (version, root, item.name) + if best is None: + return None + return best[1], best[2] + + +def probe_dotnet() -> tuple[bool, str]: + if importlib.util.find_spec("pythonnet") is None: + return False, "pythonnet is not installed (pip install pythonnet)" + missing = [ + name for name in DOTNET_LIBRARIES if not (DOTNET_LIB_DIR / name).is_file() + ] + if missing: + return False, f"bundled .NET libraries missing: {', '.join(missing)}" + found = find_runtime() + if found is None: + return ( + False, + f"no .NET runtime >= {MIN_RUNTIME_MAJOR} found (install e.g. " + "dotnet-runtime-10.0, or set DOTNET_ROOT)", + ) + return True, f".NET {found[1]} at {found[0]}" + + +def load_runtime() -> dict[str, Any]: + """Load CoreCLR and the bundled assemblies once per process.""" + global _runtime + if _runtime is not None: + return _runtime + ok, reason = probe_dotnet() + if not ok: + raise ToCodeError(f".NET input requires the .NET backend: {reason}") + found = find_runtime() + assert found is not None + root, version = found + os.environ.setdefault("DOTNET_ROOT", str(root)) + # Keep the runtime quiet and lean inside a Python host. + os.environ.setdefault("DOTNET_CLI_TELEMETRY_OPTOUT", "1") + os.environ.setdefault("DOTNET_gcServer", "0") + import pythonnet + + try: + pythonnet.load("coreclr", dotnet_root=str(root)) + except RuntimeError as exc: + if "already" not in str(exc).lower(): + raise ToCodeError(f"cannot load the .NET runtime: {exc}") from exc + import clr # type: ignore[import-not-found] + + for name in DOTNET_LIBRARIES: + clr.AddReference(str(DOTNET_LIB_DIR / name)) + import System # type: ignore[import-not-found] + + actual = str(System.Environment.Version) + if _version_tuple(actual)[:1] < (MIN_RUNTIME_MAJOR,): + raise ToCodeError( + f".NET runtime {actual} was loaded; ToCode needs {MIN_RUNTIME_MAJOR}+" + ) + _runtime = { + "clr": clr, + "System": System, + "root": root, + "version": actual, + "framework_dir": root / "shared" / "Microsoft.NETCore.App" / version, + } + return _runtime + + +# -------------------------------------------------------------------------- +# Inventory records +# -------------------------------------------------------------------------- + + +@dataclass(slots=True) +class FieldInfo: + token: int + name: str + type: str + flags: list[str] + constant: str | None = None + + +@dataclass(slots=True) +class PropertyInfo: + token: int + name: str + type: str + getter: int | None + setter: int | None + + +@dataclass(slots=True) +class EventInfo: + token: int + name: str + type: str + add: int | None + remove: int | None + + +@dataclass(slots=True) +class MethodInfo: + id: int + assembly: int + token: int + name: str + type_token: int + type_name: str + full_name: str + return_type: str + params: list[tuple[str, str]] + flags: list[str] + impl_flags: list[str] + rva: int + il_size: int + is_static: bool + is_virtual: bool + is_abstract: bool + is_constructor: bool + compiler_generated: bool + pinvoke_module: str | None = None + pinvoke_entry: str | None = None + unmanaged_entry: str | None = None + state_machine_type: int | None = None + owner_method: int | None = None # state machine / closure owner (method id) + attributes: list[str] = field(default_factory=list) + callees: list[int] = field(default_factory=list) + external_calls: list[str] = field(default_factory=list) + callers: list[int] = field(default_factory=list) + string_ref_count: int = 0 + field_refs: list[str] = field(default_factory=list) + cs_start: int | None = None + cs_end: int | None = None + il_start: int | None = None + il_end: int | None = None + + @property + def address(self) -> str: + return f"{self.assembly_name}!0x{self.token:08x}" + + # Set by the session after inventory (kept out of __init__ for brevity). + assembly_name: str = "" + + @property + def prototype(self) -> str: + params = ", ".join(f"{kind} {name}" for name, kind in self.params) + return f"{self.return_type} {self.name}({params})" + + +@dataclass(slots=True) +class TypeInfo: + assembly: int + token: int + full_name: str # dnlib style, nested with '/' + reflection_name: str # nested with '+' + namespace: str + name: str + kind: str + flags: list[str] + visibility: str + base_type: str | None + interfaces: list[str] + generic_params: list[str] + declaring_type: int | None + top_level: int # token of the top-level type that owns the file + attributes: list[str] + fields: list[FieldInfo] + properties: list[PropertyInfo] + events: list[EventInfo] + methods: list[int] # method ids + compiler_generated: bool = False + cs_path: Path | None = None + il_path: Path | None = None + cs_lines: int = 0 + il_lines: int = 0 + error: str | None = None + il_error: str | None = None + + +@dataclass(slots=True) +class ResourceInfo: + name: str + kind: str # embedded | linked | assembly-linked + size: int | None + public: bool + file: str | None = None + path: Path | None = None + entries: list[dict[str, Any]] | None = None + error: str | None = None + + +@dataclass(slots=True) +class AssemblyInfo: + index: int + file: AssemblyFile + pe: PeInfo + name: str + full_name: str + version: str + culture: str + public_key_token: str | None + module_name: str + mvid: str + kind: str + target_framework: str | None + entry_point: int | None + assembly_refs: list[dict[str, Any]] + module_refs: list[str] + attributes: list[str] + resources: list[ResourceInfo] + folder: str = "" + types: list[int] = field(default_factory=list) # tokens + user_strings: list[tuple[int, str]] = field(default_factory=list) + obfuscation_hints: list[str] = field(default_factory=list) + error: str | None = None + + +def _str(value: Any) -> str: + return "" if value is None else str(value) + + +def _flag_list(value: Any) -> list[str]: + text = _str(value) + return [ + item.strip() for item in text.split(",") if item.strip() and item.strip() != "0" + ] + + +def _attribute_text(attribute: Any) -> str: + try: + name = _str(attribute.TypeFullName) + args = [] + for arg in attribute.ConstructorArguments: + args.append(_short_value(arg.Value)) + for named in attribute.NamedArguments: + args.append(f"{_str(named.Name)}={_short_value(named.Argument.Value)}") + return f"{name}({', '.join(args)})" if args else name + except Exception: # pragma: no cover - malformed blobs + return "" + + +def _short_value(value: Any) -> str: + text = _str(value) + return text if len(text) <= 200 else text[:197] + "..." + + +def _type_sig_token(signature: Any) -> int | None: + """TypeDef token behind a ``typeof(X)`` attribute argument (a TypeSig).""" + target = getattr(signature, "TypeDefOrRef", None) + if target is None: + return None + token = int(target.MDToken.Raw) + if token >> 24 == 0x02: + return token + try: + resolved = target.ResolveTypeDef() + except Exception: + return None + return int(resolved.MDToken.Raw) if resolved is not None else None + + +def type_kind(type_def: Any) -> str: + if type_def.IsInterface: + return "interface" + if type_def.IsEnum: + return "enum" + if type_def.IsDelegate: + return "delegate" + if type_def.IsValueType: + return "struct" + if type_def.IsAbstract and type_def.IsSealed: + return "static class" + return "class" + + +_COMPILER_GENERATED = "System.Runtime.CompilerServices.CompilerGeneratedAttribute" +_GENERATED_MEMBER_RX = re.compile(r"^<([^>]+)>(?:[bg]__|$)") +_STATE_MACHINE_ATTRIBUTES = { + "System.Runtime.CompilerServices.AsyncStateMachineAttribute", + "System.Runtime.CompilerServices.IteratorStateMachineAttribute", + "System.Runtime.CompilerServices.AsyncIteratorStateMachineAttribute", +} + + +class DotnetSession: + """dnlib-backed inventory over every included assembly.""" + + backend_name = "dotnet" + backend_label = ".NET" + decompiler_label = "ICSharpCode.Decompiler + dnlib" + + def __init__(self, found: DotnetInput) -> None: + self.input = found + self.assemblies: list[AssemblyInfo] = [] + self.types: dict[tuple[int, int], TypeInfo] = {} + self.methods: list[MethodInfo] = [] + self._method_by_token: dict[tuple[int, int], MethodInfo] = {} + self._method_by_name: dict[str, MethodInfo] = {} + # (assembly index, #US offset) -> ids of the methods that load it + self.string_xrefs: dict[tuple[int, int], list[int]] = {} + self.external_refs: dict[str, dict[str, list[int]]] = {} + self.pinvoke_modules: set[str] = set() + + # -- loading ----------------------------------------------------------- + + def load(self, progress: Any = None) -> None: + runtime = load_runtime() + from dnlib.DotNet import ModuleDefMD # type: ignore[import-not-found] + + modules: list[Any] = [] + for index, item in enumerate(self.input.included): + pe = read_pe_info(item.path) + if pe is None: + continue + try: + module = ModuleDefMD.Load(str(item.path)) + except Exception as exc: + raise ToCodeError(f"dnlib cannot load {item.entry}: {exc}") from exc + info = self._assembly_info(len(self.assemblies), item, pe, module) + self.assemblies.append(info) + modules.append(module) + self._types_and_methods(info, module) + del runtime + for info, module in zip(self.assemblies, modules): + self._link_owners(info) + self._scan_references(info, module) + self._modules = modules + + def _assembly_info( + self, index: int, item: AssemblyFile, pe: PeInfo, module: Any + ) -> AssemblyInfo: + assembly = module.Assembly + target_framework = None + attributes: list[str] = [] + if assembly is not None: + for attribute in assembly.CustomAttributes: + text = _attribute_text(attribute) + attributes.append(text) + if _str(attribute.TypeFullName).endswith("TargetFrameworkAttribute"): + args = list(attribute.ConstructorArguments) + if args: + target_framework = _str(args[0].Value) + for attribute in module.CustomAttributes: + attributes.append(f"module: {_attribute_text(attribute)}") + refs = [] + for ref in module.GetAssemblyRefs(): + refs.append( + { + "name": _str(ref.Name), + "version": _str(ref.Version), + "culture": _str(ref.Culture) or None, + "public_key_token": _str(ref.PublicKeyOrToken.Token) + if ref.PublicKeyOrToken is not None + else None, + "full_name": _str(ref.FullName), + } + ) + module_refs = sorted({_str(ref.Name) for ref in module.GetModuleRefs()}) + entry = module.ManagedEntryPoint + entry_token = int(entry.MDToken.Raw) if entry is not None else None + name = _str(assembly.Name) if assembly is not None else _str(module.Name) + info = AssemblyInfo( + index=index, + file=item, + pe=pe, + name=name or item.path.stem, + full_name=_str(assembly.FullName) + if assembly is not None + else _str(module.Name), + version=_str(assembly.Version) if assembly is not None else "", + culture=_str(assembly.Culture) if assembly is not None else "", + public_key_token=_str(assembly.PublicKeyToken) + if assembly is not None and assembly.PublicKeyToken is not None + else None, + module_name=_str(module.Name), + mvid=_str(module.Mvid), + kind=_str(module.Kind), + target_framework=target_framework, + entry_point=entry_token, + assembly_refs=refs, + module_refs=module_refs, + attributes=attributes, + resources=[], + ) + for resource in module.Resources: + kind = _str(resource.ResourceType).lower() + size = None + file_name = None + if "embedded" in kind: + try: + size = int(resource.Length) + except Exception: + size = None + elif "linked" in kind and "assembly" not in kind: + file_name = _str(getattr(resource, "FileName", None)) or None + info.resources.append( + ResourceInfo( + name=_str(resource.Name), + kind="embedded" + if "embedded" in kind + else ("assembly-linked" if "assembly" in kind else "linked"), + size=size, + public=bool(resource.IsPublic), + file=file_name, + ) + ) + heap = pe.stream("#US") + if heap is not None: + data = item.path.read_bytes()[heap.offset : heap.offset + heap.size] + info.user_strings = iter_user_strings(data) + return info + + def _types_and_methods(self, info: AssemblyInfo, module: Any) -> None: + odd_names = 0 + total = 0 + for type_def in module.GetTypes(): + token = int(type_def.MDToken.Raw) + top = type_def + while top.DeclaringType is not None: + top = top.DeclaringType + name = _str(type_def.Name) + total += 1 + if name and not re.match(r"^[A-Za-z_<][\w`<>.$-]*$", name): + odd_names += 1 + attributes = [_attribute_text(item) for item in type_def.CustomAttributes] + compiler_generated = name.startswith("<") or any( + item.startswith(_COMPILER_GENERATED) for item in attributes + ) + record = TypeInfo( + assembly=info.index, + token=token, + full_name=_str(type_def.FullName), + reflection_name=_str(type_def.ReflectionFullName), + namespace=_str(top.Namespace), + name=name, + kind=type_kind(type_def), + flags=_flag_list(type_def.Attributes), + visibility=_str(type_def.Visibility), + base_type=_str(type_def.BaseType.FullName) + if type_def.BaseType is not None + else None, + interfaces=[ + _str(item.Interface.FullName) + for item in type_def.Interfaces + if item.Interface is not None + ], + generic_params=[_str(item.Name) for item in type_def.GenericParameters], + declaring_type=int(type_def.DeclaringType.MDToken.Raw) + if type_def.DeclaringType is not None + else None, + top_level=int(top.MDToken.Raw), + attributes=attributes, + fields=[self._field(item) for item in type_def.Fields], + properties=[self._property(item) for item in type_def.Properties], + events=[self._event(item) for item in type_def.Events], + methods=[], + compiler_generated=compiler_generated, + ) + self.types[(info.index, token)] = record + info.types.append(token) + for method in type_def.Methods: + self._method(info, record, method) + if total and odd_names / total > 0.2: + info.obfuscation_hints.append( + f"{odd_names}/{total} type names are not valid C# identifiers" + ) + for attribute in info.attributes: + lowered = attribute.lower() + if any( + marker in lowered + for marker in ( + "confusedby", + "dotfuscator", + "obfuscat", + "suppressildasm", + ) + ): + info.obfuscation_hints.append(f"attribute: {attribute}") + + def _field(self, item: Any) -> FieldInfo: + constant = None + if item.HasConstant and item.Constant is not None: + constant = _short_value(item.Constant.Value) + return FieldInfo( + token=int(item.MDToken.Raw), + name=_str(item.Name), + type=_str(item.FieldType.FullName) if item.FieldType is not None else "?", + flags=_flag_list(item.Attributes), + constant=constant, + ) + + def _property(self, item: Any) -> PropertyInfo: + getter = item.GetMethod + setter = item.SetMethod + signature = item.PropertySig + return PropertyInfo( + token=int(item.MDToken.Raw), + name=_str(item.Name), + type=_str(signature.RetType.FullName) + if signature is not None and signature.RetType is not None + else "?", + getter=int(getter.MDToken.Raw) if getter is not None else None, + setter=int(setter.MDToken.Raw) if setter is not None else None, + ) + + def _event(self, item: Any) -> EventInfo: + add = item.AddMethod + remove = item.RemoveMethod + return EventInfo( + token=int(item.MDToken.Raw), + name=_str(item.Name), + type=_str(item.EventType.FullName) if item.EventType is not None else "?", + add=int(add.MDToken.Raw) if add is not None else None, + remove=int(remove.MDToken.Raw) if remove is not None else None, + ) + + def _method(self, info: AssemblyInfo, owner: TypeInfo, method: Any) -> None: + signature = method.MethodSig + params: list[tuple[str, str]] = [] + if signature is not None: + names = [ + _str(item.Name) + for item in method.Parameters + if not item.IsHiddenThisParameter + ] + for position, kind in enumerate(signature.Params): + label = ( + names[position] + if position < len(names) and names[position] + else f"p{position}" + ) + params.append((label, _str(kind.FullName))) + attributes = [_attribute_text(item) for item in method.CustomAttributes] + state_machine = None + unmanaged_entry = None + for attribute in method.CustomAttributes: + type_name = _str(attribute.TypeFullName) + if type_name in _STATE_MACHINE_ATTRIBUTES: + args = list(attribute.ConstructorArguments) + if args and args[0].Value is not None: + state_machine = _type_sig_token(args[0].Value) + elif type_name.endswith("UnmanagedCallersOnlyAttribute"): + for named in attribute.NamedArguments: + if _str(named.Name) == "EntryPoint": + unmanaged_entry = _str(named.Argument.Value) or None + impl_map = method.ImplMap + pinvoke_module = None + pinvoke_entry = None + if impl_map is not None: + pinvoke_module = ( + _str(impl_map.Module.Name) if impl_map.Module is not None else None + ) + pinvoke_entry = _str(impl_map.Name) or _str(method.Name) + if pinvoke_module: + self.pinvoke_modules.add(pinvoke_module) + body_size = 0 + rva = int(method.RVA) + name = _str(method.Name) + record = MethodInfo( + id=len(self.methods), + assembly=info.index, + token=int(method.MDToken.Raw), + name=name, + type_token=owner.token, + type_name=owner.full_name, + full_name=_str(method.FullName), + return_type=_str(signature.RetType.FullName) + if signature is not None and signature.RetType is not None + else "System.Void", + params=params, + flags=_flag_list(method.Attributes), + impl_flags=_flag_list(method.ImplAttributes), + rva=rva, + il_size=body_size, + is_static=bool(method.IsStatic), + is_virtual=bool(method.IsVirtual), + is_abstract=bool(method.IsAbstract), + is_constructor=bool(method.IsConstructor), + compiler_generated=owner.compiler_generated + or name.startswith("<") + or any(item.startswith(_COMPILER_GENERATED) for item in attributes), + pinvoke_module=pinvoke_module, + pinvoke_entry=pinvoke_entry, + unmanaged_entry=unmanaged_entry, + state_machine_type=state_machine, + attributes=attributes, + ) + record.assembly_name = info.name + self.methods.append(record) + owner.methods.append(record.id) + self._method_by_token[(info.index, record.token)] = record + self._method_by_name.setdefault(f"{info.name}|{record.full_name}", record) + + def _link_owners(self, info: AssemblyInfo) -> None: + """Point compiler-generated methods back to the method that owns them. + + State machines come from [AsyncStateMachine]/[IteratorStateMachine]; + lambdas (``b__N``) and local functions (``g__Name|N``) + are matched by the C# compiler's naming scheme against the enclosing + type and, for closure classes (``<>c``, ``<>c__DisplayClass``), its + declaring type. + """ + for method in self.methods: + if method.assembly != info.index or method.state_machine_type is None: + continue + machine = self.types.get((info.index, method.state_machine_type)) + if machine is None: + continue + for member in machine.methods: + self.methods[member].owner_method = method.id + for method in self.methods: + if method.assembly != info.index or method.owner_method is not None: + continue + match = _GENERATED_MEMBER_RX.match(method.name) + if match is None: + continue + owner_name = match.group(1) + record = self.types.get((info.index, method.type_token)) + while record is not None: + candidates = [ + self.methods[item] + for item in record.methods + if self.methods[item].name == owner_name and item != method.id + ] + if candidates: + method.owner_method = candidates[0].id + break + parent = record.declaring_type + record = self.types.get((info.index, parent)) if parent else None + + # -- references --------------------------------------------------------- + + def _scan_references(self, info: AssemblyInfo, module: Any) -> None: + data = info.file.path.read_bytes() + member_refs = self._member_refs(info, module) + method_specs = self._method_specs(module) + field_names = { + field_info.token: f"{record.full_name}::{field_info.name}" + for (assembly, _), record in self.types.items() + if assembly == info.index + for field_info in record.fields + } + for method in self.methods: + if method.assembly != info.index or method.rva == 0: + continue + offset = info.pe.rva_to_offset(method.rva) + if offset is None: + continue + code = method_body(data, offset) + if code is None: + continue + method.il_size = len(code) + callees: list[int] = [] + external: list[str] = [] + fields: list[str] = [] + seen: set[int] = set() + strings = 0 + for kind, token in scan_il_tokens(code): + if kind == "string": + strings += 1 + bucket = self.string_xrefs.setdefault( + (info.index, token & 0xFFFFFF), [] + ) + if len(bucket) < MAX_STRING_XREFS and method.id not in bucket: + bucket.append(method.id) + continue + if token in seen: + continue + seen.add(token) + if kind == "method": + table = token >> 24 + if table == 0x2B: + token = method_specs.get(token & 0xFFFFFF, token) + table = token >> 24 + if table == 0x06: + target = self._method_by_token.get((info.index, token)) + if target is not None: + callees.append(target.id) + target.callers.append(method.id) + continue + if table == 0x0A: + ref = member_refs.get(token) + if ref is None: + continue + full_name, scope, internal = ref + target = internal + if target is None: + target = self._method_by_name.get(f"{scope}|{full_name}") + if target is not None: + callees.append(target.id) + target.callers.append(method.id) + else: + external.append(f"[{scope}]{full_name}") + refs = self.external_refs.setdefault(scope, {}).setdefault( + full_name, [] + ) + if len(refs) < 32: + refs.append(method.id) + elif kind == "field" and len(fields) < MAX_FIELD_REFS: + table = token >> 24 + if table == 0x04: + name = field_names.get(token) + if name: + fields.append(name) + elif table == 0x0A: + ref = member_refs.get(token) + if ref is not None: + fields.append(f"[{ref[1]}]{ref[0]}") + method.callees = callees + method.external_calls = external + method.string_ref_count = strings + method.field_refs = fields + + def _member_refs( + self, info: AssemblyInfo, module: Any + ) -> dict[int, tuple[str, str, MethodInfo | None]]: + result: dict[int, tuple[str, str, MethodInfo | None]] = {} + rows = int(module.Metadata.TablesStream.MemberRefTable.Rows) + for rid in range(1, rows + 1): + try: + ref = module.ResolveMemberRef(rid) + except Exception: + continue + if ref is None: + continue + scope = info.name + internal: MethodInfo | None = None + declaring = ref.DeclaringType + try: + if declaring is not None and declaring.DefinitionAssembly is not None: + scope = _str(declaring.DefinitionAssembly.Name) + elif ( + ref.Class is not None + and _str(ref.Class.GetType().Name) == "ModuleRefUser" + ): + scope = _str(ref.Class.Name) + except Exception: + pass + if scope == info.name and ref.IsMethodRef: + try: + resolved = ref.ResolveMethod() + except Exception: + resolved = None + if ( + resolved is not None + and resolved.Module is not None + and _str(resolved.Module.Name) == info.module_name + ): + internal = self._method_by_token.get( + (info.index, int(resolved.MDToken.Raw)) + ) + result[0x0A000000 | rid] = (_str(ref.FullName), scope, internal) + return result + + def _method_specs(self, module: Any) -> dict[int, int]: + result: dict[int, int] = {} + rows = int(module.Metadata.TablesStream.MethodSpecTable.Rows) + for rid in range(1, rows + 1): + try: + spec = module.ResolveMethodSpec(rid) + except Exception: + continue + if spec is not None and spec.Method is not None: + result[rid] = int(spec.Method.MDToken.Raw) + return result + + # -- resources ---------------------------------------------------------- + + def save_resource(self, info: AssemblyInfo, name: str, target: Path) -> int | None: + """Write an embedded resource's bytes to ``target``; its size or None.""" + import System # type: ignore[import-not-found] + + module = self._modules[info.index] + for resource in module.Resources: + if _str(resource.Name) != name: + continue + try: + raw = resource.CreateReader().ToArray() + except Exception: + return None + target.parent.mkdir(parents=True, exist_ok=True) + System.IO.File.WriteAllBytes(str(target), raw) + return int(raw.Length) + return None + + def decode_resources_file( + self, info: AssemblyInfo, name: str + ) -> list[dict[str, Any]]: + """Entries of a ``.resources`` blob via dnlib's ResourceReader.""" + from dnlib.DotNet.Resources import ResourceReader # type: ignore[import-not-found] + + module = self._modules[info.index] + resource = next(item for item in module.Resources if _str(item.Name) == name) + elements = ResourceReader.Read(module, resource.CreateReader()) + rows: list[dict[str, Any]] = [] + for element in elements.ResourceElements: + data = element.ResourceData + code = _str(data.Code) + value: Any + built_in = getattr(data, "Data", None) + if built_in is not None and not hasattr(built_in, "Length"): + value = _short_value(built_in) + else: + raw = getattr(data, "Data", None) + try: + value = ( + f"<{len(bytes(raw))} bytes>" if raw is not None else _str(data) + ) + except Exception: + value = _str(data) + rows.append({"name": _str(element.Name), "type": code, "value": value}) + return rows + + def close(self) -> None: + for module in getattr(self, "_modules", []): + try: + module.Dispose() + except Exception: + pass + self._modules = [] + + # -- lookups ------------------------------------------------------------ + + def method(self, assembly: int, token: int) -> MethodInfo | None: + return self._method_by_token.get((assembly, token)) + + def _type_index( + self, + ) -> tuple[dict[int, list[TypeInfo]], dict[tuple[int, int], list[TypeInfo]]]: + cached = getattr(self, "_index_cache", None) + if cached is not None and cached[0] == len(self.types): + return cached[1], cached[2] + tops: dict[int, list[TypeInfo]] = {} + groups: dict[tuple[int, int], list[TypeInfo]] = {} + for (index, _), record in self.types.items(): + if record.declaring_type is None: + tops.setdefault(index, []).append(record) + groups.setdefault((index, record.top_level), []).append(record) + self._index_cache = (len(self.types), tops, groups) + return tops, groups + + def top_level_types(self, assembly: int) -> list[TypeInfo]: + return self._type_index()[0].get(assembly, []) + + def nested_types(self, assembly: int, top: int) -> list[TypeInfo]: + """The top-level type ``top`` and every type nested in it.""" + return self._type_index()[1].get((assembly, top), []) + + +# -------------------------------------------------------------------------- +# Worker-side decompilation +# -------------------------------------------------------------------------- + +_METHOD_IL_RX = re.compile(r"^\s*\.method /\* (06[0-9A-Fa-f]{6}) \*/") +_END_METHOD_RX = re.compile(r"^\s*\} // end of method ") +_DECLARATION_NODES = { + "MethodDeclaration", + "ConstructorDeclaration", + "DestructorDeclaration", + "OperatorDeclaration", + "PropertyDeclaration", + "IndexerDeclaration", + "EventDeclaration", + "CustomEventDeclaration", + "Accessor", + "LambdaExpression", + "AnonymousMethodExpression", + "LocalFunctionDeclarationStatement", +} + + +@dataclass(slots=True) +class DecompiledType: + assembly: int + token: int + cs: str | None + cs_error: str | None + il: str | None + il_error: str | None + # metadata token -> (first line, last line) in the respective text + cs_ranges: dict[int, tuple[int, int]] + il_ranges: dict[int, tuple[int, int]] + + +@dataclass(slots=True) +class WorkItem: + assembly: int + path: str + token: int # 0 = assembly/module attributes + manifest + + +_worker: dict[str, Any] = {} + + +def init_dotnet_worker(search_dirs: list[str]) -> None: + load_runtime() + _worker["search_dirs"] = search_dirs + _worker["decompilers"] = {} + _worker["files"] = {} + + +def _decompiler_for(path: str) -> tuple[Any, Any, Any]: + cached = _worker["decompilers"].get(path) + if cached is not None: + return cached + from ICSharpCode.Decompiler import DecompilerSettings # type: ignore[import-not-found] + from ICSharpCode.Decompiler.CSharp import CSharpDecompiler # type: ignore[import-not-found] + from ICSharpCode.Decompiler.Metadata import ( # type: ignore[import-not-found] + DotNetCorePathFinderExtensions, + PEFile, + UniversalAssemblyResolver, + ) + + settings = DecompilerSettings() + settings.ThrowOnAssemblyResolveErrors = False + settings.ShowXmlDocumentation = False + pe = PEFile(path) + try: + target = DotNetCorePathFinderExtensions.DetectTargetFrameworkId(pe) + except Exception: + target = "" + resolver = UniversalAssemblyResolver(path, False, target) + for directory in _worker.get("search_dirs", []): + resolver.AddSearchDirectory(directory) + framework_dir = (_runtime or {}).get("framework_dir") + if framework_dir is not None: + resolver.AddSearchDirectory(str(framework_dir)) + decompiler = CSharpDecompiler(pe, resolver, settings) + cached = (decompiler, pe, settings) + _worker["decompilers"][path] = cached + return cached + + +def decompile_batch_in_worker(items: list[WorkItem]) -> list[DecompiledType]: + return [decompile_item(item) for item in items] + + +def decompile_item(item: WorkItem) -> DecompiledType: + result = DecompiledType(item.assembly, item.token, None, None, None, None, {}, {}) + try: + decompiler, pe, settings = _decompiler_for(item.path) + except Exception as exc: + result.cs_error = result.il_error = f"cannot open assembly: {_exc_text(exc)}" + return result + if item.token == 0: + try: + result.cs = str(decompiler.DecompileModuleAndAssemblyAttributesToString()) + except Exception as exc: + result.cs_error = _exc_text(exc) + try: + result.il = _manifest_il(pe) + except Exception as exc: + result.il_error = _exc_text(exc) + return result + try: + result.cs, result.cs_ranges = _decompile_type(decompiler, settings, item.token) + except Exception as exc: + result.cs_error = _exc_text(exc) + try: + result.il, result.il_ranges = _disassemble_type(pe, item.token) + except Exception as exc: + result.il_error = _exc_text(exc) + return result + + +def _exc_text(exc: BaseException) -> str: + text = str(exc).strip().splitlines() + first = text[0] if text else type(exc).__name__ + return first[:500] + + +def _decompile_type( + decompiler: Any, settings: Any, token: int +) -> tuple[str, dict[int, tuple[int, int]]]: + from System.IO import StringWriter # type: ignore[import-not-found] + from System.Reflection.Metadata.Ecma335 import MetadataTokens # type: ignore[import-not-found] + from ICSharpCode.Decompiler.CSharp import AnnotationExtensions # type: ignore[import-not-found] + from ICSharpCode.Decompiler.CSharp.OutputVisitor import ( # type: ignore[import-not-found] + CSharpOutputVisitor, + TokenWriter, + ) + from ICSharpCode.Decompiler.TypeSystem import IEntity, IEvent, IProperty # type: ignore[import-not-found] + + # DecompileTypes (unlike Decompile(EntityHandle)) wraps the type in its + # namespace declaration, as a project export would. + from System.Collections.Generic import List # type: ignore[import-not-found] + from System.Reflection.Metadata import TypeDefinitionHandle # type: ignore[import-not-found] + + handles = List[TypeDefinitionHandle]() + handles.Add(MetadataTokens.TypeDefinitionHandle(token & 0xFFFFFF)) + tree = decompiler.DecompileTypes(handles) + writer = StringWriter() + tree.AcceptVisitor( + CSharpOutputVisitor( + TokenWriter.CreateWriterThatSetsLocationsInAST(writer, " "), + settings.CSharpFormattingOptions, + ) + ) + ranges: dict[int, tuple[int, int]] = {} + + def record(entity: Any, start: int, end: int) -> None: + try: + value = int(MetadataTokens.GetToken(IEntity(entity).MetadataToken)) + except Exception: + return + if value and value not in ranges: + ranges[value] = (start, end) + + for node in tree.Descendants: + kind = type(node).__name__ + if kind not in _DECLARATION_NODES: + continue + start = int(node.StartLocation.Line) + end = int(node.EndLocation.Line) + if start <= 0: + continue + symbol = AnnotationExtensions.GetSymbol(node) + if symbol is None: + for annotation in node.Annotations: + if type(annotation).__name__ == "ILFunction": + method = getattr(annotation, "Method", None) + if method is not None: + record(method, start, end) + continue + record(symbol, start, end) + # Properties/events: map their accessor methods to the same span when + # the accessors have no separate Accessor node (expression bodies). + try: + prop = IProperty(symbol) + for accessor in (prop.Getter, prop.Setter): + if accessor is not None: + record(accessor, start, end) + except Exception: + pass + try: + event = IEvent(symbol) + for accessor in ( + event.AddAccessor, + event.RemoveAccessor, + event.InvokeAccessor, + ): + if accessor is not None: + record(accessor, start, end) + except Exception: + pass + text = str(writer.ToString()).replace("\r\n", "\n") + return text, ranges + + +def _new_disassembler(output: Any) -> Any: + from System.Threading import CancellationToken # type: ignore[import-not-found] + from ICSharpCode.Decompiler.Disassembler import ReflectionDisassembler # type: ignore[import-not-found] + + disassembler = ReflectionDisassembler(output, getattr(CancellationToken, "None")) + disassembler.ShowMetadataTokens = True + disassembler.ShowRawRVAOffsetAndBytes = True + disassembler.DetectControlStructure = True + disassembler.ExpandMemberDefinitions = True + return disassembler + + +def _disassemble_type(pe: Any, token: int) -> tuple[str, dict[int, tuple[int, int]]]: + from System.Reflection.Metadata.Ecma335 import MetadataTokens # type: ignore[import-not-found] + from ICSharpCode.Decompiler import PlainTextOutput # type: ignore[import-not-found] + + output = PlainTextOutput() + output.IndentationString = " " + _new_disassembler(output).DisassembleType( + pe, MetadataTokens.TypeDefinitionHandle(token & 0xFFFFFF) + ) + text = str(output.ToString()).replace("\r\n", "\n") + return text, il_method_ranges(text) + + +def _manifest_il(pe: Any) -> str: + from ICSharpCode.Decompiler import PlainTextOutput # type: ignore[import-not-found] + + output = PlainTextOutput() + output.IndentationString = " " + disassembler = _new_disassembler(output) + disassembler.WriteAssemblyReferences(pe.Metadata) + if pe.Metadata.IsAssembly: + disassembler.WriteAssemblyHeader(pe) + output.WriteLine() + disassembler.WriteModuleHeader(pe) + return str(output.ToString()).replace("\r\n", "\n") + + +def il_method_ranges(text: str) -> dict[int, tuple[int, int]]: + """Line ranges of every ``.method`` block in ReflectionDisassembler output.""" + ranges: dict[int, tuple[int, int]] = {} + current: tuple[int, int] | None = None + for number, line in enumerate(text.split("\n"), start=1): + match = _METHOD_IL_RX.match(line) + if match: + current = (int(match.group(1), 16), number) + continue + if current is not None and _END_METHOD_RX.match(line): + ranges[current[0]] = (current[1], number) + current = None + return ranges + + +def runtime_description() -> str: + runtime = _runtime or {} + return f".NET {runtime.get('version', '?')} ({sys.platform})" + + +__all__ = [ + "PE_MACHINES", + "AssemblyFile", + "AssemblyInfo", + "DecompiledType", + "DotnetInput", + "DotnetSession", + "MethodInfo", + "NativeFile", + "TypeInfo", + "WorkItem", + "decompile_batch_in_worker", + "discover_dotnet_input", + "init_dotnet_worker", + "is_dotnet_input", + "load_runtime", + "pinvoke_neighbours", + "probe_dotnet", +] diff --git a/src/tocode/backends/dotnet_pe.py b/src/tocode/backends/dotnet_pe.py new file mode 100644 index 0000000..0cfe765 --- /dev/null +++ b/src/tocode/backends/dotnet_pe.py @@ -0,0 +1,589 @@ +"""Pure-Python readers for .NET containers: PE/CLR headers, single-file bundles. + +Nothing here loads the .NET runtime, so input routing (``is_dotnet_input``) +stays cheap for every binary ToCode is pointed at, and the readers are +testable without a .NET install. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +import mmap +from pathlib import Path +import struct +import zlib + +# IMAGE_DIRECTORY_ENTRY_COM_DESCRIPTOR +_CLR_DIRECTORY = 14 +COMIMAGE_FLAGS_ILONLY = 0x1 +COMIMAGE_FLAGS_32BITREQUIRED = 0x2 +COMIMAGE_FLAGS_STRONGNAMESIGNED = 0x8 +COMIMAGE_FLAGS_NATIVE_ENTRYPOINT = 0x10 +COMIMAGE_FLAGS_32BITPREFERRED = 0x20000 + +PE_MACHINES = { + 0x14C: "x86", + 0x8664: "x86_64", + 0x1C0: "arm", + 0x1C4: "arm", + 0xAA64: "arm64", + 0x200: "ia64", +} +IMAGE_SCN_MEM_EXECUTE = 0x20000000 +IMAGE_SCN_MEM_READ = 0x40000000 +IMAGE_SCN_MEM_WRITE = 0x80000000 + +# Marker the .NET host (apphost / singlefilehost) uses to find the bundle +# manifest; the 8 bytes before it hold the manifest offset (0 = not a bundle). +BUNDLE_SIGNATURE = bytes( + [ + 0x8B, 0x12, 0x02, 0xB9, 0x6A, 0x61, 0x20, 0x38, + 0x72, 0x7B, 0x93, 0x02, 0x14, 0xD7, 0xA0, 0x32, + 0x13, 0xF5, 0xB9, 0xE6, 0xEF, 0xAE, 0x33, 0x18, + 0xEE, 0x3B, 0x2D, 0xCE, 0x24, 0xB3, 0x6A, 0xAE, + ] +) # fmt: skip +BUNDLE_ENTRY_KINDS = { + 0: "unknown", + 1: "assembly", + 2: "native", + 3: "deps-json", + 4: "runtime-config", + 5: "symbols", +} +_EXECUTABLE_MAGICS = ( + b"MZ", + b"\x7fELF", + b"\xcf\xfa\xed\xfe", # Mach-O 64 + b"\xce\xfa\xed\xfe", # Mach-O 32 + b"\xca\xfe\xba\xbe", # Mach-O fat +) + + +@dataclass(slots=True) +class PeSection: + name: str + virtual_address: int + virtual_size: int + raw_offset: int + raw_size: int + characteristics: int + + @property + def perms(self) -> str: + flags = self.characteristics + return ( + ("r" if flags & IMAGE_SCN_MEM_READ else "-") + + ("w" if flags & IMAGE_SCN_MEM_WRITE else "-") + + ("x" if flags & IMAGE_SCN_MEM_EXECUTE else "-") + ) + + +@dataclass(slots=True) +class MetadataStream: + name: str + offset: int # file offset + size: int + + +@dataclass(slots=True) +class PeInfo: + machine: str + pe32_plus: bool + image_base: int + sections: list[PeSection] + clr_rva: int = 0 + clr_size: int = 0 + runtime_version: str | None = None + cor_flags: int = 0 + entry_point_token: int = 0 + metadata_rva: int = 0 + metadata_size: int = 0 + resources_rva: int = 0 + resources_size: int = 0 + strong_name_rva: int = 0 + managed_native_header_rva: int = 0 + streams: list[MetadataStream] = field(default_factory=list) + + @property + def is_managed(self) -> bool: + return self.clr_rva != 0 and self.clr_size != 0 + + @property + def il_only(self) -> bool: + return bool(self.cor_flags & COMIMAGE_FLAGS_ILONLY) + + @property + def mixed_mode(self) -> bool: + # C++/CLI images clear ILONLY because they carry native code. So do + # ReadyToRun images, but their native code is precompiled IL that the + # IL decompiler already covers, so they are not "mixed" for our purpose. + return self.is_managed and not self.il_only and not self.ready_to_run + + @property + def ready_to_run(self) -> bool: + return self.managed_native_header_rva != 0 + + @property + def cor_flag_names(self) -> list[str]: + names = [] + for bit, name in ( + (COMIMAGE_FLAGS_ILONLY, "ILOnly"), + (COMIMAGE_FLAGS_32BITREQUIRED, "32BitRequired"), + (COMIMAGE_FLAGS_STRONGNAMESIGNED, "StrongNameSigned"), + (COMIMAGE_FLAGS_NATIVE_ENTRYPOINT, "NativeEntryPoint"), + (COMIMAGE_FLAGS_32BITPREFERRED, "32BitPreferred"), + ): + if self.cor_flags & bit: + names.append(name) + return names + + def rva_to_offset(self, rva: int) -> int | None: + for section in self.sections: + span = max(section.virtual_size, section.raw_size) + if section.virtual_address <= rva < section.virtual_address + span: + delta = rva - section.virtual_address + if delta >= section.raw_size: + return None + return section.raw_offset + delta + return None + + def stream(self, name: str) -> MetadataStream | None: + return next((item for item in self.streams if item.name == name), None) + + +def parse_pe(data: bytes | mmap.mmap) -> PeInfo | None: + """Parse the PE headers we need; ``None`` for anything that is not a PE.""" + try: + if data[:2] != b"MZ": + return None + pe_offset = struct.unpack_from(" _CLR_DIRECTORY: + clr_rva, clr_size = struct.unpack_from( + " None: + offset = info.rva_to_offset(info.clr_rva) + if offset is None: + info.clr_rva = info.clr_size = 0 + return + ( + _cb, + _major, + _minor, + info.metadata_rva, + info.metadata_size, + info.cor_flags, + info.entry_point_token, + info.resources_rva, + info.resources_size, + info.strong_name_rva, + ) = struct.unpack_from(" PeInfo | None: + try: + with path.open("rb") as handle: + head = handle.read(2) + if head != b"MZ": + return None + handle.seek(0) + with mmap.mmap(handle.fileno(), 0, access=mmap.ACCESS_READ) as view: + return parse_pe(view) + except (OSError, ValueError): + return None + + +def is_managed_pe(path: Path) -> bool: + info = read_pe_info(path) + return info is not None and info.is_managed + + +# -------------------------------------------------------------------------- +# User string heap (#US) +# -------------------------------------------------------------------------- + + +def _compressed_uint(data: bytes, offset: int) -> tuple[int, int]: + first = data[offset] + if first & 0x80 == 0: + return first, 1 + if first & 0xC0 == 0x80: + return ((first & 0x3F) << 8) | data[offset + 1], 2 + return ( + ((first & 0x1F) << 24) + | (data[offset + 1] << 16) + | (data[offset + 2] << 8) + | data[offset + 3], + 4, + ) + + +def iter_user_strings(heap: bytes) -> list[tuple[int, str]]: + """``(heap offset, value)`` for every string in a #US heap.""" + result: list[tuple[int, str]] = [] + offset = 1 # offset 0 is the empty blob + length = len(heap) + while offset < length: + try: + size, used = _compressed_uint(heap, offset) + except IndexError: + break + start = offset + used + if size == 0: + offset = start + continue + chars = (size - 1) & ~1 # drop the trailing flag byte + if start + chars > length: + break + value = heap[start : start + chars].decode("utf-16-le", errors="replace") + result.append((offset, value)) + offset = start + size + return result + + +# -------------------------------------------------------------------------- +# Method bodies and IL token scan +# -------------------------------------------------------------------------- + +# Operand kinds; the scan only needs sizes plus the token-bearing kinds. +_N, _B1, _B2, _B4, _B8 = "none", "i1", "i2", "i4", "i8" +TOK_METHOD, TOK_FIELD, TOK_TYPE, TOK_STRING, TOK_SIG, TOK_TOKEN = ( + "method", + "field", + "type", + "string", + "sig", + "token", +) +_SWITCH = "switch" + + +def _one_byte_table() -> dict[int, str]: + table: dict[int, str] = {} + for code in range(0x00, 0x0E): # nop .. stloc.3 + table[code] = _N + for code in range(0x0E, 0x14): # ldarg.s .. stloc.s + table[code] = _B1 + for code in range(0x14, 0x1F): # ldnull, ldc.i4.m1 .. ldc.i4.8 + table[code] = _N + table[0x1F] = _B1 # ldc.i4.s + table[0x20] = _B4 # ldc.i4 + table[0x21] = _B8 # ldc.i8 + table[0x22] = _B4 # ldc.r4 + table[0x23] = _B8 # ldc.r8 + table[0x25] = _N # dup + table[0x26] = _N # pop + table[0x27] = TOK_METHOD # jmp + table[0x28] = TOK_METHOD # call + table[0x29] = TOK_SIG # calli + table[0x2A] = _N # ret + for code in range(0x2B, 0x38): # short branches + table[code] = _B1 + for code in range(0x38, 0x45): # long branches + table[code] = _B4 + table[0x45] = _SWITCH + for code in range(0x46, 0x6F): # ldind/stind/arith/conv + table[code] = _N + table[0x6F] = TOK_METHOD # callvirt + table[0x70] = TOK_TYPE # cpobj + table[0x71] = TOK_TYPE # ldobj + table[0x72] = TOK_STRING # ldstr + table[0x73] = TOK_METHOD # newobj + table[0x74] = TOK_TYPE # castclass + table[0x75] = TOK_TYPE # isinst + table[0x76] = _N # conv.r.un + table[0x79] = TOK_TYPE # unbox + table[0x7A] = _N # throw + for code in range(0x7B, 0x81): # ldfld .. stsfld + table[code] = TOK_FIELD + table[0x81] = TOK_TYPE # stobj + for code in range(0x82, 0x8C): # conv.ovf.*.un + table[code] = _N + table[0x8C] = TOK_TYPE # box + table[0x8D] = TOK_TYPE # newarr + table[0x8E] = _N # ldlen + table[0x8F] = TOK_TYPE # ldelema + for code in range(0x90, 0xA3): # ldelem.* / stelem.* + table[code] = _N + table[0xA3] = TOK_TYPE # ldelem + table[0xA4] = TOK_TYPE # stelem + table[0xA5] = TOK_TYPE # unbox.any + for code in range(0xB3, 0xBB): # conv.ovf.* + table[code] = _N + table[0xC2] = TOK_TYPE # refanyval + table[0xC3] = _N # ckfinite + table[0xC6] = TOK_TYPE # mkrefany + table[0xD0] = TOK_TOKEN # ldtoken + for code in range(0xD1, 0xDD): # conv.u2 .. endfinally + table[code] = _N + table[0xDD] = _B4 # leave + table[0xDE] = _B1 # leave.s + table[0xDF] = _N # stind.i + table[0xE0] = _N # conv.u + return table + + +def _two_byte_table() -> dict[int, str]: + table: dict[int, str] = {code: _N for code in range(0x00, 0x06)} + table[0x06] = TOK_METHOD # ldftn + table[0x07] = TOK_METHOD # ldvirtftn + for code in range(0x09, 0x0F): # ldarg .. stloc + table[code] = _B2 + table[0x0F] = _N # localloc + table[0x11] = _N # endfilter + table[0x12] = _B1 # unaligned. + table[0x13] = _N # volatile. + table[0x14] = _N # tail. + table[0x15] = TOK_TYPE # initobj + table[0x16] = TOK_TYPE # constrained. + table[0x17] = _N # cpblk + table[0x18] = _N # initblk + table[0x19] = _B1 # no. + table[0x1A] = _N # rethrow + table[0x1C] = TOK_TYPE # sizeof + table[0x1D] = _N # refanytype + table[0x1E] = _N # readonly. + return table + + +ONE_BYTE_OPCODES = _one_byte_table() +TWO_BYTE_OPCODES = _two_byte_table() +_OPERAND_SIZES = {_N: 0, _B1: 1, _B2: 2, _B4: 4, _B8: 8} +_TOKEN_KINDS = {TOK_METHOD, TOK_FIELD, TOK_TYPE, TOK_STRING, TOK_SIG, TOK_TOKEN} + + +def method_body(data: bytes | mmap.mmap, offset: int) -> bytes | None: + """IL code bytes of the method body at ``offset`` (tiny or fat header).""" + try: + first = data[offset] + except IndexError: + return None + if first & 0x3 == 0x2: # tiny + size = first >> 2 + return bytes(data[offset + 1 : offset + 1 + size]) + if first & 0x3 == 0x3: # fat + flags_size = struct.unpack_from("> 12) * 4 + code_size = struct.unpack_from(" list[tuple[str, int]]: + """Every ``(kind, metadata token)`` an IL stream references, in order.""" + refs: list[tuple[str, int]] = [] + pc = 0 + length = len(code) + while pc < length: + opcode = code[pc] + pc += 1 + if opcode == 0xFE: + if pc >= length: + break + kind = TWO_BYTE_OPCODES.get(code[pc]) + pc += 1 + else: + kind = ONE_BYTE_OPCODES.get(opcode) + if kind is None: + break # invalid/encrypted body: stop rather than misread + if kind == _SWITCH: + if pc + 4 > length: + break + count = struct.unpack_from(" length: + break + refs.append((kind, struct.unpack_from(" tuple[str, int]: + length = 0 + shift = 0 + while True: + byte = data[offset] + offset += 1 + length |= (byte & 0x7F) << shift + shift += 7 + if byte < 0x80: + break + raw = bytes(data[offset : offset + length]) + return raw.decode("utf-8", errors="replace"), offset + length + + +def find_bundle_header(data: bytes | mmap.mmap) -> tuple[int, int] | None: + """``(signature offset, manifest offset)``; manifest 0 = plain apphost.""" + position = data.find(BUNDLE_SIGNATURE) + if position < 8: + return None + header = struct.unpack_from(" BundleManifest | None: + found = find_bundle_header(data) + if found is None: + return None + _signature, header = found + if header <= 0 or header >= len(data): + return None + try: + major, minor, count = struct.unpack_from("= 2: + cursor += 40 # deps.json / runtimeconfig.json locations + flags + entries: list[BundleEntry] = [] + for _ in range(max(0, count)): + offset, size = struct.unpack_from("= 6: + compressed = struct.unpack_from(" BundleManifest | None: + try: + with path.open("rb") as handle: + with mmap.mmap(handle.fileno(), 0, access=mmap.ACCESS_READ) as view: + if not bytes(view[:4]).startswith(_EXECUTABLE_MAGICS): + return None + return parse_bundle(view) + except (OSError, ValueError): + return None + + +def is_apphost_launcher(path: Path) -> bool: + """A non-bundled .NET apphost: the real code is ``.dll`` beside it.""" + try: + with path.open("rb") as handle: + with mmap.mmap(handle.fileno(), 0, access=mmap.ACCESS_READ) as view: + if not bytes(view[:4]).startswith(_EXECUTABLE_MAGICS): + return False + found = find_bundle_header(view) + return found is not None and found[1] == 0 + except (OSError, ValueError): + return False + + +def extract_bundle_entry(source: Path, entry: BundleEntry, target: Path) -> None: + target.parent.mkdir(parents=True, exist_ok=True) + with source.open("rb") as handle: + handle.seek(entry.offset) + if entry.compressed_size: + payload = handle.read(entry.compressed_size) + data = zlib.decompress(payload, -15) + else: + data = handle.read(entry.size) + target.write_bytes(data) diff --git a/src/tocode/cli.py b/src/tocode/cli.py index 8f8b546..be06b9f 100644 --- a/src/tocode/cli.py +++ b/src/tocode/cli.py @@ -10,6 +10,7 @@ from .apk import ApkExportOptions, export_apk from .apk_native import NativeOptions from .backends.asc import is_apk_input +from .backends.dotnet import is_dotnet_input from .errors import ToCodeError from .exporter import export_binary from .naming import default_output_name @@ -78,6 +79,16 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="APK input: only export the DEX/Android side; native .so libraries are extracted but not decompiled.", ) + parser.add_argument( + "--as-native", + action="store_true", + help=".NET input: skip the .NET backend and export the file with the native backend (IDA/r2/angr) like any PE/ELF.", + ) + parser.add_argument( + "--include-framework", + action="store_true", + help=".NET input: also decompile .NET framework/runtime assemblies and native runtime libraries found in bundles and packages.", + ) parser.add_argument( "--no-splits", action="store_true", @@ -187,6 +198,16 @@ def main(argv: list[str] | None = None) -> int: "--backend binja is not supported for APK input; native libraries " "use ida, r2, or angr" ) + if ( + args.binary is not None + and not args.as_native + and args.binary.is_file() + and is_dotnet_input(args.binary) + ): + parser.error( + "--backend binja is not supported for .NET input; native code " + "uses ida, r2, or angr (or pass --as-native)" + ) return _run_binja(args, progress, parser, argv) if args.binary is None: @@ -199,6 +220,8 @@ def main(argv: list[str] | None = None) -> int: ) if is_apk_input(binary): return _run_apk(binary, args=args, progress=progress, parser=parser) + if not args.as_native and binary.is_file() and is_dotnet_input(binary): + return _run_dotnet(binary, args=args, progress=progress) log_root = ( args.out_dir if args.out_dir is not None @@ -242,16 +265,7 @@ def _run_apk( jobs=args.jobs, native=not args.no_native, splits=not args.no_splits, - native_options=NativeOptions( - backend=args.backend, - analysis_command=args.analysis, - idadir=args.idadir, - ida_domain_path=args.ida_domain_path, - purge_cache=args.purge_cache, - jobs=args.jobs, - tree=args.tree, - entropy=args.entropy, - ), + native_options=_native_options(args), ) started = time.monotonic() try: @@ -277,6 +291,54 @@ def _run_apk( return 0 +def _native_options(args: argparse.Namespace) -> NativeOptions: + return NativeOptions( + backend=args.backend, + analysis_command=args.analysis, + idadir=args.idadir, + ida_domain_path=args.ida_domain_path, + purge_cache=args.purge_cache, + jobs=args.jobs, + tree=args.tree, + entropy=args.entropy, + ) + + +def _run_dotnet(binary: Path, *, args: argparse.Namespace, progress: Progress) -> int: + from .dotnet import DotnetExportOptions, export_dotnet + + options = DotnetExportOptions( + out_dir=args.out_dir, + jobs=args.jobs, + native=not args.no_native, + include_framework=args.include_framework, + native_options=_native_options(args), + ) + started = time.monotonic() + try: + summary = export_dotnet(binary, options=options, progress=progress) + except KeyboardInterrupt: + progress.log("tocode: interrupted") + print("tocode: interrupted", file=sys.stderr) + return 130 + except ToCodeError as exc: + progress.log(f"tocode: {exc}") + print(f"tocode: {exc}", file=sys.stderr) + return 1 + if not args.quiet: + print(f"Project: {summary.root_dir}") + print( + f"Summary: kind={summary.kind} assemblies={len(summary.assemblies)} " + f"types={summary.type_count} methods={summary.method_count} " + f"failures={len(summary.failed_types)} " + f"natives={summary.native_done}/{summary.native_total}" + ) + for line in summary.native_errors: + print(f"native: {line}", file=sys.stderr) + print(f"Exported in {_format_duration(time.monotonic() - started)}") + return 0 + + def _format_duration(seconds: float) -> str: if seconds >= 60: minutes, secs = divmod(int(round(seconds)), 60) diff --git a/src/tocode/dotnet.py b/src/tocode/dotnet.py new file mode 100644 index 0000000..bbe17b1 --- /dev/null +++ b/src/tocode/dotnet.py @@ -0,0 +1,1078 @@ +""".NET export pipeline (dnlib + ICSharpCode.Decompiler backend). + +Writes one project tree for a managed assembly, an apphost launcher, a +single-file bundle, or a NuGet package: C# and IL per top-level type under +``src/raw///``, assembly attributes, a project file, +resources, JSON metadata, and (unless disabled) a nested native ToCode export +for every native library found (mixed-mode code, bundled libraries, and +P/Invoke targets shipped next to the input). +""" + +from __future__ import annotations + +from collections import deque +from concurrent.futures import FIRST_COMPLETED, Future, ProcessPoolExecutor, wait +from concurrent.futures.process import BrokenProcessPool +from dataclasses import dataclass, field +import multiprocessing +import os +from pathlib import Path +import re +import shutil +import tempfile +import time +from typing import Any, Callable + +from . import __version__ +from . import dotnet_metadata as meta +from .apk_metadata import write_json_rows +from .apk_native import ( + NATIVE_HEARTBEAT_SECONDS, + NativeExporter, + NativeLib, + NativeOptions, + NativeRunner, + default_native_exporter, + native_libs_json, + native_min_free_mb, +) +from .backends.asc import sha256_file +from .backends.dotnet import ( + AssemblyInfo, + DecompiledType, + DotnetInput, + DotnetSession, + NativeFile, + TypeInfo, + WorkItem, + decompile_batch_in_worker, + discover_dotnet_input, + init_dotnet_worker, + is_framework_native, + native_arch, + pinvoke_neighbours, +) +from .errors import ToCodeError +from .exporter import _terminate_executor +from .metadata import write_json, write_text_atomic +from .naming import clean_path_component +from .parallel import available_memory_mb, choose_jobs +from .progress import Progress +from .schema import DotnetExportSummary + +DECOMPILE_BATCH_SIZE = 16 +# A worker holds the CLR, the decompiler's type system for the assemblies it +# touches, and the current syntax tree; large assemblies reach ~600 MB. +DOTNET_WORKER_MEMORY_MB = 1024 +DOTNET_BATCHES_PER_ROUND = 64 +DEFAULT_TYPE_TIMEOUT_SECONDS = 300 +_INVALID_FILE_CHARS = re.compile(r"[^0-9A-Za-z_.\-$@+]") +_TFM_ATTRIBUTE_RX = re.compile( + r"^\.(NETCoreApp|NETStandard|NETFramework),Version=v(\d+)\.(\d+)(?:\.(\d+))?" +) + + +@dataclass(slots=True) +class DotnetExportOptions: + out_dir: Path | None = None + jobs: int | None = None + native: bool = True + include_framework: bool = False + native_options: NativeOptions = field(default_factory=NativeOptions) + + +@dataclass +class DotnetContext: + input_path: Path + options: DotnetExportOptions + progress: Progress + found: DotnetInput | None = None + session: DotnetSession | None = None + root: Path | None = None + raw_dir: Path | None = None + data_dir: Path | None = None + workdir: Path | None = None + title: str = "" + libs: list[NativeLib] = field(default_factory=list) + native_runner: NativeRunner | None = None + source_files: list[Path] = field(default_factory=list) + il_files: list[Path] = field(default_factory=list) + project_files: list[Path] = field(default_factory=list) + failures: list[tuple[str, str]] = field(default_factory=list) + extracted: dict[str, str | None] = field(default_factory=dict) + reachable: dict[str, Any] = field(default_factory=dict) + roots: set[int] = field(default_factory=set) + worker_count: int = 1 + render_mode: str = "single" + started: float = 0.0 + + +def export_dotnet( + input_path: Path, + *, + options: DotnetExportOptions | None = None, + progress: Progress | None = None, + native_exporter: NativeExporter | None = None, + decompiler_factory: Callable[[DotnetSession], Any] | None = None, + session_factory: Callable[[DotnetInput], DotnetSession] | None = None, +) -> DotnetExportSummary: + options = options or DotnetExportOptions() + progress = progress or Progress() + context = DotnetContext( + input_path=Path(input_path).resolve(), + options=options, + progress=progress, + started=time.monotonic(), + ) + try: + _discover(context) + _prepare_root(context) + _inventory(context, session_factory) + _collect_natives(context) + _start_native(context, native_exporter) + _assign_paths(context) + _decompile(context, decompiler_factory) + _write_projects(context) + _write_resources(context) + _write_metadata(context) + _join_native(context) + _write_final_documents(context) + except BaseException: + if context.native_runner is not None: + context.native_runner.stop() + raise + finally: + if context.session is not None: + context.session.close() + if context.workdir is not None: + shutil.rmtree(context.workdir, ignore_errors=True) + return _summary(context) + + +# -------------------------------------------------------------------------- +# Steps +# -------------------------------------------------------------------------- + + +def _work_root() -> str | None: + explicit = os.environ.get("TOCODE_WORKER_TMP_DIR", "").strip() + if not explicit: + return None + candidate = Path(explicit).expanduser() + try: + candidate.mkdir(parents=True, exist_ok=True) + except OSError: + return None + return str(candidate) + + +def _discover(context: DotnetContext) -> None: + path = context.input_path + if not path.is_file(): + raise ToCodeError(f".NET input not found: {path}") + context.workdir = Path(tempfile.mkdtemp(prefix="tocode-dotnet-", dir=_work_root())) + context.found = discover_dotnet_input( + path, + workdir=context.workdir, + include_framework=context.options.include_framework, + ) + if not context.found.included: + raise ToCodeError( + f"no assemblies to decompile in {path.name} (all are framework " + "assemblies; pass --include-framework)" + ) + stem = path.name + for suffix in (".dll", ".exe", ".nupkg", ".snupkg", ".winmd"): + if stem.lower().endswith(suffix): + stem = stem[: -len(suffix)] + context.title = stem or path.stem + + +def _root_dir(context: DotnetContext) -> Path: + out_dir = context.options.out_dir + source = context.input_path + if out_dir is not None: + out = Path(out_dir).expanduser() + return out.resolve() if out.is_absolute() else (source.parent / out).resolve() + name = f"{clean_path_component(context.title)}_decompiler" + default_root = os.environ.get("TOCODE_DEFAULT_OUT_ROOT", "").strip() + if default_root: + return (Path(default_root).expanduser().resolve() / name).resolve() + return (source.parent / name).resolve() + + +def _prepare_root(context: DotnetContext) -> None: + found = _need(context.found) + root = _root_dir(context) + context.root = root + context.raw_dir = root / "src" / "raw" + context.data_dir = root / "data" + if context.progress.log_path is None: + context.progress.set_log_path(root / "tocode.log") + context.raw_dir.mkdir(parents=True, exist_ok=True) + context.data_dir.mkdir(parents=True, exist_ok=True) + included = found.included + skipped = len(found.assemblies) - len(included) + context.progress.log("Export run started") + context.progress.log( + f"Using .NET (dnlib + ICSharpCode.Decompiler) as backend for {context.title} " + f"({found.kind})" + ) + context.progress.log( + f"Assemblies: {len(included)} to decompile" + + (f", {skipped} skipped (framework/duplicates)" if skipped else "") + + (f", {len(found.natives)} native libraries" if found.natives else "") + ) + for item in found.assemblies: + if not item.include: + context.progress.file_log(f"skip {item.entry}: {item.reason}") + + +def _inventory( + context: DotnetContext, + session_factory: Callable[[DotnetInput], DotnetSession] | None, +) -> None: + found = _need(context.found) + session = session_factory(found) if session_factory else DotnetSession(found) + with context.progress.bar(total=1, desc="inventory", unit="step") as bar: + session.load() + bar.update(1) + context.session = session + context.progress.log( + f"Inventory: {len(session.assemblies)} assemblies, {len(session.types)} types, " + f"{len(session.methods)} methods, " + f"{sum(len(item.user_strings) for item in session.assemblies)} strings" + ) + for info in session.assemblies: + for hint in info.obfuscation_hints: + context.progress.log(f"warning: {info.name}: possible obfuscation ({hint})") + + +def _native_origin(context: DotnetContext, native: NativeFile) -> str: + found = _need(context.found) + container = { + "bundle": "single-file bundle", + "nupkg": "NuGet package", + "apphost": "app", + "assembly": "assembly", + }.get(found.kind, found.kind) + if native.path == found.input or native.entry.startswith("(native code"): + how = "the native (C++/CLI) half of the mixed-mode assembly" + elif native.source.startswith("beside "): + how = f"a native library next to the input that P/Invoke declarations name (`{native.entry}`)" + else: + how = f"`{native.entry}` inside `{native.source}`" + return ( + f"This is a decompiled native library from the `{context.title}` .NET " + f"{container}: `{native.name}` ({native.arch}), {how}. " + "The .NET export that owns it is three directories up (`../../..`): its " + "`imports.json` lists the P/Invoke declarations (module and entry point) " + "that call into native code, `functions.json` marks those managed methods " + "with `pinvoke`, `native-libs.json` lists the other native libraries, and " + "`src/raw//` holds the decompiled C# that uses them." + ) + + +def _collect_natives(context: DotnetContext) -> None: + found = _need(context.found) + session = _need(context.session) + root = _need(context.root) + natives = list(found.natives) + if found.kind in {"assembly", "apphost"}: + known = {native.path.resolve() for native in natives} + for module, path in pinvoke_neighbours( + found.input.parent, session.pinvoke_modules + ): + if path.resolve() in known: + continue + known.add(path.resolve()) + natives.append( + NativeFile( + name=path.name, + path=path, + entry=module, + source=f"beside {found.input.name}", + arch=native_arch(path), + framework=is_framework_native(path.name), + ) + ) + libs: list[NativeLib] = [] + for native in natives: + arch = clean_path_component(native.arch or "unknown") + target = root / "lib" / arch / native.name + if native.path != found.input: + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(native.path, target) + path = target + else: + path = native.path # mixed-mode: decompile the assembly file itself + lib = NativeLib( + abi=native.arch or "unknown", + name=native.name, + entry=native.entry, + source_apk=native.source, + size=path.stat().st_size, + sha256=sha256_file(path), + path=path, + package=context.title, + origin=_native_origin(context, native), + ) + if native.framework and not context.options.include_framework: + lib.status = "skipped: .NET runtime library (use --include-framework)" + libs.append(lib) + context.extracted[native.entry] = ( + target.relative_to(root).as_posix() if native.path != found.input else None + ) + context.libs = libs + if libs: + context.progress.log( + f"Native libraries: {len(libs)} found, " + f"{sum(1 for lib in libs if lib.status == 'pending')} to decompile" + ) + _write_native_libs(context) + + +def _start_native(context: DotnetContext, exporter: NativeExporter | None) -> None: + pending = [lib for lib in context.libs if lib.status == "pending"] + if not pending: + return + if not context.options.native: + for lib in pending: + lib.status = "skipped: --no-native" + context.progress.log( + f"Native decompilation skipped (--no-native) for {len(pending)} libraries" + ) + _write_native_libs(context) + return + runner = NativeRunner( + libs=pending, + root=_need(context.root), + progress=context.progress, + exporter=exporter or default_native_exporter(context.options.native_options), + min_free_mb=native_min_free_mb(), + available_memory=available_memory_mb, + ) + context.progress.log( + f"Native decompilation started in background for {len(pending)} libraries " + f"(backend: {context.options.native_options.backend})" + ) + runner.start() + context.native_runner = runner + + +def _join_native(context: DotnetContext) -> None: + runner = context.native_runner + if runner is None: + return + thread = runner.thread + if thread is not None and thread.is_alive(): + context.progress.log( + "Managed export done; waiting for native library export(s) " + "(progress below is mirrored from native///tocode.log)" + ) + while thread.is_alive(): + thread.join(timeout=NATIVE_HEARTBEAT_SECONDS) + if thread.is_alive(): + status = runner.describe_current() + if status: + context.progress.log(status) + runner.join() + done = sum(1 for lib in context.libs if lib.status == "done") + context.progress.log( + f"Native decompilation finished: {done}/{len(context.libs)} libraries exported" + ) + _write_native_libs(context) + + +def _write_native_libs(context: DotnetContext) -> None: + root = _need(context.root) + write_json(root / "native-libs.json", native_libs_json(context.libs, root)) + + +# -- paths --------------------------------------------------------------------- + + +def safe_file_name(name: str) -> str: + cleaned = _INVALID_FILE_CHARS.sub("_", name).strip(".") + return cleaned[:120] or "_" + + +def type_file_stem(record: TypeInfo) -> str: + """``List`1`` -> ``List-1``; compiler/obfuscator names made filesystem-safe.""" + name = record.name + if "`" in name: + base, _, arity = name.partition("`") + name = f"{base}-{arity}" if arity.isdigit() else name.replace("`", "-") + return safe_file_name(name) + + +def namespace_parts(namespace: str) -> list[str]: + return [safe_file_name(part) for part in namespace.split(".") if part] + + +def _assembly_folders(session: DotnetSession) -> None: + used: dict[str, int] = {} + for info in session.assemblies: + base = safe_file_name(info.name) + count = used.get(base.lower(), 0) + used[base.lower()] = count + 1 + info.folder = base if count == 0 else f"{base}~{count}" + + +def _assign_paths(context: DotnetContext) -> None: + session = _need(context.session) + raw_dir = _need(context.raw_dir) + _assembly_folders(session) + seen: dict[str, int] = {} + for info in session.assemblies: + base = raw_dir / info.folder + for record in session.top_level_types(info.index): + folder = base.joinpath(*namespace_parts(record.namespace)) + stem = type_file_stem(record) + key = (folder / stem).as_posix().lower() + count = seen.get(key, 0) + seen[key] = count + 1 + if count: + stem = f"{stem}~{count}" + record.cs_path = folder / f"{stem}.cs" + record.il_path = folder / f"{stem}.il" + for nested in session.nested_types(info.index, record.token): + nested.cs_path = record.cs_path + nested.il_path = record.il_path + + +# -- decompilation --------------------------------------------------------------- + + +class _PoolBroken(RuntimeError): + pass + + +def _work_items(context: DotnetContext) -> list[WorkItem]: + session = _need(context.session) + items: list[WorkItem] = [] + for info in session.assemblies: + items.append(WorkItem(info.index, str(info.file.path), 0)) + for record in session.top_level_types(info.index): + if record.name == "" and not record.methods and not record.fields: + continue + items.append(WorkItem(info.index, str(info.file.path), record.token)) + return items + + +def _select_workers(context: DotnetContext, count: int) -> int: + chosen = choose_jobs( + function_count=count, + analysis_seconds=0.0, + requested=context.options.jobs, + backend="dotnet", + ) + memory = available_memory_mb() + if memory is not None and chosen > 1: + ceiling = max(1, memory // DOTNET_WORKER_MEMORY_MB) + if ceiling < chosen: + context.progress.log( + f"Limiting decompile workers to {ceiling} ({memory} MB available, " + f"{DOTNET_WORKER_MEMORY_MB} MB per worker)" + ) + chosen = ceiling + context.worker_count = max(1, chosen) + context.render_mode = "process" if context.worker_count > 1 else "single" + return context.worker_count + + +def _type_timeout() -> int: + raw = os.environ.get("TOCODE_DOTNET_TYPE_TIMEOUT_SECONDS", "").strip() + try: + return max(10, int(raw)) if raw else DEFAULT_TYPE_TIMEOUT_SECONDS + except ValueError: + return DEFAULT_TYPE_TIMEOUT_SECONDS + + +def _decompile( + context: DotnetContext, decompiler_factory: Callable[[DotnetSession], Any] | None +) -> None: + session = _need(context.session) + items = _work_items(context) + type_items = sum(1 for item in items if item.token) + _remove_stale_sources(context) + if decompiler_factory is not None: + context.progress.log(f"Decompiling {type_items} types in-process") + decompiler = decompiler_factory(session) + with context.progress.bar( + total=len(items), desc="decompile", unit="type" + ) as bar: + for item in items: + _store_result(context, decompiler.decompile(item)) + bar.update(1) + else: + workers = _select_workers(context, type_items) + context.progress.log( + f"Decompiling {type_items} types to C# and IL with {workers} worker(s)" + ) + with context.progress.bar( + total=len(items), desc="decompile", unit="type" + ) as bar: + _decompile_pool(context, items, workers, bar) + _inherit_owner_ranges(session) + context.progress.log( + f"Decompiled {type_items - len(context.failures)} types, " + f"{len(context.failures)} failed" + ) + + +def _decompile_pool( + context: DotnetContext, items: list[WorkItem], workers: int, bar: Any +) -> None: + found = _need(context.found) + search_dirs = [str(path) for path in found.search_dirs] + queue: deque[tuple[list[WorkItem], bool]] = deque( + (items[start : start + DECOMPILE_BATCH_SIZE], False) + for start in range(0, len(items), DECOMPILE_BATCH_SIZE) + ) + timeout = _type_timeout() + while queue: + # A fresh pool per round recycles worker memory (cached type systems). + round_batches = [ + queue.popleft() + for _ in range(min(len(queue), workers * DOTNET_BATCHES_PER_ROUND)) + ] + failed = _run_round(context, round_batches, workers, search_dirs, timeout, bar) + for batch, isolated, reason in failed: + if len(batch) > 1: + # Re-run each type alone to find the one that crashed or hung. + context.progress.file_log( + f"decompile: batch of {len(batch)} failed ({reason}); isolating" + ) + queue.extendleft((([item], True) for item in reversed(batch))) + continue + item = batch[0] + if not isolated: + queue.appendleft(([item], True)) + continue + _store_result( + context, + DecompiledType( + item.assembly, item.token, None, reason, None, reason, {}, {} + ), + ) + bar.update(1) + + +def _run_round( + context: DotnetContext, + batches: list[tuple[list[WorkItem], bool]], + workers: int, + search_dirs: list[str], + timeout: int, + bar: Any, +) -> list[tuple[list[WorkItem], bool, str]]: + executor = ProcessPoolExecutor( + max_workers=max(1, min(workers, len(batches))), + mp_context=multiprocessing.get_context("spawn"), + initializer=init_dotnet_worker, + initargs=(search_dirs,), + ) + futures: dict[Future[list[DecompiledType]], tuple[list[WorkItem], bool, float]] = {} + failed: list[tuple[list[WorkItem], bool, str]] = [] + pending = deque(batches) + broken = False + + def submit() -> None: + while pending and len(futures) < max(1, workers) * 2: + batch, isolated = pending.popleft() + futures[executor.submit(decompile_batch_in_worker, batch)] = ( + batch, + isolated, + time.monotonic(), + ) + + try: + submit() + while futures: + done, _ = wait(list(futures), timeout=5, return_when=FIRST_COMPLETED) + for future in done: + batch, isolated, _started = futures.pop(future) + try: + results = future.result() + except BrokenProcessPool: + failed.append((batch, isolated, "decompiler process crashed")) + broken = True + continue + except Exception as exc: + failed.append((batch, isolated, f"{type(exc).__name__}: {exc}")) + continue + for result in results: + _store_result(context, result) + bar.update(1) + now = time.monotonic() + overdue = [ + future + for future, (batch, _isolated, started) in futures.items() + if future.running() and now - started > timeout + 30 * len(batch) + ] + if overdue or broken: + for future in list(futures): + batch, isolated, _started = futures.pop(future) + reason = ( + f"decompilation timed out after {timeout}s" + if future in overdue + else "decompiler process crashed" + ) + if future not in overdue and not broken: + reason = "worker pool restarted" + failed.append((batch, isolated, reason)) + failed.extend( + (batch, isolated, "worker pool restarted") + for batch, isolated in pending + ) + pending.clear() + break + submit() + finally: + if broken or failed: + _terminate_executor(executor) + else: + executor.shutdown(wait=True, cancel_futures=True) + return failed + + +def _store_result(context: DotnetContext, result: DecompiledType) -> None: + session = _need(context.session) + info = session.assemblies[result.assembly] + if result.token == 0: + _store_assembly_attributes(context, info, result) + return + record = session.types.get((result.assembly, result.token)) + if record is None or record.cs_path is None or record.il_path is None: + return + header = f"// ToCode: {record.full_name.replace('/', '+')} (0x{record.token:08x}) from {info.name} {info.version}\n" + if result.cs is not None: + cs_text = header + result.cs.rstrip("\n") + "\n" + cs_offset = 1 + else: + record.error = result.cs_error or "unknown decompilation error" + context.failures.append((record.full_name, record.error)) + cs_text = header + _failure_stub(session, record) + cs_offset = 0 + context.progress.file_log( + f"export {info.name}!{record.full_name} failed: {record.error}" + ) + if result.il is not None: + il_text = header + result.il.rstrip("\n") + "\n" + else: + record.il_error = result.il_error or "unknown disassembly error" + il_text = header + f"// IL disassembly failed: {record.il_error}\n" + write_text_atomic(record.cs_path, cs_text) + write_text_atomic(record.il_path, il_text) + record.cs_lines = cs_text.count("\n") + record.il_lines = il_text.count("\n") + context.source_files.append(record.cs_path) + context.il_files.append(record.il_path) + for nested in session.nested_types(record.assembly, record.token): + nested.error = record.error + nested.il_error = record.il_error + nested.cs_lines = record.cs_lines + nested.il_lines = record.il_lines + for method_id in nested.methods: + method = session.methods[method_id] + span = result.cs_ranges.get(method.token) + if span is not None and result.cs is not None: + method.cs_start, method.cs_end = ( + span[0] + cs_offset, + span[1] + cs_offset, + ) + elif result.cs is None: + method.cs_start, method.cs_end = 1, record.cs_lines + il_span = result.il_ranges.get(method.token) + if il_span is not None: + method.il_start, method.il_end = il_span[0] + 1, il_span[1] + 1 + if record.error is None: + context.progress.file_log( + f"export {info.name}!{record.full_name} - {record.cs_lines} C# / " + f"{record.il_lines} IL lines done" + ) + + +def _store_assembly_attributes( + context: DotnetContext, info: AssemblyInfo, result: DecompiledType +) -> None: + folder = _need(context.raw_dir) / info.folder / "Properties" + header = f"// ToCode: assembly and module attributes of {info.full_name}\n" + cs = header + (result.cs or f"// decompilation failed: {result.cs_error}\n") + il = header + (result.il or f"// IL manifest failed: {result.il_error}\n") + write_text_atomic( + folder / "AssemblyInfo.cs", cs if cs.endswith("\n") else cs + "\n" + ) + write_text_atomic(folder / "Manifest.il", il if il.endswith("\n") else il + "\n") + context.source_files.append(folder / "AssemblyInfo.cs") + context.il_files.append(folder / "Manifest.il") + + +def _failure_stub(session: DotnetSession, record: TypeInfo) -> str: + lines = [ + f"// decompilation failed: {record.error}", + "// The IL next to this file (.il) is complete; read it instead.", + ] + if record.namespace: + lines.append(f"namespace {record.namespace};") + lines.append(f"// {record.visibility} {record.kind} {record.name}") + for member in session.nested_types(record.assembly, record.token): + for method_id in member.methods: + method = session.methods[method_id] + lines.append(f"// {member.name}::{method.prototype}") + return "\n".join(lines) + "\n" + + +def _inherit_owner_ranges(session: DotnetSession) -> None: + """State machine / closure methods without their own C# span point at the + method that the decompiler folded them into.""" + for method in session.methods: + if method.cs_start is not None or method.owner_method is None: + continue + owner = session.methods[method.owner_method] + if owner.cs_start is not None: + method.cs_start, method.cs_end = owner.cs_start, owner.cs_end + + +def _remove_stale_sources(context: DotnetContext) -> None: + raw_dir = _need(context.raw_dir) + session = _need(context.session) + expected = { + path + for record in session.types.values() + for path in (record.cs_path, record.il_path) + if path is not None + } + for info in session.assemblies: + expected.add(raw_dir / info.folder / "Properties" / "AssemblyInfo.cs") + expected.add(raw_dir / info.folder / "Properties" / "Manifest.il") + expected.add(raw_dir / info.folder / f"{info.name}.csproj") + removed = 0 + for pattern in ("*.cs", "*.il", "*.csproj"): + for path in raw_dir.rglob(pattern): + if path not in expected: + path.unlink(missing_ok=True) + removed += 1 + if removed: + context.progress.file_log(f"removed {removed} stale source files") + + +# -- project files / resources ----------------------------------------------------- + + +def target_framework_moniker(attribute: str | None) -> str | None: + if not attribute: + return None + match = _TFM_ATTRIBUTE_RX.match(attribute) + if not match: + return None + family, major, minor = match.group(1), int(match.group(2)), int(match.group(3)) + patch = match.group(4) + if family == "NETCoreApp": + return f"net{major}.{minor}" if major >= 5 else f"netcoreapp{major}.{minor}" + if family == "NETStandard": + return f"netstandard{major}.{minor}" + return f"net{major}{minor}{patch or ''}" + + +def build_csproj(info: AssemblyInfo, *, internal_refs: set[str]) -> str: + framework = target_framework_moniker(info.target_framework) or "net10.0" + output = "Exe" if info.entry_point is not None else "Library" + if output == "Exe" and "Windows" in info.kind: + output = "WinExe" + references = [ + ref["name"] + for ref in info.assembly_refs + if ref["name"] in internal_refs + or not re.match( + r"^(System|Microsoft|mscorlib|netstandard|WindowsBase)\b", ref["name"] + ) + ] + lines = [ + "", + '', + " ", + f" {info.name}", + f" {output}", + f" {framework}", + " latest", + " true", + " annotations", + " false", + " true", + " ", + ] + if references: + lines.append(" ") + for name in sorted(set(references)): + if name in internal_refs: + lines.append( + f' ' + ) + else: + lines.append(f' ') + lines.append(" ") + lines.append("") + return "\n".join(lines) + "\n" + + +def _write_projects(context: DotnetContext) -> None: + session = _need(context.session) + raw_dir = _need(context.raw_dir) + internal = {info.name for info in session.assemblies} + for info in session.assemblies: + path = raw_dir / info.folder / f"{info.name}.csproj" + write_text_atomic( + path, build_csproj(info, internal_refs=internal - {info.name}) + ) + context.project_files.append(path) + + +def _write_resources(context: DotnetContext) -> None: + session = _need(context.session) + found = _need(context.found) + root = _need(context.root) + data_dir = _need(context.data_dir) + count = 0 + for info in session.assemblies: + folder = data_dir / "resources" / info.folder + for resource in info.resources: + if resource.kind != "embedded": + continue + target = folder / safe_file_name(resource.name) + try: + size = session.save_resource(info, resource.name, target) + except Exception as exc: # pragma: no cover - backend specific + resource.error = f"{type(exc).__name__}: {exc}" + continue + if size is None: + resource.error = "cannot read resource data" + continue + resource.path = target + resource.size = size + count += 1 + if resource.name.endswith(".resources"): + try: + resource.entries = session.decode_resources_file( + info, resource.name + ) + write_json( + target.with_name(target.name + ".json"), + {"resource": resource.name, "entries": resource.entries}, + ) + except Exception as exc: + resource.error = f"cannot decode .resources: {exc}" + for extra in found.extras: + target = data_dir / found.kind / safe_file_name(extra.entry.replace("/", "__")) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(extra.path, target) + context.extracted[extra.entry] = target.relative_to(root).as_posix() + for item in found.assemblies: + if item.path.is_relative_to(context.workdir or Path("/nonexistent")): + target = ( + data_dir / "assemblies" / safe_file_name(item.entry.replace("/", "__")) + ) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(item.path, target) + context.extracted[item.entry] = target.relative_to(root).as_posix() + else: + context.extracted.setdefault(item.entry, None) + if count: + context.progress.log(f"Extracted {count} embedded resources") + + +# -- metadata --------------------------------------------------------------------- + + +def _write_metadata(context: DotnetContext) -> None: + root = _need(context.root) + session = _need(context.session) + found = _need(context.found) + with context.progress.bar(total=11, desc="metadata", unit="doc") as bar: + context.reachable, context.roots = meta.reachable_json(session) + write_json(root / "reachable.json", context.reachable) + bar.update(1) + write_json(root / "assemblies.json", meta.assemblies_json(session, found, root)) + bar.update(1) + write_json_rows( + root / "types.json", + "types", + meta.iter_type_rows(session, root), + header={"count": len(session.types)}, + ) + bar.update(1) + write_json_rows( + root / "functions.json", + "functions", + meta.iter_function_rows(session, root, context.roots), + ) + bar.update(1) + write_json_rows( + root / "function-index.json", + "functions", + meta.iter_function_index_rows(session, root), + header={"schema_version": 2, "language": "csharp", "asm_language": "il"}, + ) + bar.update(1) + write_json_rows( + root / "strings.json", "strings", meta.iter_string_rows(session) + ) + bar.update(1) + write_json(root / "imports.json", meta.imports_json(session)) + bar.update(1) + write_json(root / "exports.json", meta.exports_json(session)) + bar.update(1) + write_json(root / "sections.json", meta.sections_json(session)) + bar.update(1) + write_json(root / "namespace-graph.json", meta.namespace_graph_json(session)) + bar.update(1) + write_json(root / "resources.json", meta.resources_json(session, root)) + bar.update(1) + + +def _write_final_documents(context: DotnetContext) -> None: + root = _need(context.root) + session = _need(context.session) + found = _need(context.found) + libs_doc = native_libs_json(context.libs, root) + write_json(root / "container.json", meta.container_json(found, context.extracted)) + write_json( + root / "triage.json", + meta.triage_json(session, found, context.reachable, libs_doc["libs"]), + ) + native_exports = [ + { + "abi": lib.abi, + "name": lib.name, + "export_dir": str(lib.export_dir) if lib.export_dir else None, + "status": lib.status, + "backend": lib.backend, + } + for lib in context.libs + ] + common: dict[str, Any] = { + "binary": str(context.input_path), + "kind": found.kind, + "assemblies": [info.name for info in session.assemblies], + "backend": "dotnet", + "decompiler": DotnetSession.decompiler_label, + "root_dir": str(root), + "src_dir": str(_need(context.raw_dir)), + "raw_src_dir": str(_need(context.raw_dir)), + "tree_src_dir": None, + "include_dir": None, + "data_dir": str(_need(context.data_dir)), + "header": None, + "agents": str(root / "AGENTS.md"), + "claude": str(root / "CLAUDE.md"), + "log": str(root / "tocode.log"), + "ida_database": None, + "source_files": [str(item) for item in context.source_files], + "il_files": [str(item) for item in context.il_files], + "project_files": [str(item) for item in context.project_files], + "tree_source_files": [], + "summary_files": [], + "asm_files": [str(item) for item in context.il_files], + "function_index": str(root / "function-index.json"), + "tree_function_index": None, + "function_count": len(session.methods), + "type_count": len(session.types), + "cluster_count": 0, + "failure_count": len(context.failures), + "requested_worker_count": context.options.jobs + if context.options.jobs is not None + else "auto", + "worker_count": context.worker_count, + "parallel_mode": context.render_mode, + "native_enabled": context.options.native, + "include_framework": context.options.include_framework, + "native_exports": native_exports, + } + write_json(root / "project.json", common) + manifest: dict[str, Any] = dict(common) + manifest.update( + { + "schema_version": 2, + "tocode_version": __version__, + "format": f"dotnet-{found.kind}", + "arch": "cil", + "bits": 64, + "entrypoints": [ + method.address + for kind, method in meta.export_roots(session) + if kind == "entrypoint" + ], + "raw_source_files": [str(item) for item in context.source_files], + "data_variable_count": 0, + "triage": str(root / "triage.json"), + "imports": str(root / "imports.json"), + "exports": str(root / "exports.json"), + "reachable": str(root / "reachable.json"), + "cluster_graph": None, + "namespace_graph": str(root / "namespace-graph.json"), + "assemblies_index": str(root / "assemblies.json"), + "types": str(root / "types.json"), + "type_count": len(session.types), + "strings": str(root / "strings.json"), + "sections": str(root / "sections.json"), + "resources": str(root / "resources.json"), + "container": str(root / "container.json"), + "native_libs": str(root / "native-libs.json"), + "variables_interesting": None, + "failures": [ + {"address": None, "name": name, "error": error} + for name, error in context.failures + ], + "native_errors": context.native_runner.errors + if context.native_runner + else [], + } + ) + write_json(root / "export-manifest.json", manifest) + write_text_atomic( + root / "AGENTS.md", + meta.build_dotnet_agents( + title=context.title, + kind=found.kind, + assemblies=[info.name for info in session.assemblies], + native=context.options.native + and any( + lib.status not in {"pending"} and not lib.status.startswith("skipped") + for lib in context.libs + ), + native_count=sum( + 1 for lib in context.libs if not lib.status.startswith("skipped") + ), + ) + + "\n", + ) + write_text_atomic(root / "CLAUDE.md", "@./AGENTS.md\n") + context.progress.log("Wrote project.json, export-manifest.json, AGENTS.md") + + +def _summary(context: DotnetContext) -> DotnetExportSummary: + session = context.session + found = _need(context.found) + return DotnetExportSummary( + root_dir=_need(context.root), + kind=found.kind, + assemblies=[info.name for info in session.assemblies] if session else [], + type_count=len(session.types) if session else 0, + method_count=len(session.methods) if session else 0, + source_files=list(context.source_files), + failed_types=list(context.failures), + native_total=len(context.libs), + native_done=sum(1 for lib in context.libs if lib.status == "done"), + native_errors=list(context.native_runner.errors) + if context.native_runner + else [], + seconds=time.monotonic() - context.started, + ) + + +def _need(value: Any) -> Any: + if value is None: + raise RuntimeError("export context not initialized") + return value diff --git a/src/tocode/dotnet_metadata.py b/src/tocode/dotnet_metadata.py new file mode 100644 index 0000000..20d08a3 --- /dev/null +++ b/src/tocode/dotnet_metadata.py @@ -0,0 +1,877 @@ +"""JSON metadata documents for .NET exports. + +Shapes follow ``metadata.py``/``apk_metadata.py`` wherever a native or Android +concept has a .NET equivalent (functions, function-index, strings, imports, +exports, sections, reachable, triage), so tooling that reads other ToCode +exports keeps working. .NET-only facts live in ``assemblies.json``, +``types.json``, ``namespace-graph.json``, ``resources.json`` and +``container.json``. Big per-row documents are generators for +``apk_metadata.write_json_rows``. +""" + +from __future__ import annotations + +from collections import deque +from pathlib import Path +import re +from typing import Any, Iterable, Iterator + +from .apk_metadata import INTERESTING_STRING_TIERS +from .backends.dotnet import ( + AssemblyInfo, + DotnetInput, + DotnetSession, + MethodInfo, + TypeInfo, +) + +# External API families worth surfacing first when triaging a .NET binary. +SUSPICIOUS_API_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = ( + ( + "dynamic code loading / reflection", + re.compile( + r"System\.Reflection\.Assembly::(Load|LoadFrom|LoadFile|UnsafeLoadFrom)\b|" + # CreateInstance() is what the compiler emits for `new T()`. + r"System\.AppDomain::(Load|ExecuteAssembly)|System\.Activator::CreateInstance(?!<)|" + r"System\.Reflection\.MethodBase::Invoke|System\.Type::InvokeMember|" + r"System\.Reflection\.Emit\.|AssemblyLoadContext::Load" + ), + ), + ("process execution", re.compile(r"System\.Diagnostics\.Process::Start")), + ( + "network", + re.compile( + r"System\.Net\.(Http\.HttpClient|WebClient|WebRequest|Sockets\.|Dns::|" + r"HttpWebRequest|WebSockets\.)" + ), + ), + ("cryptography", re.compile(r"System\.Security\.Cryptography\.")), + ("registry", re.compile(r"Microsoft\.Win32\.Registry")), + ( + "native interop", + re.compile( + r"System\.Runtime\.InteropServices\.(Marshal::(GetDelegateForFunctionPointer|" + r"Copy|AllocHGlobal|PtrToStructure)|NativeLibrary::)" + ), + ), + ("file system", re.compile(r"System\.IO\.(File|Directory|FileStream)::")), + ( + "unsafe deserialization", + re.compile( + r"BinaryFormatter|NetDataContractSerializer|SoapFormatter|LosFormatter|" + r"ObjectStateFormatter" + ), + ), + ( + "environment / host info", + re.compile( + r"System\.Environment::(get_MachineName|get_UserName|get_UserDomainName|" + r"get_OSVersion|GetEnvironmentVariable|SetEnvironmentVariable|GetFolderPath|" + r"get_CommandLine|GetCommandLineArgs|Exit|FailFast)" + ), + ), +) + + +def _rel(path: Path | None, root: Path) -> str | None: + if path is None: + return None + try: + return path.relative_to(root).as_posix() + except ValueError: + return str(path) + + +def _method_label(method: MethodInfo) -> str: + return f"{method.type_name.replace('/', '.')}.{method.name}" + + +# -------------------------------------------------------------------------- +# assemblies.json / container.json +# -------------------------------------------------------------------------- + + +def assemblies_json( + session: DotnetSession, found: DotnetInput, root: Path +) -> dict[str, Any]: + rows: list[dict[str, Any]] = [] + loaded = {info.file.path: info for info in session.assemblies} + for item in found.assemblies: + info = loaded.get(item.path) + base: dict[str, Any] = { + "file": item.entry, + "source": item.source, + "framework": item.framework, + "decompiled": info is not None, + "skip_reason": item.reason, + "target_framework_folder": item.tfm, + } + if info is None: + base["name"] = Path(item.entry).stem + rows.append(base) + continue + method_count = sum( + len(session.types[(info.index, token)].methods) for token in info.types + ) + base.update( + { + "name": info.name, + "full_name": info.full_name, + "version": info.version, + "culture": info.culture or None, + "public_key_token": info.public_key_token, + "module": info.module_name, + "mvid": info.mvid, + "kind": info.kind, + "target_framework": info.target_framework, + "runtime_version": info.pe.runtime_version, + "machine": info.pe.machine, + "pe32_plus": info.pe.pe32_plus, + "cor_flags": info.pe.cor_flag_names, + "ready_to_run": info.pe.ready_to_run, + "mixed_mode": info.pe.mixed_mode, + "strong_name_signed": "StrongNameSigned" in info.pe.cor_flag_names, + "entry_point": _entry_label(session, info), + "folder": info.folder, + "project": f"{info.folder}/{info.name}.csproj" if info.folder else None, + "type_count": len(info.types), + "method_count": method_count, + "user_string_count": len(info.user_strings), + "assembly_refs": info.assembly_refs, + "module_refs": info.module_refs, + "attributes": info.attributes, + "resources": [ + { + "name": resource.name, + "kind": resource.kind, + "size": resource.size, + "public": resource.public, + "path": _rel(resource.path, root), + } + for resource in info.resources + ], + "obfuscation_hints": info.obfuscation_hints, + } + ) + rows.append(base) + return {"count": len(rows), "assemblies": rows} + + +def _entry_label(session: DotnetSession, info: AssemblyInfo) -> dict[str, Any] | None: + if info.entry_point is None: + return None + method = session.method(info.index, info.entry_point) + if method is None: + return {"token": f"0x{info.entry_point:08x}", "name": None, "address": None} + return { + "token": f"0x{method.token:08x}", + "name": method.full_name, + "address": method.address, + } + + +def container_json( + found: DotnetInput, extracted: dict[str, str | None] +) -> dict[str, Any]: + entries: list[dict[str, Any]] = [] + if found.bundle is not None: + for entry in found.bundle.entries: + entries.append( + { + "path": entry.path, + "kind": entry.kind, + "offset": entry.offset, + "size": entry.size, + "compressed_size": entry.compressed_size or None, + "exported_path": extracted.get(entry.path), + } + ) + else: + for item in found.assemblies: + entries.append( + { + "path": item.entry, + "kind": "assembly", + "exported_path": extracted.get(item.entry), + } + ) + for native in found.natives: + entries.append( + { + "path": native.entry, + "kind": "native", + "exported_path": extracted.get(native.entry), + } + ) + for extra in found.extras: + entries.append( + { + "path": extra.entry, + "kind": extra.kind, + "exported_path": extracted.get(extra.entry), + } + ) + return { + "kind": found.kind, + "input": str(found.input), + "bundle": { + "version": f"{found.bundle.major}.{found.bundle.minor}", + "bundle_id": found.bundle.bundle_id, + "manifest_offset": found.bundle.header_offset, + } + if found.bundle is not None + else None, + "entries": entries, + } + + +# -------------------------------------------------------------------------- +# types.json / functions.json / function-index.json +# -------------------------------------------------------------------------- + + +def iter_type_rows(session: DotnetSession, root: Path) -> Iterator[dict[str, Any]]: + for record in session.types.values(): + assembly = session.assemblies[record.assembly] + yield { + "assembly": assembly.name, + "token": f"0x{record.token:08x}", + "name": record.full_name.replace("/", "+"), + "namespace": record.namespace, + "simple_name": record.name, + "kind": record.kind, + "size": None, + "visibility": record.visibility, + "flags": record.flags, + "base_type": record.base_type, + "interfaces": record.interfaces, + "generic_parameters": record.generic_params, + "declaring_type": f"0x{record.declaring_type:08x}" + if record.declaring_type + else None, + "compiler_generated": record.compiler_generated, + "attributes": record.attributes, + "cs_file": _rel(record.cs_path, root), + "il_file": _rel(record.il_path, root), + "decompile_error": record.error, + "disassemble_error": record.il_error, + "members": [ + { + "name": item.name, + "token": f"0x{item.token:08x}", + "type": item.type, + "flags": item.flags, + "constant": item.constant, + } + for item in record.fields + ], + "properties": [ + { + "name": item.name, + "token": f"0x{item.token:08x}", + "type": item.type, + "getter": f"0x{item.getter:08x}" if item.getter else None, + "setter": f"0x{item.setter:08x}" if item.setter else None, + } + for item in record.properties + ], + "events": [ + { + "name": item.name, + "token": f"0x{item.token:08x}", + "type": item.type, + } + for item in record.events + ], + "methods": [ + _method_summary(session.methods[method_id]) + for method_id in record.methods + ], + "c_decl": None, + } + + +def _method_summary(method: MethodInfo) -> dict[str, Any]: + return { + "name": method.name, + "token": f"0x{method.token:08x}", + "address": method.address, + "prototype": method.prototype, + "flags": method.flags, + "rva": f"0x{method.rva:x}" if method.rva else None, + "il_size": method.il_size, + "cs_lines": [method.cs_start, method.cs_end] if method.cs_start else None, + "il_lines": [method.il_start, method.il_end] if method.il_start else None, + } + + +def _source_paths( + session: DotnetSession, method: MethodInfo, root: Path +) -> tuple[str | None, str | None]: + record = session.types.get((method.assembly, method.type_token)) + if record is None: + return None, None + return _rel(record.cs_path, root), _rel(record.il_path, root) + + +def is_dead(session: DotnetSession, method: MethodInfo, roots: set[int]) -> bool: + if method.callers or method.id in roots or method.owner_method is not None: + return False + if method.is_virtual or method.is_constructor or method.unmanaged_entry: + return False + return method.name not in {".cctor", "Finalize", "Dispose"} + + +def iter_function_rows( + session: DotnetSession, root: Path, roots: set[int] +) -> Iterator[dict[str, Any]]: + by_id = session.methods + for method in session.methods: + cs_file, il_file = _source_paths(session, method, root) + yield { + "address": method.address, + "name": method.full_name, + "c_name": _method_label(method), + "prototype": method.prototype, + "size": method.il_size, + "calltype": "static" + if method.is_static + else ("virtual" if method.is_virtual else "instance"), + "return_type": method.return_type, + "params": [{"name": name, "type": kind} for name, kind in method.params], + "locals": [], + "decl_file": None, + "decl_dir": None, + "decl_line": None, + "nargs": len(method.params), + "nlocals": None, + "stackframe": None, + "callees": [by_id[item].address for item in method.callees], + "callees_imports": method.external_calls, + "callee_count": len(method.callees) + len(method.external_calls), + "callers": [by_id[item].address for item in method.callers], + "caller_count": len(method.callers), + "dead_code": is_dead(session, method, roots), + "source_file": cs_file, + "source_line_start": method.cs_start, + "source_line_end": method.cs_end, + "tree_source_file": None, + "tree_source_line_start": None, + "tree_source_line_end": None, + "asm_file": il_file, + "asm_line_start": method.il_start, + "asm_line_end": method.il_end, + "assembly": method.assembly_name, + "token": f"0x{method.token:08x}", + "rva": f"0x{method.rva:x}" if method.rva else None, + "flags": method.flags, + "impl_flags": method.impl_flags, + "attributes": method.attributes, + "compiler_generated": method.compiler_generated, + "owner": by_id[method.owner_method].address + if method.owner_method is not None + else None, + "pinvoke": {"module": method.pinvoke_module, "entry": method.pinvoke_entry} + if method.pinvoke_module + else None, + "unmanaged_export": method.unmanaged_entry, + "string_ref_count": method.string_ref_count, + "field_refs": method.field_refs, + } + + +def iter_function_index_rows( + session: DotnetSession, root: Path +) -> Iterator[dict[str, Any]]: + for method in session.methods: + cs_file, il_file = _source_paths(session, method, root) + yield { + "address": method.address, + "name": method.full_name, + "c": { + "path": cs_file, + "line_start": method.cs_start, + "line_end": method.cs_end, + } + if cs_file and method.cs_start + else None, + "asm": { + "path": il_file, + "line_start": method.il_start, + "line_end": method.il_end, + } + if il_file and method.il_start + else None, + "origin": None, + } + + +# -------------------------------------------------------------------------- +# strings / imports / exports / sections +# -------------------------------------------------------------------------- + + +def iter_string_rows(session: DotnetSession) -> Iterator[dict[str, Any]]: + by_id = session.methods + for info in session.assemblies: + heap = info.pe.stream("#US") + base = heap.offset if heap is not None else 0 + for offset, value in info.user_strings: + xrefs = [ + { + "function": by_id[method_id].full_name, + "address": by_id[method_id].address, + "access": "read", + } + for method_id in session.string_xrefs.get((info.index, offset), ()) + ] + yield { + "vaddr": f"{info.name}!0x{0x70000000 | offset:08x}", + "paddr": f"0x{base + offset:x}", + "size": len(value.encode("utf-16-le")), + "length": len(value), + "section": f"{info.name}#US", + "type": "utf16", + "value": value, + "xrefs": xrefs, + } + + +def imports_json(session: DotnetSession) -> dict[str, Any]: + groups: list[dict[str, Any]] = [] + for scope, members in sorted(session.external_refs.items()): + groups.append( + { + "dll": scope, + "kind": "assembly", + "functions": [ + { + "name": name, + "address": None, + "delay": False, + "callers": [session.methods[item].address for item in callers], + } + for name, callers in sorted(members.items()) + ], + } + ) + pinvoke: dict[str, list[MethodInfo]] = {} + for method in session.methods: + if method.pinvoke_module: + pinvoke.setdefault(method.pinvoke_module, []).append(method) + for module, methods in sorted(pinvoke.items()): + groups.append( + { + "dll": module, + "kind": "pinvoke", + "functions": [ + { + "name": method.pinvoke_entry or method.name, + "address": method.address, + "delay": True, + "managed": method.full_name, + "callers": [ + session.methods[item].address for item in method.callers + ], + } + for method in methods + ], + } + ) + return {"imports": groups} + + +def _type_visible(session: DotnetSession, record: TypeInfo) -> bool: + current: TypeInfo | None = record + while current is not None: + if current.visibility not in { + "Public", + "NestedPublic", + "NestedFamily", + "NestedFamORAssem", + }: + return False + parent = current.declaring_type + current = session.types.get((current.assembly, parent)) if parent else None + return True + + +def export_roots(session: DotnetSession) -> list[tuple[str, MethodInfo]]: + """(kind, method) for entry points, native exports, and public API.""" + rows: list[tuple[str, MethodInfo]] = [] + for info in session.assemblies: + if info.entry_point is not None: + method = session.method(info.index, info.entry_point) + if method is not None: + rows.append(("entrypoint", method)) + for method in session.methods: + if method.unmanaged_entry: + rows.append(("unmanaged-export", method)) + elif method.type_name == "" and method.name == ".cctor": + rows.append(("module-initializer", method)) + for method in session.methods: + info = session.assemblies[method.assembly] + if info.entry_point is not None: + continue # executables: the entry point is the API + if "Public" not in method.flags or method.compiler_generated: + continue + record = session.types.get((method.assembly, method.type_token)) + if record is not None and _type_visible(session, record): + rows.append(("public", method)) + return rows + + +def exports_json(session: DotnetSession) -> dict[str, Any]: + rows = [] + for ordinal, (kind, method) in enumerate(export_roots(session)): + rows.append( + { + "ordinal": ordinal, + "name": method.full_name, + "address": method.address, + "is_forwarder": False, + "forwarder_target": method.unmanaged_entry, + "kind": kind, + "assembly": method.assembly_name, + } + ) + return {"exports": rows} + + +def sections_json(session: DotnetSession) -> dict[str, Any]: + rows: list[dict[str, Any]] = [] + for info in session.assemblies: + for section in info.pe.sections: + rows.append( + { + "assembly": info.name, + "name": section.name, + "vaddr": f"0x{info.pe.image_base + section.virtual_address:x}", + "paddr": f"0x{section.raw_offset:x}", + "size": section.raw_size, + "vsize": section.virtual_size, + "type": "pe-section", + "permissions": section.perms, + "rwx": section.perms == "rwx", + "entropy": None, + } + ) + for stream in info.pe.streams: + rows.append( + { + "assembly": info.name, + "name": stream.name, + "vaddr": None, + "paddr": f"0x{stream.offset:x}", + "size": stream.size, + "vsize": stream.size, + "type": "metadata-stream", + "permissions": "r--", + "rwx": False, + "entropy": None, + } + ) + return {"sections": rows} + + +# -------------------------------------------------------------------------- +# reachable / namespace graph / triage +# -------------------------------------------------------------------------- + + +def reachable_json(session: DotnetSession) -> tuple[dict[str, Any], set[int]]: + roots = [method.id for _, method in export_roots(session)] + owned: dict[int, list[int]] = {} + for method in session.methods: + if method.owner_method is not None: + owned.setdefault(method.owner_method, []).append(method.id) + depths: dict[int, int] = {} + queue: deque[int] = deque() + for root in roots: + if root not in depths: + depths[root] = 0 + queue.append(root) + while queue: + current = queue.popleft() + depth = depths[current] + method = session.methods[current] + for target in [*method.callees, *owned.get(current, ())]: + if target not in depths: + depths[target] = depth + 1 + queue.append(target) + rows = [ + { + "address": session.methods[item].address, + "name": session.methods[item].full_name, + "depth": depth, + } + for item, depth in sorted(depths.items(), key=lambda pair: (pair[1], pair[0])) + ] + return ( + { + "reachable": rows, + "unreachable_count": len(session.methods) - len(depths), + "seed_count": len(roots), + "note": "virtual/interface dispatch and reflection are not followed", + }, + set(roots), + ) + + +def namespace_graph_json(session: DotnetSession) -> dict[str, Any]: + def key(method: MethodInfo) -> tuple[str, str]: + record = session.types.get((method.assembly, method.type_token)) + return method.assembly_name, record.namespace if record else "" + + counts: dict[tuple[str, str], int] = {} + folders: dict[tuple[str, str], str] = {} + for record in session.types.values(): + name = (session.assemblies[record.assembly].name, record.namespace) + counts[name] = counts.get(name, 0) + 1 + if record.cs_path is not None and name not in folders: + folders[name] = record.cs_path.parent.as_posix() + calls: dict[tuple[str, str], dict[str, int]] = {} + external: dict[tuple[str, str], dict[str, int]] = {} + for method in session.methods: + source = key(method) + for callee in method.callees: + target = key(session.methods[callee]) + if target != source: + label = f"{target[0]}:{target[1] or ''}" + bucket = calls.setdefault(source, {}) + bucket[label] = bucket.get(label, 0) + 1 + for call in method.external_calls: + scope = call[1:].split("]", 1)[0] if call.startswith("[") else "?" + bucket = external.setdefault(source, {}) + bucket[scope] = bucket.get(scope, 0) + 1 + rows = [] + for name in sorted(counts): + rows.append( + { + "assembly": name[0], + "namespace": name[1] or "", + "types": counts[name], + "folder": folders.get(name), + "calls_namespaces": [ + {"namespace": target, "calls": count} + for target, count in sorted( + calls.get(name, {}).items(), key=lambda pair: -pair[1] + ) + ], + "calls_assemblies": [ + {"assembly": target, "calls": count} + for target, count in sorted( + external.get(name, {}).items(), key=lambda pair: -pair[1] + ) + ], + } + ) + return {"namespaces": rows} + + +def interesting_strings( + values: Iterable[tuple[str, str]], *, limit: int = 100 +) -> list[dict[str, Any]]: + buckets: dict[int, list[dict[str, Any]]] = { + tier: [] for tier, _ in INTERESTING_STRING_TIERS + } + seen: set[str] = set() + for address, value in values: + if len(value) < 6 or len(value) > 512 or value in seen: + continue + for tier, pattern in INTERESTING_STRING_TIERS: + if pattern.search(value): + seen.add(value) + buckets[tier].append({"value": value, "address": address, "tier": tier}) + break + rows: list[dict[str, Any]] = [] + for tier in sorted(buckets): + rows.extend(buckets[tier]) + return rows[:limit] + + +def suspicious_apis(session: DotnetSession) -> list[dict[str, Any]]: + found: dict[str, dict[str, set[str]]] = {} + for method in session.methods: + for name in method.external_calls: + for label, pattern in SUSPICIOUS_API_PATTERNS: + if pattern.search(name): + found.setdefault(label, {}).setdefault(name, set()).add( + method.address + ) + break + return [ + { + "category": label, + "apis": [ + { + "name": name, + "caller_count": len(callers), + "callers": sorted(callers)[:10], + } + for name, callers in sorted(apis.items()) + ], + } + for label, apis in found.items() + ] + + +def triage_json( + session: DotnetSession, + found: DotnetInput, + reachable: dict[str, Any], + native_libs: list[dict[str, Any]], +) -> dict[str, Any]: + strings = ( + (f"{info.name}!0x{0x70000000 | offset:08x}", value) + for info in session.assemblies + for offset, value in info.user_strings + ) + pinvoke: dict[str, list[str]] = {} + for method in session.methods: + if method.pinvoke_module: + pinvoke.setdefault(method.pinvoke_module, []).append( + method.pinvoke_entry or method.name + ) + main = session.assemblies[0] if session.assemblies else None + return { + "binary_type": f"dotnet-{found.kind}", + "arch": "cil", + "bits": 64 if main is not None and main.pe.pe32_plus else 32, + "assemblies": [ + { + "name": info.name, + "version": info.version, + "kind": info.kind, + "target_framework": info.target_framework, + "entry_point": _entry_label(session, info), + "ready_to_run": info.pe.ready_to_run, + "mixed_mode": info.pe.mixed_mode, + "strong_name_signed": "StrongNameSigned" in info.pe.cor_flag_names, + "types": len(info.types), + "obfuscation_hints": info.obfuscation_hints, + } + for info in session.assemblies + ], + "skipped_assemblies": [ + {"file": item.entry, "reason": item.reason} + for item in found.assemblies + if not item.include + ][:200], + "type_count": len(session.types), + "method_count": len(session.methods), + "pinvoke": [ + {"module": module, "functions": sorted(set(names))[:50]} + for module, names in sorted(pinvoke.items()) + ], + "suspicious_apis": suspicious_apis(session), + "resources": [ + {"assembly": info.name, "name": resource.name, "size": resource.size} + for info in session.assemblies + for resource in info.resources + ][:100], + "native_libs": [ + { + "arch": item["abi"], + "name": item["name"], + "status": item["status"], + "export_dir": item.get("export_dir"), + } + for item in native_libs + ], + "strings_of_interest": interesting_strings(strings), + "reachable_count": len(reachable.get("reachable", [])), + "unreachable_count": reachable.get("unreachable_count", 0), + } + + +def resources_json(session: DotnetSession, root: Path) -> dict[str, Any]: + return { + "resources": [ + { + "assembly": info.name, + "name": resource.name, + "kind": resource.kind, + "size": resource.size, + "public": resource.public, + "linked_file": resource.file, + "path": _rel(resource.path, root), + "entries": resource.entries, + "error": resource.error, + } + for info in session.assemblies + for resource in info.resources + ] + } + + +# -------------------------------------------------------------------------- +# Generated AGENTS.md +# -------------------------------------------------------------------------- + + +def build_dotnet_agents( + *, title: str, kind: str, assemblies: list[str], native: bool, native_count: int +) -> str: + shown = ", ".join(f"`{name}`" for name in assemblies[:8]) + if len(assemblies) > 8: + shown += ", ..." + lines = [ + "# AGENTS", + "", + f"You are working inside a ToCode .NET export of `{title}` ({kind}; assemblies: {shown or 'none'}).", + "Treat the recovered C#, the IL, the metadata, and the extracted resources as evidence for reverse engineering.", + "", + "## Mission", + "", + "Reverse the program, answer user questions, or serve as an oracle/helper to the user regarding this recovered source code.", + "Write a report only when the user asks for one; otherwise keep findings focused on the question or task at hand.", + "Do not refactor or modify the generated export unless the user explicitly asks for edits.", + "Use subagents only for narrow evidence-gathering tasks such as strings, entry paths, or one namespace.", + "", + "## Files", + "", + "- `src/raw///.../.cs`: decompiled C# (ICSharpCode.Decompiler), one file per top-level type with its nested types, folders follow namespaces like a dnSpy/ILSpy project export.", + "- `src/raw///.../.il`: IL disassembly of the same type with RVA, raw bytecode, and metadata tokens per instruction. Read it when the C# is ambiguous, obfuscated, or failed to decompile.", + "- `src/raw//Properties/AssemblyInfo.cs` and `Manifest.il`: assembly/module attributes, references, and the IL manifest; `.csproj`: a project file for the recovered sources.", + "- `assemblies.json`: every assembly found (version, target framework, entry point, flags, ReadyToRun/mixed-mode, references, resources, obfuscation hints, and why skipped assemblies were not decompiled).", + "- `types.json`: every type with base type, interfaces, attributes, fields, properties, events, and methods with their C#/IL line ranges.", + "- `functions.json`: per-method callers/callees, external (framework/library) calls, P/Invoke targets, field references, and C#/IL line ranges. `address` is `!0x`.", + "- `function-index.json`: exact C# (`c`) and IL (`asm`) file and line range for each method.", + "- `strings.json`: the user-string heap (`ldstr` literals) with the methods that load each string.", + "- `imports.json`: members referenced from other assemblies (grouped by assembly) and P/Invoke imports (grouped by native library).", + "- `exports.json`: entry points, module initializers, `UnmanagedCallersOnly` native exports, and the public API of libraries.", + "- `reachable.json`: methods reachable from those roots with call depth (virtual dispatch and reflection are not followed).", + "- `namespace-graph.json`: inter-namespace and inter-assembly call graph.", + "- `sections.json`: PE sections and metadata streams of every assembly.", + "- `resources.json` and `data/resources//`: embedded resources, with `.resources` files decoded to JSON.", + "- `container.json`: the single-file bundle / NuGet package / assembly layout and where each entry was extracted.", + "- `triage.json`: first-read summary: assemblies, entry points, P/Invoke, suspicious API families, resources, native libraries, strings of interest, reachability counts.", + ] + if native: + lines.append( + f"- `native///`: a complete nested ToCode export for each of the {native_count} native libraries (mixed-mode code, bundled or P/Invoke'd libraries). `native-libs.json` lists their status; `lib//` holds the extracted files.", + ) + else: + lines.append( + "- `native/`: not generated (no native code found, or `--no-native`); see `native-libs.json`.", + ) + lines.extend( + [ + "- `project.json` and `export-manifest.json`: top-level export paths and artifact inventory.", + "- `tocode.log`: export history including per-type decompilation and native export status.", + "- `CLAUDE.md`: Claude entrypoint that references `AGENTS.md`.", + "", + "## Working Style", + "", + "- Start with `triage.json`, `assemblies.json`, `strings.json`, and `reachable.json`.", + "- Use `functions.json` and `function-index.json` to open only the method line ranges you need, in C# or IL.", + "- Compiler-generated types (`<>c`, `d__N`) are folded back into their owners in the C#; `owner` in `functions.json` links state machines and lambdas to the method that created them.", + "- Follow P/Invoke methods into `native/` exports when the logic lives in native code.", + "- Treat obfuscated names as opaque; rely on strings, framework calls, and resources to recover intent.", + "- Cite file paths, type and method names, metadata tokens, and line numbers for every major claim.", + "- Be explicit about uncertainty, decompilation failures (`types.json` `decompile_error`), and unresolved references.", + "", + ] + ) + return "\n".join(lines) diff --git a/src/tocode/schema.py b/src/tocode/schema.py index 62a8c73..4a08054 100644 --- a/src/tocode/schema.py +++ b/src/tocode/schema.py @@ -279,3 +279,18 @@ class ApkExportSummary: native_errors: list[str] = field(default_factory=list) manifest_path: Path | None = None seconds: float = 0.0 + + +@dataclass(slots=True) +class DotnetExportSummary: + root_dir: Path + kind: str + assemblies: list[str] + type_count: int + method_count: int + source_files: list[Path] + failed_types: list[tuple[str, str]] + native_total: int + native_done: int + native_errors: list[str] = field(default_factory=list) + seconds: float = 0.0 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..e014a93 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +import pytest + + +@pytest.fixture(autouse=True) +def _no_native_memory_wait(monkeypatch: pytest.MonkeyPatch) -> None: + """Exports in tests must not wait for free RAM on the host running them.""" + monkeypatch.setenv("TOCODE_NATIVE_MIN_FREE_MB", "0") diff --git a/tests/dotnet_fixtures.py b/tests/dotnet_fixtures.py new file mode 100644 index 0000000..3e538d2 --- /dev/null +++ b/tests/dotnet_fixtures.py @@ -0,0 +1,153 @@ +"""Synthetic .NET containers for tests (no .NET SDK or runtime needed).""" + +from __future__ import annotations + +from pathlib import Path +import struct +import zlib + +from tocode.backends.dotnet_pe import BUNDLE_SIGNATURE + +TEXT_RVA = 0x2000 +TEXT_OFFSET = 0x200 +COR20_RVA = TEXT_RVA + 0x8 +METADATA_RVA = TEXT_RVA + 0x100 + + +def _compressed(value: int) -> bytes: + if value < 0x80: + return bytes([value]) + if value < 0x4000: + return struct.pack(">H", value | 0x8000) + return struct.pack(">I", value | 0xC0000000) + + +def user_string_heap(values: list[str]) -> bytes: + heap = bytearray(b"\x00") + for value in values: + encoded = value.encode("utf-16-le") + heap += _compressed(len(encoded) + 1) + encoded + b"\x00" + while len(heap) % 4: + heap += b"\x00" + return bytes(heap) + + +def managed_pe( + *, + strings: list[str] | None = None, + cor_flags: int = 0x1, + ready_to_run: bool = False, + machine: int = 0x8664, +) -> bytes: + """A minimal PE32+ image with a CLR header, metadata root, and #US heap.""" + heap = user_string_heap(strings or ["hello", "https://example.invalid/api"]) + table_stream = b"\x00" * 8 + version = b"v4.0.30319\x00\x00" + streams = [("#~", table_stream), ("#US", heap)] + header = bytearray(b"BSJB" + struct.pack(" bytes: + data = bytearray(b"\x7fELF\x02\x01\x01" + b"\x00" * 57) + struct.pack_into(" bytes: + raw = value.encode() + length = len(raw) + prefix = bytearray() + while True: + byte = length & 0x7F + length >>= 7 + prefix.append(byte | (0x80 if length else 0)) + if not length: + break + return bytes(prefix) + raw + + +def single_file_bundle( + entries: list[tuple[str, int, bytes, bool]], *, launcher: bool = False +) -> bytes: + """An ELF "apphost" with a v6 bundle manifest. + + ``entries`` are ``(path, kind, data, compress)``; ``launcher=True`` writes + a plain apphost (signature with a zero manifest offset). + """ + host = bytearray(native_elf()) + host += b"\x00" * 64 + marker_at = len(host) + host += b"\x00" * 8 + BUNDLE_SIGNATURE + b"\x00" * 32 + if launcher: + return bytes(host) + payload = bytearray(host) + records: list[tuple[int, int, int, int, str]] = [] + for path, kind, data, compress in entries: + stored = data + compressed = 0 + if compress: + deflater = zlib.compressobj(9, zlib.DEFLATED, -15) + stored = deflater.compress(data) + deflater.flush() + compressed = len(stored) + records.append((len(payload), len(data), compressed, kind, path)) + payload += stored + manifest_at = len(payload) + manifest = bytearray(struct.pack(" Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return path diff --git a/tests/test_apk_export.py b/tests/test_apk_export.py index 1541dfc..658f5f5 100644 --- a/tests/test_apk_export.py +++ b/tests/test_apk_export.py @@ -987,3 +987,31 @@ def exporter( ) assert seen == ["com.example.app"] * 3 + + +def test_native_runner_heartbeat_reports_memory_wait(tmp_path: Path) -> None: + lib = NativeLib( + "arm64", + "libfoo.so", + "lib/arm64/libfoo.so", + "base.apk", + 1, + "h", + tmp_path / "a.so", + ) + runner = NativeRunner( + libs=[lib], + root=tmp_path, + progress=Progress(enabled=False), + exporter=_fake_native_exporter(), + min_free_mb=2048, + ) + runner.waiting = (lib, 900) + + status = runner.describe_current() + + assert status is not None + assert ( + "waiting for memory before libfoo.so (arm64): 900 MB free, need 2048 MB" + in status + ) diff --git a/tests/test_cli.py b/tests/test_cli.py index d8167e3..dfe9b6b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -113,3 +113,81 @@ def test_package_installs_a_tocode_console_script() -> None: } assert scripts == {"tocode": "tocode.cli:main"} + + +def test_parser_has_dotnet_flags() -> None: + args = build_parser().parse_args(["--as-native", "--include-framework", "app.dll"]) + + assert args.as_native is True and args.include_framework is True + assert build_parser().parse_args(["app.dll"]).as_native is False + + +def test_main_routes_dotnet_input_to_dotnet_export(tmp_path, monkeypatch) -> None: + from types import SimpleNamespace + + from dotnet_fixtures import managed_pe + + from tocode import cli, dotnet + + binary = tmp_path / "App.dll" + binary.write_bytes(managed_pe()) + seen = {} + + def fake_export(path, *, options, progress): + seen["path"] = path + seen["options"] = options + return SimpleNamespace( + root_dir=tmp_path / "out", + kind="assembly", + assemblies=["App"], + type_count=1, + method_count=1, + failed_types=[], + native_total=0, + native_done=0, + native_errors=[], + ) + + monkeypatch.setattr(dotnet, "export_dotnet", fake_export) + + assert ( + cli.main(["--include-framework", "--no-native", "-j", "2", "-q", str(binary)]) + == 0 + ) + assert seen["path"] == binary.resolve() + assert seen["options"].include_framework is True + assert seen["options"].native is False + assert seen["options"].jobs == 2 + + +def test_main_as_native_skips_the_dotnet_backend(tmp_path, monkeypatch) -> None: + from dotnet_fixtures import managed_pe + + from tocode import cli + + binary = tmp_path / "App.dll" + binary.write_bytes(managed_pe()) + called = {} + + def fake_run_one(path, *, args, progress, out_dir): + called["path"] = path + raise cli.ToCodeError("stop here") + + monkeypatch.setattr(cli, "_run_one", fake_run_one) + + assert cli.main(["--as-native", "-q", str(binary)]) == 1 + assert called["path"] == binary.resolve() + + +def test_main_rejects_binja_backend_for_dotnet_input(tmp_path) -> None: + from dotnet_fixtures import managed_pe + + from tocode.cli import main + + binary = tmp_path / "App.dll" + binary.write_bytes(managed_pe()) + + with pytest.raises(SystemExit) as info: + main(["--backend", "binja", str(binary)]) + + assert info.value.code == 2 diff --git a/tests/test_dotnet_backend.py b/tests/test_dotnet_backend.py new file mode 100644 index 0000000..ee04a13 --- /dev/null +++ b/tests/test_dotnet_backend.py @@ -0,0 +1,516 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +import struct +import zipfile + +import pytest + +from dotnet_fixtures import managed_pe, native_elf, single_file_bundle, write +from tocode.backends import dotnet as backend +from tocode.backends.dotnet import ( + DOTNET_LIB_DIR, + discover_dotnet_input, + il_method_ranges, + is_dotnet_input, + is_framework_assembly, + is_framework_native, + native_arch, + pinvoke_neighbours, +) +from tocode.backends.dotnet_pe import ( + COMIMAGE_FLAGS_32BITREQUIRED, + COMIMAGE_FLAGS_ILONLY, + COMIMAGE_FLAGS_STRONGNAMESIGNED, + ONE_BYTE_OPCODES, + TWO_BYTE_OPCODES, + extract_bundle_entry, + is_apphost_launcher, + iter_user_strings, + method_body, + parse_bundle, + parse_pe, + read_bundle, + read_pe_info, + scan_il_tokens, +) +from tocode.errors import ToCodeError + + +# -------------------------------------------------------------------------- +# PE / CLR headers +# -------------------------------------------------------------------------- + + +def test_parse_pe_reads_clr_header_and_metadata_streams() -> None: + data = managed_pe( + strings=["hello", "https://example.invalid/api"], + cor_flags=COMIMAGE_FLAGS_ILONLY | COMIMAGE_FLAGS_STRONGNAMESIGNED, + ) + + info = parse_pe(data) + + assert info is not None + assert info.is_managed and info.il_only and not info.mixed_mode + assert info.machine == "x86_64" and info.pe32_plus + assert info.image_base == 0x180000000 + assert info.runtime_version == "v4.0.30319" + assert info.entry_point_token == 0x06000001 + assert info.cor_flag_names == ["ILOnly", "StrongNameSigned"] + assert [stream.name for stream in info.streams] == ["#~", "#US"] + assert info.sections[0].name == ".text" and info.sections[0].perms == "r-x" + heap = info.stream("#US") + assert heap is not None + assert [ + value + for _, value in iter_user_strings(data[heap.offset : heap.offset + heap.size]) + ] == [ + "hello", + "https://example.invalid/api", + ] + + +def test_mixed_mode_versus_ready_to_run() -> None: + mixed = parse_pe(managed_pe(cor_flags=COMIMAGE_FLAGS_32BITREQUIRED)) + r2r = parse_pe(managed_pe(cor_flags=0, ready_to_run=True)) + + assert mixed is not None and mixed.mixed_mode and not mixed.ready_to_run + # ReadyToRun clears ILONLY too, but its native code is precompiled IL. + assert r2r is not None and r2r.ready_to_run and not r2r.mixed_mode + + +def test_parse_pe_rejects_non_pe_input() -> None: + assert parse_pe(b"\x7fELF" + b"\x00" * 64) is None + assert parse_pe(b"MZ" + b"\x00" * 10) is None + assert parse_pe(b"") is None + + +def test_rva_to_offset_maps_into_sections() -> None: + info = parse_pe(managed_pe()) + assert info is not None + assert info.rva_to_offset(0x2010) == 0x210 + assert info.rva_to_offset(0x9000) is None + + +def test_user_strings_handle_multibyte_lengths() -> None: + from dotnet_fixtures import user_string_heap + + long_value = "x" * 300 # 601 bytes -> 2-byte compressed length + heap = user_string_heap(["a", long_value, "ñandú"]) + values = iter_user_strings(heap) + + assert [value for _, value in values] == ["a", long_value, "ñandú"] + assert values[0][0] == 1 + + +# -------------------------------------------------------------------------- +# IL bodies and token scan +# -------------------------------------------------------------------------- + + +def test_method_body_reads_tiny_and_fat_headers() -> None: + tiny = bytes([(3 << 2) | 0x2, 0x02, 0x2A, 0x00]) + fat = struct.pack(" None: + code = bytes.fromhex( + "72" + "01000070" # ldstr string 1 + "28" + "0a00000a" # call memberref + "45" + "02000000" + "00000000" + "00000000" # switch with 2 targets + "fe06" + "0300002b" # ldftn methodspec + "7b" + "05000004" # ldfld field + "20" + "ffffffff" # ldc.i4 (no token) + "fe0c" + "0100" # ldloc 1 (2-byte operand) + "8d" + "0100001b" # newarr typespec + "d0" + "02000002" # ldtoken typedef + "2a" # ret + ) + + assert scan_il_tokens(code) == [ + ("string", 0x70000001), + ("method", 0x0A00000A), + ("method", 0x2B000003), + ("field", 0x04000005), + ("type", 0x1B000001), + ("token", 0x02000002), + ] + + +def test_scan_il_tokens_stops_at_invalid_opcode() -> None: + # 0x24 is unassigned: an encrypted/garbled body must not be misread. + assert scan_il_tokens(bytes.fromhex("72010000702428020000062a")) == [ + ("string", 0x70000001) + ] + + +def test_opcode_tables_leave_only_the_unassigned_ecma_codes_out() -> None: + unassigned = ( + {0x24, 0x77, 0x78, 0xC4, 0xC5} + | set(range(0xA6, 0xB3)) + | set(range(0xBB, 0xC2)) + | set(range(0xC7, 0xD0)) + ) + assert {code for code in range(0xE1) if code not in ONE_BYTE_OPCODES} == unassigned + assert {code for code in range(0x1F) if code not in TWO_BYTE_OPCODES} == { + 0x08, + 0x10, + 0x1B, + } + assert TWO_BYTE_OPCODES[0x06] == "method" and TWO_BYTE_OPCODES[0x16] == "type" + + +# -------------------------------------------------------------------------- +# Single-file bundles +# -------------------------------------------------------------------------- + + +def _bundle(tmp_path: Path) -> Path: + deps = { + "targets": { + ".NETCoreApp,Version=v10.0/linux-arm64": { + "runtimepack.Microsoft.NETCore.App.Runtime.linux-arm64/10.0.0": { + "runtime": {"System.Private.CoreLib.dll": {}}, + "native": {"libcoreclr.so": {}}, + } + } + }, + "libraries": { + "runtimepack.Microsoft.NETCore.App.Runtime.linux-arm64/10.0.0": { + "type": "runtimepack" + } + }, + } + return write( + tmp_path / "App", + single_file_bundle( + [ + ("App.dll", 1, managed_pe(strings=["app"]), True), + ( + "System.Private.CoreLib.dll", + 1, + managed_pe(strings=["corelib"]), + False, + ), + ("Vendor.Lib.dll", 1, managed_pe(strings=["vendor"]), True), + ("libcoreclr.so", 2, native_elf(), False), + ("libcustom.so", 2, native_elf(0x3E), True), + ("App.deps.json", 3, json.dumps(deps).encode(), False), + ("App.runtimeconfig.json", 4, b"{}", False), + ] + ), + ) + + +def test_parse_bundle_lists_entries_and_extracts_compressed_data( + tmp_path: Path, +) -> None: + path = _bundle(tmp_path) + + manifest = read_bundle(path) + + assert manifest is not None + assert (manifest.major, manifest.minor, manifest.bundle_id) == ( + 6, + 0, + "testbundle01", + ) + assert [(entry.path, entry.kind) for entry in manifest.entries][:4] == [ + ("App.dll", "assembly"), + ("System.Private.CoreLib.dll", "assembly"), + ("Vendor.Lib.dll", "assembly"), + ("libcoreclr.so", "native"), + ] + first = manifest.entries[0] + assert first.compressed_size and first.compressed_size < first.size + extract_bundle_entry(path, first, tmp_path / "out" / "App.dll") + assert (tmp_path / "out" / "App.dll").read_bytes() == managed_pe(strings=["app"]) + assert not is_apphost_launcher(path) + + +def test_plain_apphost_is_a_launcher_not_a_bundle(tmp_path: Path) -> None: + host = write(tmp_path / "Tool", single_file_bundle([], launcher=True)) + + assert parse_bundle(host.read_bytes()) is None + assert is_apphost_launcher(host) + assert not is_dotnet_input(host) # no Tool.dll next to it + write(tmp_path / "Tool.dll", managed_pe()) + assert is_dotnet_input(host) + + found = discover_dotnet_input(host, workdir=tmp_path / "work") + assert found.kind == "apphost" + assert [item.path.name for item in found.assemblies] == ["Tool.dll"] + + +def test_discover_bundle_splits_app_framework_and_natives(tmp_path: Path) -> None: + path = _bundle(tmp_path) + + found = discover_dotnet_input(path, workdir=tmp_path / "work") + + assert found.kind == "bundle" + assert {item.entry: item.include for item in found.assemblies} == { + "App.dll": True, + "System.Private.CoreLib.dll": False, + "Vendor.Lib.dll": True, + } + assert {item.name: item.framework for item in found.natives} == { + "libcoreclr.so": True, + "libcustom.so": False, + } + assert {item.name: item.arch for item in found.natives}["libcustom.so"] == "x86_64" + assert {extra.kind for extra in found.extras} == {"deps-json", "runtime-config"} + assert found.search_dirs == [tmp_path / "work" / "bundle"] + + everything = discover_dotnet_input( + path, workdir=tmp_path / "work2", include_framework=True + ) + assert all(item.include for item in everything.assemblies) + + +# -------------------------------------------------------------------------- +# NuGet packages and plain assemblies +# -------------------------------------------------------------------------- + + +def test_discover_nupkg_prefers_the_newest_target_framework(tmp_path: Path) -> None: + package = tmp_path / "Vendor.Lib.1.0.0.nupkg" + with zipfile.ZipFile(package, "w") as archive: + archive.writestr("Vendor.Lib.nuspec", "") + archive.writestr( + "lib/netstandard2.0/Vendor.Lib.dll", managed_pe(strings=["ns20"]) + ) + archive.writestr("lib/net8.0/Vendor.Lib.dll", managed_pe(strings=["net8"])) + archive.writestr("lib/net48/Vendor.Lib.dll", managed_pe(strings=["net48"])) + archive.writestr("ref/net8.0/Vendor.Lib.dll", managed_pe(strings=["ref"])) + archive.writestr("runtimes/linux-x64/native/libvendor.so", native_elf(0x3E)) + + found = discover_dotnet_input(package, workdir=tmp_path / "work") + + assert found.kind == "nupkg" + chosen = [item.entry for item in found.included] + assert chosen == ["lib/net8.0/Vendor.Lib.dll"] + skipped = {item.entry: item.reason for item in found.assemblies if not item.include} + assert set(skipped) == { + "lib/netstandard2.0/Vendor.Lib.dll", + "lib/net48/Vendor.Lib.dll", + "ref/net8.0/Vendor.Lib.dll", + } + assert [(item.name, item.arch) for item in found.natives] == [ + ("libvendor.so", "linux-x64") + ] + assert [extra.entry for extra in found.extras] == ["Vendor.Lib.nuspec"] + + +def test_tfm_ranking_prefers_modern_targets() -> None: + from tocode.backends.dotnet import _tfm_rank + + newtonsoft = [ + "net20", + "net35", + "net40", + "net45", + "net6.0", + "netstandard1.0", + "netstandard1.3", + "netstandard2.0", + ] + assert max(newtonsoft, key=_tfm_rank) == "net6.0" + assert sorted( + ["net48", "net472", "netcoreapp3.1", "net10.0-windows", "netstandard2.1"], + key=_tfm_rank, + ) == [ + "net472", + "net48", + "netstandard2.1", + "netcoreapp3.1", + "net10.0-windows", + ] + + +def test_discover_plain_assembly_and_mixed_mode(tmp_path: Path) -> None: + plain = write(tmp_path / "Plain.dll", managed_pe()) + mixed = write( + tmp_path / "Mixed.dll", managed_pe(cor_flags=COMIMAGE_FLAGS_32BITREQUIRED) + ) + + assert discover_dotnet_input(plain, workdir=tmp_path / "w1").natives == [] + found = discover_dotnet_input(mixed, workdir=tmp_path / "w2") + assert [(item.name, item.path) for item in found.natives] == [("Mixed.dll", mixed)] + + +def test_is_dotnet_input_rejects_native_and_garbage(tmp_path: Path) -> None: + assert not is_dotnet_input(write(tmp_path / "libfoo.so", native_elf())) + assert not is_dotnet_input(write(tmp_path / "junk.bin", b"hello")) + assert is_dotnet_input(write(tmp_path / "App.exe", managed_pe())) + with pytest.raises(ToCodeError): + discover_dotnet_input(tmp_path / "libfoo.so", workdir=tmp_path / "w") + + +def test_pinvoke_neighbours_finds_platform_named_libraries(tmp_path: Path) -> None: + write(tmp_path / "libnative_helper.so", native_elf()) + write(tmp_path / "crypto.dll", managed_pe()) # managed: not a native target + write(tmp_path / "libzstd.so", native_elf()) + + found = pinvoke_neighbours( + tmp_path, {"native_helper", "crypto", "libzstd.so", "libc"} + ) + + assert sorted((module, path.name) for module, path in found) == [ + ("libzstd.so", "libzstd.so"), + ("native_helper", "libnative_helper.so"), + ] + + +def test_framework_name_heuristics() -> None: + assert is_framework_assembly("System.Private.CoreLib.dll") + assert is_framework_assembly("mscorlib.dll") + assert is_framework_assembly("Microsoft.CSharp.dll") + assert not is_framework_assembly("Microsoft.Build.dll") + assert not is_framework_assembly("Newtonsoft.Json.dll") + assert is_framework_native("libcoreclr.so") + assert is_framework_native("libSystem.Native.so") + assert is_framework_native("clrjit.dll") + assert not is_framework_native("libsqlite3.so") + + +def test_native_arch_reads_elf_and_pe_machines(tmp_path: Path) -> None: + assert native_arch(write(tmp_path / "a.so", native_elf(0xB7))) == "arm64" + assert native_arch(write(tmp_path / "b.so", native_elf(0x3E))) == "x86_64" + assert native_arch(write(tmp_path / "c.dll", managed_pe(machine=0x14C))) == "x86" + + +def test_read_pe_info_on_missing_file(tmp_path: Path) -> None: + assert read_pe_info(tmp_path / "missing.dll") is None + + +# -------------------------------------------------------------------------- +# Decompiler output parsing and runtime discovery +# -------------------------------------------------------------------------- + + +def test_il_method_ranges_follow_method_blocks() -> None: + text = "\n".join( + [ + ".class /* 02000002 */ public C", + "{", + " .method /* 06000001 */ public hidebysig", + " instance void M () cil managed", + " {", + " IL_0000: ret", + " } // end of method C::M", + "", + " .method /* 0600000A */ private static", + " void N () cil managed", + " {", + " } // end of method C::N", + "} // end of class C", + ] + ) + + assert il_method_ranges(text) == {0x06000001: (3, 7), 0x0600000A: (9, 12)} + + +def test_find_runtime_requires_net9_or_newer( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = tmp_path / "dotnet" + for version in ("8.0.10", "9.0.4", "10.0.12"): + folder = root / "shared" / "Microsoft.NETCore.App" / version + folder.mkdir(parents=True) + (folder / "System.Private.CoreLib.dll").write_bytes(b"MZ") + monkeypatch.setattr(backend, "dotnet_root_candidates", lambda: [root]) + assert backend.find_runtime() == (root, "10.0.12") + + for version in ("9.0.4", "10.0.12"): + ( + root + / "shared" + / "Microsoft.NETCore.App" + / version + / "System.Private.CoreLib.dll" + ).unlink() + assert backend.find_runtime() is None + + +def test_vendored_libraries_match_the_documented_hashes() -> None: + expected = { + "dnlib.dll": "566fdab59c91a3c2eab14a22b67f01cb3c0cdb48fa9b9b6677e2b32998636efb", + "ICSharpCode.Decompiler.dll": "38e6abf7497845d79d9d01d56351a3203991356de1d3496b8c51142f495e9f9c", + } + for name, digest in expected.items(): + path = DOTNET_LIB_DIR / name + assert hashlib.sha256(path.read_bytes()).hexdigest() == digest, name + assert digest in (DOTNET_LIB_DIR / "README.md").read_text(encoding="utf-8") + assert (DOTNET_LIB_DIR / "LICENSE-dnlib.txt").is_file() + assert (DOTNET_LIB_DIR / "LICENSE-ICSharpCode.Decompiler.txt").is_file() + + +def _runtime_available() -> bool: + try: + return backend.probe_dotnet()[0] + except Exception: # pragma: no cover + return False + + +@pytest.mark.skipif( + not _runtime_available(), reason="needs a .NET 9+ runtime and pythonnet" +) +def test_opcode_tables_agree_with_dnlib() -> None: + backend.load_runtime() + from dnlib.DotNet.Emit import OpCodes, OperandType # type: ignore[import-not-found] + + token_kinds = { + int(OperandType.InlineMethod): "method", + int(OperandType.InlineField): "field", + int(OperandType.InlineType): "type", + int(OperandType.InlineString): "string", + int(OperandType.InlineSig): "sig", + int(OperandType.InlineTok): "token", + int(OperandType.InlineSwitch): "switch", + } + sizes = { + int(OperandType.InlineNone): "none", + int(OperandType.ShortInlineBrTarget): "i1", + int(OperandType.ShortInlineI): "i1", + int(OperandType.ShortInlineVar): "i1", + int(OperandType.InlineVar): "i2", + int(OperandType.InlineBrTarget): "i4", + int(OperandType.InlineI): "i4", + int(OperandType.ShortInlineR): "i4", + int(OperandType.InlineI8): "i8", + int(OperandType.InlineR): "i8", + } + for table, ours in ( + (OpCodes.OneByteOpCodes, ONE_BYTE_OPCODES), + (OpCodes.TwoByteOpCodes, TWO_BYTE_OPCODES), + ): + for index, opcode in enumerate(table): + if str(opcode.Name).startswith("prefix"): + continue # reserved prefix bytes, not instructions + if str(opcode.Name).startswith("UNKNOWN"): + assert index not in ours, hex(index) + continue + if index == 0xFE and table is OpCodes.OneByteOpCodes: + continue # prefix byte + kind = int(opcode.OperandType) + assert ours.get(index) == token_kinds.get(kind, sizes.get(kind)), ( + str(opcode.Name), + hex(index), + ) diff --git a/tests/test_dotnet_export.py b/tests/test_dotnet_export.py new file mode 100644 index 0000000..b25ae57 --- /dev/null +++ b/tests/test_dotnet_export.py @@ -0,0 +1,751 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from dotnet_fixtures import managed_pe, native_elf, single_file_bundle, write +from tocode import dotnet as pipeline +from tocode import dotnet_metadata as meta +from tocode.apk_native import NativeLib +from tocode.backends.dotnet import ( + AssemblyInfo, + DecompiledType, + DotnetSession, + MethodInfo, + ResourceInfo, + TypeInfo, + WorkItem, +) +from tocode.backends.dotnet_pe import parse_pe +from tocode.dotnet import ( + DotnetExportOptions, + build_csproj, + export_dotnet, + namespace_parts, + safe_file_name, + target_framework_moniker, + type_file_stem, +) +from tocode.errors import ToCodeError +from tocode.progress import Progress + +PROGRAM = 0x02000002 +CLOSURE = 0x02000003 +CLIENT = 0x02000004 +BROKEN = 0x02000005 +BOX = 0x02000006 +MODULE = 0x02000001 + +PROGRAM_CS = """using System; + +namespace Contoso.App; + +public static class Program +{ + public static void Main(string[] args) + { + var client = new Client(); + Action a = () => client.Send(); + a(); + } +} +""" + + +def _type( + token: int, + name: str, + namespace: str, + *, + declaring: int | None = None, + top: int | None = None, + kind: str = "class", + visibility: str = "Public", + parent_name: str | None = None, +) -> TypeInfo: + full = f"{namespace}.{name}" if namespace else name + if parent_name is not None: + full = f"{parent_name}/{name}" # dnlib's nested-type naming + return TypeInfo( + assembly=0, + token=token, + full_name=full, + reflection_name=full, + namespace=namespace, + name=name, + kind=kind, + flags=["Public"], + visibility=visibility, + base_type="System.Object", + interfaces=[], + generic_params=["T"] if "`" in name else [], + declaring_type=declaring, + top_level=top if top is not None else token, + attributes=[], + fields=[], + properties=[], + events=[], + methods=[], + compiler_generated=name.startswith("<"), + ) + + +class FakeSession(DotnetSession): + """Inventory built by hand: no .NET runtime involved.""" + + def load(self, progress: Any = None) -> None: + item = self.input.included[0] + pe = parse_pe(item.path.read_bytes()) + assert pe is not None + info = AssemblyInfo( + index=0, + file=item, + pe=pe, + name="Contoso.App", + full_name="Contoso.App, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null", + version="1.0.0.0", + culture="", + public_key_token=None, + module_name="Contoso.App.dll", + mvid="00000000-0000-0000-0000-000000000001", + kind="Console", + target_framework=".NETCoreApp,Version=v10.0", + entry_point=0x06000001, + assembly_refs=[ + {"name": "System.Runtime", "version": "10.0.0.0"}, + {"name": "Vendor.Lib", "version": "2.0.0.0"}, + ], + module_refs=["native_helper"], + attributes=[ + "System.Runtime.Versioning.TargetFrameworkAttribute(.NETCoreApp,Version=v10.0)" + ], + resources=[ + ResourceInfo("Contoso.App.Strings.resources", "embedded", 10, True), + ResourceInfo("config.json", "embedded", 2, False), + ResourceInfo("Other.dll", "linked", None, True, file="Other.dll"), + ], + user_strings=[(1, "https://api.example.invalid/v1"), (64, "hello world")], + ) + self.assemblies = [info] + types = [ + _type(MODULE, "", ""), + _type(PROGRAM, "Program", "Contoso.App", kind="static class"), + _type( + CLOSURE, + "<>c", + "Contoso.App", + declaring=PROGRAM, + top=PROGRAM, + visibility="NestedPrivate", + parent_name="Contoso.App.Program", + ), + _type(CLIENT, "Client", "Contoso.App.Net"), + _type(BROKEN, "Broken", "Contoso.App"), + _type(BOX, "Box`1", "Contoso.App"), + ] + for record in types: + self.types[(0, record.token)] = record + info.types.append(record.token) + + def method(token: int, owner: int, name: str, **extra: Any) -> MethodInfo: + record = self.types[(0, owner)] + item = MethodInfo( + id=len(self.methods), + assembly=0, + token=token, + name=name, + type_token=owner, + type_name=record.full_name, + full_name=f"System.Void {record.full_name}::{name}()", + return_type="System.Void", + params=[], + flags=["Public"] if not name.startswith("<") else ["Private"], + impl_flags=[], + rva=0x2050 + token % 0x100, + il_size=12, + is_static=True, + is_virtual=False, + is_abstract=False, + is_constructor=name in {".ctor", ".cctor"}, + compiler_generated=name.startswith("<") or record.compiler_generated, + **extra, + ) + item.assembly_name = "Contoso.App" + self.methods.append(item) + record.methods.append(item.id) + self._method_by_token[(0, token)] = item + return item + + main = method(0x06000001, PROGRAM, "Main") + lambda_ = method(0x06000002, CLOSURE, "
b__0_0") + send = method(0x06000003, CLIENT, "Send") + native = method( + 0x06000004, + CLIENT, + "compute", + pinvoke_module="native_helper", + pinvoke_entry="helper_compute", + ) + method(0x06000005, BROKEN, "Explode") + cctor = method(0x06000006, MODULE, ".cctor") + method(0x06000007, BOX, "Get") + lambda_.owner_method = main.id + main.callees = [send.id] + send.callers = [main.id] + lambda_.callees = [send.id] + send.callers.append(lambda_.id) + send.callees = [native.id] + native.callers = [send.id] + send.external_calls = [ + "[System.Net.Http]System.Net.Http.HttpResponseMessage System.Net.Http.HttpClient::Send(System.Net.Http.HttpRequestMessage)" + ] + self.external_refs = { + "System.Net.Http": { + send.external_calls[0][len("[System.Net.Http]") :]: [send.id] + } + } + main.string_ref_count = 1 + self.string_xrefs = {(0, 1): [main.id]} + self.pinvoke_modules = {"native_helper"} + del cctor + + def save_resource(self, info: AssemblyInfo, name: str, target: Path) -> int | None: + payload = b"{}" if name == "config.json" else b"\x00resources" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(payload) + return len(payload) + + def decode_resources_file( + self, info: AssemblyInfo, name: str + ) -> list[dict[str, Any]]: + return [{"name": "Greeting", "type": "String", "value": "hi"}] + + def close(self) -> None: + return + + +class FakeDecompiler: + def __init__(self, session: DotnetSession) -> None: + self.session = session + self.items: list[WorkItem] = [] + + def decompile(self, item: WorkItem) -> DecompiledType: + self.items.append(item) + if item.token == 0: + return DecompiledType( + 0, + 0, + "[assembly: Fake]\n", + None, + ".assembly Contoso.App\n", + None, + {}, + {}, + ) + if item.token == BROKEN: + return DecompiledType( + 0, BROKEN, None, "boom", ".class Broken\n", None, {}, {} + ) + if item.token == PROGRAM: + il = "\n".join( + [ + ".class Program", + "{", + " .method /* 06000001 */ public static void Main", + " {", + " } // end of method Program::Main", + "}", + ] + ) + return DecompiledType( + 0, + PROGRAM, + PROGRAM_CS, + None, + il, + None, + {0x06000001: (7, 12), 0x06000002: (10, 10)}, + {0x06000001: (3, 5)}, + ) + return DecompiledType( + 0, + item.token, + f"// type {item.token:#x}\n", + None, + f"// il {item.token:#x}\n", + None, + {}, + {}, + ) + + +def _fake_native_exporter(calls: list[str]) -> Any: + def run( + lib: NativeLib, out_dir: Path, progress: Progress + ) -> tuple[str, str, int, int]: + calls.append(lib.name) + out_dir.mkdir(parents=True, exist_ok=True) + (out_dir / "AGENTS.md").write_text(lib.origin or "", encoding="utf-8") + return ("ida", "Hex-Rays", 3, 0) + + return run + + +def _export(tmp_path: Path, **options: Any) -> Any: + calls: list[str] = [] + summary = export_dotnet( + tmp_path / "Contoso.App.dll", + options=DotnetExportOptions(out_dir=tmp_path / "export", **options), + progress=Progress(enabled=False), + native_exporter=_fake_native_exporter(calls), + decompiler_factory=FakeDecompiler, + session_factory=FakeSession, + ) + return summary, calls + + +@pytest.fixture +def assembly(tmp_path: Path) -> Path: + write(tmp_path / "libnative_helper.so", native_elf()) + return write(tmp_path / "Contoso.App.dll", managed_pe(strings=["x"])) + + +# -------------------------------------------------------------------------- +# helpers +# -------------------------------------------------------------------------- + + +def test_file_naming_helpers() -> None: + assert ( + safe_file_name("") + == "_PrivateImplementationDetails_" + ) + assert safe_file_name("..hidden") == "hidden" + assert namespace_parts("Contoso.App.Net") == ["Contoso", "App", "Net"] + assert namespace_parts("") == [] + assert type_file_stem(_type(1, "Dictionary`2", "System")) == "Dictionary-2" + assert type_file_stem(_type(1, "\u0001\u0002", "")) == "__" + + +def test_target_framework_monikers() -> None: + assert target_framework_moniker(".NETCoreApp,Version=v10.0") == "net10.0" + assert target_framework_moniker(".NETCoreApp,Version=v3.1") == "netcoreapp3.1" + assert target_framework_moniker(".NETStandard,Version=v2.0") == "netstandard2.0" + assert target_framework_moniker(".NETFramework,Version=v4.7.2") == "net472" + assert target_framework_moniker(".NETFramework,Version=v4.8") == "net48" + assert target_framework_moniker(None) is None + assert target_framework_moniker("Unity") is None + + +# -------------------------------------------------------------------------- +# end-to-end with fakes +# -------------------------------------------------------------------------- + + +def test_export_dotnet_writes_a_project_tree(tmp_path: Path, assembly: Path) -> None: + summary, calls = _export(tmp_path, jobs=1) + + root = summary.root_dir + assert root == tmp_path / "export" + assert summary.kind == "assembly" and summary.assemblies == ["Contoso.App"] + assert summary.failed_types == [("Contoso.App.Broken", "boom")] + assert summary.native_total == 1 and summary.native_done == 1 + assert calls == ["libnative_helper.so"] + + raw = root / "src" / "raw" / "Contoso.App" + for name in ( + "Contoso/App/Program.cs", + "Contoso/App/Program.il", + "Contoso/App/Net/Client.cs", + "Contoso/App/Net/Client.il", + "Contoso/App/Broken.cs", + "Contoso/App/Box-1.cs", + "_Module_.cs", + "Properties/AssemblyInfo.cs", + "Properties/Manifest.il", + "Contoso.App.csproj", + ): + assert (raw / name).is_file(), name + program = (raw / "Contoso/App/Program.cs").read_text(encoding="utf-8") + assert program.startswith( + "// ToCode: Contoso.App.Program (0x02000002) from Contoso.App 1.0.0.0\n" + ) + assert "namespace Contoso.App;" in program + broken = (raw / "Contoso/App/Broken.cs").read_text(encoding="utf-8") + assert "decompilation failed: boom" in broken and "Broken::System.Void" in broken + csproj = (raw / "Contoso.App.csproj").read_text(encoding="utf-8") + assert "Exe" in csproj + assert "net10.0" in csproj + assert '' in csproj + assert "System.Runtime" not in csproj + + for name in ( + "AGENTS.md", + "CLAUDE.md", + "assemblies.json", + "types.json", + "functions.json", + "function-index.json", + "strings.json", + "imports.json", + "exports.json", + "sections.json", + "reachable.json", + "namespace-graph.json", + "resources.json", + "container.json", + "triage.json", + "native-libs.json", + "project.json", + "export-manifest.json", + "tocode.log", + ): + assert (root / name).is_file(), name + + functions = { + row["c_name"]: row + for row in json.loads((root / "functions.json").read_text())["functions"] + } + main = functions["Contoso.App.Program.Main"] + # +1 for the ToCode header line written above the decompiler output. + assert (main["source_line_start"], main["source_line_end"]) == (8, 13) + assert (main["asm_line_start"], main["asm_line_end"]) == (4, 6) + assert main["source_file"] == "src/raw/Contoso.App/Contoso/App/Program.cs" + assert main["asm_file"] == "src/raw/Contoso.App/Contoso/App/Program.il" + assert main["callees"] == ["Contoso.App!0x06000003"] + assert main["dead_code"] is False + lambda_ = functions["Contoso.App.Program.<>c.
b__0_0"] + assert lambda_["source_line_start"] == 11 + assert lambda_["owner"] == "Contoso.App!0x06000001" + native = functions["Contoso.App.Net.Client.compute"] + assert native["pinvoke"] == {"module": "native_helper", "entry": "helper_compute"} + explode = functions["Contoso.App.Broken.Explode"] + assert explode["source_line_start"] == 1 # failure stub spans the file + + index = json.loads((root / "function-index.json").read_text()) + assert index["language"] == "csharp" and index["asm_language"] == "il" + entry = next( + row for row in index["functions"] if row["address"] == "Contoso.App!0x06000001" + ) + assert entry["c"]["line_start"] == 8 and entry["asm"]["line_start"] == 4 + + imports = json.loads((root / "imports.json").read_text())["imports"] + assert [(group["dll"], group["kind"]) for group in imports] == [ + ("System.Net.Http", "assembly"), + ("native_helper", "pinvoke"), + ] + assert imports[1]["functions"][0]["name"] == "helper_compute" + + exports = json.loads((root / "exports.json").read_text())["exports"] + assert [(row["kind"], row["address"]) for row in exports] == [ + ("entrypoint", "Contoso.App!0x06000001"), + ("module-initializer", "Contoso.App!0x06000006"), + ] + + reachable = json.loads((root / "reachable.json").read_text()) + depths = {row["address"]: row["depth"] for row in reachable["reachable"]} + assert depths["Contoso.App!0x06000003"] == 1 + assert depths["Contoso.App!0x06000002"] == 1 # lambda via its owner + assert depths["Contoso.App!0x06000004"] == 2 + + strings = json.loads((root / "strings.json").read_text())["strings"] + assert strings[0]["value"] == "https://api.example.invalid/v1" + assert strings[0]["vaddr"] == "Contoso.App!0x70000001" + assert strings[0]["xrefs"][0]["address"] == "Contoso.App!0x06000001" + + types = { + row["name"]: row + for row in json.loads((root / "types.json").read_text())["types"] + } + assert ( + types["Contoso.App.Program+<>c"]["cs_file"] + == "src/raw/Contoso.App/Contoso/App/Program.cs" + ) + assert types["Contoso.App.Broken"]["decompile_error"] == "boom" + + assemblies = json.loads((root / "assemblies.json").read_text())["assemblies"] + assert assemblies[0]["entry_point"]["address"] == "Contoso.App!0x06000001" + assert assemblies[0]["project"] == "Contoso.App/Contoso.App.csproj" + + resources = json.loads((root / "resources.json").read_text())["resources"] + by_name = {row["name"]: row for row in resources} + assert by_name["config.json"]["path"] == "data/resources/Contoso.App/config.json" + assert by_name["Contoso.App.Strings.resources"]["entries"][0]["value"] == "hi" + assert by_name["Other.dll"]["kind"] == "linked" + assert ( + root / "data/resources/Contoso.App/Contoso.App.Strings.resources.json" + ).is_file() + + triage = json.loads((root / "triage.json").read_text()) + assert triage["binary_type"] == "dotnet-assembly" + assert triage["pinvoke"] == [ + {"module": "native_helper", "functions": ["helper_compute"]} + ] + assert triage["suspicious_apis"][0]["category"] == "network" + assert triage["strings_of_interest"][0]["value"] == "https://api.example.invalid/v1" + + libs = json.loads((root / "native-libs.json").read_text())["libs"] + assert libs[0]["status"] == "done" and libs[0]["path"].startswith("lib/arm64/") + origin = (root / libs[0]["export_dir"] / "AGENTS.md").read_text(encoding="utf-8") + assert "`Contoso.App` .NET assembly" in origin + assert "P/Invoke declarations name (`native_helper`)" in origin + + graph = json.loads((root / "namespace-graph.json").read_text())["namespaces"] + app = next(row for row in graph if row["namespace"] == "Contoso.App") + assert app["calls_namespaces"] == [ + {"namespace": "Contoso.App:Contoso.App.Net", "calls": 2} + ] + + agents = (root / "AGENTS.md").read_text(encoding="utf-8") + assert "Contoso.App" in agents and "native///" in agents + assert (root / "CLAUDE.md").read_text(encoding="utf-8") == "@./AGENTS.md\n" + manifest = json.loads((root / "export-manifest.json").read_text()) + assert manifest["format"] == "dotnet-assembly" + assert manifest["failures"] == [ + {"address": None, "name": "Contoso.App.Broken", "error": "boom"} + ] + log = (root / "tocode.log").read_text(encoding="utf-8") + assert "export Contoso.App!Contoso.App.Broken failed: boom" in log + + +def test_export_dotnet_no_native_and_stale_cleanup( + tmp_path: Path, assembly: Path +) -> None: + stale = tmp_path / "export" / "src" / "raw" / "Old" / "Gone.cs" + stale.parent.mkdir(parents=True) + stale.write_text("stale", encoding="utf-8") + + summary, calls = _export(tmp_path, native=False) + + assert calls == [] + assert not stale.exists() + libs = json.loads((summary.root_dir / "native-libs.json").read_text())["libs"] + assert libs[0]["status"] == "skipped: --no-native" + assert (summary.root_dir / "lib" / "arm64" / "libnative_helper.so").is_file() + assert not (summary.root_dir / "native").exists() + + +def test_export_dotnet_default_out_dir( + tmp_path: Path, assembly: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("TOCODE_DEFAULT_OUT_ROOT", raising=False) + summary = export_dotnet( + assembly, + options=DotnetExportOptions(native=False), + progress=Progress(enabled=False), + decompiler_factory=FakeDecompiler, + session_factory=FakeSession, + ) + assert summary.root_dir == tmp_path / "Contoso_App_decompiler" + + +def test_export_dotnet_bundle_skips_runtime_natives(tmp_path: Path) -> None: + bundle = write( + tmp_path / "App", + single_file_bundle( + [ + ("Contoso.App.dll", 1, managed_pe(strings=["x"]), True), + ("libcoreclr.so", 2, native_elf(), False), + ("libcustom.so", 2, native_elf(), False), + ] + ), + ) + calls: list[str] = [] + summary = export_dotnet( + bundle, + options=DotnetExportOptions(out_dir=tmp_path / "export"), + progress=Progress(enabled=False), + native_exporter=_fake_native_exporter(calls), + decompiler_factory=FakeDecompiler, + session_factory=FakeSession, + ) + + assert summary.kind == "bundle" + assert calls == ["libcustom.so"] + libs = { + row["name"]: row["status"] + for row in json.loads((summary.root_dir / "native-libs.json").read_text())[ + "libs" + ] + } + assert libs == { + "libcoreclr.so": "skipped: .NET runtime library (use --include-framework)", + "libcustom.so": "done", + } + container = json.loads((summary.root_dir / "container.json").read_text()) + assert ( + container["kind"] == "bundle" + and container["bundle"]["bundle_id"] == "testbundle01" + ) + assert [entry["path"] for entry in container["entries"]] == [ + "Contoso.App.dll", + "libcoreclr.so", + "libcustom.so", + ] + assert (summary.root_dir / "data" / "assemblies" / "Contoso.App.dll").is_file() + + +def test_export_dotnet_rejects_missing_input(tmp_path: Path) -> None: + with pytest.raises(ToCodeError): + export_dotnet(tmp_path / "missing.dll", progress=Progress(enabled=False)) + + +def test_decompile_pool_isolates_crashing_types( + tmp_path: Path, assembly: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A batch that kills its worker is re-run type by type; only the culprit fails.""" + rounds: list[list[list[int]]] = [] + + def fake_round( + context: Any, + batches: list[tuple[list[WorkItem], bool]], + workers: int, + search_dirs: list[str], + timeout: int, + bar: Any, + ) -> list[tuple[list[WorkItem], bool, str]]: + rounds.append([[item.token for item in batch] for batch, _ in batches]) + failed = [] + fake = FakeDecompiler(context.session) + for batch, isolated in batches: + if any(item.token == CLIENT for item in batch): + failed.append((batch, isolated, "decompiler process crashed")) + continue + for item in batch: + pipeline._store_result(context, fake.decompile(item)) + return failed + + monkeypatch.setattr(pipeline, "_run_round", fake_round) + summary = export_dotnet( + assembly, + options=DotnetExportOptions(out_dir=tmp_path / "export", native=False), + progress=Progress(enabled=False), + session_factory=FakeSession, + ) + + assert ( + "Contoso.App.Net.Client", + "decompiler process crashed", + ) in summary.failed_types + assert ("Contoso.App.Program", "boom") not in summary.failed_types + # first round: one batch; then each item alone; then the culprit once more alone. + assert len(rounds[0]) == 1 and all(len(batch) == 1 for batch in rounds[1]) + client = ( + summary.root_dir / "src/raw/Contoso.App/Contoso/App/Net/Client.il" + ).read_text() + assert "IL disassembly failed: decompiler process crashed" in client + + +# -------------------------------------------------------------------------- +# metadata helpers +# -------------------------------------------------------------------------- + + +def test_build_csproj_for_library_with_internal_refs(tmp_path: Path) -> None: + pe = parse_pe(managed_pe()) + assert pe is not None + from tocode.backends.dotnet import AssemblyFile + + info = AssemblyInfo( + index=0, + file=AssemblyFile(tmp_path / "A.dll", "A.dll", "A.dll"), + pe=pe, + name="A", + full_name="A", + version="1.0.0.0", + culture="", + public_key_token=None, + module_name="A.dll", + mvid="", + kind="Dll", + target_framework=".NETStandard,Version=v2.0", + entry_point=None, + assembly_refs=[ + {"name": "B"}, + {"name": "netstandard"}, + {"name": "Newtonsoft.Json"}, + ], + module_refs=[], + attributes=[], + resources=[], + ) + text = build_csproj(info, internal_refs={"B"}) + assert "Library" in text + assert "netstandard2.0" in text + assert '' in text + assert '' in text + assert 'netstandard"' not in text + + +def test_interesting_strings_rank_secrets_first() -> None: + rows = meta.interesting_strings( + [ + ("a", "AES/CBC/PKCS7Padding"), + ("b", "connect to https://c2.example.invalid/x"), + ("c", "short"), + ("d", "/proc/self/maps"), + ] + ) + assert [row["address"] for row in rows] == ["b", "d", "a"] + + +def _real_runtime() -> bool: + from tocode.backends.dotnet import probe_dotnet + + return probe_dotnet()[0] + + +@pytest.mark.skipif( + not __import__("os").environ.get("TOCODE_TEST_DOTNET") or not _real_runtime(), + reason="set TOCODE_TEST_DOTNET to a managed assembly (needs .NET 9+ and pythonnet)", +) +def test_export_real_assembly(tmp_path: Path) -> None: + import os + + source = Path(os.environ["TOCODE_TEST_DOTNET"]) + summary = export_dotnet( + source, + options=DotnetExportOptions(out_dir=tmp_path / "export", native=False, jobs=1), + progress=Progress(enabled=False), + ) + + assert summary.type_count > 0 and not summary.failed_types + index = json.loads((summary.root_dir / "function-index.json").read_text()) + with_body = [row for row in index["functions"] if row["asm"]] + assert with_body and all(row["asm"]["line_start"] > 1 for row in with_body) + cs_files = list((summary.root_dir / "src" / "raw").rglob("*.cs")) + il_files = list((summary.root_dir / "src" / "raw").rglob("*.il")) + assert cs_files and len(il_files) == len(cs_files) + + +def test_suspicious_apis_ignore_compiler_generated_calls( + tmp_path: Path, assembly: Path +) -> None: + from tocode.backends.dotnet import discover_dotnet_input + + session = FakeSession(discover_dotnet_input(assembly, workdir=tmp_path / "work")) + session.load() + session.methods[0].external_calls = [ + "[System.Runtime]!!0 System.Activator::CreateInstance()", + "[System.Runtime]System.Int32 System.Environment::get_CurrentManagedThreadId()", + "[System.Runtime]System.Object System.Activator::CreateInstance(System.Type)", + "[System.Runtime]System.String System.Environment::GetEnvironmentVariable(System.String)", + ] + session.methods[2].external_calls = [] + + found = { + row["category"]: [api["name"] for api in row["apis"]] + for row in meta.suspicious_apis(session) + } + + assert found == { + "dynamic code loading / reflection": [ + "[System.Runtime]System.Object System.Activator::CreateInstance(System.Type)" + ], + "environment / host info": [ + "[System.Runtime]System.String System.Environment::GetEnvironmentVariable(System.String)" + ], + } diff --git a/uv.lock b/uv.lock index 5aaaffb..e69bc57 100644 --- a/uv.lock +++ b/uv.lock @@ -554,6 +554,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl", hash = "sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360", size = 125251, upload-time = "2026-08-26T13:33:12.928Z" }, ] +[[package]] +name = "clr-loader" +version = "0.2.10" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11'", +] +dependencies = [ + { name = "cffi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/24/c12faf3f61614b3131b5c98d3bf0d376b49c7feaa73edca559aeb2aee080/clr_loader-0.2.10.tar.gz", hash = "sha256:81f114afbc5005bafc5efe5af1341d400e22137e275b042a8979f3feb9fc9446", size = 83605, upload-time = "2026-01-03T23:13:06.984Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/61/cf819f8e8bb4d4c74661acf2498ba8d4a296714be3478d21eaabf64f5b9b/clr_loader-0.2.10-py3-none-any.whl", hash = "sha256:ebbbf9d511a7fe95fa28a95a4e04cd195b097881dfe66158dc2c281d3536f282", size = 56483, upload-time = "2026-01-03T23:13:05.439Z" }, +] + +[[package]] +name = "clr-loader" +version = "0.3.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", +] +dependencies = [ + { name = "cffi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/46/7eea92b6aa2d68af78e049cbecec5f757f1aad44ecdecdc16bbad7eead51/clr_loader-0.3.1.tar.gz", hash = "sha256:2e073e9aaf49d1ae2f56ecba27987ad5fb68be4bcd9dd34a5bed8f0e4e128366", size = 86805, upload-time = "2026-04-18T17:49:44.287Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/da/ec1a6e36624000b6df0dd61183c42342ee5814c073315e802cadaad04d2f/clr_loader-0.3.1-py3-none-any.whl", hash = "sha256:cbad189de20d202a7d621956b0fc38049e13c9bf7ca2923441eff725cd121aa1", size = 55730, upload-time = "2026-04-18T17:49:42.99Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -1736,6 +1767,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" }, ] +[[package]] +name = "pythonnet" +version = "3.0.5" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11'", +] +dependencies = [ + { name = "clr-loader", version = "0.2.10", source = { registry = "https://pypi.org/simple" } }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9a/d6/1afd75edd932306ae9bd2c2d961d603dc2b52fcec51b04afea464f1f6646/pythonnet-3.0.5.tar.gz", hash = "sha256:48e43ca463941b3608b32b4e236db92d8d40db4c58a75ace902985f76dac21cf", size = 239212, upload-time = "2024-12-13T08:30:44.393Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/f1/bfb6811df4745f92f14c47a29e50e89a36b1533130fcc56452d4660bd2d6/pythonnet-3.0.5-py3-none-any.whl", hash = "sha256:f6702d694d5d5b163c9f3f5cc34e0bed8d6857150237fae411fefb883a656d20", size = 297506, upload-time = "2024-12-13T08:30:40.661Z" }, +] + +[[package]] +name = "pythonnet" +version = "3.2.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", +] +dependencies = [ + { name = "clr-loader", version = "0.3.1", source = { registry = "https://pypi.org/simple" } }, +] +sdist = { url = "https://files.pythonhosted.org/packages/17/e7/ee7af9cebbb4992fdd32ffc363f8ab7d22609665745a46b1d0b4d9d4ccc9/pythonnet-3.2.0.tar.gz", hash = "sha256:880bed9452646624f2cca1627d463424d15abefa16933c46ba00dc10da78d184", size = 257801, upload-time = "2026-09-29T10:51:37.932Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/42/75f9429ae418f92e0366a334fd07d8313a13c8e8c0ce6ba650f6ebfb0cf6/pythonnet-3.2.0-cp311.cp312.cp313.cp314.cp315-none-any.whl", hash = "sha256:c8c7946d291156ffc1d5f22c001a63c6776947ce3c64da800ccea8181ddb1abe", size = 222406, upload-time = "2026-09-29T10:51:58.583Z" }, +] + [[package]] name = "pyvex" version = "9.2.213" @@ -2084,6 +2146,8 @@ dependencies = [ { name = "droidasc" }, { name = "ida-domain" }, { name = "idapro" }, + { name = "pythonnet", version = "3.0.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "pythonnet", version = "3.2.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "r2pipe" }, { name = "rpyc" }, { name = "tqdm" }, @@ -2105,6 +2169,8 @@ requires-dist = [ { name = "ida-domain", specifier = "==0.5.0" }, { name = "idapro", specifier = "==0.0.9" }, { name = "pytest", marker = "extra == 'dev'", specifier = "==8.4.2" }, + { name = "pythonnet", marker = "python_full_version < '3.11'", specifier = "==3.0.5" }, + { name = "pythonnet", marker = "python_full_version >= '3.11'", specifier = "==3.2.0" }, { name = "r2pipe", specifier = "==1.9.8" }, { name = "rpyc", specifier = "==6.0.2" }, { name = "tqdm", specifier = "==4.67.3" }, From 42f28b0aa6241bba09c5c0c421cc7a8c7cb4765c Mon Sep 17 00:00:00 2001 From: buzzer-re <22428720+buzzer-re@users.noreply.github.com> Date: Mon, 5 Oct 2026 11:06:56 +0900 Subject: [PATCH 4/6] Bound long names in checkpoint and source file names NativeAOT symbols (mangled generic instantiations) and long C++ template names exceed the 255-byte path component limit, so exporting a NativeAOT binary crashed while writing the checkpoint record of a cluster named after such a function. Names longer than 120 characters are now truncated and suffixed with a short hash so they stay unique. Co-Authored-By: Claude Opus 5.5 --- src/tocode/exporter.py | 3 ++- src/tocode/naming.py | 21 ++++++++++++++++++++- tests/test_algorithms.py | 18 ++++++++++++++++++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/src/tocode/exporter.py b/src/tocode/exporter.py index 646dbbb..68cfcb5 100644 --- a/src/tocode/exporter.py +++ b/src/tocode/exporter.py @@ -43,6 +43,7 @@ SHARED_CLUSTER_ID, NameBook, asm_file_name, + bounded_path_component, build_name_book, c_file_name, clean_c_identifier, @@ -265,7 +266,7 @@ def covers_all(self, addresses: list[int]) -> bool: def cluster_record_path(self, cluster: Cluster) -> Path: return ( self.clusters_dir - / f"{cluster.root:016x}_{clean_path_component(cluster.label)}.json" + / f"{cluster.root:016x}_{bounded_path_component(cluster.label)}.json" ) def load_cluster_record(self, cluster: Cluster) -> dict[str, Any] | None: diff --git a/src/tocode/naming.py b/src/tocode/naming.py index ad50377..b56577a 100644 --- a/src/tocode/naming.py +++ b/src/tocode/naming.py @@ -1,6 +1,7 @@ from __future__ import annotations from dataclasses import dataclass, field +import hashlib from pathlib import Path import re @@ -24,6 +25,24 @@ def clean_path_component(value: str) -> str: return "".join(parts).strip("_") or "unnamed" +# Filesystems cap a name at 255 bytes; leave room for prefixes and suffixes +# such as "
_" and ".json..tmp". +MAX_NAME_COMPONENT = 120 + + +def bounded_path_component(value: str, limit: int = MAX_NAME_COMPONENT) -> str: + """``clean_path_component`` capped at ``limit`` chars, kept unique by a hash. + + Mangled names (NativeAOT generics, C++ templates) can run to hundreds of + characters, longer than a single path component may be. + """ + cleaned = clean_path_component(value) + if len(cleaned) <= limit: + return cleaned + digest = hashlib.sha1(value.encode("utf-8", "surrogatepass")).hexdigest()[:10] + return f"{cleaned[: limit - 11]}_{digest}" + + def clean_c_identifier(value: str) -> str: text = _IDENT_BAD.sub("_", value) if not re.search(r"[0-9A-Za-z_]", text): @@ -37,7 +56,7 @@ def default_output_name(binary: Path) -> str: def c_file_name(cluster: Cluster) -> str: if cluster.file_base: - return f"{clean_path_component(cluster.file_base)}.c" + return f"{bounded_path_component(cluster.file_base)}.c" if cluster.root == SHARED_CLUSTER_ID: return "utils.c" return f"cluster_{cluster.root:016x}.c" diff --git a/tests/test_algorithms.py b/tests/test_algorithms.py index 3a24e1a..5fd722a 100644 --- a/tests/test_algorithms.py +++ b/tests/test_algorithms.py @@ -275,3 +275,21 @@ def cmd(self, command: str) -> str: with pytest.raises(BackendError, match="r2ghidra SLEIGH languages"): session.ensure_decompiler() + + +def test_bounded_path_component_caps_long_mangled_names() -> None: + from tocode.naming import MAX_NAME_COMPONENT, bounded_path_component + + short = bounded_path_component("sub_1000") + long_name = ( + "S_P_CoreLib_System_Collections_Generic_Dictionary_2<" + + "Int32__" * 60 + + ">__TryInsert" + ) + first = bounded_path_component(long_name) + second = bounded_path_component(long_name + "_other") + + assert short == "sub_1000" + assert len(first) == MAX_NAME_COMPONENT and len(second) == MAX_NAME_COMPONENT + assert first != second # the hash keeps distinct long names distinct + assert first.startswith("S_P_CoreLib_System_Collections_Generic_Dictionary_2_") From 36f68c53a764189665ccb2d7b202beef246c5d0d Mon Sep 17 00:00:00 2001 From: buzzer-re <22428720+buzzer-re@users.noreply.github.com> Date: Tue, 6 Oct 2026 08:43:17 +0900 Subject: [PATCH 5/6] Download the .NET libraries on demand instead of vendoring them ToCode no longer ships any DLL. The first .NET export asks once whether to download dnlib 4.5.0 and ICSharpCode.Decompiler 11.1 from nuget.org and remembers the answer; non-interactive runs are never prompted and point to `tocode --setup-dotnet`, which downloads without asking or re-verifies and repairs an existing install. Each download is checked against SHA-256 hashes pinned in backends/dotnet_libs.py, for the .nupkg and for the extracted DLL, and the file is re-hashed after it is written. Normal runs only check that the files are present. Libraries live per user (XDG data dir, Application Support, or LOCALAPPDATA; TOCODE_DOTNET_LIB_DIR overrides). Also: the --no-native help now says it applies to .NET input too. Co-Authored-By: Claude Opus 5.5 --- .gitattributes | 1 - AGENTS.md | 4 +- README.md | 4 +- pyproject.toml | 5 - src/tocode/backends/dotnet.py | 18 +- src/tocode/backends/dotnet_libs.py | 352 +++++++++++++++++++++++++++++ src/tocode/cli.py | 39 +++- tests/test_cli.py | 35 +++ tests/test_dotnet_backend.py | 19 +- tests/test_dotnet_export.py | 3 +- tests/test_dotnet_libs.py | 246 ++++++++++++++++++++ 11 files changed, 688 insertions(+), 38 deletions(-) create mode 100644 src/tocode/backends/dotnet_libs.py create mode 100644 tests/test_dotnet_libs.py diff --git a/.gitattributes b/.gitattributes index 10a98db..6313b56 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1 @@ * text=auto eol=lf -*.dll binary diff --git a/AGENTS.md b/AGENTS.md index c72139e..ffc41ac 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,7 +16,7 @@ This repository contains ToCode, a Python-only binary exporter. ToCode takes one - `src/tocode/__init__.py`: package version and `export_from_binaryview()`, the in-UI Binary Ninja library entry. - `src/tocode/analysis.py`: backend-neutral binary inventory and call graph normalization. - `src/tocode/backends/`: IDA Domain, radare2, angr, and Binary Ninja (`binja.py`) session adapters, plus the ASC/droidasc APK backend (`asc.py`: APK set discovery, DEX inventory, worker-side class decompilation). -- `src/tocode/backends/dotnet.py` / `dotnet_pe.py`: .NET backend (input discovery for assemblies, apphosts, single-file bundles and NuGet packages; dnlib inventory; pure-Python PE/CLR/bundle parsing and IL token scan; worker-side C#/IL decompilation). `backends/dotnet_lib/` holds the bundled dnlib and ICSharpCode.Decompiler DLLs (MIT) with their provenance. +- `src/tocode/backends/dotnet.py` / `dotnet_pe.py`: .NET backend (input discovery for assemblies, apphosts, single-file bundles and NuGet packages; dnlib inventory; pure-Python PE/CLR/bundle parsing and IL token scan; worker-side C#/IL decompilation). `backends/dotnet_libs.py` downloads dnlib and ICSharpCode.Decompiler on demand (ask once, hash-pinned) into a per-user folder. - `src/tocode/dotnet.py`: .NET export pipeline; `dotnet_metadata.py`: .NET JSON documents and the generated `AGENTS.md`. - `src/tocode/apk.py`: APK export pipeline (extraction, decompile pool, resource decoding, metadata); `apk_metadata.py`: manifest parsing and Android JSON documents; `apk_native.py`: extraction and background native export of every `.so`. - `src/tocode/exporter.py`: project writer, function rendering, worker-session rendering, generated export `AGENTS.md`. @@ -101,7 +101,7 @@ Framework/runtime assemblies and native runtime libraries inside bundles and pac - angr is the optional pure-Python fallback backend (`[angr]` extra). - ASC (`droidasc`, PyPI) is the APK/DEX backend and a core runtime dependency. It pulls androguard. `backends/asc.py` is the only module that imports it; it pre-imports the modules ASC would otherwise stub in `sys.modules` and silences androguard's loguru logging. Class decompilation is not thread-safe, so it runs in spawned worker processes (recycled in rounds; do not use `max_tasks_per_child`, it deadlocks spawn pools on some CPython builds). The big per-method/per-class/per-string JSON documents are streamed row by row (`apk_metadata.write_json_rows`), never built as one object. - APK native libraries are exported by `apk_native.py` on a background thread, each `export_binary` in its own spawned process so a backend OOM-kill or crash only fails that library (`native-libs.json` status). The thread waits for `TOCODE_NATIVE_MIN_FREE_MB` (default 1024) of free memory before starting each library so it does not starve the DEX pool. `TOCODE_WORKER_TMP_DIR` also places the unpacked `.apks` bundle. -- The .NET backend needs `pythonnet` (core dependency, pinned per Python version) and a .NET 9+ runtime on the host. dnlib 4.5.0 and ICSharpCode.Decompiler 11.1 are vendored under `src/tocode/backends/dotnet_lib/` (package data; hashes in its README and checked by `tests/test_dotnet_backend.py`); ICSharpCode.Decompiler 11 needs System.Reflection.Metadata 9, hence .NET 9+. Decompilation runs in spawned worker processes (recycled per round; a batch that crashes or hangs a worker is retried type by type so only the culprit fails). Tests must not require .NET: use `tests/dotnet_fixtures.py` (synthetic PE/bundles) and fake sessions; runtime-backed tests skip without it. +- The .NET backend needs `pythonnet` (core dependency, pinned per Python version) and a .NET 9+ runtime on the host. Never commit or package DLLs: dnlib 4.5.0 and ICSharpCode.Decompiler 11.1 are downloaded from nuget.org by `backends/dotnet_libs.py` after the user agrees (asked once, remembered; `tocode --setup-dotnet` for non-interactive use). Their `.nupkg` and DLL SHA-256 hashes are pinned in `LIBRARIES` and checked at download time only (normal runs check presence; `--setup-dotnet` re-verifies). To bump a version, update both hashes. ICSharpCode.Decompiler 11 needs System.Reflection.Metadata 9, hence .NET 9+. Decompilation runs in spawned worker processes (recycled per round; a batch that crashes or hangs a worker is retried type by type so only the culprit fails). Tests must not require .NET: use `tests/dotnet_fixtures.py` (synthetic PE/bundles) and fake sessions; runtime-backed tests skip without it. - Binary Ninja is an opt-in backend (`--backend binja`, never auto-selected). The `binaryninja` module is supplied by the Binary Ninja install (in-UI) or the remote VM, so it is not a pip dependency. `rpyc`, the client used for the diff --git a/README.md b/README.md index 4f6c8c8..cf67311 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,9 @@ Other disassemblers may be added in the future. ### .NET assemblies -`tocode App.dll` (also `.exe`, an apphost launcher next to its `.dll`, single-file bundles, and `.nupkg` packages) uses a built-in .NET backend: [dnlib](https://github.com/0xd4d/dnlib) for metadata and [ICSharpCode.Decompiler](https://github.com/icsharpcode/ILSpy) (ILSpy's engine) for C#, both bundled with ToCode and loaded through [pythonnet](https://pypi.org/project/pythonnet/). No dnSpy/ILSpy install is needed; only a **.NET 9+ runtime** (e.g. `sudo apt install dotnet-runtime-10.0`, or the SDK; `DOTNET_ROOT` is honoured). +`tocode App.dll` (also `.exe`, an apphost launcher next to its `.dll`, single-file bundles, and `.nupkg` packages) uses a built-in .NET backend: [dnlib](https://github.com/0xd4d/dnlib) for metadata and [ICSharpCode.Decompiler](https://github.com/icsharpcode/ILSpy) (ILSpy's engine) for C#, loaded through [pythonnet](https://pypi.org/project/pythonnet/). No dnSpy/ILSpy install is needed, only a **.NET 9+ runtime** (e.g. `sudo apt install dotnet-runtime-10.0`, or the SDK; `DOTNET_ROOT` is honoured). + +ToCode does not ship these two libraries (both MIT). The first .NET export asks once whether to download them from nuget.org and remembers the answer. Non-interactive runs (agents, CI) are never prompted: run `tocode --setup-dotnet` once instead, which downloads without asking, or re-verifies and repairs an existing install. Each download is checked against SHA-256 hashes pinned in `src/tocode/backends/dotnet_libs.py`, for both the `.nupkg` and the extracted DLL, and the files are re-hashed after being written. They are stored per user in `~/.local/share/tocode/dotnet` (macOS: `~/Library/Application Support/tocode/dotnet`, Windows: `%LOCALAPPDATA%\tocode\dotnet`, override: `TOCODE_DOTNET_LIB_DIR`). Delete that folder to undo the install or reset the remembered answer. - `src/raw///.../.cs`: C# per top-level type, folders follow assemblies and namespaces like a dnSpy/ILSpy project export; `.il` next to it keeps the bytecode (IL with RVA, raw bytes, and metadata tokens per instruction). - Per-method C# and IL line ranges in `functions.json`/`function-index.json` (address `!0x`), with lambdas and async/iterator state machines linked to the method that owns them. diff --git a/pyproject.toml b/pyproject.toml index 7397b14..c353bf5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -113,8 +113,3 @@ version = { attr = "tocode.__version__" } [tool.setuptools.packages.find] where = ["src"] - -# dnlib and ICSharpCode.Decompiler (MIT) ship inside the package so .NET input -# needs only a .NET runtime; see src/tocode/backends/dotnet_lib/README.md. -[tool.setuptools.package-data] -"tocode.backends" = ["dotnet_lib/*.dll", "dotnet_lib/*.txt", "dotnet_lib/*.md"] diff --git a/src/tocode/backends/dotnet.py b/src/tocode/backends/dotnet.py index 0d945b7..3a3ef1b 100644 --- a/src/tocode/backends/dotnet.py +++ b/src/tocode/backends/dotnet.py @@ -1,6 +1,7 @@ """.NET backend: dnlib inventory plus ICSharpCode.Decompiler C#/IL output. -Both libraries ship with ToCode (``dotnet_lib/``) and are loaded in-process +Neither library ships with ToCode: ``dotnet_libs`` downloads them on first use +(with consent, hash-pinned) into a per-user folder, and they are loaded in-process through pythonnet, so exporting a .NET binary needs a .NET 9+ runtime and no other tool. The parent process inventories every assembly with dnlib (types, members, P/Invoke, resources, references) and scans IL bytes in pure Python for @@ -22,6 +23,7 @@ import zipfile from ..errors import ToCodeError +from . import dotnet_libs from .dotnet_pe import ( PE_MACHINES, BundleManifest, @@ -35,8 +37,6 @@ scan_il_tokens, ) -DOTNET_LIB_DIR = Path(__file__).with_name("dotnet_lib") -DOTNET_LIBRARIES = ("dnlib.dll", "ICSharpCode.Decompiler.dll") MIN_RUNTIME_MAJOR = 9 NUPKG_SUFFIXES = frozenset({".nupkg", ".snupkg"}) MAX_STRING_XREFS = 64 @@ -505,11 +505,6 @@ def find_runtime() -> tuple[Path, str] | None: def probe_dotnet() -> tuple[bool, str]: if importlib.util.find_spec("pythonnet") is None: return False, "pythonnet is not installed (pip install pythonnet)" - missing = [ - name for name in DOTNET_LIBRARIES if not (DOTNET_LIB_DIR / name).is_file() - ] - if missing: - return False, f"bundled .NET libraries missing: {', '.join(missing)}" found = find_runtime() if found is None: return ( @@ -521,13 +516,14 @@ def probe_dotnet() -> tuple[bool, str]: def load_runtime() -> dict[str, Any]: - """Load CoreCLR and the bundled assemblies once per process.""" + """Load CoreCLR and the downloaded libraries once per process.""" global _runtime if _runtime is not None: return _runtime ok, reason = probe_dotnet() if not ok: raise ToCodeError(f".NET input requires the .NET backend: {reason}") + library_dir = dotnet_libs.require_installed() found = find_runtime() assert found is not None root, version = found @@ -544,8 +540,8 @@ def load_runtime() -> dict[str, Any]: raise ToCodeError(f"cannot load the .NET runtime: {exc}") from exc import clr # type: ignore[import-not-found] - for name in DOTNET_LIBRARIES: - clr.AddReference(str(DOTNET_LIB_DIR / name)) + for spec in dotnet_libs.LIBRARIES: + clr.AddReference(str(library_dir / spec.file_name)) import System # type: ignore[import-not-found] actual = str(System.Environment.Version) diff --git a/src/tocode/backends/dotnet_libs.py b/src/tocode/backends/dotnet_libs.py new file mode 100644 index 0000000..bb679bf --- /dev/null +++ b/src/tocode/backends/dotnet_libs.py @@ -0,0 +1,352 @@ +"""On-demand installer for the third-party .NET libraries the .NET backend uses. + +ToCode does not ship any DLL. The first time a .NET input is exported, the user +is asked (once) whether to download dnlib and ICSharpCode.Decompiler from +nuget.org; the answer is remembered. Each download is checked against hashes +pinned in this file (the ``.nupkg`` and each extracted DLL) and the files are +re-hashed after being written to disk. Normal runs only check that the files +are present; ``tocode --setup-dotnet`` re-verifies and repairs an install. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +import hashlib +import io +import json +import os +from pathlib import Path +import sys +import tempfile +from typing import Callable +import urllib.error +import urllib.request +import zipfile + +from ..errors import ToCodeError + +NUGET_FLAT_CONTAINER = "https://api.nuget.org/v3-flatcontainer" +CONSENT_FILE = "consent.json" +INSTALLED_FILE = "installed.json" +DOWNLOAD_TIMEOUT_SECONDS = 60 + + +@dataclass(frozen=True, slots=True) +class LibrarySpec: + package: str # NuGet package id + version: str + nupkg_sha256: str # pinned hash of the whole .nupkg + member: str # path of the DLL inside the package + dll_sha256: str # pinned hash of the extracted DLL + license: str # SPDX identifier, shown when asking + license_member: str | None = None # license file to keep next to the DLL + + @property + def file_name(self) -> str: + return self.member.rsplit("/", 1)[-1] + + @property + def url(self) -> str: + lower = self.package.lower() + return f"{NUGET_FLAT_CONTAINER}/{lower}/{self.version}/{lower}.{self.version}.nupkg" + + +# Changing a version means updating both hashes; download the .nupkg from +# nuget.org, hash it, extract ``member``, and hash that too. +LIBRARIES: tuple[LibrarySpec, ...] = ( + LibrarySpec( + package="dnlib", + version="4.5.0", + nupkg_sha256="63bc2f9579568204cc8b30fa9f6700a231bcf868a8032e09118887af3eafee58", + member="lib/netstandard2.0/dnlib.dll", + dll_sha256="566fdab59c91a3c2eab14a22b67f01cb3c0cdb48fa9b9b6677e2b32998636efb", + license="MIT", + license_member="LICENSE.txt", + ), + LibrarySpec( + package="ICSharpCode.Decompiler", + version="11.1.0.9782", + nupkg_sha256="1f76df4d35193ba4eeb50ffcd84eae381a2c48c67f95c1104dc406759448a811", + member="lib/netstandard2.0/ICSharpCode.Decompiler.dll", + dll_sha256="38e6abf7497845d79d9d01d56351a3203991356de1d3496b8c51142f495e9f9c", + license="MIT", + ), +) + + +def library_dir() -> Path: + """Per-user location of the downloaded libraries (outside any repository).""" + explicit = os.environ.get("TOCODE_DOTNET_LIB_DIR", "").strip() + if explicit: + return Path(explicit).expanduser() + if sys.platform == "win32": + base = Path(os.environ.get("LOCALAPPDATA") or Path.home() / "AppData" / "Local") + elif sys.platform == "darwin": + base = Path.home() / "Library" / "Application Support" + else: + xdg = os.environ.get("XDG_DATA_HOME", "").strip() + base = Path(xdg).expanduser() if xdg else Path.home() / ".local" / "share" + return base / "tocode" / "dotnet" + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +# -------------------------------------------------------------------------- +# Verification +# -------------------------------------------------------------------------- + + +def verify_installed(directory: Path | None = None) -> list[str]: + """Problems with the installed DLLs (empty list = present and intact).""" + directory = directory or library_dir() + problems: list[str] = [] + for spec in LIBRARIES: + path = directory / spec.file_name + if not path.is_file(): + problems.append(f"{spec.file_name} is not installed") + continue + actual = sha256_file(path) + if actual != spec.dll_sha256: + problems.append( + f"{path} has SHA-256 {actual}, expected {spec.dll_sha256} " + "(modified or from another version)" + ) + return problems + + +def is_installed(directory: Path | None = None) -> bool: + """Cheap presence check used on every run (hashes are checked at download).""" + directory = directory or library_dir() + return all((directory / spec.file_name).is_file() for spec in LIBRARIES) + + +def require_installed(directory: Path | None = None) -> Path: + directory = directory or library_dir() + if not is_installed(directory): + raise ToCodeError( + "the .NET decompiler libraries are not installed; run " + "`tocode --setup-dotnet` to download and verify them" + ) + return directory + + +# -------------------------------------------------------------------------- +# Download and install +# -------------------------------------------------------------------------- + +Fetcher = Callable[[str], bytes] + + +def _fetch(url: str) -> bytes: + request = urllib.request.Request(url, headers={"User-Agent": "tocode-dotnet-setup"}) + try: + with urllib.request.urlopen( + request, timeout=DOWNLOAD_TIMEOUT_SECONDS + ) as response: + return bytes(response.read()) + except (urllib.error.URLError, OSError) as exc: + raise ToCodeError(f"cannot download {url}: {exc}") from exc + + +def _write_atomic(path: Path, data: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + dir=path.parent, prefix=f".{path.name}.", suffix=".tmp", delete=False + ) as handle: + handle.write(data) + temp = Path(handle.name) + try: + temp.replace(path) + except OSError: + temp.unlink(missing_ok=True) + raise + + +def install( + *, + directory: Path | None = None, + fetch: Fetcher = _fetch, + log: Callable[[str], None] = lambda _message: None, +) -> Path: + """Download, verify (package and DLL hashes), and install every library.""" + directory = directory or library_dir() + directory.mkdir(parents=True, exist_ok=True) + record: dict[str, object] = { + "installed_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "libraries": [], + } + for spec in LIBRARIES: + log(f"Downloading {spec.package} {spec.version} from nuget.org") + package = fetch(spec.url) + actual = sha256_bytes(package) + if actual != spec.nupkg_sha256: + raise ToCodeError( + f"{spec.package} {spec.version}: package SHA-256 {actual} does not " + f"match the pinned {spec.nupkg_sha256}; nothing was installed" + ) + try: + with zipfile.ZipFile(io.BytesIO(package)) as archive: + dll = archive.read(spec.member) + license_text = ( + archive.read(spec.license_member) if spec.license_member else None + ) + except (zipfile.BadZipFile, KeyError) as exc: + raise ToCodeError( + f"{spec.package}: unexpected package layout: {exc}" + ) from exc + dll_hash = sha256_bytes(dll) + if dll_hash != spec.dll_sha256: + raise ToCodeError( + f"{spec.package} {spec.version}: {spec.member} SHA-256 {dll_hash} does " + f"not match the pinned {spec.dll_sha256}; nothing was installed" + ) + _write_atomic(directory / spec.file_name, dll) + if license_text is not None: + _write_atomic(directory / f"LICENSE-{spec.package}.txt", license_text) + # Checksum what is now on disk, not only what was downloaded. + on_disk = sha256_file(directory / spec.file_name) + if on_disk != spec.dll_sha256: + (directory / spec.file_name).unlink(missing_ok=True) + raise ToCodeError( + f"{spec.file_name}: SHA-256 changed while writing to disk" + ) + log(f"Verified {spec.file_name} (SHA-256 {on_disk[:16]}…)") + libraries = record["libraries"] + assert isinstance(libraries, list) + libraries.append( + { + "package": spec.package, + "version": spec.version, + "license": spec.license, + "url": spec.url, + "nupkg_sha256": spec.nupkg_sha256, + "file": spec.file_name, + "sha256": spec.dll_sha256, + } + ) + _write_atomic( + directory / INSTALLED_FILE, (json.dumps(record, indent=2) + "\n").encode() + ) + record_decision(True, directory=directory) + return directory + + +# -------------------------------------------------------------------------- +# Consent ("ask once") +# -------------------------------------------------------------------------- + + +def read_decision(directory: Path | None = None) -> bool | None: + """True/False once the user answered; None when never asked.""" + path = (directory or library_dir()) / CONSENT_FILE + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + decision = payload.get("download") if isinstance(payload, dict) else None + return decision if isinstance(decision, bool) else None + + +def record_decision(accepted: bool, *, directory: Path | None = None) -> None: + directory = directory or library_dir() + payload = { + "download": accepted, + "decided_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), + } + _write_atomic( + directory / CONSENT_FILE, (json.dumps(payload, indent=2) + "\n").encode() + ) + + +def consent_prompt() -> str: + lines = [ + ".NET support needs two open-source libraries that ToCode does not ship:", + ] + for spec in LIBRARIES: + lines.append( + f" - {spec.package} {spec.version} ({spec.license}) from nuget.org" + ) + lines.extend( + [ + f"They are verified against pinned SHA-256 hashes and stored in {library_dir()}.", + "Download them now? This is asked only once. [Y/n] ", + ] + ) + return "\n".join(lines) + + +Asker = Callable[[str], str] + + +def ensure_libraries( + *, + interactive: bool, + ask: Asker = input, + fetch: Fetcher = _fetch, + log: Callable[[str], None] = lambda _message: None, + directory: Path | None = None, +) -> Path: + """Make the libraries available, asking once; raises when the user declined.""" + directory = directory or library_dir() + if is_installed(directory): + return directory + decision = read_decision(directory) + if decision is False: + raise ToCodeError( + "the .NET backend is disabled because the library download was " + "declined; run `tocode --setup-dotnet` to enable it, or pass " + "--as-native to export the file with the native backend" + ) + if decision is None: + if not interactive: + raise ToCodeError( + "the .NET backend needs dnlib and ICSharpCode.Decompiler, which " + "are downloaded on first use; run `tocode --setup-dotnet` once " + "(non-interactive sessions are never prompted), or pass " + "--as-native to export the file with the native backend" + ) + try: + answer = ask(consent_prompt()).strip().lower() + except EOFError: + answer = "n" + accepted = answer in {"", "y", "yes"} + record_decision(accepted, directory=directory) + if not accepted: + raise ToCodeError( + "download declined (remembered); run `tocode --setup-dotnet` to " + "enable .NET support later, or pass --as-native" + ) + return install(directory=directory, fetch=fetch, log=log) + + +def setup( + *, + force: bool = False, + fetch: Fetcher = _fetch, + log: Callable[[str], None] = print, + directory: Path | None = None, +) -> Path: + """`tocode --setup-dotnet`: verify an existing install, or (re)download it.""" + directory = directory or library_dir() + if not force: + problems = verify_installed(directory) + if not problems: + record_decision(True, directory=directory) + log(f".NET libraries already installed and verified in {directory}") + return directory + for problem in problems: + log(problem) + install(directory=directory, fetch=fetch, log=log) + log(f".NET libraries installed in {directory}") + return directory diff --git a/src/tocode/cli.py b/src/tocode/cli.py index be06b9f..5b9bb41 100644 --- a/src/tocode/cli.py +++ b/src/tocode/cli.py @@ -77,13 +77,18 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument( "--no-native", action="store_true", - help="APK input: only export the DEX/Android side; native .so libraries are extracted but not decompiled.", + help="APK and .NET input: only export the DEX/managed side; native libraries are extracted and listed but not decompiled.", ) parser.add_argument( "--as-native", action="store_true", help=".NET input: skip the .NET backend and export the file with the native backend (IDA/r2/angr) like any PE/ELF.", ) + parser.add_argument( + "--setup-dotnet", + action="store_true", + help="Download and verify the .NET decompiler libraries (dnlib, ICSharpCode.Decompiler) from nuget.org without prompting, or re-verify an existing install, then exit. .NET exports otherwise ask once on first use.", + ) parser.add_argument( "--include-framework", action="store_true", @@ -184,6 +189,8 @@ def main(argv: list[str] | None = None) -> int: parser = build_parser() args = parser.parse_args(argv) progress = Progress(enabled=not args.quiet) + if args.setup_dotnet: + return _setup_dotnet() binja_only = args.list_binja or args.all_views or args.binja_view is not None if args.backend != "binja" and binja_only: @@ -304,9 +311,31 @@ def _native_options(args: argparse.Namespace) -> NativeOptions: ) +def _setup_dotnet() -> int: + from .backends import dotnet_libs + + try: + dotnet_libs.setup(log=lambda message: print(message, file=sys.stderr)) + except ToCodeError as exc: + print(f"tocode: {exc}", file=sys.stderr) + return 1 + return 0 + + def _run_dotnet(binary: Path, *, args: argparse.Namespace, progress: Progress) -> int: + from .backends import dotnet_libs from .dotnet import DotnetExportOptions, export_dotnet + try: + dotnet_libs.ensure_libraries( + interactive=sys.stdin.isatty() and sys.stderr.isatty(), + ask=_ask_on_stderr, + log=progress.log, + ) + except ToCodeError as exc: + print(f"tocode: {exc}", file=sys.stderr) + return 1 + options = DotnetExportOptions( out_dir=args.out_dir, jobs=args.jobs, @@ -339,6 +368,14 @@ def _run_dotnet(binary: Path, *, args: argparse.Namespace, progress: Progress) - return 0 +def _ask_on_stderr(prompt: str) -> str: + print(prompt, end="", file=sys.stderr, flush=True) + line = sys.stdin.readline() + if not line: # end of input is a "no", never an implicit yes + raise EOFError + return line + + def _format_duration(seconds: float) -> str: if seconds >= 60: minutes, secs = divmod(int(round(seconds)), 60) diff --git a/tests/test_cli.py b/tests/test_cli.py index dfe9b6b..40e5631 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -149,6 +149,9 @@ def fake_export(path, *, options, progress): ) monkeypatch.setattr(dotnet, "export_dotnet", fake_export) + from tocode.backends import dotnet_libs + + monkeypatch.setattr(dotnet_libs, "ensure_libraries", lambda **_kwargs: tmp_path) assert ( cli.main(["--include-framework", "--no-native", "-j", "2", "-q", str(binary)]) @@ -191,3 +194,35 @@ def test_main_rejects_binja_backend_for_dotnet_input(tmp_path) -> None: main(["--backend", "binja", str(binary)]) assert info.value.code == 2 + + +def test_parser_has_setup_dotnet_flag() -> None: + assert build_parser().parse_args(["--setup-dotnet"]).setup_dotnet is True + assert build_parser().parse_args(["app.dll"]).setup_dotnet is False + + +def test_main_setup_dotnet_runs_without_an_input(monkeypatch) -> None: + from tocode import cli + from tocode.backends import dotnet_libs + + calls = [] + monkeypatch.setattr(dotnet_libs, "setup", lambda **kwargs: calls.append(kwargs)) + + assert cli.main(["--setup-dotnet"]) == 0 + assert len(calls) == 1 + + +def test_main_dotnet_input_without_libraries_explains_setup( + tmp_path, monkeypatch, capsys +) -> None: + from dotnet_fixtures import managed_pe + + from tocode import cli + + monkeypatch.setenv("TOCODE_DOTNET_LIB_DIR", str(tmp_path / "libs")) + binary = tmp_path / "App.dll" + binary.write_bytes(managed_pe()) + + # pytest's stdin is not a terminal, so this must fail fast, not prompt. + assert cli.main(["-q", str(binary)]) == 1 + assert "tocode --setup-dotnet" in capsys.readouterr().err diff --git a/tests/test_dotnet_backend.py b/tests/test_dotnet_backend.py index ee04a13..d1f1f79 100644 --- a/tests/test_dotnet_backend.py +++ b/tests/test_dotnet_backend.py @@ -1,6 +1,5 @@ from __future__ import annotations -import hashlib import json from pathlib import Path import struct @@ -11,7 +10,6 @@ from dotnet_fixtures import managed_pe, native_elf, single_file_bundle, write from tocode.backends import dotnet as backend from tocode.backends.dotnet import ( - DOTNET_LIB_DIR, discover_dotnet_input, il_method_ranges, is_dotnet_input, @@ -449,22 +447,11 @@ def test_find_runtime_requires_net9_or_newer( assert backend.find_runtime() is None -def test_vendored_libraries_match_the_documented_hashes() -> None: - expected = { - "dnlib.dll": "566fdab59c91a3c2eab14a22b67f01cb3c0cdb48fa9b9b6677e2b32998636efb", - "ICSharpCode.Decompiler.dll": "38e6abf7497845d79d9d01d56351a3203991356de1d3496b8c51142f495e9f9c", - } - for name, digest in expected.items(): - path = DOTNET_LIB_DIR / name - assert hashlib.sha256(path.read_bytes()).hexdigest() == digest, name - assert digest in (DOTNET_LIB_DIR / "README.md").read_text(encoding="utf-8") - assert (DOTNET_LIB_DIR / "LICENSE-dnlib.txt").is_file() - assert (DOTNET_LIB_DIR / "LICENSE-ICSharpCode.Decompiler.txt").is_file() - - def _runtime_available() -> bool: + from tocode.backends import dotnet_libs + try: - return backend.probe_dotnet()[0] + return backend.probe_dotnet()[0] and dotnet_libs.is_installed() except Exception: # pragma: no cover return False diff --git a/tests/test_dotnet_export.py b/tests/test_dotnet_export.py index b25ae57..230f9ed 100644 --- a/tests/test_dotnet_export.py +++ b/tests/test_dotnet_export.py @@ -693,9 +693,10 @@ def test_interesting_strings_rank_secrets_first() -> None: def _real_runtime() -> bool: + from tocode.backends import dotnet_libs from tocode.backends.dotnet import probe_dotnet - return probe_dotnet()[0] + return probe_dotnet()[0] and dotnet_libs.is_installed() @pytest.mark.skipif( diff --git a/tests/test_dotnet_libs.py b/tests/test_dotnet_libs.py new file mode 100644 index 0000000..3d008bb --- /dev/null +++ b/tests/test_dotnet_libs.py @@ -0,0 +1,246 @@ +from __future__ import annotations + +import hashlib +import io +from pathlib import Path +import zipfile + +import pytest + +from tocode.backends import dotnet_libs +from tocode.backends.dotnet_libs import LibrarySpec +from tocode.errors import ToCodeError + + +def _package(member: str, payload: bytes, license_text: bytes | None = None) -> bytes: + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr(member, payload) + if license_text is not None: + archive.writestr("LICENSE.txt", license_text) + return buffer.getvalue() + + +class FakeNuGet: + """Serves fake packages and records requests (no network in tests).""" + + def __init__(self, packages: dict[str, bytes]) -> None: + self.packages = packages + self.requests: list[str] = [] + + def __call__(self, url: str) -> bytes: + self.requests.append(url) + return self.packages[url] + + +@pytest.fixture +def libraries(monkeypatch: pytest.MonkeyPatch) -> FakeNuGet: + """Two fake libraries whose pinned hashes match the fake packages.""" + specs = [] + packages = {} + for package, payload, license_text in ( + ("Fake.Metadata", b"metadata-dll", b"MIT license text"), + ("Fake.Decompiler", b"decompiler-dll", None), + ): + member = f"lib/netstandard2.0/{package}.dll" + nupkg = _package(member, payload, license_text) + spec = LibrarySpec( + package=package, + version="1.0.0", + nupkg_sha256=hashlib.sha256(nupkg).hexdigest(), + member=member, + dll_sha256=hashlib.sha256(payload).hexdigest(), + license="MIT", + license_member="LICENSE.txt" if license_text else None, + ) + specs.append(spec) + packages[spec.url] = nupkg + monkeypatch.setattr(dotnet_libs, "LIBRARIES", tuple(specs)) + return FakeNuGet(packages) + + +def test_library_dir_honours_override_and_xdg( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("TOCODE_DOTNET_LIB_DIR", str(tmp_path / "custom")) + assert dotnet_libs.library_dir() == tmp_path / "custom" + monkeypatch.delenv("TOCODE_DOTNET_LIB_DIR") + monkeypatch.setenv("XDG_DATA_HOME", str(tmp_path / "xdg")) + monkeypatch.setattr(dotnet_libs.sys, "platform", "linux") + assert dotnet_libs.library_dir() == tmp_path / "xdg" / "tocode" / "dotnet" + + +def test_pinned_specs_point_at_nuget() -> None: + for spec in dotnet_libs.LIBRARIES: + assert spec.url.startswith("https://api.nuget.org/v3-flatcontainer/") + assert len(spec.nupkg_sha256) == 64 and len(spec.dll_sha256) == 64 + assert spec.member.endswith(".dll") + + +def test_install_verifies_and_writes_files( + tmp_path: Path, libraries: FakeNuGet +) -> None: + messages: list[str] = [] + + dotnet_libs.install(directory=tmp_path, fetch=libraries, log=messages.append) + + assert (tmp_path / "Fake.Metadata.dll").read_bytes() == b"metadata-dll" + assert (tmp_path / "Fake.Decompiler.dll").read_bytes() == b"decompiler-dll" + assert (tmp_path / "LICENSE-Fake.Metadata.txt").read_bytes() == b"MIT license text" + assert (tmp_path / "installed.json").is_file() + assert dotnet_libs.read_decision(tmp_path) is True + assert dotnet_libs.verify_installed(tmp_path) == [] + assert any(message.startswith("Verified Fake.Metadata.dll") for message in messages) + + +def test_install_rejects_a_package_with_the_wrong_hash( + tmp_path: Path, libraries: FakeNuGet +) -> None: + url = dotnet_libs.LIBRARIES[0].url + libraries.packages[url] = _package(dotnet_libs.LIBRARIES[0].member, b"tampered") + + with pytest.raises( + ToCodeError, match="package SHA-256 .* does not match the pinned" + ): + dotnet_libs.install(directory=tmp_path, fetch=libraries) + + assert not (tmp_path / "Fake.Metadata.dll").exists() + + +def test_install_rejects_a_dll_with_the_wrong_hash( + tmp_path: Path, libraries: FakeNuGet, monkeypatch: pytest.MonkeyPatch +) -> None: + good = dotnet_libs.LIBRARIES[0] + # A package that matches its pinned hash but carries an unexpected DLL. + bad = LibrarySpec( + package=good.package, + version=good.version, + nupkg_sha256=good.nupkg_sha256, + member=good.member, + dll_sha256="0" * 64, + license="MIT", + ) + monkeypatch.setattr(dotnet_libs, "LIBRARIES", (bad,)) + + with pytest.raises(ToCodeError, match="does not match the pinned"): + dotnet_libs.install(directory=tmp_path, fetch=libraries) + + assert not (tmp_path / "Fake.Metadata.dll").exists() + + +def test_ensure_asks_once_and_installs_on_yes( + tmp_path: Path, libraries: FakeNuGet +) -> None: + prompts: list[str] = [] + + def ask(prompt: str) -> str: + prompts.append(prompt) + return "" # Enter = default yes + + dotnet_libs.ensure_libraries( + interactive=True, ask=ask, fetch=libraries, directory=tmp_path + ) + dotnet_libs.ensure_libraries( + interactive=True, ask=ask, fetch=libraries, directory=tmp_path + ) + + assert len(prompts) == 1 and "Fake.Metadata 1.0.0 (MIT)" in prompts[0] + assert len(libraries.requests) == 2 # one download per library, once + assert dotnet_libs.is_installed(tmp_path) + + +def test_ensure_remembers_a_no(tmp_path: Path, libraries: FakeNuGet) -> None: + prompts: list[str] = [] + + def ask(prompt: str) -> str: + prompts.append(prompt) + return "n" + + with pytest.raises(ToCodeError, match="declined"): + dotnet_libs.ensure_libraries( + interactive=True, ask=ask, fetch=libraries, directory=tmp_path + ) + with pytest.raises(ToCodeError, match="--setup-dotnet"): + dotnet_libs.ensure_libraries( + interactive=True, ask=ask, fetch=libraries, directory=tmp_path + ) + + assert len(prompts) == 1 + assert libraries.requests == [] + assert dotnet_libs.read_decision(tmp_path) is False + + +def test_ensure_treats_end_of_input_as_no(tmp_path: Path, libraries: FakeNuGet) -> None: + def ask(prompt: str) -> str: + raise EOFError + + with pytest.raises(ToCodeError, match="declined"): + dotnet_libs.ensure_libraries( + interactive=True, ask=ask, fetch=libraries, directory=tmp_path + ) + assert libraries.requests == [] + + +def test_ensure_never_prompts_when_not_interactive( + tmp_path: Path, libraries: FakeNuGet +) -> None: + def ask(prompt: str) -> str: + raise AssertionError("must not prompt") + + with pytest.raises(ToCodeError, match="tocode --setup-dotnet"): + dotnet_libs.ensure_libraries( + interactive=False, ask=ask, fetch=libraries, directory=tmp_path + ) + assert libraries.requests == [] + assert dotnet_libs.read_decision(tmp_path) is None # still undecided + + +def test_ensure_reinstalls_silently_after_a_previous_yes( + tmp_path: Path, libraries: FakeNuGet +) -> None: + dotnet_libs.install(directory=tmp_path, fetch=libraries) + (tmp_path / "Fake.Decompiler.dll").unlink() + + def ask(prompt: str) -> str: + raise AssertionError("already accepted") + + dotnet_libs.ensure_libraries( + interactive=False, ask=ask, fetch=libraries, directory=tmp_path + ) + assert dotnet_libs.is_installed(tmp_path) + + +def test_setup_verifies_and_repairs_a_modified_install( + tmp_path: Path, libraries: FakeNuGet +) -> None: + dotnet_libs.install(directory=tmp_path, fetch=libraries) + libraries.requests.clear() + messages: list[str] = [] + + dotnet_libs.setup(directory=tmp_path, fetch=libraries, log=messages.append) + assert libraries.requests == [] # intact: nothing downloaded + assert "already installed and verified" in messages[-1] + + (tmp_path / "Fake.Metadata.dll").write_bytes(b"patched") + assert dotnet_libs.is_installed(tmp_path) # normal runs only check presence + assert dotnet_libs.verify_installed(tmp_path) # setup re-hashes + + dotnet_libs.setup(directory=tmp_path, fetch=libraries, log=messages.append) + assert (tmp_path / "Fake.Metadata.dll").read_bytes() == b"metadata-dll" + assert len(libraries.requests) == 2 + + +def test_setup_after_a_no_enables_dotnet(tmp_path: Path, libraries: FakeNuGet) -> None: + dotnet_libs.record_decision(False, directory=tmp_path) + + dotnet_libs.setup(directory=tmp_path, fetch=libraries, log=lambda _message: None) + + assert dotnet_libs.read_decision(tmp_path) is True + assert dotnet_libs.is_installed(tmp_path) + + +def test_require_installed_explains_how_to_install( + tmp_path: Path, libraries: FakeNuGet +) -> None: + with pytest.raises(ToCodeError, match="tocode --setup-dotnet"): + dotnet_libs.require_installed(tmp_path) From ca2935716b143417a62eb8e49209b1e30c3c9c35 Mon Sep 17 00:00:00 2001 From: buzzer-re <22428720+buzzer-re@users.noreply.github.com> Date: Tue, 6 Oct 2026 08:52:28 +0900 Subject: [PATCH 6/6] Move development notes from the README to DEVELOPMENT.md The README now only points to DEVELOPMENT.md, which covers setup, local backends for testing, running tests and the quality gate, checking changes on real exports, dependency rules, and releasing. Co-Authored-By: Claude Opus 5.5 --- DEVELOPMENT.md | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 20 +---------- 2 files changed, 99 insertions(+), 19 deletions(-) create mode 100644 DEVELOPMENT.md diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 0000000..5be96a4 --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,98 @@ +# Developing ToCode + +ToCode was built with agentic coding, and contributing the same way works +well. Agents (and humans) should read [AGENTS.md](AGENTS.md) first: it is the +project's instruction file, with the layout, export contract, dependency +rules, and per-backend notes. + +## Setup + +You need Python 3.10+ and [uv](https://docs.astral.sh/uv/). + +```bash +git clone https://github.com/buzzer-re/ToCode +cd ToCode +uv sync --locked --extra dev # project + pytest in .venv +uv run tocode --help # run the CLI from the checkout +``` + +`uv tool install --force --editable .` installs a `tocode` command that +follows your working tree. + +## Backends for local testing + +The unit tests need none of these; real exports do. Install what matches the +area you are changing: + +| Backend | What to install | Notes | +| --- | --- | --- | +| IDA | IDA Pro 9 with idalib | `ida-domain`/`idapro` come with ToCode; IDA is found via `IDADIR`, `--idadir`, or common install paths. | +| radare2 | `r2` plus r2ghidra (`r2pm -ci r2ghidra r2ghidra-sleigh`) | Used by `--backend r2` and as the `auto` fallback. | +| angr | `uv sync --extra angr` | Large; tests in `tests/test_angr_backend.py` skip without it. | +| Binary Ninja | Binary Ninja with binja-headless | See the Binary Ninja section of the README. | +| Android | nothing extra (`droidasc` is a core dependency) | Real APK test: set `TOCODE_TEST_APK=/path/to/app.apk`. | +| .NET | a .NET 9+ runtime, then `uv run tocode --setup-dotnet` | The SDK (`dotnet-sdk-10.0`) lets you build test assemblies. Real test: `TOCODE_TEST_DOTNET=/path/to/App.dll`. | + +Tests that need a backend, runtime, or sample skip themselves when it is +missing, so the suite always runs. CI installs only the Python dependencies +(no IDA, radare2, angr, Binary Ninja, or .NET runtime). + +## Tests and the quality gate + +```bash +uv run --extra dev pytest -q # all tests +uv run --extra dev pytest -q tests/test_cli.py # one file +uv run --extra dev pytest -q -k dotnet # by keyword +``` + +Before opening a PR, run the same checks as CI: + +```bash +./ci-local.sh # ruff format, ruff lint, mypy, pytest, compileall +./ci-local.sh --fix # apply ruff formatting and safe lint fixes first +``` + +On Windows PowerShell: + +```powershell +powershell -ExecutionPolicy Bypass -File .\ci-local.ps1 # add -Fix to auto-fix +``` + +Write tests so they do not depend on the machine running them: no network, +no installed backend or runtime, and no particular amount of free memory. +`tests/dotnet_fixtures.py` builds synthetic .NET files, and +`tests/test_apk_export.py` / `tests/test_dotnet_export.py` show how to drive a +whole export with fake sessions. + +## Checking a change on real inputs + +After touching an export path, run a real export and read the result: + +```bash +uv run tocode /bin/true -o /tmp/tocode-check --backend auto -j 2 +``` + +Confirm the tree matches the export contract in `AGENTS.md`, `tocode.log` has +no tracebacks, and `export-manifest.json` reports no unexpected failures. Never +commit export directories (`*_decompiler/` and similar). + +## Dependencies + +- Runtime dependencies go in `pyproject.toml` with exact pins (`==`); run + `uv lock` and commit `uv.lock` (CI uses `--locked`). +- Keep dependencies few and tied to exporting a project. +- Do not commit or package binaries. The .NET libraries are downloaded on + demand with pinned hashes (`src/tocode/backends/dotnet_libs.py`); to update + one, change its version and both hashes there. + +## Releasing + +The version lives only in `src/tocode/__init__.py` (`__version__`). + +```bash +uv build # sdist + wheel in dist/ +uvx twine check dist/* # metadata and README rendering +uvx twine upload dist/* # needs a PyPI token +``` + +The PyPI package is `tocode-cli`; the installed command is `tocode`. diff --git a/README.md b/README.md index cf67311..c024798 100644 --- a/README.md +++ b/README.md @@ -246,22 +246,4 @@ export_from_binaryview(bv, "out/") ## Development -This tool was built using agentic coding, so if you plan to help, I strongly advise doing the same. - -Before changing ToCode, have Python, uv, ruff, mypy, pytest, and compileall available. For backend work, also have IDA or radare2 installed, depending on what you are touching. - -The main instructions for agents are in `AGENTS.md`. Read it before starting, and make sure the local quality gate passes before proceeding. - -### Quality Gate - -Run the local CI gate before opening a PR: - -```bash -./ci-local.sh -``` - -On Windows PowerShell: - -```powershell -powershell -ExecutionPolicy Bypass -File .\ci-local.ps1 -``` +See [DEVELOPMENT.md](DEVELOPMENT.md) for setup, tests, the local quality gate, and release steps.