diff --git a/unified/AGENTS.md b/unified/AGENTS.md index bbd271764f45..72de05d120be 100644 --- a/unified/AGENTS.md +++ b/unified/AGENTS.md @@ -16,7 +16,10 @@ by Apple's swift-syntax rather than by tree-sitter. - `extractor/src/languages/swift/adapter.rs` converts that JSON into a yeast AST. - The raw parse tree's shape is described by `extractor/swift_node_types.yml`, - which is maintained by hand. + which is generated from swift-syntax by `swift-syntax-rs/schemagen`. Do not + edit it by hand; regenerate it with `scripts/regenerate-node-types.sh` after + changing the pinned swift-syntax version, then review the diff alongside the + mapping in `extractor/src/languages/swift/swift.rs`. ## AST Mapping - The target AST shape is described by `extractor/ast_types.yml`. diff --git a/unified/extractor/src/languages/swift/adapter.rs b/unified/extractor/src/languages/swift/adapter.rs index f34e5df3d00b..ae245eb21e80 100644 --- a/unified/extractor/src/languages/swift/adapter.rs +++ b/unified/extractor/src/languages/swift/adapter.rs @@ -261,7 +261,9 @@ fn parse_range(node: &Value) -> Option { }) } -/// The authoritative swift-syntax input node-types schema. +/// The authoritative swift-syntax input node-types schema, generated from +/// swift-syntax by `swift-syntax-rs/schemagen` (run +/// `unified/scripts/regenerate-node-types.sh` to refresh it). /// [`json_to_ast`] seeds every parse with the schema built from this, /// pre-registering every input kind and field so rule matching never references /// a name absent from a given file's tree. diff --git a/unified/extractor/swift_node_types.yml b/unified/extractor/swift_node_types.yml index c98dd1d33b23..16db4cbf0c64 100644 --- a/unified/extractor/swift_node_types.yml +++ b/unified/extractor/swift_node_types.yml @@ -1,3 +1,5 @@ +# GENERATED from swift-syntax by unified/swift-syntax-rs/schemagen. +# Do not edit; run unified/scripts/regenerate-node-types.sh instead. supertypes: decl: - accessorDecl diff --git a/unified/scripts/regenerate-node-types.sh b/unified/scripts/regenerate-node-types.sh new file mode 100755 index 000000000000..e811dd3dd632 --- /dev/null +++ b/unified/scripts/regenerate-node-types.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# Regenerate `extractor/swift_node_types.yml`, the schema describing the shape +# of the trees produced by `swift_syntax_rs::parse_to_json`, from swift-syntax +# itself. +# +# Run this after changing the pinned swift-syntax version, and review the diff: +# a new or renamed node kind generally means the mapping in +# `extractor/src/languages/swift/swift.rs` needs attention too. +# +# This needs a local Swift toolchain (see `swift-syntax-rs/.swift-version` for +# the pinned version). The schema it derives from lives in `SyntaxSupport`, a +# target of swift-syntax's separate `CodeGeneration` package: it is not a +# product of swift-syntax, and Bazel's swift-syntax module does not export its +# sources, so there is no way to depend on it directly. +set -euo pipefail +IFS=$'\n\t' + +root=$(cd "$(dirname "$0")/.." && pwd) +swift_syntax_rs_dir="$root/swift-syntax-rs" +ffi_dir="$swift_syntax_rs_dir/swift" +schemagen_dir="$swift_syntax_rs_dir/schemagen" +output="$root/extractor/swift_node_types.yml" + +if ! command -v swift >/dev/null 2>&1; then + echo "error: Swift is required; install the version pinned in $swift_syntax_rs_dir/.swift-version." >&2 + exit 1 +fi + +# Codespaces sets `safe.bareRepository=explicit` through environment-based Git +# configuration, which prevents SwiftPM from using its cached bare dependency +# repositories. Relax only that injected setting, and only for Swift +# subprocesses, as `swift-syntax-rs/build.rs` does for local Cargo builds. +run_swift() { + if [[ ${GIT_CONFIG_KEY_0:-} == "safe.bareRepository" ]]; then + GIT_CONFIG_VALUE_0=all swift "$@" + else + swift "$@" + fi +} + +# `schemagen` takes swift-syntax as a path dependency on this checkout, so that +# the schema describes exactly the version the parser links. +echo "Resolving swift-syntax..." >&2 +( + cd "$ffi_dir" + run_swift package resolve >&2 +) +checkout="$ffi_dir/.build/checkouts/swift-syntax" +syntax_support="$checkout/CodeGeneration/Sources/SyntaxSupport" +if [[ ! -d $syntax_support ]]; then + echo "error: $syntax_support not found after resolving swift-syntax." >&2 + exit 1 +fi + +# Refresh rather than merge, so that sources deleted upstream do not linger. +rm -rf "$schemagen_dir/Sources/SyntaxSupport" +cp -R "$syntax_support" "$schemagen_dir/Sources/SyntaxSupport" + +echo "Generating $output..." >&2 +# Generate to a temporary file first: redirecting straight into `$output` would +# truncate the existing schema before the build has even run, leaving nothing +# behind if it fails. +tmp=$(mktemp) +trap 'rm -f "$tmp"' EXIT +( + cd "$schemagen_dir" + run_swift run schemagen +) > "$tmp" +if [[ ! -s $tmp ]]; then + echo "error: schemagen produced no output; $output left unchanged." >&2 + exit 1 +fi +mv "$tmp" "$output" +chmod 644 "$output" +echo "Regenerated $output" >&2 diff --git a/unified/swift-syntax-rs/README.md b/unified/swift-syntax-rs/README.md index 299fc40951a6..d272c551bd7c 100644 --- a/unified/swift-syntax-rs/README.md +++ b/unified/swift-syntax-rs/README.md @@ -150,6 +150,20 @@ cargo test The first build compiles `swift-syntax` and can take several minutes. +## Regenerating the extractor node types + +After updating the pinned swift-syntax version, regenerate the unified +extractor's input schema: + +```sh +../scripts/regenerate-node-types.sh +``` + +The script uses swift-syntax's authoritative `SyntaxSupport` definitions and +requires the local Swift toolchain pinned by [`.swift-version`](.swift-version). +Review the resulting `extractor/swift_node_types.yml` diff alongside the Swift +mapping rules. See [`schemagen/README.md`](schemagen/README.md) for details. + ## Building with Bazel (CI) CI builds this crate hermetically with Bazel. A Swift toolchain is downloaded diff --git a/unified/swift-syntax-rs/schemagen/.gitignore b/unified/swift-syntax-rs/schemagen/.gitignore new file mode 100644 index 000000000000..e6ee67bbf52c --- /dev/null +++ b/unified/swift-syntax-rs/schemagen/.gitignore @@ -0,0 +1,4 @@ +/.build +# Copied from swift-syntax's CodeGeneration package by +# `unified/scripts/regenerate-node-types.sh`; not ours to vendor. +/Sources/SyntaxSupport diff --git a/unified/swift-syntax-rs/schemagen/Package.swift b/unified/swift-syntax-rs/schemagen/Package.swift new file mode 100644 index 000000000000..c241713751e6 --- /dev/null +++ b/unified/swift-syntax-rs/schemagen/Package.swift @@ -0,0 +1,41 @@ +// swift-tools-version:5.9 +import PackageDescription + +// `schemagen` regenerates `unified/extractor/swift_node_types.yml`, the input +// schema describing the shape of the trees produced by +// `swift_syntax_rs::parse_to_json`. Run it through +// `unified/scripts/regenerate-node-types.sh`, which stages the sources this +// package needs; see `README.md` for the details. +// +// The tools version is deliberately older than the FFI package's: it selects +// the Swift 5 language mode, and `SyntaxSupport` (see below) is not clean under +// Swift 6 strict concurrency because its node tables are non-Sendable globals. +let package = Package( + name: "schemagen", + platforms: [ + // Matches the FFI package: swift-syntax 603 requires macOS 10.15. + .macOS(.v10_15), + ], + dependencies: [ + // Deliberately a path dependency on the checkout the neighbouring FFI + // package resolved, rather than a second URL/exact pin: the schema has + // to describe the very swift-syntax that the parser links, and a + // single pin cannot drift from itself. + .package(name: "swift-syntax", path: "../swift/.build/checkouts/swift-syntax"), + ], + targets: [ + // `SyntaxSupport` is a target of swift-syntax's separate + // `CodeGeneration` package, not a product of swift-syntax itself, so it + // cannot be depended on directly. The regeneration script copies its + // sources here (the directory is git-ignored) and this target builds + // them as if they were our own. + .target( + name: "SyntaxSupport", + dependencies: [ + .product(name: "SwiftSyntax", package: "swift-syntax"), + .product(name: "SwiftSyntaxBuilder", package: "swift-syntax"), + ] + ), + .executableTarget(name: "schemagen", dependencies: ["SyntaxSupport"]), + ] +) diff --git a/unified/swift-syntax-rs/schemagen/README.md b/unified/swift-syntax-rs/schemagen/README.md new file mode 100644 index 000000000000..727f93c48b30 --- /dev/null +++ b/unified/swift-syntax-rs/schemagen/README.md @@ -0,0 +1,71 @@ +# schemagen + +Generates [`unified/extractor/swift_node_types.yml`][schema], the schema that +describes the shape of the trees produced by `swift_syntax_rs::parse_to_json`. +The extractor seeds every parse with it, so rule matching never refers to a +node kind or field that swift-syntax can produce but the schema does not know. + +Run it through the script, which stages the sources described below: + +```console +$ unified/scripts/regenerate-node-types.sh +``` + +Do this after changing the pinned swift-syntax version, and read the resulting +diff: a new or renamed node kind usually means the mapping in +[`swift.rs`][mapping] needs attention too. + +This requires the local Swift toolchain pinned by +[`.swift-version`](../.swift-version). + +## Why the sources are copied in + +The schema is derived from `SyntaxSupport`, the module that describes +swift-syntax's own syntax tree. This is the same description swift-syntax +generates itself from, and is therefore authoritative in a way that observing +parser output never would be. The runtime `SwiftSyntax` module is not a +substitute: its `SyntaxNodeStructure` exposes layout as key paths, without the +field names, optionality, and base-kind relationships this schema records. + +`SyntaxSupport` is awkward to depend on, though. It is a target of +`CodeGeneration`, a package inside the swift-syntax repository that is +separate from swift-syntax itself, and it is not one of that package's +products. SwiftPM can only depend on products, and Bazel's swift-syntax module +does not export the `CodeGeneration` sources, so neither build system can +reach it directly. + +The regeneration script therefore copies those sources out of the resolved +swift-syntax checkout into `Sources/SyntaxSupport`, where this package builds +them as its own. That directory is git-ignored and refreshed on every run, so +it always matches the pinned version rather than drifting as a stale vendored +copy would. + +For the same reason this package takes swift-syntax as a path dependency on the +checkout the neighbouring FFI package resolved, rather than declaring a second +pinned dependency of its own. The schema must describe exactly the +swift-syntax version linked by `swift-syntax-rs`. + +## What is filtered out + +The schema describes the JSON the extractor's adapter receives, not +swift-syntax's tree verbatim, so `main.swift` mirrors what +[`adapter.rs`][adapter] does: + +- Abstract base kinds become `supertypes:` entries rather than node kinds. +- Collection nodes are dropped, and a collection-typed child is recorded as + its element kinds, because the adapter elides collections into JSON arrays. +- `unexpectedBeforeX`, `unexpectedBetweenXAndY`, and `unexpectedAfterX` + error-recovery children are dropped; no rule matches them. This filters on + the child name: `unexpectedCodeDecl` is a real node kind and is retained. +- Token-typed children become the synthetic `_token` kind. Only the varying + token kinds whose `TokenSpec` is `.other` and has no fixed text are emitted + as kinds of their own. These are derived from `Token.allCases` and should match + `VARYING_TOKEN_KINDS` in `adapter.rs`. Fixed tokens are anonymous and keyed + by their text, so no rule can name them. + +Setting `EMIT_SUPERTYPES=0` omits the `supertypes:` section, which can be useful +when diffing two versions for kind and field changes alone. + +[schema]: ../../extractor/swift_node_types.yml +[mapping]: ../../extractor/src/languages/swift/swift.rs +[adapter]: ../../extractor/src/languages/swift/adapter.rs diff --git a/unified/swift-syntax-rs/schemagen/Sources/schemagen/main.swift b/unified/swift-syntax-rs/schemagen/Sources/schemagen/main.swift new file mode 100644 index 000000000000..0fcb53035349 --- /dev/null +++ b/unified/swift-syntax-rs/schemagen/Sources/schemagen/main.swift @@ -0,0 +1,100 @@ +import Foundation +import SyntaxSupport + +// Named-leaf ("varying") token kinds, mirroring the extractor adapter's +// VARYING_TOKEN_KINDS. Fixed tokens are anonymous (keyed by text) and are not +// matched by any rule, so they are not emitted here. +let varyingTokens = Token.allCases.compactMap { token -> String? in + let spec = token.spec + guard spec.text == nil else { return nil } + // The generic `keyword` token has no `TokenSpec.text`, but each concrete + // keyword has a fixed spelling carried by its associated value. + guard case .other = spec.kind else { return nil } + return spec.identifier.text +} + +// The yeast type references that a child maps to. A collection wrapper is +// elided by the adapter, so a collection child maps to its element kinds. +func typeRefs(_ child: Child) -> [String] { + switch child.kind { + case .node(let kind): + return [kind.rawValue] + case .nodeChoices(let choices, _): + return choices.flatMap { typeRefs($0) } + case .collection(let kind, _, _, _, _): + if let collection = SYNTAX_NODES.first(where: { $0.kind == kind })?.collectionNode { + let elements = collection.elementChoices.map { $0.rawValue } + return elements.isEmpty ? [kind.rawValue] : elements + } + return [kind.rawValue] + case .token: + return ["_token"] + } +} + +func isMultiple(_ child: Child) -> Bool { + if case .collection = child.kind { + return true + } + return false +} + +var supertypes: [String: [String]] = [:] +var named: [(String, [Child])] = [] + +for node in SYNTAX_NODES { + if node.kind.isBase { + continue + } + if node.base == .syntaxCollection { + continue + } + supertypes[node.base.rawValue, default: []].append(node.kind.rawValue) + named.append((node.kind.rawValue, node.layoutNode?.children ?? [])) +} + +var output = "" +output += "# GENERATED from swift-syntax by unified/swift-syntax-rs/schemagen.\n" +output += "# Do not edit; run unified/scripts/regenerate-node-types.sh instead.\n" +let emitSupertypes = ProcessInfo.processInfo.environment["EMIT_SUPERTYPES"] != "0" +if emitSupertypes { + output += "supertypes:\n" + for base in supertypes.keys.sorted() { + output += " \(base):\n" + for member in supertypes[base]!.sorted() { + output += " - \(member)\n" + } + } +} +output += "named:\n" +for (kind, children) in named.sorted(by: { $0.0 < $1.0 }) { + output += " \(kind):\n" + for child in children { + // swift-syntax error-recovery slots (`unexpectedBeforeX`, + // `unexpectedBetweenXAndY`, and `unexpectedAfterX`) are never matched + // by rules. + if child.name.hasPrefix("unexpected") { + continue + } + var key = child.name + if isMultiple(child) { + key += "*" + } else if child.isOptional { + key += "?" + } + let refs = typeRefs(child) + let value = refs.count == 1 ? refs[0] : "[" + refs.joined(separator: ", ") + "]" + output += " \(key): \(value)\n" + } +} + +// Synthetic leaf for token-typed fields, plus the named ("varying") token +// kinds that are not already emitted as layout nodes (`stringSegment`, for +// example, is both a node and a token kind and must only be emitted once). +let namedKinds = Set(named.map { $0.0 }) +output += " _token:\n" +for token in varyingTokens.sorted() where !namedKinds.contains(token) { + output += " \(token):\n" +} + +print(output, terminator: "")