diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dea0c78e..53907f7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,6 +42,17 @@ jobs: shell: pwsh run: | ./build/Build.Windows.ps1 + - name: Upload archives + # Consumed by the publish-npm job + if: github.event_name != 'pull_request' + uses: actions/upload-artifact@v7 + with: + name: archives + path: | + artifacts/seqcli-*-*.zip + artifacts/seqcli-*-*.tar.gz + if-no-files-found: error + retention-days: 1 build-linux: name: Build (Linux) @@ -69,3 +80,45 @@ jobs: shell: pwsh run: | ./build/Build.Linux.ps1 -SeqDockerTag $env:SEQ_DOCKER_TAG + + publish-npm: + name: Publish (npm) + runs-on: ubuntu-24.04 + needs: build-windows + + # Mirrors NuGet publishing: builds from any branch this workflow targets (dev builds as + # prereleases under the `dev` dist-tag, main builds as `latest`), but never pull requests. + if: github.event_name != 'pull_request' + + permissions: + contents: read + # Required for npm trusted publishing (OIDC) and provenance + id-token: write + + steps: + - uses: actions/checkout@v6 + - name: Setup + uses: actions/setup-node@v7 + with: + node-version: 24.x + # Bootstrap only: together with NODE_AUTH_TOKEN below, authenticates using the NPM_TOKEN + # secret. Once trusted publishing is configured for every @datalust/seqcli* package + # (bound to this workflow file, ci.yml), remove `registry-url` here and `NODE_AUTH_TOKEN` + # below so that npm authenticates with the OIDC token instead. + registry-url: https://registry.npmjs.org/ + - name: Update npm + # Trusted publishing and automatic provenance require npm 11.5.1 or later + run: | + npm install -g npm@latest + npm --version + - name: Download archives + uses: actions/download-artifact@v8 + with: + name: archives + path: npm-archives + - name: Publish + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + shell: pwsh + run: | + ./build/Build.Npm.ps1 -ArchiveDir ./npm-archives diff --git a/.gitignore b/.gitignore index 4e050436..5cbf57dc 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,8 @@ x64/ x86/ bld/ [Bb]in/ +# The npm launcher package keeps its script in bin/ +!npm/seqcli/bin/ [Oo]bj/ [Ll]og/ @@ -296,3 +298,7 @@ global.json .claude/ .qwen/ .agents/ + +# npm packaging staging area (build/Build.Npm.ps1) +npm-staging/ +npm-archives/ diff --git a/README.md b/README.md index 1213183d..9886253a 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,14 @@ The Seq installer for Windows includes `seqcli`. Otherwise, download the [releas dotnet tool install --global seqcli ``` +With Node.js installed, `seqcli` can be installed from npm using: + +``` +npm install -g @datalust/seqcli +``` + +On Windows, if the Seq installation directory is on your `PATH`, the `seqcli` bundled with Seq may take precedence over the npm-installed copy; `where seqcli` shows the resolution order, and `npx @datalust/seqcli ` always runs the npm version. + To set a default server URL and API key, run: ``` @@ -1464,6 +1472,8 @@ seqcli search -f "@Exception like '%TimeoutException%'" -c 30 | ------ | ----------- | | `-f`, `--filter=VALUE` | A filter to apply to the search, for example `Host = 'xmpweb-01.example.com'` | | `-c`, `--count=VALUE` | The maximum number of events to retrieve; the default is 1 | +| `--column=VALUE` | A column to display preceding each event's message; any Seq expression can be supplied, for example `OrderId`, `@SpanKind`, or `@Resource['service.name']`; this argument can be used multiple times, adding columns in order; applies to plain-text output only | +| `--no-signal-columns` | Do not show columns associated with the specified signal expression | | `--start=VALUE` | ISO 8601 date/time to query from | | `--end=VALUE` | ISO 8601 date/time to query to | | `--json` | Print output in newline-delimited JSON (the default is plain text) | @@ -1636,6 +1646,8 @@ Stream log events matching a filter. | Option | Description | | ------ | ----------- | | `-f`, `--filter=VALUE` | An optional server-side filter to apply to the stream, for example `@Level = 'Error'` | +| `--column=VALUE` | A column to display preceding each event's message; any Seq expression can be supplied, for example `OrderId`, `@SpanKind`, or `@Resource['service.name']`; this argument can be used multiple times, adding columns in order; applies to plain-text output only | +| `--no-signal-columns` | Do not show columns associated with the specified signal expression | | `--json` | Print output in newline-delimited JSON (the default is plain text) | | `--no-color` | Don't colorize text output | | `--force-color` | Force redirected output to have ANSI color (unless `--no-color` is also specified) | diff --git a/build/Build.Common.ps1 b/build/Build.Common.ps1 index 7d9e97a1..7abf7b7d 100644 --- a/build/Build.Common.ps1 +++ b/build/Build.Common.ps1 @@ -12,3 +12,12 @@ function Get-SemVer() $base + "." + $revision } } + +function Get-NpmVersion($version) +{ + # npm requires strict semver, which forbids leading zeros in numeric identifiers; the build number + # is zero-padded (e.g. 2026.1.02616), so strip the padding from the patch component (-> 2026.1.2616). + # Prerelease suffixes are alphanumeric identifiers and are left as-is. + if ($version -notmatch '^(\d+)\.(\d+)\.(\d+)(.*)$') { throw "Unrecognized version: $version" } + "$([int]$Matches[1]).$([int]$Matches[2]).$([int]$Matches[3])$($Matches[4])" +} diff --git a/build/Build.Npm.ps1 b/build/Build.Npm.ps1 new file mode 100644 index 00000000..71e2fc37 --- /dev/null +++ b/build/Build.Npm.ps1 @@ -0,0 +1,267 @@ +# Publishes the npm packages for a seqcli build: one `@datalust/seqcli-` package per release +# archive, then the launcher package `@datalust/seqcli` (from ./npm/seqcli) with its +# optionalDependencies pinned to the same version. +# +# In CI (see publish-npm in .github/workflows/ci.yml) the archives come from the build-windows job's +# artifacts, so dev builds are published as prereleases (dist-tag `dev`) and main builds as `latest`, +# matching NuGet. Packages that already exist on the registry at the target version are skipped, so +# a partially-failed run can simply be re-run. +# +# Usage: +# ./build/Build.Npm.ps1 -ArchiveDir ./npm-archives # CI: version from Get-SemVer +# ./build/Build.Npm.ps1 -Version 2026.1.02616 # (re)publish GitHub release v2026.1.02616 +# ./build/Build.Npm.ps1 -Version 2026.1.02616 -ArchiveDir ./x -DryRun # stage and `npm pack` only +param( + # Build version as it appears in archive names, e.g. 2026.1.02616 or 2026.1.02700-dev-02700. + # Defaults to Get-SemVer, which in CI reproduces the version computed by the build jobs. + [string] $Version, + + # npm dist-tag; defaults to `latest` for release versions and `dev` for prereleases. + [string] $DistTag, + + # Directory containing seqcli--.zip|.tar.gz archives; when omitted, the archives + # are downloaded from GitHub release v with `gh release download`. + [string] $ArchiveDir, + + # GitHub repository to download release assets from. Also identifies the repository whose CI + # publishes via npm trusted publishing (forks without an NPM_TOKEN skip publishing). + [string] $Repo = 'datalust/seqcli', + + # Stage the packages and run `npm pack` instead of `npm publish`. + [switch] $DryRun +) + +Push-Location $PSScriptRoot/../ + +. ./build/Build.Common.ps1 + +$ErrorActionPreference = 'Stop' + +$scope = '@datalust' +$launcherName = "$scope/seqcli" +$staging = './npm-staging' + +if (-not $Version) { + $Version = Get-SemVer +} + +$npmVersion = Get-NpmVersion $Version + +if (-not $DistTag) { + $DistTag = @{ $true = 'dev'; $false = 'latest' }[$npmVersion.Contains('-')] +} + +Write-Host "Release version: $Version" +Write-Host "npm version: $npmVersion" +Write-Host "npm dist-tag: $DistTag" +Write-Host "Dry run: $DryRun" + +if (-not $DryRun -and -not $env:NODE_AUTH_TOKEN -and $env:GITHUB_REPOSITORY -ne $Repo) { + # Forks have neither the NPM_TOKEN secret nor a trusted publisher configuration. + Write-Host "Skipping npm publishing: no npm credentials are available in this environment" + Pop-Location + exit 0 +} + +function Get-Rids +{ + $([xml](Get-Content ./src/SeqCli/SeqCli.csproj)).Project.PropertyGroup.RuntimeIdentifiers.Split(';') +} + +function Get-PlatformSpec($rid) +{ + $os = switch -Wildcard ($rid) { + 'win-*' { 'win32' } + 'osx-*' { 'darwin' } + 'linux-*' { 'linux' } + default { throw "Unrecognized RID: $rid" } + } + + $cpu = ($rid -split '-')[-1] + + $libc = $null + if ($rid -like 'linux-musl-*') { $libc = 'musl' } + elseif ($rid -like 'linux-*') { $libc = 'glibc' } + + return @{ os = $os; cpu = $cpu; libc = $libc; isWindows = ($os -eq 'win32') } +} + +function Get-ReleaseArchive($rid) +{ + $pattern = "seqcli-$Version-$rid.*" + + if ($ArchiveDir) { + $archive = Get-ChildItem -Path $ArchiveDir -Filter $pattern | Select-Object -First 1 + if (-not $archive) { throw "No archive matching $pattern in $ArchiveDir" } + return $archive.FullName + } + + $downloads = "$staging/download" + New-Item -ItemType Directory -Force -Path $downloads | Out-Null + + & gh release download "v$Version" --repo $Repo --dir $downloads --pattern $pattern --clobber + if ($LASTEXITCODE -ne 0) { throw "Downloading $pattern from release v$Version failed" } + + $archive = Get-ChildItem -Path $downloads -Filter $pattern | Select-Object -First 1 + if (-not $archive) { throw "Release v$Version has no asset matching $pattern" } + return $archive.FullName +} + +function Expand-ReleaseArchive($archive, $destination) +{ + if (Test-Path $destination) { Remove-Item -Recurse -Force $destination } + New-Item -ItemType Directory -Force -Path $destination | Out-Null + + if ($archive -like '*.zip') { + Expand-Archive -Path $archive -DestinationPath $destination -Force + } else { + & tar -xzf $archive -C $destination + if ($LASTEXITCODE -ne 0) { throw "Extracting $archive failed" } + } + + # The archives contain a single `seqcli--/` root folder; the package needs the + # binary at its root, so lift the contents up one level. + $entries = @(Get-ChildItem -Force $destination) + if ($entries.Count -eq 1 -and $entries[0].PSIsContainer) { + $root = $entries[0].FullName + Get-ChildItem -Force $root | Move-Item -Destination $destination + Remove-Item -Force $root + } +} + +function Write-PlatformPackageJson($rid, $spec, $destination) +{ + $package = Get-Content ./npm/platform-package.json -Raw | ConvertFrom-Json -AsHashtable + $package.name = "$scope/seqcli-$rid" + $package.version = $npmVersion + $package.description = $package.description.Replace('{{rid}}', $rid) + $package.os = @($spec.os) + $package.cpu = @($spec.cpu) + if ($spec.libc) { $package.libc = @($spec.libc) } + + $package | ConvertTo-Json -Depth 5 | Set-Content -Path "$destination/package.json" -NoNewline +} + +function Test-NpmPublished($name) +{ + $output = & npm view "$name@$npmVersion" version --json 2>$null + return ($LASTEXITCODE -eq 0) -and -not [string]::IsNullOrWhiteSpace(($output -join '')) +} + +function Publish-NpmPackage($name, $directory) +{ + if ($DryRun) { + Write-Host "Packing $name@$npmVersion" + $tarballs = "$staging/tarballs" + New-Item -ItemType Directory -Force -Path $tarballs | Out-Null + & npm pack $directory --pack-destination $tarballs + if ($LASTEXITCODE -ne 0) { throw "Packing $name failed" } + return + } + + if (Test-NpmPublished $name) { + Write-Host "Skipping $name@$npmVersion; already published" + return + } + + Write-Host "Publishing $name@$npmVersion with dist-tag $DistTag" + $arguments = @('publish', $directory, '--access', 'public', '--tag', $DistTag) + if ($env:GITHUB_ACTIONS -eq 'true') { $arguments += '--provenance' } + & npm @arguments + if ($LASTEXITCODE -ne 0) { throw "Publishing $name failed" } +} + +function Assert-NpmPublished($name) +{ + # `npm publish` returns as soon as the registry accepts the upload, but the read path (the + # packument served via npm's CDN) is updated asynchronously and can lag by several minutes, + # particularly for the first-ever publish of a package name. Poll for up to ten minutes. + $timeout = [TimeSpan]::FromMinutes(10) + $interval = 15 + $started = Get-Date + + while ($true) { + if (Test-NpmPublished $name) { return } + + $elapsed = (Get-Date) - $started + if ($elapsed -ge $timeout) { break } + + Write-Host ("Waiting for {0}@{1} to become visible on the registry ({2:mm\:ss} elapsed)" -f $name, $npmVersion, $elapsed) + Start-Sleep -Seconds $interval + } + + throw "$name@$npmVersion is not visible on the registry after $($timeout.TotalMinutes) minutes" +} + +function Stage-PlatformPackage($rid) +{ + $spec = Get-PlatformSpec $rid + $directory = "$staging/seqcli-$rid" + + $archive = Get-ReleaseArchive $rid + Write-Host "Staging $scope/seqcli-$rid from $archive" + Expand-ReleaseArchive $archive $directory + + $binary = Join-Path $directory $(if ($spec.isWindows) { 'seqcli.exe' } else { 'seqcli' }) + if (-not (Test-Path $binary)) { throw "Expected $binary in $archive" } + + if (-not $spec.isWindows) { + & chmod +x $binary + if ($LASTEXITCODE -ne 0) { throw "chmod failed for $binary" } + } + + Write-PlatformPackageJson $rid $spec $directory + + return $directory +} + +function Stage-LauncherPackage($rids) +{ + $directory = "$staging/seqcli" + if (Test-Path $directory) { Remove-Item -Recurse -Force $directory } + Copy-Item -Recurse ./npm/seqcli $directory + + # The launcher package is the one users see on npmjs.com, so it carries the repository README + # and license rather than maintaining separate copies under ./npm. + Copy-Item ./README.md "$directory/README.md" + Copy-Item ./LICENSE "$directory/LICENSE" + + $package = Get-Content "$directory/package.json" -Raw | ConvertFrom-Json -AsHashtable + $package.version = $npmVersion + $package.optionalDependencies = [ordered]@{} + foreach ($rid in $rids) { + $package.optionalDependencies["$scope/seqcli-$rid"] = $npmVersion + } + + $package | ConvertTo-Json -Depth 5 | Set-Content -Path "$directory/package.json" -NoNewline + + return $directory +} + +if (Test-Path $staging) { Remove-Item -Recurse -Force $staging } +New-Item -ItemType Directory -Force -Path $staging | Out-Null + +$rids = Get-Rids + +foreach ($rid in $rids) { + $directory = Stage-PlatformPackage $rid + Publish-NpmPackage "$scope/seqcli-$rid" $directory +} + +if (-not $DryRun) { + # Never expose a launcher whose optional dependencies can't all be resolved. + foreach ($rid in $rids) { + Assert-NpmPublished "$scope/seqcli-$rid" + } +} + +$launcherDirectory = Stage-LauncherPackage $rids +Publish-NpmPackage $launcherName $launcherDirectory + +if (-not $DryRun) { + Assert-NpmPublished $launcherName + & npm view $launcherName dist-tags + Write-Host "Install with: npm install -g $launcherName@$npmVersion" +} + +Pop-Location diff --git a/build/README.md b/build/README.md new file mode 100644 index 00000000..6dd06c32 --- /dev/null +++ b/build/README.md @@ -0,0 +1,88 @@ +# Building and publishing `seqcli` + +This directory holds the PowerShell scripts that build, test, package, and publish `seqcli`. They are driven by the GitHub Actions workflow in [`.github/workflows/ci.yml`](../.github/workflows/ci.yml), but can also be run locally with `pwsh`. + +| Script | Runs on | Produces | +|---|---|---| +| `Build.Common.ps1` | (dot-sourced by the others) | Version number helpers | +| `Build.Windows.ps1` | Windows | Release archives for every platform, the `seqcli` .NET tool package, generated docs, a GitHub release, NuGet publish | +| `Build.Linux.ps1` | Linux | `datalust/seqcli` Docker images for `linux/amd64` and `linux/arm64` | +| `Build.Npm.ps1` | Linux (any OS locally) | `@datalust/seqcli` and `@datalust/seqcli-` npm packages | + +`7-zip/` contains a vendored copy of `7za.exe`, used by `Build.Windows.ps1` to produce `.zip` and `.tar.gz` archives with consistent contents on Windows. + +## Versioning + +Every artifact from one CI run shares a single version, computed by `Get-SemVer` in `Build.Common.ps1`: + +``` +.[--] +``` + +* `` is read from [`baseversion`](../baseversion) in the repository root (e.g. `2026.1`). Bump it there when starting a new release line. +* `` is `CI_BUILD_NUMBER_BASE + 2300`, zero-padded to five digits (e.g. `02616`). `CI_BUILD_NUMBER_BASE` is the GitHub Actions `run_number`; the fixed offset keeps build numbers increasing across the move from the previous CI system. Note the comment at the top of `ci.yml`: renaming the workflow file resets `run_number`, which would produce lower version numbers than already-published releases. Locally, where the variable is unset, the build number is the literal string `local`. +* The prerelease suffix is added for every branch except `main`. It is the first ten characters of the branch name, stripped of anything other than letters, digits and hyphens, followed by the build number. A `dev` build is therefore `2026.1.02700-dev-02700`; a `main` build is `2026.1.02616`. + +`CI_TARGET_BRANCH` overrides the branch detected from git. In CI it is set from `github.head_ref` (for pull requests) or `github.ref_name`. + +### npm versions + +npm enforces strict semver, which forbids leading zeros in numeric components, so `Get-NpmVersion` strips the padding from the patch component when publishing to npm: release `v2026.1.02616` becomes `2026.1.2616` on npm, and `2026.1.02700-dev-02700` becomes `2026.1.2700-dev-02700` (prerelease identifiers are left as-is). Archive names and GitHub release tags always use the padded form. + +## The CI workflow + +The workflow runs on pushes and pull requests to `dev` and `main`, and can be triggered manually with `workflow_dispatch`. Three jobs run: + +1. **Build (Windows)** runs `Build.Windows.ps1`. On non-PR builds, it uploads the release archives as a workflow artifact named `archives` for the npm job to consume. +2. **Build (Linux)** runs `Build.Linux.ps1`, after configuring `binfmt` so that ARM64 images can be built on the x64 runner. It runs in parallel with the Windows job. +3. **Publish (npm)** runs `Build.Npm.ps1` after the Windows job succeeds, on every non-PR build. + +Environment variables control what gets published: + +| Variable | Set in CI to | Effect | +|---|---|---| +| `CI_BUILD_NUMBER_BASE` | `github.run_number` | Build number component of the version | +| `CI_TARGET_BRANCH` | `github.head_ref` or `github.ref_name` | Branch component of the version | +| `CI_PUBLISH` | `True` for pushes to `main`, or when the manual `publish` input is set | Whether `Build.Windows.ps1` creates a GitHub release | +| `NUGET_API_KEY` | `secrets.NUGET_API_KEY` | When non-empty, `Build.Windows.ps1` pushes the tool package to NuGet | +| `GH_TOKEN` | `secrets.GITHUB_TOKEN` | Used by `gh release create` | +| `DOCKER_USER`, `DOCKER_TOKEN` | Docker Hub secrets | When `DOCKER_TOKEN` is non-empty, `Build.Linux.ps1` pushes images | +| `NODE_AUTH_TOKEN` | `secrets.NPM_TOKEN` | Authenticates `npm publish` (see below) | +| `SEQ_DOCKER_TAG` | e.g. `2026.1` | Seq image tag used for the Linux end-to-end tests | + +GitHub only supplies repository secrets to branch builds, not to pull requests, so PR builds run the full build and test steps but publish nothing. + +### What each branch publishes + +| Trigger | GitHub release | NuGet | Docker Hub | npm | +|---|---|---|---|---| +| Pull request | no | no | no | no | +| Push to `dev` | no (unless manual `publish`) | prerelease version | `datalust/seqcli-ci:` | prerelease, dist-tag `dev` | +| Push to `main` | yes, `v` | release version | `datalust/seqcli-ci:` | release, dist-tag `latest` | + +Note that `Build.Linux.ps1` always pushes to the `datalust/seqcli-ci` repository (the image name with a `-ci` suffix), never directly to `datalust/seqcli`. Promoting an image to the public `datalust/seqcli` repository, and tagging it `latest`, is a separate step outside this repository. + +To publish a release from `dev` (for example a preview build), run the workflow manually from the Actions tab with the **Publish a GitHub release** input checked. The release is marked as a prerelease because the branch is not `main`. + +## Running the builds locally + +All scripts need PowerShell 7 (`pwsh`) and the .NET 10 SDK, and must be run from anywhere inside the repository (they `Push-Location` to the root themselves). Without the `CI_*` variables the version is `.local`, and nothing is published because the credential variables are unset. + +* `Build.Windows.ps1` requires Windows: it uses `7za.exe`, installs Seq with Chocolatey for the end-to-end tests, and builds `win-*` RIDs. Run it as `./build/Build.Windows.ps1`. +* `Build.Linux.ps1` requires Docker with `buildx`. Building the `arm64` image on an x64 host needs `binfmt` configured, as the workflow does. Run it as `./build/Build.Linux.ps1 -SeqDockerTag 2026.1`. +* `Build.Npm.ps1` runs anywhere with `npm`, `tar`, and `gh`. To exercise it end to end without publishing, point it at a directory of archives (for example a local `artifacts/` from a Windows build, or assets downloaded from a release) and use `-DryRun`: + + ```shell + gh release download v2026.1.02616 --dir ./npm-archives --pattern 'seqcli-*-*.*' + ./build/Build.Npm.ps1 -Version 2026.1.02616 -ArchiveDir ./npm-archives -DryRun + ls npm-staging/tarballs + ``` + +`artifacts/`, `npm-staging/` and `npm-archives/` are all ignored by git. + +## Checklist for a new release line + +1. Update [`baseversion`](../baseversion). +2. Update the SDK version in [`ci.global.json`](../ci.global.json) to match the Seq release. +3. Update `SEQ_DOCKER_TAG` in [`ci.yml`](../.github/workflows/ci.yml) to the Seq image the end-to-end tests should run against. +4. If the target framework changes, update `$framework` in `Build.Windows.ps1` and `Build.Linux.ps1`, and the `COPY` paths in the Dockerfiles. diff --git a/npm/platform-package.json b/npm/platform-package.json new file mode 100644 index 00000000..eee4c44a --- /dev/null +++ b/npm/platform-package.json @@ -0,0 +1,24 @@ +{ + "name": "{{name}}", + "version": "{{version}}", + "description": "seqcli binaries for {{rid}}. Install @datalust/seqcli instead of depending on this package directly.", + "license": "Apache-2.0", + "homepage": "https://github.com/datalust/seqcli", + "repository": { + "type": "git", + "url": "git+https://github.com/datalust/seqcli.git" + }, + "engines": { + "node": ">=18" + }, + "os": [ + "{{os}}" + ], + "cpu": [ + "{{cpu}}" + ], + "preferUnplugged": true, + "publishConfig": { + "access": "public" + } +} diff --git a/npm/seqcli/bin/seqcli.js b/npm/seqcli/bin/seqcli.js new file mode 100755 index 00000000..d7078783 --- /dev/null +++ b/npm/seqcli/bin/seqcli.js @@ -0,0 +1,147 @@ +#!/usr/bin/env node +'use strict'; + +// Launcher for the platform-specific seqcli binary. The binary itself ships in one of the +// `@datalust/seqcli-` packages, installed as an optional dependency of `@datalust/seqcli` +// and selected by npm using the `os`/`cpu`/`libc` fields in each package. + +const { spawn } = require('child_process'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const SCOPE = '@datalust'; +const RELEASES_URL = 'https://github.com/datalust/seqcli/releases'; + +// `${process.platform}-${process.arch}` (with `-musl` inserted for musl-based Linux) -> .NET RID. +const RIDS = { + 'win32-x64': 'win-x64', + 'win32-arm64': 'win-arm64', + 'darwin-x64': 'osx-x64', + 'darwin-arm64': 'osx-arm64', + 'linux-x64': 'linux-x64', + 'linux-arm64': 'linux-arm64', + 'linux-musl-x64': 'linux-musl-x64', + 'linux-musl-arm64': 'linux-musl-arm64', +}; + +function isMusl() { + try { + return !process.report.getReport().header.glibcVersionRuntime; + } catch { + return false; + } +} + +// Candidate platform packages in order of preference. On Linux the detected libc variant is tried +// first, then the other one, so that package managers that ignore the `libc` field (and therefore +// install both) still run the right binary. +function candidatePackages() { + const { platform, arch } = process; + const keys = []; + if (platform === 'linux') { + const musl = isMusl(); + keys.push(musl ? `linux-musl-${arch}` : `linux-${arch}`); + keys.push(musl ? `linux-${arch}` : `linux-musl-${arch}`); + } else { + keys.push(`${platform}-${arch}`); + } + return keys.filter((k) => RIDS[k]).map((k) => `${SCOPE}/seqcli-${RIDS[k]}`); +} + +function locateBinary() { + const launcherVersion = require('../package.json').version; + const candidates = candidatePackages(); + const problems = []; + + for (const name of candidates) { + let packageJsonPath; + try { + packageJsonPath = require.resolve(`${name}/package.json`); + } catch { + problems.push(`${name} is not installed`); + continue; + } + + const installedVersion = require(packageJsonPath).version; + if (installedVersion !== launcherVersion) { + problems.push(`${name}@${installedVersion} does not match ${SCOPE}/seqcli@${launcherVersion}`); + continue; + } + + const exe = path.join(path.dirname(packageJsonPath), process.platform === 'win32' ? 'seqcli.exe' : 'seqcli'); + if (!fs.existsSync(exe)) { + problems.push(`${name} is installed but ${exe} is missing`); + continue; + } + + return exe; + } + + const lines = []; + if (candidates.length === 0) { + lines.push(`seqcli: ${process.platform}-${process.arch} is not supported by the npm package.`); + } else { + lines.push('seqcli: could not find the platform-specific seqcli package.'); + for (const p of problems) lines.push(` - ${p}`); + lines.push(''); + lines.push(`Reinstall with: npm install -g ${SCOPE}/seqcli@${launcherVersion}`); + lines.push('(optional dependencies must not be omitted; check for --omit=optional / --no-optional)'); + lines.push(`or install the platform package directly: npm install -g ${candidates[0]}@${launcherVersion}`); + } + lines.push(''); + lines.push(`Supported platforms: ${Object.values(RIDS).join(', ')}.`); + lines.push(`Other downloads: ${RELEASES_URL}`); + console.error(lines.join('\n')); + process.exit(1); +} + +function ensureExecutable(exe) { + if (process.platform === 'win32') return; + try { + fs.accessSync(exe, fs.constants.X_OK); + } catch { + try { + fs.chmodSync(exe, 0o755); + } catch { + // Reported by spawn() as EACCES below. + } + } +} + +function run() { + const exe = locateBinary(); + ensureExecutable(exe); + + const child = spawn(exe, process.argv.slice(2), { stdio: 'inherit', windowsHide: true }); + + // Ctrl+C is delivered by the terminal to the whole foreground process group (or console on + // Windows), so the child already receives it. Ignore it here so this process outlives the + // child and can report the child's exit status. + process.on('SIGINT', () => {}); + + // Signals from supervisors (kill, systemd, CI cancellation) target this process only; forward them. + for (const signal of ['SIGTERM', 'SIGHUP']) { + process.on(signal, () => child.kill(signal)); + } + + child.on('error', (err) => { + console.error(`seqcli: failed to start ${exe}: ${err.message}`); + process.exit(1); + }); + + child.on('exit', (code, signal) => { + if (signal) { + process.removeAllListeners(signal); + try { + process.kill(process.pid, signal); + } catch { + // Fall through to a conventional exit code. + } + process.exit(128 + (os.constants.signals[signal] || 0)); + } + process.exit(code === null ? 1 : code); + }); +} + +run(); diff --git a/npm/seqcli/package.json b/npm/seqcli/package.json new file mode 100644 index 00000000..23f767a2 --- /dev/null +++ b/npm/seqcli/package.json @@ -0,0 +1,46 @@ +{ + "name": "@datalust/seqcli", + "//version": "These version are replaced during the deployment process", + "version": "0.0.0", + "description": "The Seq command-line client. Administer, log, ingest, search, from any OS.", + "keywords": [ + "seq", + "seqcli", + "datalust", + "logging", + "structured-logging", + "cli" + ], + "license": "Apache-2.0", + "homepage": "https://github.com/datalust/seqcli", + "repository": { + "type": "git", + "url": "git+https://github.com/datalust/seqcli.git" + }, + "bugs": { + "url": "https://github.com/datalust/seqcli/issues" + }, + "bin": { + "seqcli": "bin/seqcli.js" + }, + "files": [ + "bin", + "README.md" + ], + "engines": { + "node": ">=18" + }, + "publishConfig": { + "access": "public" + }, + "optionalDependencies": { + "@datalust/seqcli-win-x64": "0.0.0", + "@datalust/seqcli-win-arm64": "0.0.0", + "@datalust/seqcli-linux-x64": "0.0.0", + "@datalust/seqcli-linux-arm64": "0.0.0", + "@datalust/seqcli-linux-musl-x64": "0.0.0", + "@datalust/seqcli-linux-musl-arm64": "0.0.0", + "@datalust/seqcli-osx-x64": "0.0.0", + "@datalust/seqcli-osx-arm64": "0.0.0" + } +} diff --git a/src/Roastery/Data/Database.cs b/src/Roastery/Data/Database.cs index 9234f1ce..251b67f1 100644 --- a/src/Roastery/Data/Database.cs +++ b/src/Roastery/Data/Database.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.Globalization; using System.Linq; using System.Reflection; diff --git a/src/Roastery/Util/Distribution.cs b/src/Roastery/Util/Distribution.cs index b20ffb49..5264cd14 100644 --- a/src/Roastery/Util/Distribution.cs +++ b/src/Roastery/Util/Distribution.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Runtime.CompilerServices; -using System.Threading; namespace Roastery.Util; diff --git a/src/Roastery/Web/RequestLoggingMiddleware.cs b/src/Roastery/Web/RequestLoggingMiddleware.cs index c75a5002..92d50bf3 100644 --- a/src/Roastery/Web/RequestLoggingMiddleware.cs +++ b/src/Roastery/Web/RequestLoggingMiddleware.cs @@ -1,6 +1,5 @@ using System; using System.Diagnostics; -using System.Diagnostics.Metrics; using System.Net; using System.Threading.Tasks; using Roastery.Metrics; diff --git a/src/SeqCli/Api/EventEntityJson.cs b/src/SeqCli/Api/EventEntityJson.cs new file mode 100644 index 00000000..308c0735 --- /dev/null +++ b/src/SeqCli/Api/EventEntityJson.cs @@ -0,0 +1,104 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; +using System.Text.Json.Nodes; +using Seq.Api.Model.Events; +using Seq.Api.Model.Shared; +using SeqCli.Data; +using SeqCli.Output; + +namespace SeqCli.Api; + +/// +/// Converts event entities into compact JSON format for further processing. This class is only necessary because +/// Seq.Api doesn't yet provide a simple compact-JSON based result format for searches. Once we've filled +/// that gap, this class, and can be removed. +/// +static class EventEntityJson +{ + public static JsonObject ToEventJson(EventEntity evt) + { + var eventJson = new JsonObject + { + // Earlier versions relied on Serilog output formatting to show timestamps in local time; we'll need + // to consider adding some compensating mechanism to `Seq.Syntax`. + ["@t"] = DateTimeOffset.ParseExact(evt.Timestamp, "o", CultureInfo.InvariantCulture) + .ToLocalTime().ToString("o", CultureInfo.InvariantCulture) + }; + + if (evt.MessageTemplateTokens != null) + eventJson["@mt"] = ToMessageTemplateText(evt.MessageTemplateTokens); + + if (!string.IsNullOrWhiteSpace(evt.Level) && evt.Level != "Information") + eventJson["@l"] = evt.Level; + + if (!string.IsNullOrWhiteSpace(evt.Exception)) + eventJson["@x"] = evt.Exception; + + if (!string.IsNullOrWhiteSpace(evt.TraceId)) + eventJson["@tr"] = evt.TraceId; + + if (!string.IsNullOrWhiteSpace(evt.SpanId)) + eventJson["@sp"] = evt.SpanId; + + if (!string.IsNullOrWhiteSpace(evt.ParentId)) + eventJson["@ps"] = evt.ParentId; + + if (!string.IsNullOrWhiteSpace(evt.Start)) + eventJson["@st"] = evt.Start; + + if (!string.IsNullOrWhiteSpace(evt.SpanKind)) + eventJson["@sk"] = evt.SpanKind; + + if (evt.Resource?.Count > 0) + eventJson["@ra"] = ToPropertiesObject(evt.Resource); + + if (evt.Scope?.Count > 0) + eventJson["@sa"] = ToPropertiesObject(evt.Scope); + + if (evt.Properties != null) + { + foreach (var property in evt.Properties) + eventJson[EventJsonFormat.EscapeUserPropertyName(property.Name)] = ToSystemTextJson.FromApiValue(property.Value); + } + + return eventJson; + } + + static string ToMessageTemplateText(List tokens) + { + var text = new StringBuilder(); + foreach (var token in tokens) + { + if (token.Text != null) + text.Append(token.Text.Replace("{", "{{").Replace("}", "}}")); + else + text.Append(token.RawText ?? $"{{{token.PropertyName}}}"); + } + + return text.ToString(); + } + + static JsonObject ToPropertiesObject(List properties) + { + var result = new JsonObject(); + foreach (var property in properties) + result[property.Name] = ToSystemTextJson.FromApiValue(property.Value); + return result; + } +} diff --git a/src/SeqCli/Api/LevelMapping.cs b/src/SeqCli/Api/LevelMapping.cs new file mode 100644 index 00000000..bc987977 --- /dev/null +++ b/src/SeqCli/Api/LevelMapping.cs @@ -0,0 +1,99 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using Seq.Api.Model.LogEvents; + +namespace SeqCli.Api; + +public static class LevelMapping +{ + static readonly Dictionary LevelsByName = + new(StringComparer.OrdinalIgnoreCase) + { + ["t"] = "Trace", + ["tr"] = "Trace", + ["trc"] = "Trace", + ["trce"] = "Trace", + ["trace"] = "Trace", + ["v"] = "Verbose", + ["ver"] = "Verbose", + ["vrb"] = "Verbose", + ["verb"] = "Verbose", + ["verbose"] = "Verbose", + ["d"] = "Debug", + ["de"] = "Debug", + ["dbg"] = "Debug", + ["deb"] = "Debug", + ["dbug"] = "Debug", + ["debu"] = "Debug", + ["debug"] = "Debug", + ["i"] = "Information", + ["in"] = "Information", + ["inf"] = "Information", + ["info"] = "Information", + ["information"] = "Information", + ["notice"] = "Notice", + ["w"] = "Warning", + ["wa"] = "Warning", + ["war"] = "Warning", + ["wrn"] = "Warning", + ["warn"] = "Warning", + ["warning"] = "Warning", + ["e"] = "Error", + ["er"] = "Error", + ["err"] = "Error", + ["erro"] = "Error", + ["eror"] = "Error", + ["error"] = "Error", + ["f"] = "Fatal", + ["fa"] = "Fatal", + ["ftl"] = "Fatal", + ["fat"] = "Fatal", + ["fatl"] = "Fatal", + ["fatal"] = "Fatal", + ["c"] = "Critical", + ["cr"] = "Critical", + ["crt"] = "Critical", + ["cri"] = "Critical", + ["crit"] = "Critical", + ["critical"] = "Critical", + ["emerg"] = "Emergency", + ["alert"] = "Alert", + ["panic"] = "Panic" + }; + + // Intended only for use by ingest extraction patterns. + public static string ToFullLevelName(string level) + { + return LevelsByName.TryGetValue(level, out var m) ? m : level; + } + + public static LogEventLevel ToSeqApiLogEventLevel(string level) + { + if (string.IsNullOrEmpty(level)) + return LogEventLevel.Information; + + return ToFullLevelName(level) switch + { + "Trace" or "Verbose" => LogEventLevel.Verbose, + "Debug" => LogEventLevel.Debug, + "Warning" => LogEventLevel.Warning, + "Error" => LogEventLevel.Error, + "Fatal" or "Critical" or "Emergency" or "Alert" or "Panic" => LogEventLevel.Fatal, + _ => LogEventLevel.Information + }; + } +} diff --git a/src/SeqCli/Api/ToSystemTextJson.cs b/src/SeqCli/Api/ToSystemTextJson.cs new file mode 100644 index 00000000..92428c06 --- /dev/null +++ b/src/SeqCli/Api/ToSystemTextJson.cs @@ -0,0 +1,45 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Text.Json.Nodes; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using SeqCli.Data; + +namespace SeqCli.Api; + +static class ToSystemTextJson +{ + /// + /// Convert a value deserialized by the Seq API client into its `System.Text.Json` equivalent. + /// + public static JsonNode? FromApiValue(object? value) + { + return value switch + { + null => null, + JToken token => FromNewtonsoft(token), + _ => EventJsonFormat.CreateScalar(value) + }; + } + + /// Conversion helper for values retrieved through the Seq API client. + public static JsonNode? FromNewtonsoft(JToken token) + { + if (token is JValue { Value: null }) + return null; + + return JsonNode.Parse(token.ToString(Formatting.None)); + } +} diff --git a/src/SeqCli/Apps/AppLoader.cs b/src/SeqCli/Apps/AppLoader.cs index c0a03ff5..e46cdfed 100644 --- a/src/SeqCli/Apps/AppLoader.cs +++ b/src/SeqCli/Apps/AppLoader.cs @@ -34,7 +34,8 @@ class AppLoader : IDisposable [ typeof(SeqApp).Assembly, typeof(Log).Assembly, - typeof(SerilogExpression).Assembly + // Seq.Syntax uses version-specific assembly names to improve our chances of successful loading. + typeof(SeqExpression).Assembly ]; public AppLoader(string packageBinaryPath) diff --git a/src/SeqCli/Apps/Hosting/AppContainer.cs b/src/SeqCli/Apps/Hosting/AppContainer.cs index 8a58c2bd..6949921d 100644 --- a/src/SeqCli/Apps/Hosting/AppContainer.cs +++ b/src/SeqCli/Apps/Hosting/AppContainer.cs @@ -21,7 +21,7 @@ using Newtonsoft.Json.Linq; using Seq.Apps; using Seq.Apps.LogEvents; -using SeqCli.Mapping; +using SeqCli.Api; using Serilog; using Serilog.Events; using Serilog.Formatting.Compact.Reader; @@ -109,11 +109,11 @@ async Task SendTypedEventAsync(string clef) { if (_seqApp is ISubscribeTo led) { - led.On(EventFormat.FromRaw(eventId, eventType, serilogEvent)); + led.On(EventFormat.FromSerilogLogEvent(eventId, eventType, serilogEvent)); } else if (_seqApp is ISubscribeToAsync leda) { - await leda.OnAsync(EventFormat.FromRaw(eventId, eventType, serilogEvent)); + await leda.OnAsync(EventFormat.FromSerilogLogEvent(eventId, eventType, serilogEvent)); } else if (_seqApp is ISubscribeTo sled) { @@ -143,7 +143,8 @@ LogEvent ReadSerilogEvent(string clef, out string eventId, out uint eventType) if (jobject.TryGetValue("@l", out var levelToken)) { jobject.Remove("@l"); - jobject.Add("@l", new JValue(LevelMapping.ToSerilogLevel(levelToken.Value()!).ToString())); + // The Seq.Api `LogEventLevel` enum intentionally matches the Serilog one. + jobject.Add("@l", new JValue(LevelMapping.ToSeqApiLogEventLevel(levelToken.Value()!).ToString())); } SanitizeTraceIdentifiers(jobject); diff --git a/src/SeqCli/Apps/Hosting/EventFormat.cs b/src/SeqCli/Apps/Hosting/EventFormat.cs index 1ff23253..67a37b4d 100644 --- a/src/SeqCli/Apps/Hosting/EventFormat.cs +++ b/src/SeqCli/Apps/Hosting/EventFormat.cs @@ -24,7 +24,7 @@ namespace SeqCli.Apps.Hosting; static class EventFormat { - public static Event FromRaw(string eventId, uint eventType, LogEvent raw) + public static Event FromSerilogLogEvent(string eventId, uint eventType, LogEvent raw) { var properties = new Dictionary(); foreach (var prop in raw.Properties) diff --git a/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs b/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs index 49ceba1d..d8a2182d 100644 --- a/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs @@ -17,12 +17,10 @@ using System.Linq; using System.Threading.Tasks; using Seq.Api.Model.Alerting; -using Seq.Api.Model.LogEvents; using Seq.Api.Model.Shared; using SeqCli.Api; using SeqCli.Cli.Features; using SeqCli.Config; -using SeqCli.Mapping; using SeqCli.Signals; using SeqCli.Syntax; using SeqCli.Util; @@ -178,7 +176,7 @@ protected override async Task Run() alert.Having = _having; if (_notificationLevel != null) - alert.NotificationLevel = Enum.Parse(LevelMapping.ToFullLevelName(_notificationLevel)); + alert.NotificationLevel = LevelMapping.ToSeqApiLogEventLevel(_notificationLevel); if (_suppressionTime != null) alert.SuppressionTime = DurationMoniker.ToTimeSpan(_suppressionTime); diff --git a/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs b/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs index fe514434..25375b1e 100644 --- a/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs @@ -16,13 +16,11 @@ using System.Linq; using System.Threading.Tasks; using Seq.Api; -using Seq.Api.Model.LogEvents; using Seq.Api.Model.Security; using Seq.Api.Model.Shared; using SeqCli.Api; using SeqCli.Cli.Features; using SeqCli.Config; -using SeqCli.Mapping; using SeqCli.Util; using Serilog; @@ -125,7 +123,7 @@ protected override async Task Run() if (_level != null) { - apiKey.InputSettings.MinimumLevel = Enum.Parse(LevelMapping.ToFullLevelName(_level)); + apiKey.InputSettings.MinimumLevel = LevelMapping.ToSeqApiLogEventLevel(_level); } apiKey.AssignedPermissions.Clear(); diff --git a/src/SeqCli/Cli/Commands/IngestCommand.cs b/src/SeqCli/Cli/Commands/IngestCommand.cs index b965ddb3..be413c56 100644 --- a/src/SeqCli/Cli/Commands/IngestCommand.cs +++ b/src/SeqCli/Cli/Commands/IngestCommand.cs @@ -14,18 +14,17 @@ using System; using System.Collections.Generic; +using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; using SeqCli.Api; using SeqCli.Cli.Features; using SeqCli.Config; +using SeqCli.Data; using SeqCli.Ingestion; -using SeqCli.Mapping; using SeqCli.PlainText; using SeqCli.Syntax; using Serilog; -using Serilog.Core; -using Serilog.Events; namespace SeqCli.Cli.Commands; @@ -82,66 +81,58 @@ public IngestCommand() protected override async Task Run() { - try - { - var enrichers = new List(); - - if (_level != null) - enrichers.Add(new ScalarPropertyEnricher(LevelMapping.SurrogateLevelProperty, _level)); - - foreach (var (name, value) in _properties.FlatProperties) - enrichers.Add(new ScalarPropertyEnricher(name, value)); - - Func? filter = null; - if (_filter != null) - { - var eval = SeqSyntax.CompileExpression(_filter); - filter = evt => Seq.Syntax.Expressions.ExpressionResult.IsTrue(eval(evt)); - } + var enrichers = new List(); - var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = SeqConnectionFactory.Connect(_connection, config); - - // The API key is passed through separately because `SeqConnection` doesn't expose a batched ingestion - // mechanism and so we manually construct `HttpRequestMessage`s deeper in the stack. Nice feature gap to - // close at some point! - var (_, apiKey) = SeqConnectionFactory.GetConnectionDetails(_connection, config); - var batchSize = _batchSize.Value; + if (_level != null) + enrichers.Add(new LevelEnricher(_level)); - foreach (var input in _fileInputFeature.OpenInputs()) - { - using (input) - { - ILogEventReader reader = _json - ? new JsonLogEventReader(input) - : new PlainTextLogEventReader(input, _pattern); - - reader = new EnrichingReader(reader, enrichers); - - if (_message != null) - reader = new StaticMessageTemplateReader(reader, _message); - - var exit = await LogShipper.ShipEventsAsync( - connection, - apiKey, - reader, - _invalidDataHandlingFeature.InvalidDataHandling, - _sendFailureHandlingFeature.SendFailureHandling, - batchSize, - filter, - CancellationToken.None); - - if (exit != 0) - return exit; - } - } + foreach (var (name, value) in _properties.FlatProperties) + enrichers.Add(new ScalarPropertyEnricher(name, value)); - return 0; + Func? filter = null; + if (_filter != null) + { + var eval = SeqSyntax.CompileExpression(_filter); + filter = evt => eval(evt).IsTrue(); } - catch (Exception ex) + + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = SeqConnectionFactory.Connect(_connection, config); + + // The API key is passed through separately because `SeqConnection` doesn't expose a batched ingestion + // mechanism and so we manually construct `HttpRequestMessage`s deeper in the stack. Nice feature gap to + // close at some point! + var (_, apiKey) = SeqConnectionFactory.GetConnectionDetails(_connection, config); + var batchSize = _batchSize.Value; + + foreach (var input in _fileInputFeature.OpenInputs()) { - Log.Error(ex, "Ingestion failed: {ErrorMessage}", ex.Message); - return 1; + using (input) + { + IEventReader reader = _json + ? new JsonEventReader(input) + : new PlainTextEventReader(input, _pattern); + + reader = new EnrichingReader(reader, enrichers); + + if (_message != null) + reader = new StaticMessageTemplateReader(reader, _message); + + var exit = await LogShipper.ShipEventsAsync( + connection, + apiKey, + reader, + _invalidDataHandlingFeature.InvalidDataHandling, + _sendFailureHandlingFeature.SendFailureHandling, + batchSize, + filter, + CancellationToken.None); + + if (exit != 0) + return exit; + } } + + return 0; } } \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Metrics/SearchCommand.cs b/src/SeqCli/Cli/Commands/Metrics/SearchCommand.cs index 5cc1c3ee..9b86cd6b 100644 --- a/src/SeqCli/Cli/Commands/Metrics/SearchCommand.cs +++ b/src/SeqCli/Cli/Commands/Metrics/SearchCommand.cs @@ -22,7 +22,6 @@ using SeqCli.Cli.Features; using SeqCli.Config; using SeqCli.Util; -using Serilog; namespace SeqCli.Cli.Commands.Metrics; @@ -69,56 +68,48 @@ public SearchCommand() protected override async Task Run() { - try - { - var config = RuntimeConfigurationLoader.Load(_storagePath); - var output = _output.GetOutputFormat(config); - var connection = SeqConnectionFactory.Connect(_connection, config); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var output = _output.GetOutputFormat(config); + var connection = SeqConnectionFactory.Connect(_connection, config); - string? filter = null; - if (!string.IsNullOrWhiteSpace(_filter)) - filter = (await connection.Expressions.ToStrictAsync(_filter)).StrictExpression; + string? filter = null; + if (!string.IsNullOrWhiteSpace(_filter)) + filter = (await connection.Expressions.ToStrictAsync(_filter)).StrictExpression; - var result = await connection.Metrics.SearchAsync( - _groups, - filter, - _count, - rangeStartUtc: _range.Start, - rangeEndUtc: _range.End, - trace: _trace); - - // We convert the metric into a query result to improve formatting consistency. Room for an abstraction of - // some kind here. - var rows = new List(); - foreach (var metric in result.Metrics) - { - var row = new List - { - metric.Name ?? metric.Accessor, - metric.Kind, - metric.Unit, - metric.Description - }; - - foreach (var value in metric.GroupKey) - row.Add(value); - - rows.Add(row.ToArray()); - } - var asRowset = new QueryResultPart + var result = await connection.Metrics.SearchAsync( + _groups, + filter, + _count, + rangeStartUtc: _range.Start, + rangeEndUtc: _range.End, + trace: _trace); + + // We convert the metric into a query result to improve formatting consistency. Room for an abstraction of + // some kind here. + var rows = new List(); + foreach (var metric in result.Metrics) + { + var row = new List { - Columns = new[] { "Name", "Kind", "Unit", "Description" }.Concat(_groups).ToArray(), - Rows = rows.ToArray() + metric.Name ?? metric.Accessor, + metric.Kind, + metric.Unit, + metric.Description }; - output.WriteQueryResult(asRowset); - - return 0; + foreach (var value in metric.GroupKey) + row.Add(value); + + rows.Add(row.ToArray()); } - catch (Exception ex) + var asRowset = new QueryResultPart { - Log.Error(ex, "Could not retrieve metrics: {ErrorMessage}", ex.Message); - return 1; - } + Columns = new[] { "Name", "Kind", "Unit", "Description" }.Concat(_groups).ToArray(), + Rows = rows.ToArray() + }; + + output.WriteQueryResult(asRowset); + + return 0; } } \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/PrintCommand.cs b/src/SeqCli/Cli/Commands/PrintCommand.cs index 2740297f..0bc67f59 100644 --- a/src/SeqCli/Cli/Commands/PrintCommand.cs +++ b/src/SeqCli/Cli/Commands/PrintCommand.cs @@ -14,16 +14,16 @@ using System; using System.IO; +using System.Text.Json; +using System.Text.Json.Nodes; using System.Threading.Tasks; -using Newtonsoft.Json; -using Seq.Syntax.Expressions; using SeqCli.Cli.Features; using SeqCli.Config; using SeqCli.Ingestion; using SeqCli.Output; +using SeqCli.Syntax; using SeqCli.Util; using Serilog; -using Serilog.Events; namespace SeqCli.Cli.Commands; @@ -61,16 +61,16 @@ protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storage); - Func? filter = null; + Func? filter = null; if (_filter != null) { - if (!SerilogExpression.TryCompile(_filter, out var compiled, out var error)) + if (!SeqSyntax.TryCompileExpression(_filter, out var compiled, out var error)) { Log.Error("The specified filter could not be compiled: {Error}", error); return 1; } - filter = evt => ExpressionResult.IsTrue(compiled(evt)); + filter = evt => compiled(evt).IsTrue(); } var template = _template == null ? null : PrintTemplate.InterpretEscapeChars(_template); @@ -80,7 +80,7 @@ protected override async Task Run() { using (input) { - var reader = new JsonLogEventReader(input); + var reader = new JsonEventReader(input); var isAtEnd = false; do @@ -90,12 +90,12 @@ protected override async Task Run() var result = await reader.TryReadAsync(); isAtEnd = result.IsAtEnd; - if (result.LogEvent != null && (filter == null || filter(result.LogEvent))) - output.WriteLogEvent(result.LogEvent); + if (result.Document != null && (filter == null || filter(result.Document))) + output.WriteEvent(result.Document); } catch (Exception ex) { - if (ex is not JsonReaderException && ex is not InvalidDataException || + if (ex is not JsonException && ex is not InvalidDataException || _invalidDataHandlingFeature.InvalidDataHandling != InvalidDataHandling.Ignore) throw; } diff --git a/src/SeqCli/Cli/Commands/SearchCommand.cs b/src/SeqCli/Cli/Commands/SearchCommand.cs index 365725b7..1ce602e7 100644 --- a/src/SeqCli/Cli/Commands/SearchCommand.cs +++ b/src/SeqCli/Cli/Commands/SearchCommand.cs @@ -18,6 +18,7 @@ using SeqCli.Api; using SeqCli.Cli.Features; using SeqCli.Config; +using SeqCli.Output; using Serilog; // ReSharper disable UnusedType.Global @@ -33,6 +34,7 @@ class SearchCommand : Command readonly DateRangeFeature _range; readonly SignalExpressionFeature _signal; readonly StoragePathFeature _storagePath; + readonly EventColumnsFeature _eventColumns; string? _filter; int _count = 1; int _httpClientTimeout = 100000; @@ -44,11 +46,13 @@ public SearchCommand() "f=|filter=", "A filter to apply to the search, for example `Host = 'xmpweb-01.example.com'`", v => _filter = v); + Options.Add( "c=|count=", $"The maximum number of events to retrieve; the default is {_count}", v => _count = int.Parse(v, CultureInfo.InvariantCulture)); + _eventColumns = Enable(); _range = Enable(); _output = Enable(new OutputFormatFeature(supportNative: true, supportJson: true)); _storagePath = Enable(); @@ -68,59 +72,54 @@ public SearchCommand() protected override async Task Run() { - try - { - var config = RuntimeConfigurationLoader.Load(_storagePath); - var output = _output.GetOutputFormat(config); - var connection = SeqConnectionFactory.Connect(_connection, config); - connection.Client.HttpClient.Timeout = TimeSpan.FromMilliseconds(_httpClientTimeout); + var config = RuntimeConfigurationLoader.Load(_storagePath); + + var connection = SeqConnectionFactory.Connect(_connection, config); + connection.Client.HttpClient.Timeout = TimeSpan.FromMilliseconds(_httpClientTimeout); + + var columns = await _eventColumns.GetColumns(connection, _signal.Signal); + var output = _output.GetOutputFormat(config, TextFormatters.PlainOutputTemplate(columns)); - string? filter = null; - if (!string.IsNullOrWhiteSpace(_filter)) - filter = (await connection.Expressions.ToStrictAsync(_filter)).StrictExpression; + string? filter = null; + if (!string.IsNullOrWhiteSpace(_filter)) + filter = (await connection.Expressions.ToStrictAsync(_filter)).StrictExpression; - try + try + { + if (!_noWebSockets) { - if (!_noWebSockets) + await foreach (var evt in connection.Events.EnumerateAsync(null, + _signal.Signal, + filter, + _count, + fromDateUtc: _range.Start, + toDateUtc: _range.End, + trace: _trace, + render: output.RequiresRender)) { - await foreach (var evt in connection.Events.EnumerateAsync(null, - _signal.Signal, - filter, - _count, - fromDateUtc: _range.Start, - toDateUtc: _range.End, - trace: _trace, - render: output.RequiresRender)) - { - output.WriteEventEntity(evt); - } - - return 0; + output.WriteEventEntity(evt); } - } - catch (NotSupportedException nse) - { - Log.Information(nse, "WebSockets not supported; falling back to paged search"); - } - - await foreach (var evt in connection.Events.PagedEnumerateAsync(null, - _signal.Signal, - filter, - _count, - fromDateUtc: _range.Start, - toDateUtc: _range.End, - trace: _trace, - render: output.RequiresRender)) - { - output.WriteEventEntity(evt); - } - return 0; + return 0; + } } - catch (Exception ex) + catch (NotSupportedException nse) { - Log.Error(ex, "Could not retrieve search result: {ErrorMessage}", ex.Message); - return 1; + Log.Information(nse, "WebSockets not supported; falling back to paged search"); } + + await foreach (var evt in connection.Events.PagedEnumerateAsync(null, + _signal.Signal, + filter, + _count, + fromDateUtc: _range.Start, + toDateUtc: _range.End, + trace: _trace, + render: output.RequiresRender)) + { + output.WriteEventEntity(evt); + } + + return 0; } } \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/TailCommand.cs b/src/SeqCli/Cli/Commands/TailCommand.cs index 9d4d4957..b4670112 100644 --- a/src/SeqCli/Cli/Commands/TailCommand.cs +++ b/src/SeqCli/Cli/Commands/TailCommand.cs @@ -13,11 +13,14 @@ // limitations under the License. using System; +using System.IO; +using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; using SeqCli.Api; using SeqCli.Cli.Features; using SeqCli.Config; +using SeqCli.Output; namespace SeqCli.Cli.Commands; @@ -29,6 +32,7 @@ class TailCommand : Command readonly OutputFormatFeature _output; readonly SignalExpressionFeature _signal; readonly StoragePathFeature _storagePath; + readonly EventColumnsFeature _eventColumns; string? _filter; public TailCommand() @@ -38,6 +42,7 @@ public TailCommand() "An optional server-side filter to apply to the stream, for example `@Level = 'Error'`", v => _filter = v); + _eventColumns = Enable(); _output = Enable(new OutputFormatFeature(supportNative: true, supportJson: true)); _storagePath = Enable(); _signal = Enable(); @@ -59,17 +64,20 @@ protected override async Task Run() strict = converted.StrictExpression; } - var output = _output.GetOutputFormat(config); + var columns = await _eventColumns.GetColumns(connection, _signal.Signal); + var output = _output.GetOutputFormat(config, TextFormatters.PlainOutputTemplate(columns)); try { - await foreach (var evt in connection.Events.StreamAsync( + await foreach (var evt in connection.Events.StreamDocumentsAsync( filter: strict, signal: _signal.Signal, render: true, + clef: true, cancellationToken: cancel.Token)) { - output.WriteEventEntity(evt); + var eventJson = JsonNode.Parse(evt)?.AsObject() ?? throw new InvalidDataException("Non-JSON document received."); + output.WriteEvent(eventJson); } } catch (OperationCanceledException) diff --git a/src/SeqCli/Cli/Commands/TraceCommand.cs b/src/SeqCli/Cli/Commands/TraceCommand.cs index 6e255069..348e40f9 100644 --- a/src/SeqCli/Cli/Commands/TraceCommand.cs +++ b/src/SeqCli/Cli/Commands/TraceCommand.cs @@ -81,86 +81,78 @@ public TraceCommand() protected override async Task Run() { - try + if (_id == null) { - if (_id == null) - { - Log.Error("A trace id must be specified"); - return 1; - } - - var traceId = _id.ToLowerInvariant(); - if (!TraceQuery.IsValidTraceId(traceId)) - { - Log.Error("The trace id {TraceId} is not valid; trace ids are 32 hexadecimal digits", _id); - return 1; - } + Log.Error("A trace id must be specified"); + return 1; + } - var spanId = _spanId?.ToLowerInvariant(); - if (spanId != null && !TraceQuery.IsValidSpanId(spanId)) - { - Log.Error("The span id {SpanId} is not valid; span ids are 16 hexadecimal digits", _spanId); - return 1; - } + var traceId = _id.ToLowerInvariant(); + if (!TraceQuery.IsValidTraceId(traceId)) + { + Log.Error("The trace id {TraceId} is not valid; trace ids are 32 hexadecimal digits", _id); + return 1; + } - var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = SeqConnectionFactory.Connect(_connection, config); + var spanId = _spanId?.ToLowerInvariant(); + if (spanId != null && !TraceQuery.IsValidSpanId(spanId)) + { + Log.Error("The span id {SpanId} is not valid; span ids are 16 hexadecimal digits", _spanId); + return 1; + } - var result = await connection.Data.TryQueryAsync(TraceQuery.Build(traceId, _includeLogs, _includeExceptions, _columns)); - if (!string.IsNullOrWhiteSpace(result.Error)) - { - Log.Error("Could not retrieve trace: {ErrorMessage}", result.Error); - foreach (var reason in result.Reasons) - Log.Error("{Reason}", reason); - return 1; - } + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = SeqConnectionFactory.Connect(_connection, config); - var traceEvents = TraceQuery.ReadEvents(result, _includeExceptions, _columns); - if (traceEvents.Count == 0) - { - Log.Error("No events found for trace {TraceId}", traceId); - return 1; - } + var result = await connection.Data.TryQueryAsync(TraceQuery.Build(traceId, _includeLogs, _includeExceptions, _columns)); + if (!string.IsNullOrWhiteSpace(result.Error)) + { + Log.Error("Could not retrieve trace: {ErrorMessage}", result.Error); + foreach (var reason in result.Reasons) + Log.Error("{Reason}", reason); + return 1; + } - var complete = traceEvents.Count != TraceQuery.MaxEvents; - if (!complete) - Log.Warning("Only the first {Count} events in the trace were retrieved; the tree may be incomplete", - TraceQuery.MaxEvents); + var traceEvents = TraceQuery.ReadEvents(result, _includeExceptions, _columns); + if (traceEvents.Count == 0) + { + Log.Error("No events found for trace {TraceId}", traceId); + return 1; + } - var roots = TraceTreeBuilder.Build(traceEvents); + var complete = traceEvents.Count != TraceQuery.MaxEvents; + if (!complete) + Log.Warning("Only the first {Count} events in the trace were retrieved; the tree may be incomplete", + TraceQuery.MaxEvents); - TraceTreeNode? subtreeRoot = null; - if (spanId != null) - { - subtreeRoot = TraceTreeBuilder.FindSpan(roots, spanId); - if (subtreeRoot == null) - { - Log.Error("The span {SpanId} does not appear in trace {TraceId}", spanId, traceId); - return 1; - } - } + var roots = TraceTreeBuilder.Build(traceEvents); - var output = _output.GetOutputFormat(config, TraceFormatter.OutputTemplate(_columns.Count)); - if (output.Json) - { - var document = subtreeRoot != null ? - TraceTreeJObjectConverter.FromSubtree(traceId, subtreeRoot, complete, _includeLogs, _columns) : - TraceTreeJObjectConverter.FromRoots(traceId, roots, complete, _includeLogs, _columns); - - output.WriteObject(document); - } - else + TraceTreeNode? subtreeRoot = null; + if (spanId != null) + { + subtreeRoot = TraceTreeBuilder.FindSpan(roots, spanId); + if (subtreeRoot == null) { - foreach (var logEvent in TraceFormatter.ToLogEvents(subtreeRoot != null ? [subtreeRoot] : roots)) - output.WriteLogEvent(logEvent); + Log.Error("The span {SpanId} does not appear in trace {TraceId}", spanId, traceId); + return 1; } + } - return 0; + var output = _output.GetOutputFormat(config, TraceFormatter.OutputTemplate(_columns.Count)); + if (output.Json) + { + var document = subtreeRoot != null ? + TraceTreeJObjectConverter.FromSubtree(traceId, subtreeRoot, complete, _includeLogs, _columns) : + TraceTreeJObjectConverter.FromRoots(traceId, roots, complete, _includeLogs, _columns); + + output.WriteObject(document); } - catch (Exception ex) + else { - Log.Error(ex, "Could not retrieve trace: {ErrorMessage}", ex.Message); - return 1; + foreach (var eventJson in TraceFormatter.ToEventJson(subtreeRoot != null ? [subtreeRoot] : roots)) + output.WriteEvent(eventJson); } + + return 0; } } diff --git a/src/SeqCli/Cli/Features/EventColumnsFeature.cs b/src/SeqCli/Cli/Features/EventColumnsFeature.cs new file mode 100644 index 00000000..68ee7116 --- /dev/null +++ b/src/SeqCli/Cli/Features/EventColumnsFeature.cs @@ -0,0 +1,72 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Seq.Api; +using Seq.Api.Model.Signals; +using SeqCli.Signals; +using SeqCli.Syntax; +using SeqCli.Util; + +namespace SeqCli.Cli.Features; + +class EventColumnsFeature : CommandFeature +{ + readonly List _columns = []; + bool _noSignalColumns; + + public override void Enable(OptionSet options) + { + options.Add( + "column=", + "A column to display preceding each event's message; any Seq expression can be supplied, for " + + "example `OrderId`, `@SpanKind`, or `@Resource.service.name`; this argument can be used multiple " + + "times, adding columns in order; applies to plain-text output only", + c => _columns.Add(ArgumentString.Normalize(c) ?? throw new ArgumentException("Columns require a value."))); + + options.Add( + "no-signal-columns", + "Do not show columns associated with the specified signal expression", + _ => _noSignalColumns = true); + } + + public async Task> GetColumns(SeqConnection connection, SignalExpressionPart? signal) + { + var columns = new List(); + if (!_noSignalColumns && signal is { } signalExpression) + { + foreach (var signalId in signalExpression.ReferencedSignalIds()) + { + var signalEntity = await connection.Signals.FindAsync(signalId); + foreach (var column in signalEntity.Columns) + { + columns.Add(column.Expression); + } + } + } + + columns.AddRange(_columns); + + foreach (var column in columns) + { + // A better error than a failed output template parse. + if (!SeqSyntax.TryCompileExpression(column, out _, out var error)) + throw new ArgumentException($"The column expression `{column}` could not be compiled: {error}"); + } + + return columns; + } +} diff --git a/src/SeqCli/Cli/Features/SignalExpressionFeature.cs b/src/SeqCli/Cli/Features/SignalExpressionFeature.cs index 54522c15..f7c45467 100644 --- a/src/SeqCli/Cli/Features/SignalExpressionFeature.cs +++ b/src/SeqCli/Cli/Features/SignalExpressionFeature.cs @@ -13,6 +13,7 @@ // limitations under the License. using Seq.Api.Model.Signals; +using SeqCli.Signals; namespace SeqCli.Cli.Features; @@ -27,9 +28,7 @@ public SignalExpressionPart? Signal if (string.IsNullOrWhiteSpace(_signalExpression)) return null; - // This is a hack that just happens to work because of the way - // signal ids are passed through ToString() as literals - return SignalExpressionPart.Signal(_signalExpression.Trim()); + return SignalExpressionParser.ParseExpression(_signalExpression); } } diff --git a/src/SeqCli/Csv/CsvWriter.cs b/src/SeqCli/Csv/CsvWriter.cs index 75f6553a..f87ae2a3 100644 --- a/src/SeqCli/Csv/CsvWriter.cs +++ b/src/SeqCli/Csv/CsvWriter.cs @@ -1,22 +1,34 @@ using System; -using System.Collections.Generic; using System.IO; using Seq.Api.Model.Data; +using Seq.Syntax.Templates.Themes; using SeqCli.Mcp.Data; -using SeqCli.Output; -using Serilog.Templates.Themes; namespace SeqCli.Csv; static class CsvWriter { + // Delimited output is written directly rather than rendered through a template, so styled + // runs are opened and closed here. + static void SetStyle(TextWriter output, TemplateTheme? theme, TemplateThemeStyle style) + { + if (theme?.Open(style) is { } open) + output.Write(open); + } + + static void ResetStyle(TextWriter output, TemplateTheme? theme, TemplateThemeStyle style) + { + if (theme?.Close(style) is { } close) + output.Write(close); + } + public static void WriteQueryResult(QueryResultPart result, Func stringify, TemplateTheme? theme, TextWriter output) { if (!string.IsNullOrWhiteSpace(result.Error)) { - theme?.Set(output, TemplateThemeStyle.Text); + SetStyle(output, theme, TemplateThemeStyle.Text); QueryResultHelper.WriteErrorResult(output, result); - theme?.Reset(output); + ResetStyle(output, theme, TemplateThemeStyle.Text); } var first = true; @@ -40,39 +52,39 @@ static void WriteCell(TextWriter output, TemplateTheme? theme, object? value, Fu } else { - theme?.Set(output, TemplateThemeStyle.TertiaryText); + SetStyle(output, theme, TemplateThemeStyle.TertiaryText); output.Write(','); - theme?.Reset(output); + ResetStyle(output, theme, TemplateThemeStyle.TertiaryText); } - - theme?.Set(output, TemplateThemeStyle.TertiaryText); + + SetStyle(output, theme, TemplateThemeStyle.TertiaryText); output.Write('"'); - theme?.Reset(output); + ResetStyle(output, theme, TemplateThemeStyle.TertiaryText); var valueAsString = stringify(value); - + var dataStyle = isHeadingRow ? TemplateThemeStyle.Name : TemplateThemeStyle.Text; var doubleQuote = valueAsString.IndexOf('"'); while (doubleQuote != -1) { - theme?.Set(output, dataStyle); + SetStyle(output, theme, dataStyle); output.Write(valueAsString[..doubleQuote]); - theme?.Reset(output); - - theme?.Set(output, TemplateThemeStyle.Scalar); + ResetStyle(output, theme, dataStyle); + + SetStyle(output, theme, TemplateThemeStyle.Scalar); output.Write("\"\""); - theme?.Reset(output); + ResetStyle(output, theme, TemplateThemeStyle.Scalar); valueAsString = valueAsString[(doubleQuote + 1)..]; doubleQuote = valueAsString.IndexOf('"'); } - - theme?.Set(output, dataStyle); + + SetStyle(output, theme, dataStyle); output.Write(valueAsString); - theme?.Reset(output); - - theme?.Set(output, TemplateThemeStyle.TertiaryText); + ResetStyle(output, theme, dataStyle); + + SetStyle(output, theme, TemplateThemeStyle.TertiaryText); output.Write('"'); - theme?.Reset(output); + ResetStyle(output, theme, TemplateThemeStyle.TertiaryText); } } \ No newline at end of file diff --git a/src/SeqCli/Data/EventJsonFormat.cs b/src/SeqCli/Data/EventJsonFormat.cs new file mode 100644 index 00000000..a862e4ff --- /dev/null +++ b/src/SeqCli/Data/EventJsonFormat.cs @@ -0,0 +1,55 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Text.Json.Nodes; + +namespace SeqCli.Data; + +static class EventJsonFormat +{ + public static string EscapeUserPropertyName(string name) + { + return name.StartsWith('@') ? $"@{name}" : name; + } + + /// + /// Use this function when converting a value of uncertain or non-primitive type into a . It's + /// okay to use for strongly-typed primitives. + /// + public static JsonNode? CreateScalar(object? value) + { + return value switch + { + null => null, + string s => JsonValue.Create(s), + bool b => JsonValue.Create(b), + byte n => JsonValue.Create(n), + sbyte n => JsonValue.Create(n), + short n => JsonValue.Create(n), + ushort n => JsonValue.Create(n), + int n => JsonValue.Create(n), + uint n => JsonValue.Create(n), + long n => JsonValue.Create(n), + ulong n => JsonValue.Create(n), + float n => JsonValue.Create(n), + double n => JsonValue.Create(n), + decimal n => JsonValue.Create(n), + TimeSpan ts => JsonValue.Create(ts.ToString("c")), + DateTime dt => JsonValue.Create(dt), + DateTimeOffset dto => JsonValue.Create(dto), + _ => JsonValue.Create(value.ToString()) + }; + } +} diff --git a/src/SeqCli/Util/TextException.cs b/src/SeqCli/Data/IEventEnricher.cs similarity index 60% rename from src/SeqCli/Util/TextException.cs rename to src/SeqCli/Data/IEventEnricher.cs index 2017129d..55c8584b 100644 --- a/src/SeqCli/Util/TextException.cs +++ b/src/SeqCli/Data/IEventEnricher.cs @@ -1,4 +1,4 @@ -// Copyright 2013-2015 Serilog Contributors +// Copyright © Datalust and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,22 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; +using System.Text.Json.Nodes; -namespace SeqCli.Util; +namespace SeqCli.Data; -class TextException : Exception +/// +/// Adds or updates fields on an event JSON document; the equivalent, in Seq's data model, of a +/// Serilog enricher. +/// +interface IEventEnricher { - readonly string _text; - - public TextException(string text) - : base("This exception type provides ToString() access to details only.") - { - _text = text; - } - - public override string ToString() - { - return _text; - } -} \ No newline at end of file + void Enrich(JsonObject eventJson); +} diff --git a/src/SeqCli/Output/RedundantEventTypeRemovalEnricher.cs b/src/SeqCli/Data/LevelEnricher.cs similarity index 64% rename from src/SeqCli/Output/RedundantEventTypeRemovalEnricher.cs rename to src/SeqCli/Data/LevelEnricher.cs index d32e6666..b50df51c 100644 --- a/src/SeqCli/Output/RedundantEventTypeRemovalEnricher.cs +++ b/src/SeqCli/Data/LevelEnricher.cs @@ -1,4 +1,4 @@ -// Copyright © Datalust and contributors. +// Copyright © Datalust and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,15 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -using Serilog.Core; -using Serilog.Events; +using System.Text.Json.Nodes; -namespace SeqCli.Output; +namespace SeqCli.Data; -public class RedundantEventTypeRemovalEnricher : ILogEventEnricher +/// +/// Overrides the event's @l level with a fixed value. +/// +class LevelEnricher(string level) : IEventEnricher { - public void Enrich(LogEvent logEvent, ILogEventPropertyFactory propertyFactory) + public void Enrich(JsonObject eventJson) { - logEvent.RemovePropertyIfPresent("@i"); + eventJson["@l"] = level; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Ingestion/ScalarPropertyEnricher.cs b/src/SeqCli/Data/ScalarPropertyEnricher.cs similarity index 59% rename from src/SeqCli/Ingestion/ScalarPropertyEnricher.cs rename to src/SeqCli/Data/ScalarPropertyEnricher.cs index 7146c7e7..0d5f2a4c 100644 --- a/src/SeqCli/Ingestion/ScalarPropertyEnricher.cs +++ b/src/SeqCli/Data/ScalarPropertyEnricher.cs @@ -1,4 +1,4 @@ -// Copyright © Datalust and contributors. +// Copyright © Datalust and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,23 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. -using SeqCli.Util; -using Serilog.Core; -using Serilog.Events; +using System.Text.Json.Nodes; -namespace SeqCli.Ingestion; +namespace SeqCli.Data; -class ScalarPropertyEnricher : ILogEventEnricher +class ScalarPropertyEnricher : IEventEnricher { - readonly LogEventProperty _property; + readonly string _name; + readonly object? _scalarValue; public ScalarPropertyEnricher(string name, object? scalarValue) { - _property = LogEventPropertyFactory.SafeCreate(name, new ScalarValue(scalarValue)); + _name = EventJsonFormat.EscapeUserPropertyName(name); + _scalarValue = scalarValue; } - public void Enrich(LogEvent logEvent, ILogEventPropertyFactory propertyFactory) + public void Enrich(JsonObject eventJson) { - logEvent.AddOrUpdateProperty(_property); + eventJson[_name] = EventJsonFormat.CreateScalar(_scalarValue); } -} \ No newline at end of file +} diff --git a/src/SeqCli/Forwarder/ForwarderModule.cs b/src/SeqCli/Forwarder/ForwarderModule.cs index 6bb7ef67..9ed614b4 100644 --- a/src/SeqCli/Forwarder/ForwarderModule.cs +++ b/src/SeqCli/Forwarder/ForwarderModule.cs @@ -22,8 +22,6 @@ using SeqCli.Forwarder.Web.Api; using SeqCli.Forwarder.Web.Host; using Serilog; -using Serilog.Formatting; -using Serilog.Templates; namespace SeqCli.Forwarder; @@ -66,25 +64,17 @@ protected override void Load(ContainerBuilder builder) if (_config.Forwarder.Diagnostics.ExposeIngestionLog) { Log.ForContext().Warning("Configured to expose ingestion log via HTTP API"); - builder.RegisterType().As(); - - var ingestionLogTemplate = $"[{{@t:o}} {{@l:u3}}] {{@m}}{Environment.NewLine}"; if (_config.Forwarder.Diagnostics.IngestionLogShowDetail) { Log.ForContext().Warning("Including full client, payload, and error detail in the ingestion log"); - ingestionLogTemplate += - $"{{#if ClientHostIP is not null}}Client IP address: {{ClientHostIP}}{Environment.NewLine}{{#end}}" + - $"{{#if DocumentStart is not null}}First {{StartToLog}} characters of payload: {{DocumentStart:l}}{Environment.NewLine}{{#end}}" + - "{@x}"; } - - builder.Register(_ => new ExpressionTemplate(ingestionLogTemplate)).As(); + + builder.Register(_ => new IngestionLogEndpoints(_config.Forwarder.Diagnostics.IngestionLogShowDetail)).As(); } - builder.Register(c => + builder.Register(_ => { - var config = c.Resolve(); - var baseUri = config.Connection.ServerUrl; + var baseUri = _config.Connection.ServerUrl; if (string.IsNullOrWhiteSpace(baseUri)) throw new ArgumentException("The destination Seq server URL must be configured in `SeqCli.json`."); @@ -95,13 +85,13 @@ protected override void Load(ContainerBuilder builder) // this expression, using an "or" operator. var hasSocketHandlerOption = - config.Connection.PooledConnectionLifetimeMilliseconds.HasValue; + _config.Connection.PooledConnectionLifetimeMilliseconds.HasValue; if (hasSocketHandlerOption) { var httpMessageHandler = new SocketsHttpHandler { - PooledConnectionLifetime = config.Connection.PooledConnectionLifetimeMilliseconds.HasValue ? TimeSpan.FromMilliseconds(config.Connection.PooledConnectionLifetimeMilliseconds.Value) : Timeout.InfiniteTimeSpan, + PooledConnectionLifetime = _config.Connection.PooledConnectionLifetimeMilliseconds.HasValue ? TimeSpan.FromMilliseconds(_config.Connection.PooledConnectionLifetimeMilliseconds.Value) : Timeout.InfiniteTimeSpan, }; return new HttpClient(httpMessageHandler) { BaseAddress = new Uri(baseUri) }; diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs index cf30acfb..eba1c743 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs @@ -12,23 +12,25 @@ // See the License for the specific language governing permissions and // limitations under the License. +using System; +using System.Globalization; using System.IO; using System.Text; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; using SeqCli.Forwarder.Diagnostics; -using Serilog.Formatting; +using Serilog.Events; namespace SeqCli.Forwarder.Web.Api; class IngestionLogEndpoints : IMapEndpoints { - readonly ITextFormatter _formatter; + readonly bool _showDetail; readonly Encoding _utf8 = new UTF8Encoding(false); - public IngestionLogEndpoints(ITextFormatter formatter) + public IngestionLogEndpoints(bool showDetail) { - _formatter = formatter; + _showDetail = showDetail; } public void MapEndpoints(WebApplication app) @@ -45,10 +47,55 @@ public void MapEndpoints(WebApplication app) using var log = new StringWriter(); foreach (var logEvent in events) { - _formatter.Format(logEvent, log); + Format(logEvent, log); } return Results.Content(log.ToString(), "text/plain", _utf8); }); } + + void Format(LogEvent logEvent, TextWriter log) + { + log.Write($"[{logEvent.Timestamp:o} {Abbreviate(logEvent.Level)}] "); + + static string Abbreviate(LogEventLevel logEventLevel) + { + // Here because we don't want Serilog level conversion routines, or any other Serilog model conversion + // routines, to propagate. + return logEventLevel switch + { + LogEventLevel.Verbose => "VRB", + LogEventLevel.Debug => "DBG", + LogEventLevel.Information => "INF", + LogEventLevel.Warning => "WAR", + LogEventLevel.Error => "ERR", + LogEventLevel.Fatal => "FTL", + _ => throw new ArgumentOutOfRangeException(nameof(logEventLevel), logEventLevel, null) + }; + } + + logEvent.RenderMessage(log, CultureInfo.InvariantCulture); + log.WriteLine(); + if (_showDetail) + { + if (logEvent.Properties.TryGetValue("ClientHostIP", out var clientHostIPProperty) && + clientHostIPProperty is ScalarValue { Value: string clientHostIP}) + { + log.WriteLine($"Client IP address: {clientHostIP}"); + } + + if (logEvent.Properties.TryGetValue("DocumentStart", out var documentStartProperty) && + documentStartProperty is ScalarValue { Value: string documentStart} && + logEvent.Properties.TryGetValue("StartToLog", out var startToLogProperty) && + startToLogProperty is ScalarValue { Value: {} startToLog }) + { + log.WriteLine($"First {startToLog} characters of payload: {documentStart}"); + } + + if (logEvent.Exception is { } exception) + { + log.WriteLine(exception); + } + } + } } diff --git a/src/SeqCli/Ingestion/BatchResult.cs b/src/SeqCli/Ingestion/BatchResult.cs index 0c4b52ec..daef0697 100644 --- a/src/SeqCli/Ingestion/BatchResult.cs +++ b/src/SeqCli/Ingestion/BatchResult.cs @@ -1,15 +1,15 @@ -using Serilog.Events; +using System.Text.Json.Nodes; namespace SeqCli.Ingestion; struct BatchResult { - public LogEvent[] LogEvents { get; } + public JsonObject[] Documents { get; } public bool IsLast { get; } - public BatchResult(LogEvent[] logEvents, bool isLast) + public BatchResult(JsonObject[] documents, bool isLast) { - LogEvents = logEvents; + Documents = documents; IsLast = isLast; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Ingestion/EnrichingReader.cs b/src/SeqCli/Ingestion/EnrichingReader.cs index 198ab234..6207bf4c 100644 --- a/src/SeqCli/Ingestion/EnrichingReader.cs +++ b/src/SeqCli/Ingestion/EnrichingReader.cs @@ -1,18 +1,18 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; -using Serilog.Core; +using SeqCli.Data; namespace SeqCli.Ingestion; -class EnrichingReader : ILogEventReader +class EnrichingReader : IEventReader { - readonly ILogEventReader _inner; - readonly IReadOnlyCollection _enrichers; + readonly IEventReader _inner; + readonly IReadOnlyCollection _enrichers; public EnrichingReader( - ILogEventReader inner, - IReadOnlyCollection enrichers) + IEventReader inner, + IReadOnlyCollection enrichers) { _inner = inner ?? throw new ArgumentNullException(nameof(inner)); _enrichers = enrichers ?? throw new ArgumentNullException(nameof(enrichers)); @@ -22,13 +22,12 @@ public async Task TryReadAsync() { var result = await _inner.TryReadAsync(); - if (result.LogEvent != null) + if (result.Document != null) { foreach (var enricher in _enrichers) - // We're breaking the nullability contract of `ILogEventEnricher.Enrich()`, here. - enricher.Enrich(result.LogEvent, null!); + enricher.Enrich(result.Document); } return result; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Ingestion/ILogEventReader.cs b/src/SeqCli/Ingestion/IEventReader.cs similarity index 79% rename from src/SeqCli/Ingestion/ILogEventReader.cs rename to src/SeqCli/Ingestion/IEventReader.cs index a92b09b9..0ca24530 100644 --- a/src/SeqCli/Ingestion/ILogEventReader.cs +++ b/src/SeqCli/Ingestion/IEventReader.cs @@ -2,7 +2,7 @@ namespace SeqCli.Ingestion; -interface ILogEventReader +interface IEventReader { Task TryReadAsync(); } \ No newline at end of file diff --git a/src/SeqCli/Ingestion/JsonEventReader.cs b/src/SeqCli/Ingestion/JsonEventReader.cs new file mode 100644 index 00000000..689710bd --- /dev/null +++ b/src/SeqCli/Ingestion/JsonEventReader.cs @@ -0,0 +1,61 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.IO; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using SeqCli.PlainText.Framing; +using Superpower; +using Superpower.Model; + +namespace SeqCli.Ingestion; + +class JsonEventReader : IEventReader +{ + static readonly TimeSpan TrailingLineArrivalDeadline = TimeSpan.FromMilliseconds(10); + + readonly FrameReader _reader; + + public JsonEventReader(TextReader input) + { + _reader = new FrameReader( + input ?? throw new ArgumentNullException(nameof(input)), + Parse.Return(TextSpan.None), + TrailingLineArrivalDeadline); + } + + public async Task TryReadAsync() + { + var frame = await _reader.TryReadAsync(); + if (!frame.HasValue) + return new ReadResult(null, frame.IsAtEnd); + + if (frame.IsOrphan) + throw new InvalidDataException($"A line arrived late or could not be parsed: `{frame.Value.Trim()}`."); + + return new ReadResult(ReadFromJson(frame.Value), frame.IsAtEnd); + } + + static JsonObject ReadFromJson(string json) + { + if (JsonNode.Parse(json) is not JsonObject eventJson) + throw new InvalidDataException($"The line is not a JSON object: `{json.Trim()}`."); + + if (!eventJson.ContainsKey("@t")) + eventJson["@t"] = DateTime.UtcNow; + + return eventJson; + } +} diff --git a/src/SeqCli/Ingestion/JsonLogEventReader.cs b/src/SeqCli/Ingestion/JsonLogEventReader.cs deleted file mode 100644 index 719da10c..00000000 --- a/src/SeqCli/Ingestion/JsonLogEventReader.cs +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright © Datalust and contributors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Globalization; -using System.IO; -using System.Threading.Tasks; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using SeqCli.Mapping; -using SeqCli.PlainText.Framing; -using Serilog.Events; -using Serilog.Formatting.Compact.Reader; -using Superpower; -using Superpower.Model; - -namespace SeqCli.Ingestion; - -class JsonLogEventReader : ILogEventReader -{ - static readonly TimeSpan TrailingLineArrivalDeadline = TimeSpan.FromMilliseconds(10); - static readonly JsonSerializer _serializer = JsonSerializer.Create(new JsonSerializerSettings - { - DateParseHandling = DateParseHandling.None, - Culture = CultureInfo.InvariantCulture - }); - - readonly FrameReader _reader; - - public JsonLogEventReader(TextReader input) - { - _reader = new FrameReader( - input ?? throw new ArgumentNullException(nameof(input)), - Parse.Return(TextSpan.None), - TrailingLineArrivalDeadline); - } - - public async Task TryReadAsync() - { - var frame = await _reader.TryReadAsync(); - if (!frame.HasValue) - return new ReadResult(null, frame.IsAtEnd); - - if (frame.IsOrphan) - throw new InvalidDataException($"A line arrived late or could not be parsed: `{frame.Value.Trim()}`."); - - var frameValue = new JsonTextReader(new StringReader(frame.Value)); - if (!(_serializer.Deserialize(frameValue) is JObject jobject)) - throw new InvalidDataException($"The line is not a JSON object: `{frame.Value.Trim()}`."); - - var evt = ReadFromJObject(jobject); - return new ReadResult(evt, frame.IsAtEnd); - } - - public static LogEvent ReadFromJson(string json) - { - var frameValue = new JsonTextReader(new StringReader(json)); - if (_serializer.Deserialize(frameValue) is not JObject jObject) - throw new InvalidDataException($"The line is not a JSON object: `{json.Trim()}`."); - - return ReadFromJObject(jObject); - } - - static LogEvent ReadFromJObject(JObject jObject) - { - if (!jObject.TryGetValue("@t", out _)) - jObject.Add("@t", new JValue(DateTime.UtcNow.ToString("O"))); - - if (jObject.TryGetValue("@l", out var levelToken)) - { - var originalLevel = levelToken.Value()!; - jObject.Remove("@l"); - - var serilogLevel = LevelMapping.ToSerilogLevel(originalLevel); - if (serilogLevel != LogEventLevel.Information) - jObject.Add("@l", new JValue(serilogLevel.ToString())); - - jObject.Add(LevelMapping.SurrogateLevelProperty, originalLevel); - } - - return LogEventReader.ReadFromJObject(jObject); - } -} \ No newline at end of file diff --git a/src/SeqCli/Ingestion/LogShipper.cs b/src/SeqCli/Ingestion/LogShipper.cs index f0a19741..313d6ceb 100644 --- a/src/SeqCli/Ingestion/LogShipper.cs +++ b/src/SeqCli/Ingestion/LogShipper.cs @@ -19,22 +19,17 @@ using System.Net.Http; using System.Net.Http.Headers; using System.Text; +using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; -using Newtonsoft.Json; using Seq.Api; using SeqCli.Api; -using SeqCli.Output; using Serilog; -using Serilog.Events; -using Serilog.Formatting; namespace SeqCli.Ingestion; static class LogShipper { - static readonly ITextFormatter JsonFormatter = TextFormatters.Json(null); - public static async Task ShipBufferAsync( SeqConnection connection, string? apiKey, @@ -49,7 +44,7 @@ public static async Task ShipBufferAsync( ContentType = new MediaTypeHeaderValue(ApiConstants.ClefMediaType, "utf-8") } }; - + var retries = 0; while (true) { @@ -87,22 +82,22 @@ public static async Task ShipBufferAsync( { sendFailureLog.Error(ex, "Failed to ship a batch"); } - + var millisecondsDelay = (int)Math.Min(Math.Pow(2, retries) * 2000, 60000); sendFailureLog.Information("Backing off connection schedule; will retry in {MillisecondsDelay}", millisecondsDelay); await Task.Delay(millisecondsDelay, cancellationToken); retries += 1; } } - + public static async Task ShipEventsAsync( SeqConnection connection, string? apiKey, - ILogEventReader reader, + IEventReader reader, InvalidDataHandling invalidDataHandling, SendFailureHandling sendFailureHandling, int batchSize, - Func? filter, + Func? filter, CancellationToken cancellationToken) { const int maxEmptyBatchWaitMS = 2000; @@ -116,10 +111,10 @@ public static async Task ShipEventsAsync( var statusCode = await SendBatchAsync( connection, apiKey, - batch.LogEvents, + batch.Documents, sendFailureHandling != SendFailureHandling.Ignore ? Log.Logger : null, cancellationToken); - + sendSucceeded = (int)statusCode is >= 200 and < 300; } catch (Exception ex) @@ -136,7 +131,7 @@ public static async Task ShipEventsAsync( if (sendFailureHandling == SendFailureHandling.Retry) { var millisecondsDelay = (int)Math.Min(Math.Pow(2, retries) * 2000, 60000); - await Task.Delay(millisecondsDelay); + await Task.Delay(millisecondsDelay, cancellationToken); retries += 1; continue; } @@ -146,7 +141,7 @@ public static async Task ShipEventsAsync( if (batch.IsLast) break; - + batch = await ReadBatchAsync(reader, filter, batchSize, invalidDataHandling, maxEmptyBatchWaitMS); } @@ -154,15 +149,15 @@ public static async Task ShipEventsAsync( } static async Task ReadBatchAsync( - ILogEventReader reader, - Func? filter, + IEventReader reader, + Func? filter, int count, InvalidDataHandling invalidDataHandling, int maxWaitMS) { - var batch = new List(); + var batch = new List(); var isLast = false; - + // Avoid consuming stacks of CPU unnecessarily when there's no work to do. We do eventually yield // an empty batch, because level switching relies on this. var totalWaitMS = 0; @@ -175,7 +170,7 @@ static async Task ReadBatchAsync( { var rr = await reader.TryReadAsync(); isLast = rr.IsAtEnd; - var evt = rr.LogEvent; + var evt = rr.Document; if (evt == null) { if (isLast || batch.Count != 0 || totalWaitMS > maxWaitMS) @@ -195,7 +190,7 @@ static async Task ReadBatchAsync( } catch (Exception ex) { - if (ex is JsonReaderException || ex is InvalidDataException) + if (ex is System.Text.Json.JsonException or InvalidDataException) { if (invalidDataHandling == InvalidDataHandling.Ignore) continue; @@ -204,14 +199,14 @@ static async Task ReadBatchAsync( throw; } - return new BatchResult(batch.ToArray(), isLast); + return new BatchResult([.. batch], isLast); } while (true); } static async Task SendBatchAsync( SeqConnection connection, string? apiKey, - IReadOnlyCollection batch, + IReadOnlyCollection batch, ILogger? sendFailureLog, CancellationToken cancellationToken) { @@ -223,7 +218,8 @@ static async Task SendBatchAsync( using (var builder = new StringWriter()) { foreach (var evt in batch) - JsonFormatter.Format(evt, builder); + // ReSharper disable once MethodHasAsyncOverload + builder.WriteLine(evt.ToJsonString()); content = new StringContent(builder.ToString(), Encoding.UTF8, ApiConstants.ClefMediaType); } @@ -247,7 +243,7 @@ static async Task SendAsync(SeqConnection connection, string? ap { try { - var error = JsonConvert.DeserializeObject(resultJson)!; + var error = Newtonsoft.Json.JsonConvert.DeserializeObject(resultJson)!; sendFailureLog.Error("Shipping failed with status code {StatusCode}: {ErrorMessage}", result.StatusCode, @@ -264,4 +260,4 @@ static async Task SendAsync(SeqConnection connection, string? ap sendFailureLog.Error("Shipping failed with status code {StatusCode} ({ReasonPhrase})", result.StatusCode, result.ReasonPhrase); return result.StatusCode; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Ingestion/ReadResult.cs b/src/SeqCli/Ingestion/ReadResult.cs index 87e10076..0e074de5 100644 --- a/src/SeqCli/Ingestion/ReadResult.cs +++ b/src/SeqCli/Ingestion/ReadResult.cs @@ -1,15 +1,30 @@ -using Serilog.Events; +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Text.Json.Nodes; namespace SeqCli.Ingestion; readonly struct ReadResult { - public LogEvent? LogEvent { get; } + public JsonObject? Document { get; } + public bool IsAtEnd { get; } - public ReadResult(LogEvent? logEvent, bool isAtEnd) + public ReadResult(JsonObject? document, bool isAtEnd) { - LogEvent = logEvent; + Document = document; IsAtEnd = isAtEnd; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Ingestion/StaticMessageTemplateReader.cs b/src/SeqCli/Ingestion/StaticMessageTemplateReader.cs index 973bff60..d5d62591 100644 --- a/src/SeqCli/Ingestion/StaticMessageTemplateReader.cs +++ b/src/SeqCli/Ingestion/StaticMessageTemplateReader.cs @@ -1,37 +1,29 @@ using System; -using System.Linq; using System.Threading.Tasks; -using SeqCli.Util; -using Serilog.Events; -using Serilog.Parsing; namespace SeqCli.Ingestion; -class StaticMessageTemplateReader : ILogEventReader +class StaticMessageTemplateReader : IEventReader { - readonly ILogEventReader _inner; - readonly MessageTemplate _messageTemplate; + readonly IEventReader _inner; + readonly string _messageTemplate; - public StaticMessageTemplateReader(ILogEventReader inner, string messageTemplate) + public StaticMessageTemplateReader(IEventReader inner, string messageTemplate) { _inner = inner ?? throw new ArgumentNullException(nameof(inner)); - _messageTemplate = new MessageTemplateParser().Parse(messageTemplate); + _messageTemplate = messageTemplate ?? throw new ArgumentNullException(nameof(messageTemplate)); } public async Task TryReadAsync() { var result = await _inner.TryReadAsync(); - if (result.LogEvent == null) - return result; + if (result.Document != null) + { + result.Document.Remove("@m"); + result.Document["@mt"] = _messageTemplate; + } - var evt = new LogEvent( - result.LogEvent.Timestamp, - result.LogEvent.Level, - result.LogEvent.Exception, - _messageTemplate, - result.LogEvent.Properties.Select(kv => LogEventPropertyFactory.SafeCreate(kv.Key, kv.Value))); - - return new ReadResult(evt, result.IsAtEnd); + return result; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Ingestion/TraceConstants.cs b/src/SeqCli/Ingestion/TraceConstants.cs deleted file mode 100644 index 55fe7f34..00000000 --- a/src/SeqCli/Ingestion/TraceConstants.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace SeqCli.Ingestion; - -static class TraceConstants -{ - internal const string ParentSpanIdProperty = "ParentSpanId"; - - internal const string SpanStartTimestampProperty = "SpanStartTimestamp"; -} diff --git a/src/SeqCli/Mapping/LevelMapping.cs b/src/SeqCli/Mapping/LevelMapping.cs deleted file mode 100644 index ff79087b..00000000 --- a/src/SeqCli/Mapping/LevelMapping.cs +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright © Datalust and contributors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Collections.Generic; -using Serilog.Events; - -namespace SeqCli.Mapping; - -public static class LevelMapping -{ - // Use a "hygienic" name for the original level value to avoid collisions - internal static readonly string SurrogateLevelProperty = $"_SeqcliOriginalLevel_{Guid.NewGuid():N}"; - - static readonly Dictionary LevelsByName = - new(StringComparer.OrdinalIgnoreCase) - { - ["t"] = ("Trace", LogEventLevel.Verbose), - ["tr"] = ("Trace", LogEventLevel.Verbose), - ["trc"] = ("Trace", LogEventLevel.Verbose), - ["trce"] = ("Trace", LogEventLevel.Verbose), - ["trace"] = ("Trace", LogEventLevel.Verbose), - ["v"] = ("Verbose", LogEventLevel.Verbose), - ["ver"] = ("Verbose", LogEventLevel.Verbose), - ["vrb"] = ("Verbose", LogEventLevel.Verbose), - ["verb"] = ("Verbose", LogEventLevel.Verbose), - ["verbose"] = ("Verbose", LogEventLevel.Verbose), - ["d"] = ("Debug", LogEventLevel.Debug), - ["de"] = ("Debug", LogEventLevel.Debug), - ["dbg"] = ("Debug", LogEventLevel.Debug), - ["deb"] = ("Debug", LogEventLevel.Debug), - ["dbug"] = ("Debug", LogEventLevel.Debug), - ["debu"] = ("Debug", LogEventLevel.Debug), - ["debug"] = ("Debug", LogEventLevel.Debug), - ["i"] = ("Information", LogEventLevel.Information), - ["in"] = ("Information", LogEventLevel.Information), - ["inf"] = ("Information", LogEventLevel.Information), - ["info"] = ("Information", LogEventLevel.Information), - ["information"] = ("Information", LogEventLevel.Information), - ["notice"] = ("Notice", LogEventLevel.Information), - ["w"] = ("Warning", LogEventLevel.Warning), - ["wa"] = ("Warning", LogEventLevel.Warning), - ["war"] = ("Warning", LogEventLevel.Warning), - ["wrn"] = ("Warning", LogEventLevel.Warning), - ["warn"] = ("Warning", LogEventLevel.Warning), - ["warning"] = ("Warning", LogEventLevel.Warning), - ["e"] = ("Error", LogEventLevel.Error), - ["er"] = ("Error", LogEventLevel.Error), - ["err"] = ("Error", LogEventLevel.Error), - ["erro"] = ("Error", LogEventLevel.Error), - ["eror"] = ("Error", LogEventLevel.Error), - ["error"] = ("Error", LogEventLevel.Error), - ["f"] = ("Fatal", LogEventLevel.Fatal), - ["fa"] = ("Fatal", LogEventLevel.Fatal), - ["ftl"] = ("Fatal", LogEventLevel.Fatal), - ["fat"] = ("Fatal", LogEventLevel.Fatal), - ["fatl"] = ("Fatal", LogEventLevel.Fatal), - ["fatal"] = ("Fatal", LogEventLevel.Fatal), - ["c"] = ("Critical", LogEventLevel.Fatal), - ["cr"] = ("Critical", LogEventLevel.Fatal), - ["crt"] = ("Critical", LogEventLevel.Fatal), - ["cri"] = ("Critical", LogEventLevel.Fatal), - ["crit"] = ("Critical", LogEventLevel.Fatal), - ["critical"] = ("Critical", LogEventLevel.Fatal), - ["emerg"] = ("Emergency", LogEventLevel.Fatal), - ["alert"] = ("Alert", LogEventLevel.Fatal), - ["panic"] = ("Panic", LogEventLevel.Fatal) - }; - - public static LogEventLevel ToSerilogLevel(string level) - { - if (string.IsNullOrEmpty(level)) - return LogEventLevel.Information; - - return LevelsByName.TryGetValue(level, out var m) ? m.Item2 : LogEventLevel.Information; - } - - public static string ToFullLevelName(string level) - { - return LevelsByName.TryGetValue(level, out var m) ? m.Item1 : level; - } -} \ No newline at end of file diff --git a/src/SeqCli/Mapping/MetricsMapping.cs b/src/SeqCli/Mapping/MetricsMapping.cs deleted file mode 100644 index fe104666..00000000 --- a/src/SeqCli/Mapping/MetricsMapping.cs +++ /dev/null @@ -1,8 +0,0 @@ -using System; - -namespace SeqCli.Mapping; - -public static class MetricsMapping -{ - internal static readonly string SurrogateDefinitionsProperty = $"_SeqcliMetricDefinitions_{Guid.NewGuid():N}"; -} diff --git a/src/SeqCli/Mcp/McpServerInstaller.cs b/src/SeqCli/Mcp/McpServerInstaller.cs index 1160aa33..8ada8823 100644 --- a/src/SeqCli/Mcp/McpServerInstaller.cs +++ b/src/SeqCli/Mcp/McpServerInstaller.cs @@ -15,6 +15,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using Newtonsoft.Json.Linq; using Serilog; @@ -121,9 +122,11 @@ public static void Install(string? agent, bool global, string? profileName = nul root[target.ServerMapKey] = serverMap; } + var (command, leadingArgs) = ResolveCommand(); + // A connection profile is the only connection setting we propagate; the server URL and // API key are resolved from config at runtime so they're not baked into the agent's file. - var args = new JArray("mcp", "run"); + var args = new JArray(leadingArgs.Concat(["mcp", "run"]).ToArray()); if (profileName != null) { args.Add("--profile"); @@ -132,7 +135,7 @@ public static void Install(string? agent, bool global, string? profileName = nul serverMap[ServerName] = new JObject { - ["command"] = "seqcli", + ["command"] = command, ["args"] = args, }; @@ -146,6 +149,60 @@ public static void Install(string? agent, bool global, string? profileName = nul Log.Information("Installed Seq MCP server for {Agent} to {Path}", agent, path); } + // Agents resolve `seqcli` from PATH when they start the server. On Windows, an npm-installed + // `seqcli` is a `seqcli.cmd` shim, which hosts that spawn processes without a shell can't run + // directly, so in that case the server is launched through `cmd /c` instead. + static (string Command, string[] LeadingArgs) ResolveCommand() => + ResolveCommand( + OperatingSystem.IsWindows(), + Environment.GetEnvironmentVariable("PATH"), + Environment.GetEnvironmentVariable("PATHEXT"), + File.Exists); + + internal static (string Command, string[] LeadingArgs) ResolveCommand( + bool isWindows, + string? path, + string? pathExt, + Func fileExists) + { + if (!isWindows) + return ("seqcli", []); + + var found = FindOnWindowsPath("seqcli", path, pathExt, fileExists); + if (found == null) + return ("seqcli", []); + + var extension = Path.GetExtension(found); + if (extension.Equals(".cmd", StringComparison.OrdinalIgnoreCase) || + extension.Equals(".bat", StringComparison.OrdinalIgnoreCase)) + { + Log.Information("Found `seqcli` on PATH as {ShimPath}; the MCP server will be launched via `cmd /c`", found); + return ("cmd", ["/c", "seqcli"]); + } + + return ("seqcli", []); + } + + // Mirrors how Windows locates a command: each PATH directory in turn, trying the PATHEXT + // extensions in order within it. + static string? FindOnWindowsPath(string name, string? path, string? pathExt, Func fileExists) + { + var extensions = (pathExt is { Length: > 0 } ? pathExt : ".COM;.EXE;.BAT;.CMD") + .Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + + foreach (var directory in (path ?? "").Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + foreach (var extension in extensions) + { + var candidate = Path.Combine(directory, name + extension); + if (fileExists(candidate)) + return candidate; + } + } + + return null; + } + static AgentTarget Unsupported(string message) => new(_ => throw new NotSupportedException(message), "mcpServers"); diff --git a/src/SeqCli/Mcp/Tools/Search/SearchTools.cs b/src/SeqCli/Mcp/Tools/Search/SearchTools.cs index 7dc9671d..7d62d9ef 100644 --- a/src/SeqCli/Mcp/Tools/Search/SearchTools.cs +++ b/src/SeqCli/Mcp/Tools/Search/SearchTools.cs @@ -27,11 +27,10 @@ using Seq.Api.Model.Events; using Seq.Api.Model.Signals; using Seq.Syntax.Templates; -using SeqCli.Mapping; -using SeqCli.Output; +using SeqCli.Api; using SeqCli.Signals; +using SeqCli.Syntax; using Serilog; -using Serilog.Events; using NativeFormatter = SeqCli.Output.NativeFormatter; // ReSharper disable UnusedMember.Global @@ -42,8 +41,9 @@ namespace SeqCli.Mcp.Tools.Search; class SearchTools(McpSession session, SeqConnection connection) { const string ResultIdPropertyName = "__seqcli_ResultId"; - static readonly ExpressionTemplate SearchResultFormatter = new ( - $"{{{ResultIdPropertyName}}} [{{UtcDateTime(@t)}} {{{LevelMapping.SurrogateLevelProperty}}}] {{@m}}{Environment.NewLine}{{#if @x is not null}}{{Substring(ToString(@x), 0, 512)}}...{Environment.NewLine}{{#end}}" + static readonly ExpressionTemplate SearchResultFormatter = SeqSyntax.ParseTemplate( + $"{{{ResultIdPropertyName}}} [{{UtcDateTime(@Timestamp)}} {{@Level}}] {{@Message}}{Environment.NewLine}" + + $"{{#if @Exception is not null}}{{Substring(ToString(@Exception), 0, 512)}}...{Environment.NewLine}{{#end}}" ); [McpServerTool(Name = "seq_new_session", ReadOnly = true, Title = "Begin a new Search/Query Session")] @@ -181,12 +181,10 @@ public async Task SearchEventsAsync( foreach (var result in takenResults) { var resultId = session.ImportSearchResult(result); - - var serilogEvent = OutputFormat.ToSerilogEvent(result); - OutputFormat.FlattenPropertiesUsedWithDottedNames(result, serilogEvent); - serilogEvent.AddOrUpdateProperty(new LogEventProperty(ResultIdPropertyName, new ScalarValue(resultId))); - serilogEvent.AddOrUpdateProperty(new LogEventProperty(LevelMapping.SurrogateLevelProperty, new ScalarValue(result.Level ?? "Information"))); - SearchResultFormatter.Format(serilogEvent, responseText); + + var eventJson = EventEntityJson.ToEventJson(result); + eventJson[ResultIdPropertyName] = resultId; + SearchResultFormatter.Format(eventJson, responseText); } return new CallToolResult diff --git a/src/SeqCli/Output/FlareTheme.cs b/src/SeqCli/Output/FlareTheme.cs index 38d1e578..a1026fee 100644 --- a/src/SeqCli/Output/FlareTheme.cs +++ b/src/SeqCli/Output/FlareTheme.cs @@ -13,13 +13,12 @@ // limitations under the License. using System.Collections.Generic; -using System.IO; -using Serilog.Templates.Themes; +using Seq.Syntax.Templates.Themes; namespace SeqCli.Output; /// -/// Flare is Seq's embedded stream/columnar database. This theme is derived from one build originally +/// Flare is Seq's embedded stream/columnar database. This theme is derived from one built originally /// for the flaretl command-line tooling used there. /// static class FlareTheme @@ -45,26 +44,5 @@ static class FlareTheme [TemplateThemeStyle.LevelFatal] = "\e[38;5;0197m\e[48;5;0238m" }; - public static readonly TemplateTheme SeqCli = new(FlareThemeStyles); - - // `CsvWriter` implements its own theming behavior because the required APIs are not public in Serilog.Expressions. - // The best way forward for this is likely to be porting theming to Seq.Syntax, and exposing the required APIs there. - - const string AnsiStyleResetSequence = "\e[0m"; - - // The passed-in theme is ignored because SerilogExpressions themes are opaque. All formatting uses the SeqCli theme. - // ReSharper disable once UnusedParameter.Global - extension(TemplateTheme theme) - { - public void Set(TextWriter output, TemplateThemeStyle style) - { - if (FlareThemeStyles.TryGetValue(style, out var styleSequence)) - output.Write(styleSequence); - } - - public void Reset(TextWriter output) - { - output.Write(AnsiStyleResetSequence); - } - } -} \ No newline at end of file + public static readonly TemplateTheme SeqCli = new AnsiTheme(FlareThemeStyles); +} diff --git a/src/SeqCli/Output/OutputFormat.cs b/src/SeqCli/Output/OutputFormat.cs index 2cd46d09..ae2492fb 100644 --- a/src/SeqCli/Output/OutputFormat.cs +++ b/src/SeqCli/Output/OutputFormat.cs @@ -15,24 +15,20 @@ using System; using System.Collections; using System.Collections.Generic; -using System.Diagnostics; using System.Globalization; -using System.Linq; +using System.Text.Json.Nodes; using Newtonsoft.Json; using Newtonsoft.Json.Converters; using Newtonsoft.Json.Linq; using Seq.Api.Model; using Seq.Api.Model.Data; using Seq.Api.Model.Events; +using Seq.Syntax.Templates; +using Seq.Syntax.Templates.Encoding; +using Seq.Syntax.Templates.Themes; +using SeqCli.Api; using SeqCli.Config; using SeqCli.Csv; -using SeqCli.Mapping; -using SeqCli.Util; -using Serilog; -using Serilog.Core; -using Serilog.Events; -using Serilog.Parsing; -using Serilog.Templates.Themes; namespace SeqCli.Output; @@ -40,10 +36,10 @@ sealed class OutputFormat { // See https://no-color.org for semantics. const string NoColorEnvironmentVariable = "NO_COLOR"; - + readonly OutputSyntax _syntax; - readonly string? _plainTextTemplate; - readonly Logger _formatter; + readonly ExpressionTemplate? _eventFormatter; + readonly ExpressionTemplate _jsonValueFormatter; readonly JsonSerializer _serializer = JsonSerializer.CreateDefault(new JsonSerializerSettings { @@ -92,7 +88,6 @@ internal OutputFormat( bool allowAnsiEscapes) { _syntax = syntax; - _plainTextTemplate = plainTextTemplate; var resolvedNoColor = ResolveNoColor(noColor, forceColor, outputConfig, noColorSetInEnvironment, allowAnsiEscapes); var applyThemeToRedirectedOutput = !resolvedNoColor && (forceColor ?? outputConfig.ForceColor); @@ -102,12 +97,20 @@ internal OutputFormat( ? FlareTheme.SeqCli : null; - _formatter = CreateOutputLogger(); + _eventFormatter = Json + ? TextFormatters.Json(TemplateTheme) + : Text + ? TextFormatters.Plain(TemplateTheme, plainTextTemplate) + : null; + + _jsonValueFormatter = new ExpressionTemplate( + "{Value}" + Environment.NewLine, + encoder: TemplateTheme != null ? TemplateOutputEncoder.Ansi(TemplateTheme) : null); } static bool NoColorSetInEnvironment() => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable(NoColorEnvironmentVariable)); - + internal static bool ResolveNoColor( bool? noColorFlag, bool? forceColorFlag, @@ -135,27 +138,6 @@ internal static bool ResolveNoColor( public bool RequiresRender => Native; - Logger CreateOutputLogger() - { - var outputConfiguration = new LoggerConfiguration() - .MinimumLevel.Is(LevelAlias.Minimum) - .Enrich.With(); - - if (Json) - { - outputConfiguration.WriteTo.Console(TextFormatters.Json(TemplateTheme)); - } - else if (Text) - { - outputConfiguration.WriteTo.Console(TextFormatters.Plain(TemplateTheme, _plainTextTemplate)); - } - - // The logger is not configured for Native output, which avoids it. Ideally we'll shift away from using - // Serilog here, and move Text/Json over to EventEntity-driven formatters, too. - - return outputConfiguration.CreateLogger(); - } - public void WriteEntity(Entity entity) { if (entity == null) throw new ArgumentNullException(nameof(entity)); @@ -163,17 +145,11 @@ public void WriteEntity(Entity entity) var jo = JObject.FromObject( entity, _serializer); - + if (Json) { jo.Remove("Links"); - - var writer = new LoggerConfiguration() - .Destructure.With() - .Enrich.With() - .WriteTo.Console(TextFormatters.Plain(TemplateTheme, "{@m}" + Environment.NewLine)) - .CreateLogger(); - writer.Information("{@Entity}", jo); + WriteJsonValue(ToSystemTextJson.FromNewtonsoft(jo)); } else if (Text) { @@ -189,21 +165,14 @@ public void WriteEntity(Entity entity) public void WriteObject(object value) { if (value == null) throw new ArgumentNullException(nameof(value)); - + if (Json) { var jo = value is ICollection and not (IDictionary or JToken) ? (JToken)JArray.FromObject(value, _serializer) : JObject.FromObject(value, _serializer); - // Using the same method of JSON colorization as above - - var writer = new LoggerConfiguration() - .Destructure.With() - .Enrich.With() - .WriteTo.Console(TextFormatters.Plain(TemplateTheme, "{@m}" + Environment.NewLine)) - .CreateLogger(); - writer.Information("{@Entity}", jo); + WriteJsonValue(ToSystemTextJson.FromNewtonsoft(jo)); } else if (Text) { @@ -216,6 +185,11 @@ public void WriteObject(object value) } } + void WriteJsonValue(JsonNode? value) + { + _jsonValueFormatter.Format(new JsonObject { ["Value"] = value }, Console.Out); + } + public void ListEntities(IEnumerable list) { foreach (var entity in list) @@ -223,7 +197,7 @@ public void ListEntities(IEnumerable list) WriteEntity(entity); } } - + // ReSharper disable once MemberCanBeMadeStatic.Global #pragma warning disable CA1822 public void WriteText(string? text) @@ -257,125 +231,15 @@ public void WriteEventEntity(EventEntity evt) } else { - var serilogEvent = ToSerilogEvent(evt); - - if (Text) - { - // Add flattened versions of structured properties that are referenced using dotted-name syntax in - // message templates, e.g. {user.name}. Serilog.Expressions template rendering doesn't otherwise - // support these. In text output mode, these aren't usually observable, though - // seqcli print --template="{@p}" will make them visible. - FlattenPropertiesUsedWithDottedNames(evt, serilogEvent); - } - - WriteLogEvent(serilogEvent); - } - } - - public void WriteLogEvent(LogEvent logEvent) - { - _formatter.Write(logEvent); - } - - public static LogEvent ToSerilogEvent(EventEntity evt) - { - ActivityTraceId traceId = default; - if (!string.IsNullOrWhiteSpace(evt.TraceId)) - traceId = ActivityTraceId.CreateFromString(evt.TraceId); - - ActivitySpanId spanId = default; - if (!string.IsNullOrWhiteSpace(evt.SpanId)) - spanId = ActivitySpanId.CreateFromString(evt.SpanId); - - var serilogEvent = new LogEvent( - DateTimeOffset.ParseExact(evt.Timestamp, "o", CultureInfo.InvariantCulture).ToLocalTime(), - LevelMapping.ToSerilogLevel(evt.Level), - string.IsNullOrWhiteSpace(evt.Exception) ? null : new TextException(evt.Exception), - new MessageTemplate(evt.MessageTemplateTokens.Select(ToMessageTemplateToken)), - evt.Properties - .Select(p => CreateProperty(p.Name, p.Value)), - traceId, - spanId - ); - - if (evt.Scope?.Count > 0) - serilogEvent.AddOrUpdateProperty(new("@sa", new StructureValue(evt.Scope.Select(p => CreateProperty(p.Name, p.Value))))); - - if (evt.Resource?.Count > 0) - serilogEvent.AddOrUpdateProperty(new("@ra", new StructureValue(evt.Resource.Select(p => CreateProperty(p.Name, p.Value))))); - - if (!string.IsNullOrWhiteSpace(evt.ParentId)) - serilogEvent.AddOrUpdateProperty(new("@ps", new ScalarValue(evt.ParentId))); - - if (!string.IsNullOrWhiteSpace(evt.Start)) - serilogEvent.AddOrUpdateProperty(new("@st", new ScalarValue(evt.Start))); - - if (!string.IsNullOrWhiteSpace(evt.SpanKind)) - serilogEvent.AddOrUpdateProperty(new("@sk", new ScalarValue(evt.SpanKind))); - - return serilogEvent; - } - - public static void FlattenPropertiesUsedWithDottedNames(EventEntity evt, LogEvent serilogEvent) - { - foreach (var token in evt.MessageTemplateTokens) - { - if (token.Text != null || token.PropertyName is not { } name || !name.Contains('.') || - serilogEvent.Properties.ContainsKey(name)) - { - continue; - } - - var steps = name.Split('.'); - var value = evt.Properties.FirstOrDefault(p => p.Name == steps[0])?.Value; - for (var i = 1; i < steps.Length; ++i) - { - value = (value as JObject)?.GetValue(steps[i]); - } - - if (value is JToken resolved) - { - // Existing flat-named properties, where present, win. - serilogEvent.AddPropertyIfAbsent(LogEventPropertyFactory.SafeCreate( - name, resolved is JValue scalar ? new ScalarValue(scalar.Value) : CreatePropertyValue(resolved))); - } + WriteEvent(EventEntityJson.ToEventJson(evt)); } } - static MessageTemplateToken ToMessageTemplateToken(MessageTemplateTokenPart token) - { - // Not ideal, we lose renderings, alignment etc. here. - - if (token.Text != null) - return new TextToken(token.Text); - return new PropertyToken(token.PropertyName, token.RawText ?? $"{{{token.PropertyName}}}"); - } - - static LogEventProperty CreateProperty(string name, object value) + public void WriteEvent(JsonObject eventJson) { - return LogEventPropertyFactory.SafeCreate(name, CreatePropertyValue(value)); + _eventFormatter?.Format(eventJson, Console.Out); } - internal static LogEventPropertyValue CreatePropertyValue(object value) - { - switch (value) - { - case JObject jo: - jo.TryGetValue("$typeTag", out var tt); - return new StructureValue( - jo.Properties() - .Where(kvp => kvp.Name != "$typeTag") - .Select(kvp => CreateProperty(kvp.Name, kvp.Value)), - (tt as JValue)?.Value as string); - - case JArray ja: - return new SequenceValue(ja.Select(CreatePropertyValue)); - - default: - return new ScalarValue(value); - } - } - static string Stringify(object? value) { return value switch diff --git a/src/SeqCli/Output/StripStructureTypeEnricher.cs b/src/SeqCli/Output/StripStructureTypeEnricher.cs deleted file mode 100644 index 352cd1bf..00000000 --- a/src/SeqCli/Output/StripStructureTypeEnricher.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System.Linq; -using SeqCli.Util; -using Serilog.Core; -using Serilog.Data; -using Serilog.Events; - -namespace SeqCli.Output; - -public class StripStructureTypeEnricher : LogEventPropertyValueRewriter, ILogEventEnricher -{ - public void Enrich(LogEvent logEvent, ILogEventPropertyFactory propertyFactory) - { - foreach (var property in logEvent.Properties) - { - var updated = LogEventPropertyFactory.SafeCreate(property.Key, Visit(null, property.Value)); - logEvent.AddOrUpdateProperty(updated); - } - } - - protected override LogEventPropertyValue VisitStructureValue(object? state, StructureValue structure) - { - return new StructureValue(structure.Properties.Select(p => - LogEventPropertyFactory.SafeCreate(p.Name, Visit(null, p.Value)))); - } -} \ No newline at end of file diff --git a/src/SeqCli/Output/TextFormatters.cs b/src/SeqCli/Output/TextFormatters.cs index 86cfbee7..92d07c2f 100644 --- a/src/SeqCli/Output/TextFormatters.cs +++ b/src/SeqCli/Output/TextFormatters.cs @@ -13,42 +13,48 @@ // limitations under the License. using System; -using SeqCli.Ingestion; -using SeqCli.Mapping; -using Serilog.Expressions; -using Serilog.Formatting; -using Serilog.Templates; -using Serilog.Templates.Themes; +using System.Collections.Generic; +using System.Text; +using Seq.Syntax.Templates; +using Seq.Syntax.Templates.Encoding; +using Seq.Syntax.Templates.Themes; +using SeqCli.Syntax; namespace SeqCli.Output; -// This is the only usage of Serilog.Expressions remaining in seqcli; the upstream Seq.Syntax doesn't yet support -// tracing properties or theming. static class TextFormatters { - public static ITextFormatter Json(TemplateTheme? theme) => new ExpressionTemplate( - $"{{ " + - $"if {MetricsMapping.SurrogateDefinitionsProperty} is not null then " + - // Emit a metric sample - $"{{@t, @l: undefined(), @d: {MetricsMapping.SurrogateDefinitionsProperty}, ..rest()}} " + - $"else " + - // Emit a log or span - $"{{@t, @mt, @l: coalesce({LevelMapping.SurrogateLevelProperty}, if @l = 'Information' then undefined() else @l), @x, @sp, @tr, @ps: coalesce({TraceConstants.ParentSpanIdProperty}, @ps), @st: coalesce({TraceConstants.SpanStartTimestampProperty}, @st), ..rest()}} " + - $"}}" + - Environment.NewLine, - theme: theme, - // The `OutputFormat` constructor has already decided whether to colorize. - applyThemeWhenOutputIsRedirected: true - ); - - static readonly string DefaultPlainTextOutputTemplate = - "[{@t:o} {@l:u3}] {@m}{#if IsSpan()} ({Milliseconds(Elapsed()):0.###} ms){#end}" + Environment.NewLine + "{@x}"; - - public static ITextFormatter Plain(TemplateTheme? theme, string? outputTemplate) => new ExpressionTemplate( - outputTemplate ?? DefaultPlainTextOutputTemplate, - theme: theme, - nameResolver: new StaticMemberNameResolver(typeof(TracingFunctions)), - // The `OutputFormat` constructor has already decided whether to colorize. - applyThemeWhenOutputIsRedirected: true - ); -} \ No newline at end of file + /// + /// Newline-delimited CLEF output: the event JSON document is written verbatim, with theming + /// when a theme is supplied. + /// + public static ExpressionTemplate Json(TemplateTheme? theme) => new( + "{@Data}" + Environment.NewLine, + encoder: Encoder(theme)); + + /// + /// The default plain-text template, showing ahead of each + /// event's message. + /// + /// Column expressions, evaluated against each event; any Seq expression can be + /// supplied. + internal static string PlainOutputTemplate(IEnumerable? columns = null) => + "[{@Timestamp:o} {@Level:u3}] " + ColumnsFragment(columns ?? []) + + "{@Message}{#if @Elapsed is not null} ({TotalMilliseconds(@Elapsed):0.###} ms){#end}" + + Environment.NewLine + "{@Exception}"; + + internal static string ColumnsFragment(IEnumerable columns) + { + var fragment = new StringBuilder(); + foreach (var column in columns) + fragment.Append($"{{#if ({column}) <> ''}}{{({column})}} {{#end}}"); + + return fragment.ToString(); + } + + public static ExpressionTemplate Plain(TemplateTheme? theme, string? outputTemplate) => + SeqSyntax.ParseTemplate(outputTemplate ?? PlainOutputTemplate(), Encoder(theme)); + + static TemplateOutputEncoder? Encoder(TemplateTheme? theme) => + theme != null ? TemplateOutputEncoder.Ansi(theme) : null; +} diff --git a/src/SeqCli/Output/TraceFormatter.cs b/src/SeqCli/Output/TraceFormatter.cs index d2256e3d..b74b2651 100644 --- a/src/SeqCli/Output/TraceFormatter.cs +++ b/src/SeqCli/Output/TraceFormatter.cs @@ -14,11 +14,13 @@ using System; using System.Collections.Generic; +using System.Globalization; +using System.Linq; using System.Text; -using SeqCli.Mapping; +using System.Text.Json.Nodes; +using SeqCli.Api; +using SeqCli.Data; using SeqCli.Traces; -using SeqCli.Util; -using Serilog.Events; namespace SeqCli.Output; @@ -31,36 +33,30 @@ static class TraceFormatter const string SpanConnector = "├─ ", LastSpanConnector = "└─ ", LogConnector = "┊ ", Continuation = "│ ", Gap = " "; - static string ColumnPropertyName(int index) => $"{ColumnPrefixProperty}_{index}"; + // The trace query evaluates column expressions server-side, so their results are carried in + // surrogate properties rather than being recomputed by the output template. + static string ColumnProperty(int index) => $"{ColumnPrefixProperty}_{index}"; public static string OutputTemplate(int columnCount) { - var template = new StringBuilder($"[{{@t:o}} {{@l:u3}}] {{{TreePrefixProperty}}}"); - - // `<> ''` is undefined, and hence falsy, when the property is missing; the guard thus - // drops the column, and its trailing space, for both missing and empty values. - for (var i = 0; i < columnCount; ++i) - { - var column = ColumnPropertyName(i); - template.Append($"{{#if {column} <> ''}}{{{column}}} {{#end}}"); - } - - template.Append($"{{@m}}{{#if {ElapsedProperty} is not null}} ({{Milliseconds({ElapsedProperty}):0.###}} ms){{#end}}"); - template.Append(Environment.NewLine).Append("{@x}"); + var template = new StringBuilder($"[{{@Timestamp:o}} {{@Level:u3}}] {{{TreePrefixProperty}}}"); + template.Append(TextFormatters.ColumnsFragment(Enumerable.Range(0, columnCount).Select(ColumnProperty))); + template.Append($"{{@Message}}{{#if {ElapsedProperty} is not null}} ({{TotalMilliseconds({ElapsedProperty}):0.###}} ms){{#end}}"); + template.Append(Environment.NewLine).Append("{@Exception}"); return template.ToString(); } - public static IEnumerable ToLogEvents(IReadOnlyList roots) + public static IEnumerable ToEventJson(IReadOnlyList roots) { foreach (var root in roots) { - yield return ToLogEvent(root, root.Element.IsSpan ? "" : LogConnector); + yield return ToEventJson(root, root.Element.IsSpan ? "" : LogConnector); foreach (var descendant in WalkChildren(root, "")) yield return descendant; } } - static IEnumerable WalkChildren(TraceTreeNode parent, string indent) + static IEnumerable WalkChildren(TraceTreeNode parent, string indent) { for (var i = 0; i < parent.Children.Count; ++i) { @@ -71,40 +67,43 @@ static IEnumerable WalkChildren(TraceTreeNode parent, string indent) isLast ? LastSpanConnector : SpanConnector : LogConnector; - yield return ToLogEvent(child, indent + connector); + yield return ToEventJson(child, indent + connector); foreach (var descendant in WalkChildren(child, indent + (isLast ? Gap : Continuation))) yield return descendant; } } - static LogEvent ToLogEvent(TraceTreeNode treeNode, string treePrefix) + static JsonObject ToEventJson(TraceTreeNode treeNode, string treePrefix) { var evt = treeNode.Element; - var properties = new List + // Spans are positioned and shown at their start time. + var eventJson = new JsonObject { - new(TreePrefixProperty, new ScalarValue(treePrefix)) + ["@t"] = evt.SortKey.ToLocalTime().ToString("o", CultureInfo.InvariantCulture), + ["@mt"] = evt.MessageTemplate, + [TreePrefixProperty] = treePrefix }; - properties.AddRange(evt.TemplateProperties); + if (!string.IsNullOrEmpty(evt.Level)) + eventJson["@l"] = evt.Level; + + if (!string.IsNullOrWhiteSpace(evt.Exception)) + eventJson["@x"] = evt.Exception; + + foreach (var (name, value) in evt.TemplateProperties) + eventJson[name] = value?.DeepClone(); if (evt.Elapsed is { } elapsed) - properties.Add(new(ElapsedProperty, new ScalarValue(elapsed))); + eventJson[ElapsedProperty] = EventJsonFormat.CreateScalar(elapsed); for (var i = 0; i < evt.Columns.Count; ++i) { if (evt.Columns[i] is { } value) - properties.Add(LogEventPropertyFactory.SafeCreate( - ColumnPropertyName(i), OutputFormat.CreatePropertyValue(value))); + eventJson[ColumnProperty(i)] = ToSystemTextJson.FromApiValue(value); } - // Spans are positioned and shown at their start time. - return new LogEvent( - evt.SortKey.ToLocalTime(), - LevelMapping.ToSerilogLevel(evt.Level ?? ""), - string.IsNullOrWhiteSpace(evt.Exception) ? null : new TextException(evt.Exception), - evt.MessageTemplate, - properties); + return eventJson; } } diff --git a/src/SeqCli/Output/TracingFunctions.cs b/src/SeqCli/Output/TracingFunctions.cs deleted file mode 100644 index 5e2ba112..00000000 --- a/src/SeqCli/Output/TracingFunctions.cs +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright © Datalust Pty Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Globalization; -using SeqCli.Ingestion; -using Serilog.Events; - -namespace SeqCli.Output; - -static class TracingFunctions -{ - public static LogEventPropertyValue? Elapsed(LogEvent logEvent) - { - if (logEvent.Properties.TryGetValue(TraceConstants.SpanStartTimestampProperty, out var sst) && - sst is ScalarValue { Value: DateTime spanStart }) - { - return new ScalarValue(logEvent.Timestamp - spanStart); - } - - if (logEvent.Properties.TryGetValue("@st", out var st) && - st is ScalarValue { Value: string spanStartIso } && - DateTimeOffset.TryParse(spanStartIso, CultureInfo.InvariantCulture, out var spanStartDto)) - { - return new ScalarValue(logEvent.Timestamp - spanStartDto); - } - - return null; - } - - public static LogEventPropertyValue? IsSpan(LogEvent logEvent) - { - return new ScalarValue(Elapsed(logEvent) != null); - } - - public static LogEventPropertyValue? Milliseconds(LogEventPropertyValue? timeSpan) - { - // Truncates instead of rounding. - if (timeSpan is ScalarValue { Value: TimeSpan ts }) - return new ScalarValue((decimal)ts.Ticks / TimeSpan.TicksPerMillisecond); - - return null; - } -} \ No newline at end of file diff --git a/src/SeqCli/PlainText/EventJsonBuilder.cs b/src/SeqCli/PlainText/EventJsonBuilder.cs new file mode 100644 index 00000000..7acddcb2 --- /dev/null +++ b/src/SeqCli/PlainText/EventJsonBuilder.cs @@ -0,0 +1,102 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text.Json.Nodes; +using SeqCli.Data; +using Superpower.Model; + +namespace SeqCli.PlainText; + +/// +/// Assembles the values captured by a plain-text extraction pattern into an event JSON +/// document in Seq's emission schema. +/// +static class EventJsonBuilder +{ + public static JsonObject FromProperties(IDictionary properties, string? remainder) + { + var eventJson = new JsonObject + { + ["@t"] = GetTimestamp(properties).ToString("o", CultureInfo.InvariantCulture) + }; + + if (TryGetText(properties, ReifiedProperties.Level, out var level)) + eventJson["@l"] = level; + + if (TryGetText(properties, ReifiedProperties.Message, out var message)) + eventJson["@m"] = message; + + if (TryGetText(properties, ReifiedProperties.Exception, out var exception)) + eventJson["@x"] = exception; + + if (TryGetText(properties, ReifiedProperties.TraceId, out var traceId)) + eventJson["@tr"] = traceId; + + if (TryGetText(properties, ReifiedProperties.SpanId, out var spanId)) + eventJson["@sp"] = spanId; + + if (TryGetText(properties, ReifiedProperties.StartTimestamp, out var start)) + eventJson["@st"] = start; + + foreach (var (name, value) in properties) + { + if (!ReifiedProperties.IsReifiedProperty(name)) + eventJson[EventJsonFormat.EscapeUserPropertyName(name)] = UnwrapTextSpans(value); + } + + if (remainder != null) + eventJson[EventJsonFormat.EscapeUserPropertyName("@unmatched")] = UnwrapTextSpans(remainder); + + return eventJson; + } + + static JsonNode? UnwrapTextSpans(object? value) + { + // We should consider whether text spans might also end up in extracted dictionary or array elements, though + // I don't think they will, currently. + return value is TextSpan span + ? JsonValue.Create(span.ToStringValue()) + : EventJsonFormat.CreateScalar(value); + } + + static bool TryGetText(IDictionary properties, string name, out string text) + { + if (properties.TryGetValue(name, out var value) && value is TextSpan span) + { + text = span.ToStringValue(); + return true; + } + + text = ""; + return false; + } + + static DateTimeOffset GetTimestamp(IDictionary properties) + { + if (properties.TryGetValue(ReifiedProperties.Timestamp, out var t)) + { + if (t is TextSpan span && DateTimeOffset.TryParse(span.ToStringValue(), + CultureInfo.InvariantCulture, DateTimeStyles.AssumeLocal, out var ts)) + return ts; + + if (t is DateTimeOffset dto) + return dto; + } + + return DateTimeOffset.Now; + } +} diff --git a/src/SeqCli/PlainText/Extraction/Matchers.cs b/src/SeqCli/PlainText/Extraction/Matchers.cs index c2326f01..fef3c2cc 100644 --- a/src/SeqCli/PlainText/Extraction/Matchers.cs +++ b/src/SeqCli/PlainText/Extraction/Matchers.cs @@ -3,11 +3,12 @@ using System.Globalization; using System.Linq; using System.Reflection; -using SeqCli.Mapping; +using SeqCli.Api; using SeqCli.PlainText.Parsers; using Superpower; using Superpower.Model; using Superpower.Parsers; +// ReSharper disable MemberCanBePrivate.Global namespace SeqCli.PlainText.Extraction; @@ -122,7 +123,7 @@ static class Matchers // Equivalent to :* at end-of-pattern public static TextParser MultiLineContent { get; } = - Span.WithAll(ch => true) + Span.WithAll(_ => true) .Select(span => (object?)span); [Matcher("n")] diff --git a/src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs b/src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs deleted file mode 100644 index 1362716f..00000000 --- a/src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright © Datalust and contributors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Globalization; -using System.Linq; -using SeqCli.Mapping; -using SeqCli.Util; -using Serilog.Events; -using Serilog.Parsing; -using Superpower.Model; - -namespace SeqCli.PlainText.LogEvents; - -static class LogEventBuilder -{ - public static LogEvent FromProperties(IDictionary properties, string? remainder) - { - var timestamp = GetTimestamp(properties); - var level = GetLevel(properties); - var exception = TryGetException(properties); - var messageTemplate = GetMessageTemplate(properties); - var traceId = GetTraceId(properties); - var spanId = GetSpanId(properties); - var props = GetLogEventProperties(properties, remainder); - - var fallbackMappedLevel = level != null ? LevelMapping.ToSerilogLevel(level) : LogEventLevel.Information; - properties[LevelMapping.SurrogateLevelProperty] = level; - - return new LogEvent( - timestamp, - fallbackMappedLevel, - exception, - messageTemplate, - props, - traceId ?? default, - spanId ?? default - ); - } - - static readonly MessageTemplate NoMessage = new MessageTemplateParser().Parse(""); - - static MessageTemplate GetMessageTemplate(IDictionary properties) - { - if (properties.TryGetValue(ReifiedProperties.Message, out var m) && - m is TextSpan ts) - { - var text = ts.ToStringValue(); - return new MessageTemplate([new TextToken(text)]); - } - - return NoMessage; - } - - static string? GetLevel(IDictionary properties) - { - if (properties.TryGetValue(ReifiedProperties.Level, out var l) && - l is TextSpan ts) - return ts.ToStringValue(); - - return null; - } - - static ActivityTraceId? GetTraceId(IDictionary properties) - { - if (properties.TryGetValue(ReifiedProperties.TraceId, out var tr) && - tr is TextSpan ts) - return ActivityTraceId.CreateFromString(ts.ToStringValue()); - - return null; - } - - static ActivitySpanId? GetSpanId(IDictionary properties) - { - if (properties.TryGetValue(ReifiedProperties.SpanId, out var sp) && - sp is TextSpan ts) - return ActivitySpanId.CreateFromString(ts.ToStringValue()); - - return null; - } - - static Exception? TryGetException(IDictionary properties) - { - if (properties.TryGetValue(ReifiedProperties.Exception, out var x) && - x is TextSpan ts) - return new TextOnlyException(ts.ToStringValue()); - return null; - } - - static IEnumerable GetLogEventProperties(IDictionary properties, string? remainder) - { - var payload = properties - .Where(p => !ReifiedProperties.IsReifiedProperty(p.Key)) - .Select(p => LogEventPropertyFactory.SafeCreate(p.Key, new ScalarValue(p.Value))); - - if (remainder != null) - payload = payload.Concat(new[] - { - LogEventPropertyFactory.SafeCreate("@unmatched", new ScalarValue(remainder)) - }); - return payload; - } - - static DateTimeOffset GetTimestamp(IDictionary properties) - { - if (properties.TryGetValue(ReifiedProperties.Timestamp, out var t)) - { - if (t is TextSpan span && DateTimeOffset.TryParse(span.ToStringValue(), - CultureInfo.InvariantCulture, DateTimeStyles.AssumeLocal, out var ts)) - return ts; - - if (t is DateTimeOffset dto) - return dto; - } - - return DateTimeOffset.Now; - } -} \ No newline at end of file diff --git a/src/SeqCli/PlainText/LogEvents/TextOnlyException.cs b/src/SeqCli/PlainText/LogEvents/TextOnlyException.cs deleted file mode 100644 index 614c927f..00000000 --- a/src/SeqCli/PlainText/LogEvents/TextOnlyException.cs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright © Datalust and contributors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; - -namespace SeqCli.PlainText.LogEvents; - -class TextOnlyException : Exception -{ - readonly string _toStringValue; - - public TextOnlyException(string toStringValue) - { - _toStringValue = toStringValue ?? throw new ArgumentNullException(nameof(toStringValue)); - } - - public override string ToString() - { - return _toStringValue; - } -} \ No newline at end of file diff --git a/src/SeqCli/PlainText/PlainTextLogEventReader.cs b/src/SeqCli/PlainText/PlainTextEventReader.cs similarity index 84% rename from src/SeqCli/PlainText/PlainTextLogEventReader.cs rename to src/SeqCli/PlainText/PlainTextEventReader.cs index fae2df86..21da4e89 100644 --- a/src/SeqCli/PlainText/PlainTextLogEventReader.cs +++ b/src/SeqCli/PlainText/PlainTextEventReader.cs @@ -1,23 +1,23 @@ using System; using System.IO; using System.Threading.Tasks; +using SeqCli.Data; using SeqCli.Ingestion; using SeqCli.PlainText.Extraction; using SeqCli.PlainText.Framing; -using SeqCli.PlainText.LogEvents; using SeqCli.PlainText.Parsers; using SeqCli.PlainText.Patterns; namespace SeqCli.PlainText; -class PlainTextLogEventReader : ILogEventReader +class PlainTextEventReader : IEventReader { static readonly TimeSpan TrailingLineArrivalDeadline = TimeSpan.FromMilliseconds(10); readonly NameValueExtractor _nameValueExtractor; readonly FrameReader _reader; - public PlainTextLogEventReader(TextReader input, string extractionPattern) + public PlainTextEventReader(TextReader input, string extractionPattern) { if (extractionPattern == null) throw new ArgumentNullException(nameof(extractionPattern)); _nameValueExtractor = ExtractionPatternInterpreter.CreateNameValueExtractor(ExtractionPatternParser.Parse(extractionPattern)); @@ -36,7 +36,7 @@ public async Task TryReadAsync() var (properties, remainder) = _nameValueExtractor.ExtractValues(frame.Value); - var evt = LogEventBuilder.FromProperties(properties, remainder); + var evt = EventJsonBuilder.FromProperties(properties, remainder); return new ReadResult(evt, frame.IsAtEnd); } } \ No newline at end of file diff --git a/src/SeqCli/Program.cs b/src/SeqCli/Program.cs index f6a0a11f..0c42c5ff 100644 --- a/src/SeqCli/Program.cs +++ b/src/SeqCli/Program.cs @@ -54,8 +54,11 @@ static async Task Main(string[] args) } catch (Exception ex) { - Log.Debug(ex, "Unhandled command exception"); - Log.Fatal("The command failed: {UnhandledExceptionMessage}", Presentation.FormattedMessage(ex)); + // The `--verbose` flag flips the level switch from `Error` to `Information`; we use that as a signal to + // include full stack traces, it's a bit of a sneaky backchannel but saves adding yet more infrastructure. + var reportedException = levelSwitch.MinimumLevel < LogEventLevel.Error ? ex : null; + + Log.Fatal(reportedException, "The command failed: {UnhandledExceptionMessage}", Presentation.FormattedMessage(ex)); return 1; } finally diff --git a/src/SeqCli/Ingestion/BufferingSink.cs b/src/SeqCli/Sample/Ingestion/BufferingSink.cs similarity index 50% rename from src/SeqCli/Ingestion/BufferingSink.cs rename to src/SeqCli/Sample/Ingestion/BufferingSink.cs index 879b930c..59b0225d 100644 --- a/src/SeqCli/Ingestion/BufferingSink.cs +++ b/src/SeqCli/Sample/Ingestion/BufferingSink.cs @@ -1,31 +1,41 @@ -using System; +using System; using System.Collections.Concurrent; +using System.Text.Json.Nodes; using System.Threading.Tasks; +using SeqCli.Ingestion; using Serilog.Core; using Serilog.Events; -namespace SeqCli.Ingestion; +namespace SeqCli.Sample.Ingestion; -class BufferingSink: ILogEventSink, ILogEventReader, IDisposable +/// +/// Bridges the sample simulation's Serilog-based event generation into the +/// JSON-document-based shipping pipeline. +/// +class BufferingSink: ILogEventSink, IEventReader, IDisposable { - readonly ConcurrentQueue _queue = new(); + readonly ConcurrentQueue _queue = new(); const int QueueCapacity = 10000; volatile bool _disposed; - + public void Emit(LogEvent logEvent) { // No problem if this is racy - we can afford a bit of extra queue space. if (_disposed || _queue.Count > QueueCapacity) return; - - _queue.Enqueue(logEvent); + + var document = MetricsMapping.TryGetMetricSampleJson(logEvent, out var sample) + ? sample + : SimulationEvent.ToJsonObject(logEvent); + + _queue.Enqueue(document); } public Task TryReadAsync() { - if (!_queue.TryDequeue(out var logEvent)) + if (!_queue.TryDequeue(out var document)) return Task.FromResult(new ReadResult(null, _disposed)); - return Task.FromResult(new ReadResult(logEvent, _disposed)); + return Task.FromResult(new ReadResult(document, _disposed)); } public void Dispose() @@ -34,4 +44,4 @@ public void Dispose() _disposed = true; _queue.Clear(); } -} \ No newline at end of file +} diff --git a/src/SeqCli/Sample/Ingestion/MetricsMapping.cs b/src/SeqCli/Sample/Ingestion/MetricsMapping.cs new file mode 100644 index 00000000..f50da4d7 --- /dev/null +++ b/src/SeqCli/Sample/Ingestion/MetricsMapping.cs @@ -0,0 +1,58 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Text.Json.Nodes; +using SeqCli.Data; +using Serilog.Events; + +namespace SeqCli.Sample.Ingestion; + +/// +/// The sample simulation generates metric samples as Serilog events carrying their metric +/// definitions in a surrogate property, because Serilog's data model has no @d +/// equivalent. Events marked this way ship as metric samples rather than logs. +/// +static class MetricsMapping +{ + // Use a "hygienic" name for the definitions property to avoid collisions. + internal static readonly string SurrogateDefinitionsProperty = $"_SeqcliMetricDefinitions_{Guid.NewGuid():N}"; + + public static bool TryGetMetricSampleJson(LogEvent logEvent, [NotNullWhen(true)] out JsonObject? sample) + { + if (!logEvent.Properties.TryGetValue(SurrogateDefinitionsProperty, out var definitions)) + { + sample = null; + return false; + } + + // Metric samples carry only a timestamp, definitions, and their dimension/value + // properties; no message or level. + sample = new JsonObject + { + ["@t"] = logEvent.Timestamp.ToString("o", CultureInfo.InvariantCulture), + ["@d"] = SimulationEvent.ToJsonNode(definitions) + }; + + foreach (var (name, value) in logEvent.Properties) + { + if (name != SurrogateDefinitionsProperty) + sample[EventJsonFormat.EscapeUserPropertyName(name)] = SimulationEvent.ToJsonNode(value); + } + + return true; + } +} diff --git a/src/SeqCli/Sample/Ingestion/SimulationEvent.cs b/src/SeqCli/Sample/Ingestion/SimulationEvent.cs new file mode 100644 index 00000000..988b4952 --- /dev/null +++ b/src/SeqCli/Sample/Ingestion/SimulationEvent.cs @@ -0,0 +1,105 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Globalization; +using System.Linq; +using System.Text.Json.Nodes; +using SeqCli.Data; +using Serilog.Events; + +namespace SeqCli.Sample.Ingestion; + +/// Used only in the Roastery simulation; no other event data should ever be processed using this type. +static class SimulationEvent +{ + const string ParentSpanIdProperty = "ParentSpanId", + SpanStartTimestampProperty = "SpanStartTimestamp"; + + public static JsonObject ToJsonObject(LogEvent logEvent) + { + var eventJson = new JsonObject + { + ["@t"] = logEvent.Timestamp.ToString("o", CultureInfo.InvariantCulture), + ["@mt"] = logEvent.MessageTemplate.Text + }; + + if (logEvent.Level != LogEventLevel.Information) + eventJson["@l"] = logEvent.Level.ToString(); + + if (logEvent.Exception != null) + eventJson["@x"] = logEvent.Exception.ToString(); + + if (logEvent.TraceId is { } traceId) + eventJson["@tr"] = traceId.ToHexString(); + + if (logEvent.SpanId is { } spanId) + eventJson["@sp"] = spanId.ToHexString(); + + foreach (var (name, value) in logEvent.Properties) + eventJson[EventJsonFormat.EscapeUserPropertyName(name)] = ToJsonNode(value); + + LiftSpanProperties(eventJson); + + return eventJson; + } + + public static JsonNode? ToJsonNode(LogEventPropertyValue value) + { + switch (value) + { + case ScalarValue scalar: + return EventJsonFormat.CreateScalar(scalar.Value); + + case SequenceValue sequence: + return new JsonArray(sequence.Elements.Select(ToJsonNode).ToArray()); + + case StructureValue structure: + { + var result = new JsonObject(); + foreach (var property in structure.Properties) + result[property.Name] = ToJsonNode(property.Value); + if (structure.TypeTag != null) + result["$type"] = structure.TypeTag; + return result; + } + + case DictionaryValue dictionary: + { + var result = new JsonObject(); + foreach (var (key, element) in dictionary.Elements) + result[key.Value?.ToString() ?? "null"] = ToJsonNode(element); + return result; + } + + default: + return EventJsonFormat.CreateScalar(value.ToString()); + } + } + + static void LiftSpanProperties(JsonObject eventJson) + { + LiftProperty(eventJson, SpanStartTimestampProperty, "@st"); + LiftProperty(eventJson, ParentSpanIdProperty, "@ps"); + } + + static void LiftProperty(JsonObject eventJson, string propertyName, string reifiedName) + { + if (eventJson.TryGetPropertyValue(propertyName, out var value)) + { + eventJson.Remove(propertyName); + if (!eventJson.ContainsKey(reifiedName)) + eventJson[reifiedName] = value; + } + } +} diff --git a/src/SeqCli/Sample/Loader/Simulation.cs b/src/SeqCli/Sample/Loader/Simulation.cs index a54f0a28..23828632 100644 --- a/src/SeqCli/Sample/Loader/Simulation.cs +++ b/src/SeqCli/Sample/Loader/Simulation.cs @@ -17,7 +17,7 @@ using Roastery.Metrics; using Seq.Api; using SeqCli.Ingestion; -using SeqCli.Mapping; +using SeqCli.Sample.Ingestion; using Serilog; namespace SeqCli.Sample.Loader; diff --git a/src/SeqCli/SeqCli.csproj b/src/SeqCli/SeqCli.csproj index fcb51b71..346d0ed8 100644 --- a/src/SeqCli/SeqCli.csproj +++ b/src/SeqCli/SeqCli.csproj @@ -4,7 +4,7 @@ net10.0 seqcli ..\..\asset\SeqCli.ico - win-x64;linux-x64;linux-musl-x64;osx-x64;linux-arm64;linux-musl-arm64;osx-arm64 + win-x64;linux-x64;linux-musl-x64;osx-x64;win-arm64;linux-arm64;linux-musl-arm64;osx-arm64 True True @@ -31,7 +31,6 @@ - @@ -43,13 +42,10 @@ - + - - - diff --git a/src/SeqCli/Signals/SignalExpressionPartExtensions.cs b/src/SeqCli/Signals/SignalExpressionPartExtensions.cs new file mode 100644 index 00000000..d99ddb39 --- /dev/null +++ b/src/SeqCli/Signals/SignalExpressionPartExtensions.cs @@ -0,0 +1,34 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Seq.Api.Model.Signals; + +namespace SeqCli.Signals; + +static class SignalExpressionPartExtensions +{ + public static IEnumerable ReferencedSignalIds(this SignalExpressionPart expr) + { + return expr.Kind switch + { + SignalExpressionKind.Signal => [expr.SignalId], + SignalExpressionKind.Intersection or SignalExpressionKind.Union => expr.Left.ReferencedSignalIds() + .Concat(expr.Right.ReferencedSignalIds()), + _ => throw new ArgumentOutOfRangeException(nameof(expr)) + }; + } +} \ No newline at end of file diff --git a/src/SeqCli/Syntax/SeqCliNameResolver.cs b/src/SeqCli/Syntax/SeqCliNameResolver.cs deleted file mode 100644 index 91b4abab..00000000 --- a/src/SeqCli/Syntax/SeqCliNameResolver.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System.Diagnostics.CodeAnalysis; -using Seq.Syntax.Expressions; - -namespace SeqCli.Syntax; - -class SeqCliNameResolver: NameResolver -{ - public override bool TryResolveBuiltInPropertyName(string alias, [MaybeNullWhen(false)] out string target) - { - switch (alias) - { - case "@l": - target = "coalesce(SeqCliOriginalLevel, @l)"; - return true; - default: - target = null; - return false; - } - } -} diff --git a/src/SeqCli/Syntax/SeqSyntax.cs b/src/SeqCli/Syntax/SeqSyntax.cs index 3974b4ad..4d05d078 100644 --- a/src/SeqCli/Syntax/SeqSyntax.cs +++ b/src/SeqCli/Syntax/SeqSyntax.cs @@ -1,11 +1,54 @@ -using Seq.Syntax.Expressions; +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Diagnostics.CodeAnalysis; +using Seq.Syntax.Expressions; +using Seq.Syntax.Templates; +using Seq.Syntax.Templates.Encoding; +using Seq.Syntax.Compatibility; namespace SeqCli.Syntax; +/// +/// Compiles the expressions and templates accepted on the command line. Uses the Seq.Syntax v1 +/// compatibility shim so that established seqcli syntax — abbreviated built-in names like +/// @l, and the Elapsed()/Milliseconds() functions — keeps working. +/// static class SeqSyntax { public static CompiledExpression CompileExpression(string expression) { - return SerilogExpression.Compile(expression, nameResolver: new SeqCliNameResolver()); + if (!TryCompileExpression(expression, out var compiled, out var error)) + throw new ArgumentException(error); + + return compiled; + } + + public static bool TryCompileExpression( + string expression, + [MaybeNullWhen(false)] out CompiledExpression result, + [MaybeNullWhen(true)] out string error) + { + return V1.TryCompileExpression(expression, formatProvider: null, null, out result, out error); + } + + public static ExpressionTemplate ParseTemplate(string template, TemplateOutputEncoder? encoder = null) + { + if (!V1.TryParseTemplate(template, culture: null, null, encoder, out var parsed, out var error)) + throw new ArgumentException(error); + + return parsed; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Traces/StructuredMessage.cs b/src/SeqCli/Traces/StructuredMessage.cs index 73654f04..702c9006 100644 --- a/src/SeqCli/Traces/StructuredMessage.cs +++ b/src/SeqCli/Traces/StructuredMessage.cs @@ -14,30 +14,30 @@ using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Text.Json.Nodes; using Newtonsoft.Json.Linq; -using SeqCli.Output; -using SeqCli.Util; -using Serilog.Events; -using Serilog.Parsing; +using SeqCli.Api; namespace SeqCli.Traces; static class StructuredMessage { /// - /// Reads the token array produced by the Seq `@StructuredMessage` property - /// into a Serilog message template, along with the property values needed to render it. + /// Reads the token array produced by the Seq `@StructuredMessage` property into message + /// template text, along with the property values needed to render it. Dotted hole names + /// are stored as nested structures, matching how message rendering resolves them. /// - public static (MessageTemplate Message, IReadOnlyList Properties) Read(object? structuredMessage) + public static (string MessageTemplate, JsonObject Properties) Read(object? structuredMessage) { if (structuredMessage is null or JValue { Type: JTokenType.Null }) - return (new MessageTemplate([]), []); + return ("", new JsonObject()); if (structuredMessage is not JArray tokens) throw new InvalidDataException($"Expected a structured message but found `{structuredMessage}`."); - var templateTokens = new List(); - var properties = new List(); + var templateTokens = new List<(bool IsText, string Text)>(); + var properties = new JsonObject(); var propertyNames = new HashSet(); foreach (var token in tokens) @@ -48,14 +48,14 @@ public static (MessageTemplate Message, IReadOnlyList Properti throw new InvalidDataException("A message template hole is missing its `name`."); // Currently ignores `formatted`. - templateTokens.Add(new PropertyToken(name, (hole["raw"] as JValue)?.Value as string ?? $"{{{name}}}")); + templateTokens.Add((false, (hole["raw"] as JValue)?.Value as string ?? $"{{{name}}}")); if (hole.TryGetValue("value", out var value) && propertyNames.Add(name)) - properties.Add(LogEventPropertyFactory.SafeCreate(name, CreatePropertyValue(value))); + SetPathProperty(properties, name, ToSystemTextJson.FromNewtonsoft(value)); } else if (token is JValue { Type: JTokenType.String } text) { - templateTokens.Add(new TextToken((string)text.Value!)); + templateTokens.Add((true, (string)text.Value!)); } else { @@ -65,27 +65,53 @@ public static (MessageTemplate Message, IReadOnlyList Properti TrimEnd(templateTokens); - return (new MessageTemplate(templateTokens), properties); + var templateText = string.Concat(templateTokens.Select(t => + t.IsText ? t.Text.Replace("{", "{{").Replace("}", "}}") : t.Text)); + + return (templateText, properties); + } + + // Message rendering resolves dotted hole names as paths into nested objects, so `a.b` + // becomes member `b` of object `a`. If placing a value along the path would collide with a + // non-object value, the hole is left unresolvable and renders as raw text. + static void SetPathProperty(JsonObject properties, string name, JsonNode? value) + { + var steps = name.Split('.'); + var target = properties; + for (var i = 0; i < steps.Length - 1; ++i) + { + if (target.TryGetPropertyValue(steps[i], out var next)) + { + if (next is not JsonObject nextObject) + return; + + target = nextObject; + } + else + { + var nextObject = new JsonObject(); + target[steps[i]] = nextObject; + target = nextObject; + } + } + + target[steps[^1]] = value; } - static void TrimEnd(List templateTokens) + static void TrimEnd(List<(bool IsText, string Text)> templateTokens) { - while (templateTokens.Count > 0 && templateTokens[^1] is TextToken text) + while (templateTokens.Count > 0 && templateTokens[^1] is (true, var text)) { - var trimmed = text.Text.TrimEnd(); - if (trimmed.Length == text.Text.Length) + var trimmed = text.TrimEnd(); + if (trimmed.Length == text.Length) break; templateTokens.RemoveAt(templateTokens.Count - 1); if (trimmed.Length > 0) { - templateTokens.Add(new TextToken(trimmed)); + templateTokens.Add((true, trimmed)); break; } } } - - static LogEventPropertyValue CreatePropertyValue(JToken value) => value is JValue scalar ? - new ScalarValue(scalar.Value) : - OutputFormat.CreatePropertyValue(value); } diff --git a/src/SeqCli/Traces/TraceTreeElement.cs b/src/SeqCli/Traces/TraceTreeElement.cs index 52a25a8f..df4a6756 100644 --- a/src/SeqCli/Traces/TraceTreeElement.cs +++ b/src/SeqCli/Traces/TraceTreeElement.cs @@ -14,7 +14,7 @@ using System; using System.Collections.Generic; -using Serilog.Events; +using System.Text.Json.Nodes; namespace SeqCli.Traces; @@ -22,8 +22,8 @@ record TraceTreeElement( string Id, DateTimeOffset Timestamp, string? Level, - MessageTemplate MessageTemplate, - IReadOnlyList TemplateProperties, + string MessageTemplate, + JsonObject TemplateProperties, string? Exception, string? SpanId, string? ParentId, diff --git a/src/SeqCli/Traces/TraceTreeJObjectConverter.cs b/src/SeqCli/Traces/TraceTreeJObjectConverter.cs index 4399c15b..5d170f83 100644 --- a/src/SeqCli/Traces/TraceTreeJObjectConverter.cs +++ b/src/SeqCli/Traces/TraceTreeJObjectConverter.cs @@ -15,17 +15,16 @@ using System.Collections.Generic; using System.Globalization; using System.IO; +using System.Text.Json.Nodes; using Newtonsoft.Json.Linq; -using SeqCli.Mapping; +using Seq.Syntax.Templates; using SeqCli.Output; -using Serilog.Events; -using Serilog.Formatting; namespace SeqCli.Traces; static class TraceTreeJObjectConverter { - static readonly ITextFormatter MessageFormatter = TextFormatters.Plain(theme: null, "{@m}"); + static readonly ExpressionTemplate MessageFormatter = TextFormatters.Plain(theme: null, "{@Message}"); public static JObject FromRoots(string traceId, IReadOnlyList roots, bool complete, bool includeTypeMarker, IReadOnlyList columns) { @@ -82,7 +81,7 @@ static JObject ToJson(TraceTreeNode node, bool includeTypeMarker, IReadOnlyList< json["parentSpanId"] = evt.ParentId; if (!string.IsNullOrEmpty(evt.Level)) - json["level"] = LevelMapping.ToFullLevelName(evt.Level); + json["level"] = evt.Level; if (evt.IsSpan) { @@ -131,15 +130,16 @@ static JObject ToJson(TraceTreeNode node, bool includeTypeMarker, IReadOnlyList< static string RenderMessage(TraceTreeElement evt) { - var logEvent = new LogEvent( - evt.SortKey, - LevelMapping.ToSerilogLevel(evt.Level ?? ""), - exception: null, - evt.MessageTemplate, - evt.TemplateProperties); + var eventJson = new JsonObject + { + ["@mt"] = evt.MessageTemplate + }; + + foreach (var (name, value) in evt.TemplateProperties) + eventJson[name] = value?.DeepClone(); var message = new StringWriter(); - MessageFormatter.Format(logEvent, message); + MessageFormatter.Format(eventJson, message); return message.ToString(); } } diff --git a/src/SeqCli/Util/JsonNetDestructuringPolicy.cs b/src/SeqCli/Util/JsonNetDestructuringPolicy.cs deleted file mode 100644 index 8d9bf7bc..00000000 --- a/src/SeqCli/Util/JsonNetDestructuringPolicy.cs +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2015 Destructurama Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using Newtonsoft.Json.Linq; -using Serilog.Core; -using Serilog.Events; - -namespace SeqCli.Util; - -sealed class JsonNetDestructuringPolicy : IDestructuringPolicy -{ - public bool TryDestructure(object value, ILogEventPropertyValueFactory propertyValueFactory, [NotNullWhen(true)] out LogEventPropertyValue? result) - { - switch (value) - { - case JObject jo: - result = Destructure(jo, propertyValueFactory); - return true; - case JArray ja: - result = Destructure(ja, propertyValueFactory); - return true; - case JValue jv: - result = Destructure(jv, propertyValueFactory); - return true; - } - - result = null; - return false; - } - - static LogEventPropertyValue Destructure(JValue jv, ILogEventPropertyValueFactory propertyValueFactory) - { - return propertyValueFactory.CreatePropertyValue(jv.Value!, destructureObjects: true); - } - - static SequenceValue Destructure(JArray ja, ILogEventPropertyValueFactory propertyValueFactory) - { - var elems = ja.Select(t => propertyValueFactory.CreatePropertyValue(t, destructureObjects: true)); - return new SequenceValue(elems); - } - - static LogEventPropertyValue Destructure(JObject jo, ILogEventPropertyValueFactory propertyValueFactory) - { - string? typeTag = null; - var props = new List(jo.Count); - - foreach (var prop in jo.Properties()) - { - if (prop.Name == "$type") - { - if (prop.Value is JValue typeVal && typeVal.Value is string v) - { - typeTag = v; - continue; - } - } - else if (!LogEventProperty.IsValidName(prop.Name)) - { - return DestructureToDictionaryValue(jo, propertyValueFactory); - } - - props.Add(new LogEventProperty(prop.Name, propertyValueFactory.CreatePropertyValue(prop.Value, destructureObjects: true))); - } - - return new StructureValue(props, typeTag); - } - - static DictionaryValue DestructureToDictionaryValue(JObject jo, ILogEventPropertyValueFactory propertyValueFactory) - { - var elements = jo.Properties().Select( - prop => new KeyValuePair( - new ScalarValue(prop.Name), - propertyValueFactory.CreatePropertyValue(prop.Value, destructureObjects: true)) - ); - return new DictionaryValue(elements); - } -} \ No newline at end of file diff --git a/src/SeqCli/Util/LogEventPropertyFactory.cs b/src/SeqCli/Util/LogEventPropertyFactory.cs deleted file mode 100644 index 89c23987..00000000 --- a/src/SeqCli/Util/LogEventPropertyFactory.cs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright © Datalust Pty Ltd and Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using Serilog.Events; - -namespace SeqCli.Util; - -static class LogEventPropertyFactory -{ - const string InvalidPropertyNameSubstitute = "(unnamed)"; - - public static LogEventProperty SafeCreate(string name, LogEventPropertyValue value) - { - if (value == null) throw new ArgumentNullException(nameof(value)); - - if (!LogEventProperty.IsValidName(name)) - name = InvalidPropertyNameSubstitute; - - return new LogEventProperty(name, value); - } -} \ No newline at end of file diff --git a/src/SeqCli/Util/Presentation.cs b/src/SeqCli/Util/Presentation.cs index 125ee15b..7a4df543 100644 --- a/src/SeqCli/Util/Presentation.cs +++ b/src/SeqCli/Util/Presentation.cs @@ -29,7 +29,7 @@ static class Presentation /// and causal chain. public static string FormattedMessage(Exception ex) { - if (ex == null) throw new ArgumentNullException(nameof(ex)); + ArgumentNullException.ThrowIfNull(ex); static Exception Unwrap(Exception outer) { @@ -38,8 +38,10 @@ static Exception Unwrap(Exception outer) static string Describe(Exception toDescribe) { - // :-) - return toDescribe.Message.Replace(", see inner exception", ""); + var described = toDescribe.Message.Replace(", see inner exception", "").Trim(); + if (!described.EndsWith('.')) + described += "."; + return described; } var unwrapped = Unwrap(ex); @@ -49,7 +51,7 @@ static string Describe(Exception toDescribe) { unwrapped = Unwrap(unwrapped.InnerException); - message.Append(' '); + message.Append(" → "); message.Append(Describe(unwrapped)); } diff --git a/test/SeqCli.EndToEnd/Data/trace-tree.clef b/test/SeqCli.EndToEnd/Data/trace-tree.clef index 465817fa..2f00a000 100644 --- a/test/SeqCli.EndToEnd/Data/trace-tree.clef +++ b/test/SeqCli.EndToEnd/Data/trace-tree.clef @@ -4,3 +4,7 @@ {"@t":"2023-12-20T00:50:00.2Z","@l":"Warning","@tr":"7d4dedcc73b18e449e0e4ea08cbe346d","@sp":"2222222222222222","@mt":"{RowCount} rows retrieved","RowCount":42,"@x":"System.TimeoutException: The query timeout was reached"} {"@t":"2023-12-20T00:50:00.9Z","@st":"2023-12-20T00:50:00.5Z","@tr":"7d4dedcc73b18e449e0e4ea08cbe346d","@sp":"4444444444444444","@ps":"1111111111111111","@m":"Render response"} {"@t":"2023-12-20T00:50:00.95Z","@tr":"7d4dedcc73b18e449e0e4ea08cbe346d","@sp":"9999999999999999","@m":"Orphan log"} +{"@t":"2023-12-20T00:50:00.85Z","@st":"2023-12-20T00:50:00.55Z","@tr":"7d4dedcc73b18e449e0e4ea08cbe346d","@sp":"5555555555555555","@ps":"4444444444444444","@m":"Serialize model"} +{"@t":"2023-12-20T00:50:00.8Z","@st":"2023-12-20T00:50:00.6Z","@tr":"7d4dedcc73b18e449e0e4ea08cbe346d","@sp":"6666666666666666","@ps":"5555555555555555","@m":"Serialize order"} +{"@t":"2023-12-20T00:50:00.75Z","@st":"2023-12-20T00:50:00.65Z","@tr":"7d4dedcc73b18e449e0e4ea08cbe346d","@sp":"7777777777777777","@ps":"6666666666666666","@m":"Format currency"} +{"@t":"2023-12-20T00:50:00.72Z","@st":"2023-12-20T00:50:00.7Z","@tr":"7d4dedcc73b18e449e0e4ea08cbe346d","@sp":"8888888888888888","@ps":"7777777777777777","@m":"Lookup locale"} diff --git a/test/SeqCli.EndToEnd/Events/EventsDeleteTestCase.cs b/test/SeqCli.EndToEnd/Events/EventsDeleteTestCase.cs index 29b585c9..1a28fd48 100644 --- a/test/SeqCli.EndToEnd/Events/EventsDeleteTestCase.cs +++ b/test/SeqCli.EndToEnd/Events/EventsDeleteTestCase.cs @@ -1,4 +1,3 @@ -using System; using System.IO; using System.Threading.Tasks; using Seq.Api; @@ -6,7 +5,7 @@ using Serilog; using Xunit; -namespace SeqCli.EndToEnd.Delete; +namespace SeqCli.EndToEnd.Events; public class EventsDeleteTestCase : ICliTestCase { diff --git a/test/SeqCli.EndToEnd/Events/EventsDeleteWithDateRangeAllTestCase.cs b/test/SeqCli.EndToEnd/Events/EventsDeleteWithDateRangeAllTestCase.cs index ed56ac08..72688e94 100644 --- a/test/SeqCli.EndToEnd/Events/EventsDeleteWithDateRangeAllTestCase.cs +++ b/test/SeqCli.EndToEnd/Events/EventsDeleteWithDateRangeAllTestCase.cs @@ -6,7 +6,7 @@ using Serilog; using Xunit; -namespace SeqCli.EndToEnd.Delete; +namespace SeqCli.EndToEnd.Events; public class EventsDeleteWithDateRangeAllTestCase : ICliTestCase { diff --git a/test/SeqCli.EndToEnd/Events/SearchColumnsTestCase.cs b/test/SeqCli.EndToEnd/Events/SearchColumnsTestCase.cs new file mode 100644 index 00000000..036bec56 --- /dev/null +++ b/test/SeqCli.EndToEnd/Events/SearchColumnsTestCase.cs @@ -0,0 +1,45 @@ +using System.IO; +using System.Threading.Tasks; +using Seq.Api; +using SeqCli.EndToEnd.Support; +using Serilog; +using Xunit; + +#nullable enable + +namespace SeqCli.EndToEnd.Events; + +public class SearchColumnsTestCase : ICliTestCase +{ + const string TraceId = "7d4dedcc73b18e449e0e4ea08cbe346d"; + + public Task ExecuteAsync( + SeqConnection connection, + ILogger logger, + CliCommandRunner runner) + { + var inputFile = Path.Combine("Data", "trace-tree.clef"); + Assert.True(File.Exists(inputFile)); + + var exit = runner.Exec("ingest", $"--json -i {inputFile}"); + Assert.Equal(0, exit); + + var filter = $"--filter=\"@TraceId = '{TraceId}' and Customer is not null\""; + + exit = runner.Exec("search", $"{filter} -c 10 --column Customer --column RowCount"); + Assert.Equal(0, exit); + Assert.Contains("] scott GET /orders", runner.LastRunProcess!.Output); + + // Columns apply to plain-text output only. + exit = runner.Exec("search", $"{filter} -c 10 --column Customer --json"); + Assert.Equal(0, exit); + Assert.Contains("GET {Route}", runner.LastRunProcess!.Output); + Assert.DoesNotContain("_SeqcliColumn", runner.LastRunProcess!.Output); + + exit = runner.Exec("search", $"{filter} -c 10 --column \"not a valid (\""); + Assert.Equal(1, exit); + Assert.Contains("could not be compiled", runner.LastRunProcess!.Output); + + return Task.CompletedTask; + } +} diff --git a/test/SeqCli.EndToEnd/Events/SearchSignalColumnsTestCase.cs b/test/SeqCli.EndToEnd/Events/SearchSignalColumnsTestCase.cs new file mode 100644 index 00000000..2f874c7b --- /dev/null +++ b/test/SeqCli.EndToEnd/Events/SearchSignalColumnsTestCase.cs @@ -0,0 +1,101 @@ +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Seq.Api; +using SeqCli.EndToEnd.Support; +using Serilog; +using Xunit; + +#nullable enable + +namespace SeqCli.EndToEnd.Events; + +public class SearchSignalColumnsTestCase : ICliTestCase +{ + const string TraceId = "7d4dedcc73b18e449e0e4ea08cbe346d"; + + public async Task ExecuteAsync( + SeqConnection connection, + ILogger logger, + CliCommandRunner runner) + { + var inputFile = Path.Combine("Data", "trace-tree.clef"); + Assert.True(File.Exists(inputFile)); + + var exit = runner.Exec("ingest", $"--json -i {inputFile}"); + Assert.Equal(0, exit); + + exit = runner.Exec("signal create", "-t Orders -f \"@TraceId is not null\" -c Customer -c RowCount"); + Assert.Equal(0, exit); + + exit = runner.Exec("signal create", "-t Rows -f \"RowCount is not null\" -c \"RowCount * 2\""); + Assert.Equal(0, exit); + + exit = runner.Exec("signal create", "-t Unadorned -f \"@TraceId is not null\""); + Assert.Equal(0, exit); + + var signals = await connection.Signals.ListAsync(shared: true); + var orders = signals.Single(s => s.Title == "Orders").Id; + var rows = signals.Single(s => s.Title == "Rows").Id; + var unadorned = signals.Single(s => s.Title == "Unadorned").Id; + + var filter = $"--filter=\"@TraceId = '{TraceId}'\""; + + // The signal's columns are displayed, in the order the signal declares them. + exit = runner.Exec("search", $"--signal {orders} {filter} -c 10"); + Assert.Equal(0, exit); + var output = runner.LastRunProcess!.Output; + Assert.Contains("] scott GET /orders", output); + Assert.Contains("] 42 42 rows retrieved", output); + + // Signal columns precede any specified with `--column`. + exit = runner.Exec("search", $"--signal {orders} {filter} -c 10 --column \"@Level\""); + Assert.Equal(0, exit); + output = runner.LastRunProcess!.Output; + Assert.Contains("] scott Information GET /orders", output); + Assert.Contains("] 42 Warning 42 rows retrieved", output); + + // `--no-signal-columns` drops the signal's columns, but not those specified with `--column`. + exit = runner.Exec("search", $"--signal {orders} {filter} -c 10 --no-signal-columns"); + Assert.Equal(0, exit); + output = runner.LastRunProcess!.Output; + Assert.Contains("] GET /orders", output); + Assert.DoesNotContain("scott", output); + + exit = runner.Exec("search", $"--signal {orders} {filter} -c 10 --no-signal-columns --column \"@Level\""); + Assert.Equal(0, exit); + output = runner.LastRunProcess!.Output; + Assert.Contains("] Information GET /orders", output); + Assert.DoesNotContain("scott", output); + + // Signal columns apply to plain-text output only. + exit = runner.Exec("search", $"--signal {orders} {filter} -c 10 --json"); + Assert.Equal(0, exit); + output = runner.LastRunProcess!.Output; + Assert.Contains("GET {Route}", output); + Assert.DoesNotContain("_SeqcliColumn", output); + + // Columns are collected from every signal referenced by the expression. + exit = runner.Exec("search", $"--signal {orders},{rows} {filter} -c 10"); + Assert.Equal(0, exit); + Assert.Contains("] 42 84 42 rows retrieved", runner.LastRunProcess!.Output); + + exit = runner.Exec("search", $"--signal \"{orders}~{rows}\" {filter} -c 10"); + Assert.Equal(0, exit); + output = runner.LastRunProcess!.Output; + Assert.Contains("] scott GET /orders", output); + Assert.Contains("] 42 84 42 rows retrieved", output); + + // A signal without columns contributes none. + exit = runner.Exec("search", $"--signal {unadorned} {filter} -c 10"); + Assert.Equal(0, exit); + output = runner.LastRunProcess!.Output; + Assert.Contains("] GET /orders", output); + Assert.DoesNotContain("scott", output); + + // A signal that can't be found is reported, rather than silently ignored. + exit = runner.Exec("search", $"--signal signal-999999 {filter} -c 10"); + Assert.Equal(1, exit); + Assert.Contains("The command failed", runner.LastRunProcess!.Output); + } +} diff --git a/test/SeqCli.EndToEnd/Events/TailColumnsTestCase.cs b/test/SeqCli.EndToEnd/Events/TailColumnsTestCase.cs new file mode 100644 index 00000000..02fa26de --- /dev/null +++ b/test/SeqCli.EndToEnd/Events/TailColumnsTestCase.cs @@ -0,0 +1,67 @@ +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Seq.Api; +using SeqCli.EndToEnd.Support; +using Serilog; +using Xunit; + +#nullable enable + +namespace SeqCli.EndToEnd.Events; + +public class TailColumnsTestCase : ICliTestCase +{ + public async Task ExecuteAsync( + SeqConnection connection, + ILogger logger, + CliCommandRunner runner) + { + var inputFile = Path.Combine("Data", "trace-tree.clef"); + Assert.True(File.Exists(inputFile)); + + var exit = runner.Exec("signal create", "-t Orders -f \"@TraceId is not null\" -c Customer -c RowCount"); + Assert.Equal(0, exit); + + var signals = await connection.Signals.ListAsync(shared: true); + var orders = signals.Single(s => s.Title == "Orders").Id; + + var filter = "--filter=\"Customer is not null\""; + + // A column expression that can't be compiled is reported. + exit = runner.Exec("tail", "--column \"not a valid (\""); + Assert.Equal(1, exit); + Assert.Contains("could not be compiled", runner.LastRunProcess!.Output); + + // Signal columns precede those specified with `--column`. + using (var tail = runner.Spawn("tail", $"--signal {orders} {filter} --column \"@Level\"")) + { + await IngestUntilTailWrites(runner, tail, inputFile, "] scott Information GET /orders"); + } + + // `--no-signal-columns` drops the signal's columns, but not those specified with `--column`. + using (var tail = runner.Spawn("tail", $"--signal {orders} {filter} --no-signal-columns --column \"@Level\"")) + { + await IngestUntilTailWrites(runner, tail, inputFile, "] Information GET /orders"); + Assert.DoesNotContain("scott", tail.Output); + } + } + + // Events ingested before the tail command's streaming connection is established won't be + // observed, so ingest the test data repeatedly until the expected line appears. + static async Task IngestUntilTailWrites(CliCommandRunner runner, CaptiveProcess tail, string inputFile, string expected) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (!tail.Output.Contains(expected)) + { + if (DateTime.UtcNow > deadline) + Assert.Fail($"Timed out waiting for `{expected}` in: {tail.Output}"); + + var exit = runner.Exec("ingest", $"--json -i {inputFile}"); + Assert.Equal(0, exit); + + await Task.Delay(TimeSpan.FromSeconds(1)); + } + } +} diff --git a/test/SeqCli.EndToEnd/Forwarder/ForwarderSimpleIngestionTestCase.cs b/test/SeqCli.EndToEnd/Forwarder/ForwarderSimpleIngestionTestCase.cs index bb199942..d90065e5 100644 --- a/test/SeqCli.EndToEnd/Forwarder/ForwarderSimpleIngestionTestCase.cs +++ b/test/SeqCli.EndToEnd/Forwarder/ForwarderSimpleIngestionTestCase.cs @@ -1,5 +1,4 @@ using System; -using System.Globalization; using System.Threading.Tasks; using Seq.Api; using SeqCli.EndToEnd.Support; diff --git a/test/SeqCli.EndToEnd/Mcp/McpMetricsBasicsTestCase.cs b/test/SeqCli.EndToEnd/Mcp/McpMetricsBasicsTestCase.cs index 77fc5f69..f1fa88fc 100644 --- a/test/SeqCli.EndToEnd/Mcp/McpMetricsBasicsTestCase.cs +++ b/test/SeqCli.EndToEnd/Mcp/McpMetricsBasicsTestCase.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Text.Json.Nodes; using System.Threading.Tasks; using JetBrains.Annotations; using ModelContextProtocol.Client; diff --git a/test/SeqCli.EndToEnd/Mcp/McpTraceTestCase.cs b/test/SeqCli.EndToEnd/Mcp/McpTraceTestCase.cs index c81c4882..9c7b8d81 100644 --- a/test/SeqCli.EndToEnd/Mcp/McpTraceTestCase.cs +++ b/test/SeqCli.EndToEnd/Mcp/McpTraceTestCase.cs @@ -33,7 +33,7 @@ protected override async Task ExecuteAsync(SeqConnection connection, ILogger log }); var text = AssertTextResult(loaded); - Assert.Contains("Loaded 4 span(s)", text); + Assert.Contains("Loaded 8 span(s)", text); var document = AssertStructuredObjectResult(loaded); Assert.Equal(TraceId, document.GetProperty("traceId").GetString()); diff --git a/test/SeqCli.EndToEnd/Search/SearchWithFilterTestCase.cs b/test/SeqCli.EndToEnd/Search/SearchWithFilterTestCase.cs new file mode 100644 index 00000000..0e3a2f1d --- /dev/null +++ b/test/SeqCli.EndToEnd/Search/SearchWithFilterTestCase.cs @@ -0,0 +1,35 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using Seq.Api; +using SeqCli.EndToEnd.Support; +using Serilog; +using Xunit; + +namespace SeqCli.EndToEnd.Search; + +public class SearchWithFilterTestCase : ICliTestCase +{ + public async Task ExecuteAsync( + SeqConnection connection, + ILogger logger, + CliCommandRunner runner) + { + await DirectIngestion.IngestClef(connection, "'@mt': 'Event {N}', 'N': 1, 'Host': 'xmpweb-01.example.com'"); + await DirectIngestion.IngestClef(connection, "'@mt': 'Event {N}', 'N': 2, 'Host': 'xmpweb-02.example.com'"); + await DirectIngestion.IngestClef(connection, "'@mt': 'Event {N}', 'N': 3, 'Host': 'xmpweb-02.example.com'"); + + var exit = runner.Exec("search", "--filter=\"Host = 'xmpweb-02.example.com' and N > 2\" --count=10 --json"); + Assert.Equal(0, exit); + + var results = runner.LastRunProcess!.Output + .Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries) + .Select(JObject.Parse) + .ToList(); + + var evt = Assert.Single(results); + Assert.Equal(3, evt["N"]!.Value()); + Assert.Equal("xmpweb-02.example.com", evt["Host"]!.Value()); + } +} diff --git a/test/SeqCli.EndToEnd/Settings/SettingBasicsTestCase.cs b/test/SeqCli.EndToEnd/Settings/SettingBasicsTestCase.cs index f387a400..1de07801 100644 --- a/test/SeqCli.EndToEnd/Settings/SettingBasicsTestCase.cs +++ b/test/SeqCli.EndToEnd/Settings/SettingBasicsTestCase.cs @@ -1,5 +1,4 @@ -using System; -using System.Threading.Tasks; +using System.Threading.Tasks; using Seq.Api; using SeqCli.EndToEnd.Support; using Serilog; diff --git a/test/SeqCli.EndToEnd/Skills/SkillsInstallTestCase.cs b/test/SeqCli.EndToEnd/Skills/SkillsInstallTestCase.cs index 88871bc1..5bca2a32 100644 --- a/test/SeqCli.EndToEnd/Skills/SkillsInstallTestCase.cs +++ b/test/SeqCli.EndToEnd/Skills/SkillsInstallTestCase.cs @@ -1,6 +1,5 @@ using System.IO; using System.Threading.Tasks; -using JetBrains.Annotations; using Seq.Api; using SeqCli.EndToEnd.Support; using Serilog; diff --git a/test/SeqCli.EndToEnd/Traces/TraceShowTestCase.cs b/test/SeqCli.EndToEnd/Traces/TraceShowTestCase.cs index 34707ef3..e85c24fa 100644 --- a/test/SeqCli.EndToEnd/Traces/TraceShowTestCase.cs +++ b/test/SeqCli.EndToEnd/Traces/TraceShowTestCase.cs @@ -100,6 +100,16 @@ public Task ExecuteAsync( ["SELECT * FROM orders", "42 rows retrieved"], ((JArray)query["children"]!).Select(c => (string)c["message"]!).ToArray()); + var node = root; + foreach (var message in new[] {"Render response", "Serialize model", "Serialize order", "Format currency", "Lookup locale"}) + { + var children = Assert.IsType(node["children"]); + node = Assert.IsType(Assert.Single(children, c => (string?)c["message"] == message)); + } + + Assert.Equal("8888888888888888", (string?)node["spanId"]); + Assert.Null(node["children"]); + var orphan = (JObject)Assert.Single((JArray)document["orphans"]!); Assert.Equal("log", (string?)orphan["type"]); Assert.Equal("Orphan log", (string?)orphan["message"]); diff --git a/test/SeqCli.EndToEnd/User/UserCreateRemoveTestCase.cs b/test/SeqCli.EndToEnd/User/UserCreateRemoveTestCase.cs index ddbf95d2..25e3fb99 100644 --- a/test/SeqCli.EndToEnd/User/UserCreateRemoveTestCase.cs +++ b/test/SeqCli.EndToEnd/User/UserCreateRemoveTestCase.cs @@ -4,7 +4,6 @@ using SeqCli.EndToEnd.Support; using Serilog; using Xunit; -using System.IO; using System.Linq; namespace SeqCli.EndToEnd.User; diff --git a/test/SeqCli.Tests/Csv/CsvWriterTests.cs b/test/SeqCli.Tests/Csv/CsvWriterTests.cs index cf4dbeb9..8d46d098 100644 --- a/test/SeqCli.Tests/Csv/CsvWriterTests.cs +++ b/test/SeqCli.Tests/Csv/CsvWriterTests.cs @@ -3,7 +3,7 @@ using System.IO; using Seq.Api.Model.Data; using SeqCli.Csv; -using Serilog.Templates.Themes; +using Seq.Syntax.Templates.Themes; using Xunit; namespace SeqCli.Tests.Csv; @@ -12,8 +12,8 @@ public class CsvWriterTests { const char Escape = '\x1b'; - // `CsvWriter` writes to the console without going through Serilog's console sink, so unlike the other - // output paths it has no opportunity to suppress the theme itself. + // `CsvWriter` writes delimited output directly rather than rendering a template, so unlike the + // other output paths it applies (or omits) the theme itself. [Fact] public void QueryResultsAreNotColorizedWhenOutputIsRedirected() { diff --git a/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs b/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs index 60dee141..7e6bf24e 100644 --- a/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs +++ b/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs @@ -1,5 +1,4 @@ using System.Linq; -using SeqCli.Forwarder.Filesystem.System; using SeqCli.Forwarder.Storage; using SeqCli.Tests.Forwarder.Filesystem; using Xunit; diff --git a/test/SeqCli.Tests/Mcp/McpServerInstallerTests.cs b/test/SeqCli.Tests/Mcp/McpServerInstallerTests.cs new file mode 100644 index 00000000..b6334c5f --- /dev/null +++ b/test/SeqCli.Tests/Mcp/McpServerInstallerTests.cs @@ -0,0 +1,75 @@ +using System; +using System.Collections.Generic; +using SeqCli.Mcp; +using Xunit; + +namespace SeqCli.Tests.Mcp; + +public class McpServerInstallerTests +{ + // Candidate paths are built with Path.Combine, which uses the host's separator; normalize so the + // Windows-style expectations hold when the tests run on Linux or macOS. + static Func FileSystemWith(params string[] files) + { + var set = new HashSet(files, StringComparer.OrdinalIgnoreCase); + return candidate => set.Contains(candidate.Replace('/', '\\')); + } + + [Fact] + public void OnNonWindowsPlatformsSeqCliIsLaunchedDirectly() + { + var (command, leadingArgs) = McpServerInstaller.ResolveCommand( + false, "/usr/local/bin:/usr/bin", null, _ => true); + + Assert.Equal("seqcli", command); + Assert.Empty(leadingArgs); + } + + [Fact] + public void OnWindowsAnExecutableOnPathIsLaunchedDirectly() + { + var (command, leadingArgs) = McpServerInstaller.ResolveCommand( + true, + @"C:\Program Files\Seq;C:\Users\me\AppData\Roaming\npm", + ".COM;.EXE;.BAT;.CMD", + FileSystemWith(@"C:\Program Files\Seq\seqcli.exe", @"C:\Users\me\AppData\Roaming\npm\seqcli.cmd")); + + Assert.Equal("seqcli", command); + Assert.Empty(leadingArgs); + } + + [Fact] + public void OnWindowsAnNpmShimOnPathIsLaunchedViaCmd() + { + var (command, leadingArgs) = McpServerInstaller.ResolveCommand( + true, + @"C:\Users\me\AppData\Roaming\npm;C:\Program Files\Seq", + ".COM;.EXE;.BAT;.CMD", + FileSystemWith(@"C:\Users\me\AppData\Roaming\npm\seqcli.cmd", @"C:\Program Files\Seq\seqcli.exe")); + + Assert.Equal("cmd", command); + Assert.Equal(["/c", "seqcli"], leadingArgs); + } + + [Fact] + public void OnWindowsWhenSeqCliIsNotOnPathItIsLaunchedDirectly() + { + var (command, leadingArgs) = McpServerInstaller.ResolveCommand( + true, @"C:\Windows\system32", null, _ => false); + + Assert.Equal("seqcli", command); + Assert.Empty(leadingArgs); + } + + [Fact] + public void PathExtDefaultsAreUsedWhenTheVariableIsMissing() + { + var (command, _) = McpServerInstaller.ResolveCommand( + true, + @"C:\Users\me\AppData\Roaming\npm", + null, + FileSystemWith(@"C:\Users\me\AppData\Roaming\npm\seqcli.cmd")); + + Assert.Equal("cmd", command); + } +} diff --git a/test/SeqCli.Tests/Output/OutputFormatTests.cs b/test/SeqCli.Tests/Output/OutputFormatTests.cs index 4c08a64e..e323aa60 100644 --- a/test/SeqCli.Tests/Output/OutputFormatTests.cs +++ b/test/SeqCli.Tests/Output/OutputFormatTests.cs @@ -1,10 +1,10 @@ using System.IO; using Newtonsoft.Json.Linq; using Seq.Api.Model.Events; +using SeqCli.Api; using SeqCli.Config; using SeqCli.Output; using SeqCli.Tests.Support; -using Serilog.Events; using Xunit; #nullable enable @@ -128,11 +128,10 @@ static EventEntity MakeDottedHoleEvent(params (string Name, object? Value)[] pro static string RenderMessage(EventEntity evt) { - var serilogEvent = OutputFormat.ToSerilogEvent(evt); - OutputFormat.FlattenPropertiesUsedWithDottedNames(evt, serilogEvent); + var eventJson = EventEntityJson.ToEventJson(evt); var output = new StringWriter(); - TextFormatters.Plain(theme: null, "{@m}").Format(serilogEvent, output); + TextFormatters.Plain(theme: null, "{@m}").Format(eventJson, output); return output.ToString(); } @@ -145,16 +144,6 @@ public void DottedHoleNamesResolveThroughNestedStructures() Assert.Equal("Hello Barney!", RenderMessage(evt)); } - [Fact] - public void FlatPropertiesWinOverStructureTraversal() - { - var evt = MakeDottedHoleEvent( - ("user.greeting.first", "G'day"), - ("user", JObject.Parse("""{"greeting": {"first": "Hello"}, "name": "Barney"}"""))); - - Assert.Equal("G'day Barney!", RenderMessage(evt)); - } - [Fact] public void UnresolvableDottedHolesRenderAsRawText() { @@ -162,20 +151,4 @@ public void UnresolvableDottedHolesRenderAsRawText() Assert.Equal("{user.greeting.first} {user.name}!", RenderMessage(evt)); } - - [Fact] - public void ResolvedScalarsAreUnwrappedFromTheirJsonRepresentation() - { - var evt = Some.MakeEvent(e => - { - e.MessageTemplateTokens = [new MessageTemplateTokenPart { PropertyName = "order.total" }]; - e.Properties = Some.MakeProperties(("order", JObject.Parse("""{"total": 42}"""))); - }); - - var serilogEvent = OutputFormat.ToSerilogEvent(evt); - OutputFormat.FlattenPropertiesUsedWithDottedNames(evt, serilogEvent); - - var scalar = Assert.IsType(serilogEvent.Properties["order.total"]); - Assert.Equal(42L, scalar.Value); - } } diff --git a/test/SeqCli.Tests/Output/TextFormattersTests.cs b/test/SeqCli.Tests/Output/TextFormattersTests.cs index f3548e0d..c3412725 100644 --- a/test/SeqCli.Tests/Output/TextFormattersTests.cs +++ b/test/SeqCli.Tests/Output/TextFormattersTests.cs @@ -1,11 +1,13 @@ #nullable enable using System; +using System.Globalization; using System.IO; +using System.Text.Json.Nodes; +using Seq.Api.Model.Events; +using Seq.Syntax.Templates.Themes; +using SeqCli.Api; using SeqCli.Output; using SeqCli.Tests.Support; -using Serilog.Events; -using Serilog.Parsing; -using Serilog.Templates.Themes; using Xunit; namespace SeqCli.Tests.Output; @@ -13,7 +15,7 @@ namespace SeqCli.Tests.Output; public class TextFormattersTests { const char Escape = '\x1b'; - static readonly DateTimeOffset FixedTimestamp = new(2024, 1, 1, 10, 0, 1, 250, TimeSpan.Zero); + const string FixedTimestamp = "2024-01-01T10:00:01.2500000+00:00"; [Fact] public void ThemedJsonOutputIsColorizedRegardlessOfRedirection() @@ -27,12 +29,18 @@ public void UnthemedJsonOutputIsNotColorized() Assert.DoesNotContain(Escape, RenderJson(theme: null)); } + [Fact] + public void UnthemedJsonOutputIsTheEventDocumentVerbatim() + { + Assert.Equal( + """{"@t":"2024-01-01T10:00:01.2500000+00:00","@mt":"Hello, {Name}!","Name":"world"}""" + Environment.NewLine, + RenderJson(theme: null, SomeEventJson())); + } + [Fact] public void LogEventsAreFormattedWithTheDefaultTextTemplate() { - var evt = SomeLogEvent( - level: LogEventLevel.Warning, - properties: new LogEventProperty("Name", new ScalarValue("world"))); + var evt = SomeEventJson(level: "Warning"); Assert.Equal( $"[2024-01-01T10:00:01.2500000+00:00 WRN] Hello, world!{Environment.NewLine}", @@ -42,11 +50,7 @@ public void LogEventsAreFormattedWithTheDefaultTextTemplate() [Fact] public void ExceptionsAreIncludedInTextOutput() { - var evt = SomeLogEvent( - FixedTimestamp, - LogEventLevel.Error, - new Exception("Boom!"), - new LogEventProperty("Name", new ScalarValue("world"))); + var evt = SomeEventJson(level: "Error", exception: "System.Exception: Boom!"); Assert.Equal( $"[2024-01-01T10:00:01.2500000+00:00 ERR] Hello, world!{Environment.NewLine}System.Exception: Boom!{Environment.NewLine}", @@ -54,14 +58,10 @@ public void ExceptionsAreIncludedInTextOutput() } [Fact] - public void SpanElapsedTimeIsComputedFromTheStartTimestampProperty() + public void SpanElapsedTimeIsComputedFromTheStartTimestamp() { - // Events retrieved from the Seq API carry span start timestamps in ISO-8601 `@st` properties. - var evt = SomeLogEvent(FixedTimestamp, properties: - [ - new LogEventProperty("Name", new ScalarValue("world")), - new LogEventProperty("@st", new ScalarValue("2024-01-01T10:00:00.0000000Z")) - ]); + var evt = SomeEventJson(); + evt["@st"] = "2024-01-01T10:00:00.0000000Z"; Assert.Equal( $"[2024-01-01T10:00:01.2500000+00:00 INF] Hello, world! (1250 ms){Environment.NewLine}", @@ -69,53 +69,108 @@ public void SpanElapsedTimeIsComputedFromTheStartTimestampProperty() } [Fact] - public void SpanElapsedTimeIsComputedFromTheSurrogateStartTimestampProperty() + public void ACustomOutputTemplateReplacesTheDefault() + { + Assert.Equal( + $"INF Hello, world!{Environment.NewLine}", + RenderText(SomeEventJson(), $"{{@l:u3}} {{@m}}{Environment.NewLine}")); + } + + [Fact] + public void ColumnsPrecedeTheMessageInOrder() { - // Ingested spans carry a surrogate `SpanStartTimestamp` property with a `DateTime` value. - var evt = SomeLogEvent(FixedTimestamp, properties: - [ - new LogEventProperty("Name", new ScalarValue("world")), - new LogEventProperty("SpanStartTimestamp", new ScalarValue( - FixedTimestamp.UtcDateTime.AddMilliseconds(-1.5))) - ]); + var evt = Some.MakeEvent(e => e.Properties = Some.MakeProperties(("Customer", "scott"), ("OrderId", 42))); Assert.Equal( - $"[2024-01-01T10:00:01.2500000+00:00 INF] Hello, world! (1.5 ms){Environment.NewLine}", - RenderText(evt)); + $"[{At(evt)} INF] scott 42 Hello{Environment.NewLine}", + RenderText(evt, "Customer", "OrderId")); } [Fact] - public void ACustomOutputTemplateReplacesTheDefault() + public void MissingAndEmptyColumnValuesLeaveNoRedundantSpace() + { + var evt = Some.MakeEvent(e => e.Properties = Some.MakeProperties(("Empty", ""), ("OrderId", 42))); + + Assert.Equal( + $"[{At(evt)} INF] 42 Hello{Environment.NewLine}", + RenderText(evt, "Missing", "Empty", "OrderId")); + } + + [Fact] + public void SeqStyleNamesResolveAgainstApiEvents() + { + var evt = Some.MakeEvent(e => + { + e.Properties = []; + e.Level = "Warning"; + e.SpanKind = "Server"; + e.Resource = Some.MakeProperties(("service.name", "frontend")); + }); + + Assert.Equal( + $"[{At(evt)} WRN] frontend Server Hello{Environment.NewLine}", + RenderText(evt, "@Resource['service.name']", "@SpanKind")); + } + + [Fact] + public void ComputedColumnValuesAreRendered() + { + var evt = Some.MakeEvent(e => e.Properties = Some.MakeProperties(("OrderId", 42))); + + Assert.Equal( + $"[{At(evt)} INF] order-42 Hello{Environment.NewLine}", + RenderText(evt, "concat('order-', tostring(OrderId))")); + } + + [Theory] + [InlineData("if OrderId > 40 then 'big' else 'small'", "big")] + [InlineData("{id: OrderId}.id", "42")] + [InlineData("concat('{', tostring(OrderId), '}')", "{42}")] + [InlineData("Missing or OrderId = 42", "true")] + [InlineData("[OrderId, 'x'][0]", "42")] + public void ColumnExpressionsUsingTemplateDelimitersAreRendered(string column, string expected) { - var evt = SomeLogEvent(properties: new LogEventProperty("Name", new ScalarValue("world"))); + var evt = Some.MakeEvent(e => e.Properties = Some.MakeProperties(("OrderId", 42))); - Assert.Equal($"INF Hello, world!{Environment.NewLine}", RenderText(evt, $"{{@l:u3}} {{@m}}{Environment.NewLine}")); + Assert.Equal( + $"[{At(evt)} INF] {expected} Hello{Environment.NewLine}", + RenderText(evt, column)); } - static LogEvent SomeLogEvent( - DateTimeOffset? timestamp = null, - LogEventLevel level = LogEventLevel.Information, - Exception? exception = null, - params LogEventProperty[] properties) + static string At(EventEntity evt) => + DateTimeOffset.ParseExact(evt.Timestamp, "o", CultureInfo.InvariantCulture).ToLocalTime().ToString("o"); + + static string RenderText(EventEntity evt, params string[] columns) => + RenderText(EventEntityJson.ToEventJson(evt), TextFormatters.PlainOutputTemplate(columns)); + + static JsonObject SomeEventJson(string? level = null, string? exception = null) { - return new LogEvent( - timestamp ?? FixedTimestamp, - level, - exception, - new MessageTemplateParser().Parse("Hello, {Name}!"), - properties); + var evt = new JsonObject + { + ["@t"] = FixedTimestamp, + ["@mt"] = "Hello, {Name}!", + ["Name"] = "world" + }; + + if (level != null) + evt["@l"] = level; + + if (exception != null) + evt["@x"] = exception; + + return evt; } - static string RenderText(LogEvent evt, string? outputTemplate = null) + static string RenderText(JsonObject evt, string? outputTemplate = null) { var output = new StringWriter(); TextFormatters.Plain(theme: null, outputTemplate).Format(evt, output); return output.ToString(); } - static string RenderJson(TemplateTheme? theme) + static string RenderJson(TemplateTheme? theme, JsonObject? evt = null) { - var evt = OutputFormat.ToSerilogEvent(Some.MakeEvent(e => e.Properties = [])); + evt ??= EventEntityJson.ToEventJson(Some.MakeEvent(e => e.Properties = [])); var output = new StringWriter(); TextFormatters.Json(theme).Format(evt, output); diff --git a/test/SeqCli.Tests/Output/TraceFormatterTests.cs b/test/SeqCli.Tests/Output/TraceFormatterTests.cs index fce99356..9dc3a694 100644 --- a/test/SeqCli.Tests/Output/TraceFormatterTests.cs +++ b/test/SeqCli.Tests/Output/TraceFormatterTests.cs @@ -3,10 +3,9 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text.Json.Nodes; using SeqCli.Output; using SeqCli.Traces; -using Serilog.Events; -using Serilog.Parsing; using Xunit; namespace SeqCli.Tests.Output; @@ -18,14 +17,14 @@ public class TraceFormatterTests static TraceTreeElement Span(string spanId, string? parentId, double startMs = 0, double elapsedMs = 1, string? message = null, IReadOnlyList? columns = null) => new($"event-span-{spanId}", T0.AddMilliseconds(startMs + elapsedMs), null, - new MessageTemplate([new TextToken(message ?? $"span {spanId}")]), [], + message ?? $"span {spanId}", new JsonObject(), null, spanId, parentId, T0.AddMilliseconds(startMs), TimeSpan.FromMilliseconds(elapsedMs), columns ?? []); static TraceTreeElement Log(string? spanId, double timestampMs, string message = "log", string? level = null, string? exception = null) => new($"event-log-{timestampMs}-{message}", T0.AddMilliseconds(timestampMs), level, - new MessageTemplate([new TextToken(message)]), [], exception, + message, new JsonObject(), exception, spanId, null, null, null, []); static string Render(params TraceTreeElement[] events) @@ -33,8 +32,8 @@ static string Render(params TraceTreeElement[] events) var output = new StringWriter(); var formatter = TextFormatters.Plain(theme: null, TraceFormatter.OutputTemplate(events.Max(e => e.Columns.Count))); - foreach (var logEvent in TraceFormatter.ToLogEvents(TraceTreeBuilder.Build(events))) - formatter.Format(logEvent, output); + foreach (var eventJson in TraceFormatter.ToEventJson(TraceTreeBuilder.Build(events))) + formatter.Format(eventJson, output); return output.ToString(); } @@ -115,12 +114,8 @@ public void MissingAndEmptyColumnValuesLeaveNoRedundantSpace(object? first) public void TemplateHolesAreFilledFromMessageProperties() { var evt = new TraceTreeElement("event-1", T0.AddMilliseconds(1.5), null, - new MessageTemplate([ - new TextToken("GET "), - new PropertyToken("Route", "{Route}"), - new TextToken(" as "), - new PropertyToken("User", "{User}")]), - [new LogEventProperty("Route", new ScalarValue("/orders"))], + "GET {Route} as {User}", + new JsonObject { ["Route"] = "/orders" }, null, "a", null, T0, TimeSpan.FromMilliseconds(1.5), []); Assert.Equal( diff --git a/test/SeqCli.Tests/PlainText/EventJsonBuilderTests.cs b/test/SeqCli.Tests/PlainText/EventJsonBuilderTests.cs new file mode 100644 index 00000000..0e392eb9 --- /dev/null +++ b/test/SeqCli.Tests/PlainText/EventJsonBuilderTests.cs @@ -0,0 +1,61 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Globalization; +using SeqCli.Data; +using SeqCli.PlainText; +using Superpower.Model; +using Xunit; + +namespace SeqCli.Tests.PlainText; + +public class EventJsonBuilderTests +{ + [Fact] + public void SuppliedValuesAreUsed() + { + var properties = new Dictionary + { + ["@t"] = new TextSpan("2018-02-01T13:00:00.123Z"), + ["@l"] = new TextSpan("WRN"), + ["@m"] = new TextSpan("Hello, world"), + ["@x"] = new TextSpan("EverythingFailedException"), + ["MachineName"] = new TextSpan("TP"), + ["Count"] = 42 + }; + + var remainder = "rem"; + var evt = EventJsonBuilder.FromProperties(properties, remainder); + + Assert.Equal("2018-02-01T13:00:00.1230000+00:00", + DateTimeOffset.Parse((string)evt["@t"]!, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind) + .ToUniversalTime().ToString("o")); + Assert.Equal("Hello, world", (string?)evt["@m"]); + Assert.Equal("WRN", (string?)evt["@l"]); + Assert.Equal("EverythingFailedException", (string?)evt["@x"]); + Assert.Equal(42, (int?)evt["Count"]); + Assert.Equal("TP", (string?)evt["MachineName"]); + Assert.Equal("rem", (string?)evt["@@unmatched"]); + } + + [Fact] + public void MissingValuesAreDefaulted() + { + var evt = EventJsonBuilder.FromProperties(new Dictionary(), null); + + var timestamp = DateTimeOffset.Parse((string)evt["@t"]!, CultureInfo.InvariantCulture, + DateTimeStyles.RoundtripKind); + Assert.True(timestamp > DateTimeOffset.Now.AddSeconds(-5)); + Assert.False(evt.ContainsKey("@m")); + Assert.False(evt.ContainsKey("@l")); + Assert.False(evt.ContainsKey("@x")); + } + + [Fact] + public void DateTimeOffsetTimestampsAreAccepted() + { + var then = DateTimeOffset.Now.AddDays(-5); + var evt = EventJsonBuilder.FromProperties(new Dictionary{["@t"] = then}, null); + Assert.Equal(then.ToString("o", CultureInfo.InvariantCulture), (string?)evt["@t"]); + } +} diff --git a/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs b/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs index 0993251f..fa251cb9 100644 --- a/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs +++ b/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Globalization; -using SeqCli.PlainText; using SeqCli.PlainText.Extraction; using SeqCli.PlainText.Patterns; using Xunit; diff --git a/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs b/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs index 171d6aba..fac72167 100644 --- a/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs +++ b/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs @@ -1,5 +1,4 @@ -using System; -using System.Linq; +using System.Linq; using SeqCli.PlainText.Patterns; using Superpower; using Xunit; diff --git a/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs b/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs deleted file mode 100644 index 75eaf8d5..00000000 --- a/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs +++ /dev/null @@ -1,56 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using SeqCli.PlainText.LogEvents; -using Serilog.Events; -using Superpower.Model; -using Xunit; - -namespace SeqCli.Tests.PlainText; - -public class LogEventBuilderTests -{ - [Fact] - public void SuppliedValuesAreUsed() - { - var properties = new Dictionary - { - ["@t"] = new TextSpan("2018-02-01T13:00:00.123Z"), - ["@l"] = new TextSpan("WRN"), - ["@m"] = new TextSpan("Hello, world"), - ["@x"] = new TextSpan("EverythingFailedException"), - ["MachineName"] = new TextSpan("TP"), - ["Count"] = 42 - }; - - var remainder = "rem"; - var evt = LogEventBuilder.FromProperties(properties, remainder); - - Assert.Equal("2018-02-01T13:00:00.1230000+00:00", evt.Timestamp.ToString("o")); - Assert.Equal("Hello, world", evt.RenderMessage()); - Assert.Equal(LogEventLevel.Warning, evt.Level); - Assert.Equal("EverythingFailedException", evt.Exception?.ToString()); - Assert.Equal(42, ((ScalarValue)evt.Properties["Count"]).Value); - Assert.Equal("TP", ((ScalarValue)evt.Properties["MachineName"]).Value!.ToString()); - Assert.Equal("rem", ((ScalarValue)evt.Properties["@unmatched"]).Value!.ToString()); - } - - [Fact] - public void MissingValuesAreDefaulted() - { - var evt = LogEventBuilder.FromProperties(new Dictionary(), null); - - Assert.True(evt.Timestamp > DateTimeOffset.Now.AddSeconds(-5)); - Assert.Equal("", evt.RenderMessage()); - Assert.Equal(LogEventLevel.Information, evt.Level); - Assert.Null(evt.Exception); - } - - [Fact] - public void DateTimeOffsetTimestampsAreAccepted() - { - var then = DateTimeOffset.Now.AddDays(-5); - var evt = LogEventBuilder.FromProperties(new Dictionary{["@t"] = then}, null); - Assert.Equal(then, evt.Timestamp); - } -} \ No newline at end of file diff --git a/test/SeqCli.Tests/PlainText/StaticMessageTemplateReaderTests.cs b/test/SeqCli.Tests/PlainText/StaticMessageTemplateReaderTests.cs index 29f99471..9c68a1d5 100644 --- a/test/SeqCli.Tests/PlainText/StaticMessageTemplateReaderTests.cs +++ b/test/SeqCli.Tests/PlainText/StaticMessageTemplateReaderTests.cs @@ -1,4 +1,5 @@ -using System.Threading.Tasks; +#nullable enable +using System.Threading.Tasks; using SeqCli.Ingestion; using SeqCli.Tests.Support; using Xunit; @@ -10,11 +11,13 @@ public class StaticMessageTemplateReaderTests [Fact] public async Task ReaderSubstitutesMessageTemplate() { - var evt = Some.LogEvent(); + var evt = Some.EventJson(); + evt["@m"] = "A pre-rendered message"; const string mt = "This is a message template"; - var reader = new FixedLogEventReader(new ReadResult(evt, false)); + var reader = new FixedEventReader(new ReadResult(evt, false)); var wrapper = new StaticMessageTemplateReader(reader, mt); var result = await wrapper.TryReadAsync(); - Assert.Equal(mt, result.LogEvent.MessageTemplate.Text); + Assert.Equal(mt, (string?)result.Document!["@mt"]); + Assert.False(result.Document.ContainsKey("@m")); } -} \ No newline at end of file +} diff --git a/test/SeqCli.Tests/Sample/SimulationEventTests.cs b/test/SeqCli.Tests/Sample/SimulationEventTests.cs new file mode 100644 index 00000000..0c69a6e4 --- /dev/null +++ b/test/SeqCli.Tests/Sample/SimulationEventTests.cs @@ -0,0 +1,108 @@ +#nullable enable +using System; +using System.Linq; +using SeqCli.Sample.Ingestion; +using Serilog; +using Serilog.Events; +using Xunit; + +namespace SeqCli.Tests.Sample; + +public class SimulationEventTests +{ + static LogEvent CaptureEvent(Action log) + { + LogEvent? captured = null; + var logger = new LoggerConfiguration() + .MinimumLevel.Verbose() + .WriteTo.Sink(new CapturingSink(evt => captured = evt)) + .CreateLogger(); + log(logger); + return captured ?? throw new InvalidOperationException("No event was captured."); + } + + class CapturingSink(Action capture) : Serilog.Core.ILogEventSink + { + public void Emit(LogEvent logEvent) => capture(logEvent); + } + + [Fact] + public void EventFieldsMapToTheEmissionSchema() + { + var evt = CaptureEvent(log => log.Warning(new Exception("Boom!"), "Hello, {Name}!", "world")); + var eventJson = SimulationEvent.ToJsonObject(evt); + + Assert.Equal(evt.Timestamp.ToString("o"), (string?)eventJson["@t"]); + Assert.Equal("Hello, {Name}!", (string?)eventJson["@mt"]); + Assert.Equal("Warning", (string?)eventJson["@l"]); + Assert.StartsWith("System.Exception: Boom!", (string?)eventJson["@x"]); + Assert.Equal("world", (string?)eventJson["Name"]); + } + + [Fact] + public void InformationLevelsAreOmitted() + { + var evt = CaptureEvent(log => log.Information("Hello")); + + Assert.False(SimulationEvent.ToJsonObject(evt).ContainsKey("@l")); + } + + [Fact] + public void StructuredValuesSerializeAsJson() + { + var evt = CaptureEvent(log => log.Information("{@Order} {Items}", + new { Id = 7, Total = 4.5 }, new[] { "a", "b" })); + var eventJson = SimulationEvent.ToJsonObject(evt); + + Assert.Equal(7, (int?)eventJson["Order"]!["Id"]); + Assert.Equal(4.5, (double?)eventJson["Order"]!["Total"]); + Assert.Equal(new[] { "a", "b" }, eventJson["Items"]!.AsArray().Select(i => (string?)i).ToArray()); + } + + [Fact] + public void SerilogTracingSpanPropertiesAreLifted() + { + var start = DateTime.UtcNow.AddMilliseconds(-25); + var evt = CaptureEvent(log => log + .ForContext("SpanStartTimestamp", start) + .ForContext("ParentSpanId", "8899aabbccddeeff") + .Information("GET /orders")); + var eventJson = SimulationEvent.ToJsonObject(evt); + + Assert.Equal(start, eventJson["@st"]!.GetValue()); + Assert.Equal("8899aabbccddeeff", (string?)eventJson["@ps"]); + Assert.False(eventJson.ContainsKey("SpanStartTimestamp")); + Assert.False(eventJson.ContainsKey("ParentSpanId")); + } + + [Fact] + public void MetricDefinitionsProduceMetricSamples() + { + var evt = CaptureEvent(log => log + .ForContext(MetricsMapping.SurrogateDefinitionsProperty, new { roasted_kg = new { unit = "kg" } }, destructureObjects: true) + .ForContext("roasted_kg", 42.5) + .Information("Metrics sampled")); + + Assert.True(MetricsMapping.TryGetMetricSampleJson(evt, out var eventJson)); + Assert.Equal("kg", (string?)eventJson["@d"]!["roasted_kg"]!["unit"]); + Assert.Equal(42.5, (double?)eventJson["roasted_kg"]); + Assert.False(eventJson.ContainsKey("@mt")); + Assert.False(eventJson.ContainsKey("@l")); + } + + [Fact] + public void PlainEventsAreNotMetricSamples() + { + var evt = CaptureEvent(log => log.Information("Hello")); + + Assert.False(MetricsMapping.TryGetMetricSampleJson(evt, out _)); + } + + [Fact] + public void PropertyNamesBeginningWithAtAreEscaped() + { + var evt = CaptureEvent(log => log.ForContext("@evil", "value").Information("Hello")); + + Assert.Equal("value", (string?)SimulationEvent.ToJsonObject(evt)["@@evil"]); + } +} diff --git a/test/SeqCli.Tests/Support/FixedLogEventReader.cs b/test/SeqCli.Tests/Support/FixedEventReader.cs similarity index 73% rename from test/SeqCli.Tests/Support/FixedLogEventReader.cs rename to test/SeqCli.Tests/Support/FixedEventReader.cs index 538c5b65..2c29398f 100644 --- a/test/SeqCli.Tests/Support/FixedLogEventReader.cs +++ b/test/SeqCli.Tests/Support/FixedEventReader.cs @@ -3,11 +3,11 @@ namespace SeqCli.Tests.Support; -class FixedLogEventReader : ILogEventReader +class FixedEventReader : IEventReader { readonly ReadResult _result; - public FixedLogEventReader(ReadResult result) + public FixedEventReader(ReadResult result) { _result = result; } diff --git a/test/SeqCli.Tests/Support/Some.cs b/test/SeqCli.Tests/Support/Some.cs index 7ba27d31..0a2aa24e 100644 --- a/test/SeqCli.Tests/Support/Some.cs +++ b/test/SeqCli.Tests/Support/Some.cs @@ -1,11 +1,10 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Security.Cryptography; +using System.Text.Json.Nodes; using Seq.Api.Model.Events; using Seq.Api.Model.Shared; -using Serilog.Events; -using Serilog.Parsing; namespace SeqCli.Tests.Support; @@ -15,14 +14,13 @@ static class Some { static readonly RandomNumberGenerator Rng = RandomNumberGenerator.Create(); - public static LogEvent LogEvent() + public static JsonObject EventJson() { - return new LogEvent( - DateTimeOffset.UtcNow, - LogEventLevel.Information, - null, - new MessageTemplateParser().Parse("Test"), - Enumerable.Empty()); + return new JsonObject + { + ["@t"] = DateTimeOffset.UtcNow.ToString("o"), + ["@mt"] = "Test" + }; } public static string String() @@ -41,7 +39,7 @@ public static byte[] Bytes(int count) Rng.GetBytes(bytes); return bytes; } - + public static EventEntity MakeEvent(Action? configure = null) { var evt = new EventEntity @@ -58,4 +56,4 @@ public static EventEntity MakeEvent(Action? configure = null) public static List MakeProperties(params (string Name, object? Value)[] items) => items.Select(i => new EventPropertyPart(i.Name, i.Value)).ToList(); -} \ No newline at end of file +} diff --git a/test/SeqCli.Tests/Syntax/AliasedExpressionParserTests.cs b/test/SeqCli.Tests/Syntax/AliasedExpressionParserTests.cs index 43cceeca..7699f513 100644 --- a/test/SeqCli.Tests/Syntax/AliasedExpressionParserTests.cs +++ b/test/SeqCli.Tests/Syntax/AliasedExpressionParserTests.cs @@ -1,4 +1,3 @@ -using System; using SeqCli.Syntax; using Xunit; diff --git a/test/SeqCli.Tests/Traces/StructuredMessageTests.cs b/test/SeqCli.Tests/Traces/StructuredMessageTests.cs index 731a0bb8..a89ed699 100644 --- a/test/SeqCli.Tests/Traces/StructuredMessageTests.cs +++ b/test/SeqCli.Tests/Traces/StructuredMessageTests.cs @@ -1,10 +1,8 @@ #nullable enable using System.IO; -using System.Linq; +using System.Text.Json.Nodes; using Newtonsoft.Json.Linq; using SeqCli.Traces; -using Serilog.Events; -using Serilog.Parsing; using Xunit; namespace SeqCli.Tests.Traces; @@ -24,8 +22,8 @@ public void MissingStructuredMessagesReadAsEmpty() { foreach (var cell in new object?[] { null, JValue.CreateNull() }) { - var (message, properties) = StructuredMessage.Read(cell); - Assert.Empty(message.Tokens); + var (mt, properties) = StructuredMessage.Read(cell); + Assert.Equal("", mt); Assert.Empty(properties); } } @@ -33,34 +31,38 @@ public void MissingStructuredMessagesReadAsEmpty() [Fact] public void TextTokensAreRead() { - var (message, properties) = StructuredMessage.Read(new JArray("Hello", ", ", "world")); + var (mt, properties) = StructuredMessage.Read(new JArray("Hello", ", ", "world")); - Assert.Equal("Hello, world", message.Text); - Assert.All(message.Tokens, token => Assert.IsType(token)); + Assert.Equal("Hello, world", mt); Assert.Empty(properties); } + [Fact] + public void LiteralBracesAreEscapedInTemplateText() + { + var (mt, _) = StructuredMessage.Read(new JArray("a {not-a-hole} b")); + Assert.Equal("a {{not-a-hole}} b", mt); + } + [Fact] public void HolesCarryRawTextAndValues() { - var (message, properties) = StructuredMessage.Read(new JArray( + var (mt, properties) = StructuredMessage.Read(new JArray( "Hello, ", Hole("Name", "{Name:x}", "World"), "!")); - Assert.Equal("Hello, {Name:x}!", message.Text); - var hole = Assert.IsType(message.Tokens.ElementAt(1)); - Assert.Equal("Name", hole.PropertyName); + Assert.Equal("Hello, {Name:x}!", mt); var property = Assert.Single(properties); - Assert.Equal("Name", property.Name); - Assert.Equal(new ScalarValue("World"), property.Value); + Assert.Equal("Name", property.Key); + Assert.Equal("World", (string?)property.Value); } [Fact] public void HolesWithoutValuesContributeNoProperties() { - var (message, properties) = StructuredMessage.Read(new JArray(Hole("Name"))); + var (mt, properties) = StructuredMessage.Read(new JArray(Hole("Name"))); - Assert.Equal("{Name}", message.Text); + Assert.Equal("{Name}", mt); Assert.Empty(properties); } @@ -74,46 +76,55 @@ public void DuplicateHolesContributeASingleProperty() } [Fact] - public void ScalarHoleValuesAreUnwrapped() + public void ScalarHoleValuesAreRead() { var (_, properties) = StructuredMessage.Read(new JArray(Hole("Count", value: 42L))); - var scalar = Assert.IsType(Assert.Single(properties).Value); - Assert.Equal(42L, scalar.Value); + Assert.Equal(42L, (long?)Assert.Single(properties).Value); } [Fact] - public void StructuredHoleValuesBecomeStructures() + public void StructuredHoleValuesBecomeObjects() { var (_, properties) = StructuredMessage.Read(new JArray( Hole("Order", value: new JObject(new JProperty("Id", 7))))); - var structure = Assert.IsType(Assert.Single(properties).Value); - Assert.Equal("Id", Assert.Single(structure.Properties).Name); + var structure = Assert.IsType(Assert.Single(properties).Value); + Assert.Equal(7, (int?)structure["Id"]); + } + + [Fact] + public void DottedHoleNamesBecomeNestedObjects() + { + var (mt, properties) = StructuredMessage.Read(new JArray( + Hole("user.name", value: "Barney"))); + + Assert.Equal("{user.name}", mt); + var user = Assert.IsType(properties["user"]); + Assert.Equal("Barney", (string?)user["name"]); } [Fact] public void TrailingWhitespaceIsTrimmed() { - var (message, _) = StructuredMessage.Read(new JArray("Hi ", "}", " \n")); + var (mt, _) = StructuredMessage.Read(new JArray("Hi ", "}", " \n")); - Assert.Equal("Hi }", message.Text); + Assert.Equal("Hi }}", mt); } [Fact] public void WhitespaceOnlyMessagesReadAsEmpty() { - var (message, _) = StructuredMessage.Read(new JArray(" ")); + var (mt, _) = StructuredMessage.Read(new JArray(" ")); - Assert.Empty(message.Tokens); + Assert.Equal("", mt); } [Fact] public void TrailingHolesAreNotTrimmed() { - var (message, _) = StructuredMessage.Read(new JArray("Took ", Hole("Elapsed"))); - - Assert.Equal("Took {Elapsed}", message.Text); + var (mt, _) = StructuredMessage.Read(new JArray("Took ", Hole("Elapsed"))); + Assert.Equal("Took {Elapsed}", mt); } [Fact] diff --git a/test/SeqCli.Tests/Traces/TraceQueryTests.cs b/test/SeqCli.Tests/Traces/TraceQueryTests.cs index fb282be5..84a5baab 100644 --- a/test/SeqCli.Tests/Traces/TraceQueryTests.cs +++ b/test/SeqCli.Tests/Traces/TraceQueryTests.cs @@ -3,7 +3,6 @@ using Newtonsoft.Json.Linq; using Seq.Api.Model.Data; using SeqCli.Traces; -using Serilog.Events; using Xunit; namespace SeqCli.Tests.Traces; @@ -85,7 +84,7 @@ public void SpanRowsAreRead() Assert.Equal("event-1", evt.Id); Assert.Equal(timestamp, evt.Timestamp); Assert.Equal("INFO", evt.Level); - Assert.Equal("Hello!", evt.MessageTemplate.Text); + Assert.Equal("Hello!", evt.MessageTemplate); Assert.Empty(evt.TemplateProperties); Assert.Null(evt.Exception); Assert.Equal("0011223344556677", evt.SpanId); @@ -157,10 +156,10 @@ public void StructuredMessageHolesBecomeTemplatePropertiesAndValues() var evt = Assert.Single(TraceQuery.ReadEvents(result, includeExceptions: true, [])); - Assert.Equal("Hello, {Name}!", evt.MessageTemplate.Text); + Assert.Equal("Hello, {Name}!", evt.MessageTemplate); var property = Assert.Single(evt.TemplateProperties); - Assert.Equal("Name", property.Name); - Assert.Equal(new ScalarValue("World"), property.Value); + Assert.Equal("Name", property.Key); + Assert.Equal("World", (string?)property.Value); } [Fact] diff --git a/test/SeqCli.Tests/Traces/TraceTreeBuilderTests.cs b/test/SeqCli.Tests/Traces/TraceTreeBuilderTests.cs index 2511109f..2f16c2f9 100644 --- a/test/SeqCli.Tests/Traces/TraceTreeBuilderTests.cs +++ b/test/SeqCli.Tests/Traces/TraceTreeBuilderTests.cs @@ -1,9 +1,8 @@ #nullable enable using System; using System.Linq; +using System.Text.Json.Nodes; using SeqCli.Traces; -using Serilog.Events; -using Serilog.Parsing; using Xunit; namespace SeqCli.Tests.Traces; @@ -14,12 +13,12 @@ public class TraceTreeBuilderTests static TraceTreeElement Span(string spanId, string? parentId, double startMs = 0, double elapsedMs = 1) => new($"event-span-{spanId}", T0.AddMilliseconds(startMs + elapsedMs), null, - new MessageTemplate([new TextToken($"span {spanId}")]), [], null, + $"span {spanId}", new JsonObject(), null, spanId, parentId, T0.AddMilliseconds(startMs), TimeSpan.FromMilliseconds(elapsedMs), []); static TraceTreeElement Log(string? spanId, double timestampMs, string message = "log") => new($"event-log-{timestampMs}-{message}", T0.AddMilliseconds(timestampMs), null, - new MessageTemplate([new TextToken(message)]), [], null, + message, new JsonObject(), null, spanId, null, null, null, []); [Fact] @@ -76,7 +75,7 @@ public void SiblingSpansAndLogsInterleaveChronologically() var root = Assert.Single(roots); Assert.Equal( ["first", "span c", "span b", "last"], - root.Children.Select(c => c.Element.MessageTemplate.Text).ToArray()); + root.Children.Select(c => c.Element.MessageTemplate).ToArray()); } [Fact] @@ -104,7 +103,7 @@ public void OrphanLogsBecomeRootsAlongsideSpans() Assert.Equal( ["span a", "first", "second", "span b"], - roots.Select(r => r.Element.MessageTemplate.Text).ToArray()); + roots.Select(r => r.Element.MessageTemplate).ToArray()); Assert.All(roots, r => Assert.Empty(r.Children)); } @@ -117,7 +116,7 @@ public void OrphanLogsWithNoRootSpanRemainAtRootLevel() ]); Assert.Equal(2, roots.Count); - Assert.Equal(["first", "second"], roots.Select(r => r.Element.MessageTemplate.Text).ToArray()); + Assert.Equal(["first", "second"], roots.Select(r => r.Element.MessageTemplate).ToArray()); } [Fact] diff --git a/test/SeqCli.Tests/Traces/TraceTreeJObjectConverterTests.cs b/test/SeqCli.Tests/Traces/TraceTreeJObjectConverterTests.cs index b0dfa1c9..6eb90582 100644 --- a/test/SeqCli.Tests/Traces/TraceTreeJObjectConverterTests.cs +++ b/test/SeqCli.Tests/Traces/TraceTreeJObjectConverterTests.cs @@ -2,10 +2,9 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text.Json.Nodes; using Newtonsoft.Json.Linq; using SeqCli.Traces; -using Serilog.Events; -using Serilog.Parsing; using Xunit; namespace SeqCli.Tests.Traces; @@ -20,14 +19,14 @@ static TraceTreeElement Span(string spanId, string? parentId, double startMs = 0 string? message = null, string? level = null, string? exception = null, IReadOnlyList? columns = null) => new($"event-span-{spanId}", T0.AddMilliseconds(startMs + elapsedMs), level, - new MessageTemplate([new TextToken(message ?? $"span {spanId}")]), [], + message ?? $"span {spanId}", new JsonObject(), exception, spanId, parentId, T0.AddMilliseconds(startMs), TimeSpan.FromMilliseconds(elapsedMs), columns ?? []); static TraceTreeElement Log(string? spanId, double timestampMs, string message = "log", string? level = null, string? exception = null, IReadOnlyList? columns = null) => new($"event-log-{timestampMs}-{message}", T0.AddMilliseconds(timestampMs), level, - new MessageTemplate([new TextToken(message)]), [], exception, + message, new JsonObject(), exception, spanId, null, null, null, columns ?? []); static JObject ToJson(params TraceTreeElement[] events) => ToJson([], events); @@ -142,30 +141,13 @@ public void LogsWithNoCapturedEnclosingSpanBecomeOrphans() Assert.Equal("uncaptured", (string?)orphans[0]["spanId"]); Assert.Null(orphans[2]["spanId"]); } - - [Fact] - public void LevelsAreNormalizedToFullNames() - { - var document = ToJson( - Span("a", null), - Log("a", 1, level: "warn"), - Log("a", 2, level: "Nonstandard")); - - var children = (JArray)document["root"]!["children"]!; - Assert.Equal("Warning", (string?)children[0]["level"]); - Assert.Equal("Nonstandard", (string?)children[1]["level"]); - } - + [Fact] public void TemplateHolesAreFilledFromMessageProperties() { var evt = new TraceTreeElement("event-1", T0.AddMilliseconds(1.5), null, - new MessageTemplate([ - new TextToken("GET "), - new PropertyToken("Route", "{Route}"), - new TextToken(" as "), - new PropertyToken("User", "{User}")]), - [new LogEventProperty("Route", new ScalarValue("/orders"))], + "GET {Route} as {User}", + new JsonObject { ["Route"] = "/orders" }, null, "a", null, T0, TimeSpan.FromMilliseconds(1.5), []); var document = ToJson(evt);