From adfcb19d2c8b4dc954e807d17a1f76da29804ff4 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 26 Aug 2026 16:28:04 +1000 Subject: [PATCH 01/32] Avoid accidental data trunctation when outputting JSON through Serilog. Assisted-by: Claude:claude-opus-5 --- src/SeqCli/Output/OutputFormat.cs | 4 ++++ test/SeqCli.EndToEnd/Data/trace-tree.clef | 4 ++++ test/SeqCli.EndToEnd/Mcp/McpTraceTestCase.cs | 2 +- test/SeqCli.EndToEnd/Traces/TraceShowTestCase.cs | 12 ++++++++++++ 4 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/SeqCli/Output/OutputFormat.cs b/src/SeqCli/Output/OutputFormat.cs index 2cd46d09..64a5703f 100644 --- a/src/SeqCli/Output/OutputFormat.cs +++ b/src/SeqCli/Output/OutputFormat.cs @@ -170,6 +170,8 @@ public void WriteEntity(Entity entity) var writer = new LoggerConfiguration() .Destructure.With() + // The default limit truncates deeply-nested documents, such as trace trees. + .Destructure.ToMaximumDepth(10000) .Enrich.With() .WriteTo.Console(TextFormatters.Plain(TemplateTheme, "{@m}" + Environment.NewLine)) .CreateLogger(); @@ -200,6 +202,8 @@ public void WriteObject(object value) var writer = new LoggerConfiguration() .Destructure.With() + // The default limit truncates deeply-nested documents, such as trace trees. + .Destructure.ToMaximumDepth(10000) .Enrich.With() .WriteTo.Console(TextFormatters.Plain(TemplateTheme, "{@m}" + Environment.NewLine)) .CreateLogger(); diff --git a/test/SeqCli.EndToEnd/Data/trace-tree.clef b/test/SeqCli.EndToEnd/Data/trace-tree.clef index 465817fa..2f00a000 100644 --- a/test/SeqCli.EndToEnd/Data/trace-tree.clef +++ b/test/SeqCli.EndToEnd/Data/trace-tree.clef @@ -4,3 +4,7 @@ {"@t":"2023-12-20T00:50:00.2Z","@l":"Warning","@tr":"7d4dedcc73b18e449e0e4ea08cbe346d","@sp":"2222222222222222","@mt":"{RowCount} rows retrieved","RowCount":42,"@x":"System.TimeoutException: The query timeout was reached"} {"@t":"2023-12-20T00:50:00.9Z","@st":"2023-12-20T00:50:00.5Z","@tr":"7d4dedcc73b18e449e0e4ea08cbe346d","@sp":"4444444444444444","@ps":"1111111111111111","@m":"Render response"} {"@t":"2023-12-20T00:50:00.95Z","@tr":"7d4dedcc73b18e449e0e4ea08cbe346d","@sp":"9999999999999999","@m":"Orphan log"} +{"@t":"2023-12-20T00:50:00.85Z","@st":"2023-12-20T00:50:00.55Z","@tr":"7d4dedcc73b18e449e0e4ea08cbe346d","@sp":"5555555555555555","@ps":"4444444444444444","@m":"Serialize model"} +{"@t":"2023-12-20T00:50:00.8Z","@st":"2023-12-20T00:50:00.6Z","@tr":"7d4dedcc73b18e449e0e4ea08cbe346d","@sp":"6666666666666666","@ps":"5555555555555555","@m":"Serialize order"} +{"@t":"2023-12-20T00:50:00.75Z","@st":"2023-12-20T00:50:00.65Z","@tr":"7d4dedcc73b18e449e0e4ea08cbe346d","@sp":"7777777777777777","@ps":"6666666666666666","@m":"Format currency"} +{"@t":"2023-12-20T00:50:00.72Z","@st":"2023-12-20T00:50:00.7Z","@tr":"7d4dedcc73b18e449e0e4ea08cbe346d","@sp":"8888888888888888","@ps":"7777777777777777","@m":"Lookup locale"} diff --git a/test/SeqCli.EndToEnd/Mcp/McpTraceTestCase.cs b/test/SeqCli.EndToEnd/Mcp/McpTraceTestCase.cs index c81c4882..9c7b8d81 100644 --- a/test/SeqCli.EndToEnd/Mcp/McpTraceTestCase.cs +++ b/test/SeqCli.EndToEnd/Mcp/McpTraceTestCase.cs @@ -33,7 +33,7 @@ protected override async Task ExecuteAsync(SeqConnection connection, ILogger log }); var text = AssertTextResult(loaded); - Assert.Contains("Loaded 4 span(s)", text); + Assert.Contains("Loaded 8 span(s)", text); var document = AssertStructuredObjectResult(loaded); Assert.Equal(TraceId, document.GetProperty("traceId").GetString()); diff --git a/test/SeqCli.EndToEnd/Traces/TraceShowTestCase.cs b/test/SeqCli.EndToEnd/Traces/TraceShowTestCase.cs index 34707ef3..3fd970c7 100644 --- a/test/SeqCli.EndToEnd/Traces/TraceShowTestCase.cs +++ b/test/SeqCli.EndToEnd/Traces/TraceShowTestCase.cs @@ -100,6 +100,18 @@ public Task ExecuteAsync( ["SELECT * FROM orders", "42 rows retrieved"], ((JArray)query["children"]!).Select(c => (string)c["message"]!).ToArray()); + // Deeply-nested spans must survive JSON serialization; the tree is six spans deep here. + var node = root; + foreach (var message in new[] { "Render response", "Serialize model", "Serialize order", "Format currency", "Lookup locale" }) + { + var children = node["children"] as JArray; + Assert.True(children != null, $"Expected children under `{(string?)node["message"]}` in: {runner.LastRunProcess!.Output}"); + node = (JObject)children!.Single(c => (string?)c["message"] == message); + } + + Assert.Equal("8888888888888888", (string?)node["spanId"]); + Assert.Null(node["children"]); + var orphan = (JObject)Assert.Single((JArray)document["orphans"]!); Assert.Equal("log", (string?)orphan["type"]); Assert.Equal("Orphan log", (string?)orphan["message"]); From 64300d14b5cb2c7137fa42c0be8eb97d590e8f0c Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 26 Aug 2026 16:32:38 +1000 Subject: [PATCH 02/32] Tidy up test --- test/SeqCli.EndToEnd/Traces/TraceShowTestCase.cs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/test/SeqCli.EndToEnd/Traces/TraceShowTestCase.cs b/test/SeqCli.EndToEnd/Traces/TraceShowTestCase.cs index 3fd970c7..5a241dbd 100644 --- a/test/SeqCli.EndToEnd/Traces/TraceShowTestCase.cs +++ b/test/SeqCli.EndToEnd/Traces/TraceShowTestCase.cs @@ -100,13 +100,11 @@ public Task ExecuteAsync( ["SELECT * FROM orders", "42 rows retrieved"], ((JArray)query["children"]!).Select(c => (string)c["message"]!).ToArray()); - // Deeply-nested spans must survive JSON serialization; the tree is six spans deep here. var node = root; - foreach (var message in new[] { "Render response", "Serialize model", "Serialize order", "Format currency", "Lookup locale" }) + foreach (var message in ["Render response", "Serialize model", "Serialize order", "Format currency", "Lookup locale"]) { - var children = node["children"] as JArray; - Assert.True(children != null, $"Expected children under `{(string?)node["message"]}` in: {runner.LastRunProcess!.Output}"); - node = (JObject)children!.Single(c => (string?)c["message"] == message); + var children = Assert.IsType(node["children"]); + node = (JObject)children.Single(c => (string?)c["message"] == message); } Assert.Equal("8888888888888888", (string?)node["spanId"]); From 6ac4531668b77789d6bbff91b5d6b71202baf535 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 26 Aug 2026 16:34:51 +1000 Subject: [PATCH 03/32] Drop a comment --- src/SeqCli/Output/OutputFormat.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/SeqCli/Output/OutputFormat.cs b/src/SeqCli/Output/OutputFormat.cs index 64a5703f..7b9c5ea7 100644 --- a/src/SeqCli/Output/OutputFormat.cs +++ b/src/SeqCli/Output/OutputFormat.cs @@ -170,7 +170,6 @@ public void WriteEntity(Entity entity) var writer = new LoggerConfiguration() .Destructure.With() - // The default limit truncates deeply-nested documents, such as trace trees. .Destructure.ToMaximumDepth(10000) .Enrich.With() .WriteTo.Console(TextFormatters.Plain(TemplateTheme, "{@m}" + Environment.NewLine)) @@ -202,7 +201,6 @@ public void WriteObject(object value) var writer = new LoggerConfiguration() .Destructure.With() - // The default limit truncates deeply-nested documents, such as trace trees. .Destructure.ToMaximumDepth(10000) .Enrich.With() .WriteTo.Console(TextFormatters.Plain(TemplateTheme, "{@m}" + Environment.NewLine)) From 4b64c34ff2e972f5b21b420e201af1e003659ee6 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 26 Aug 2026 16:37:29 +1000 Subject: [PATCH 04/32] More test tidiying --- test/SeqCli.EndToEnd/Traces/TraceShowTestCase.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/SeqCli.EndToEnd/Traces/TraceShowTestCase.cs b/test/SeqCli.EndToEnd/Traces/TraceShowTestCase.cs index 5a241dbd..e85c24fa 100644 --- a/test/SeqCli.EndToEnd/Traces/TraceShowTestCase.cs +++ b/test/SeqCli.EndToEnd/Traces/TraceShowTestCase.cs @@ -101,10 +101,10 @@ public Task ExecuteAsync( ((JArray)query["children"]!).Select(c => (string)c["message"]!).ToArray()); var node = root; - foreach (var message in ["Render response", "Serialize model", "Serialize order", "Format currency", "Lookup locale"]) + foreach (var message in new[] {"Render response", "Serialize model", "Serialize order", "Format currency", "Lookup locale"}) { var children = Assert.IsType(node["children"]); - node = (JObject)children.Single(c => (string?)c["message"] == message); + node = Assert.IsType(Assert.Single(children, c => (string?)c["message"] == message)); } Assert.Equal("8888888888888888", (string?)node["spanId"]); From 519f1047764679422d895e04a11df8fd813d8f45 Mon Sep 17 00:00:00 2001 From: Ashley Date: Sat, 29 Aug 2026 07:50:44 +1000 Subject: [PATCH 05/32] add arm64 support for Windows --- src/SeqCli/SeqCli.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SeqCli/SeqCli.csproj b/src/SeqCli/SeqCli.csproj index fcb51b71..f666ccdd 100644 --- a/src/SeqCli/SeqCli.csproj +++ b/src/SeqCli/SeqCli.csproj @@ -4,7 +4,7 @@ net10.0 seqcli ..\..\asset\SeqCli.ico - win-x64;linux-x64;linux-musl-x64;osx-x64;linux-arm64;linux-musl-arm64;osx-arm64 + win-x64;linux-x64;linux-musl-x64;osx-x64;win-arm64;linux-arm64;linux-musl-arm64;osx-arm64 True True From a0fb7e778bf4f3a2885fb7a0fc429a52252b52e6 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Tue, 1 Sep 2026 15:20:44 +1000 Subject: [PATCH 06/32] Migrate data handling from Serilog's `LogEvent` across to `System.Text.Json.JsonObject`, with the help of Seq.Syntax v2.0. Assisted-by: Claude:claude-fable-5 --- src/SeqCli/Apps/AppLoader.cs | 6 +- src/SeqCli/Apps/Hosting/AppContainer.cs | 3 +- .../Apps/Hosting/SerilogLevelMapping.cs | 41 ++++ src/SeqCli/Cli/Commands/IngestCommand.cs | 22 +- src/SeqCli/Cli/Commands/PrintCommand.cs | 20 +- src/SeqCli/Cli/Commands/TraceCommand.cs | 4 +- src/SeqCli/Csv/CsvWriter.cs | 56 +++-- src/SeqCli/Forwarder/ForwarderModule.cs | 9 +- .../Web/Api/IngestionLogEndpoints.cs | 9 +- src/SeqCli/Ingestion/BatchResult.cs | 10 +- src/SeqCli/Ingestion/EnrichingReader.cs | 19 +- .../{ILogEventReader.cs => IEventReader.cs} | 2 +- src/SeqCli/Ingestion/JsonEventReader.cs | 64 ++++++ src/SeqCli/Ingestion/JsonLogEventReader.cs | 94 --------- src/SeqCli/Ingestion/LogShipper.cs | 40 ++-- src/SeqCli/Ingestion/ReadResult.cs | 15 +- src/SeqCli/Ingestion/SerilogEventJson.cs | 91 ++++++++ .../Ingestion/SerilogTracingConventions.cs | 46 ++++ .../Ingestion/StaticMessageTemplateReader.cs | 32 ++- src/SeqCli/Ingestion/TraceConstants.cs | 8 - src/SeqCli/Mapping/EventEntityJson.cs | 104 +++++++++ src/SeqCli/Mapping/LevelMapping.cs | 126 ++++++----- src/SeqCli/Mapping/MetricsMapping.cs | 8 - src/SeqCli/Mcp/Tools/Search/SearchTools.cs | 17 +- src/SeqCli/Output/FlareTheme.cs | 30 +-- src/SeqCli/Output/OutputFormat.cs | 197 +++--------------- .../Output/StripStructureTypeEnricher.cs | 25 --- src/SeqCli/Output/TextFormatters.cs | 52 ++--- src/SeqCli/Output/TraceFormatter.cs | 49 +++-- src/SeqCli/Output/TracingFunctions.cs | 55 ----- .../PlainText/LogEvents/EventJsonBuilder.cs | 100 +++++++++ .../PlainText/LogEvents/LogEventBuilder.cs | 131 ------------ .../PlainText/LogEvents/TextOnlyException.cs | 32 --- ...EventReader.cs => PlainTextEventReader.cs} | 6 +- .../{ => Sample}/Ingestion/BufferingSink.cs | 30 ++- src/SeqCli/Sample/Ingestion/MetricsMapping.cs | 59 ++++++ src/SeqCli/Sample/Loader/Simulation.cs | 2 +- src/SeqCli/SeqCli.csproj | 6 +- src/SeqCli/Syntax/EventJson.cs | 66 ++++++ .../IEventEnricher.cs} | 27 +-- .../LevelEnricher.cs} | 18 +- .../ScalarPropertyEnricher.cs | 22 +- src/SeqCli/Syntax/SeqCliNameResolver.cs | 20 -- src/SeqCli/Syntax/SeqSyntax.cs | 50 ++++- src/SeqCli/Syntax/V1/TracingFunctions.cs | 58 ++++++ src/SeqCli/Traces/StructuredMessage.cs | 70 +++++-- src/SeqCli/Traces/TraceTreeElement.cs | 6 +- .../Traces/TraceTreeJObjectConverter.cs | 24 +-- src/SeqCli/Util/JsonNetDestructuringPolicy.cs | 91 -------- src/SeqCli/Util/JsonNodes.cs | 45 ++++ src/SeqCli/Util/LogEventPropertyFactory.cs | 33 --- test/SeqCli.Tests/Csv/CsvWriterTests.cs | 6 +- .../Ingestion/SerilogEventJsonTests.cs | 109 ++++++++++ test/SeqCli.Tests/Output/OutputFormatTests.cs | 60 ++++-- .../Output/TextFormattersTests.cs | 86 ++++---- .../Output/TraceFormatterTests.cs | 19 +- .../PlainText/EventJsonBuilderTests.cs | 60 ++++++ .../PlainText/LogEventBuilderTests.cs | 56 ----- .../StaticMessageTemplateReaderTests.cs | 13 +- ...dLogEventReader.cs => FixedEventReader.cs} | 4 +- test/SeqCli.Tests/Support/Some.cs | 22 +- .../Traces/StructuredMessageTests.cs | 55 +++-- test/SeqCli.Tests/Traces/TraceQueryTests.cs | 9 +- .../Traces/TraceTreeBuilderTests.cs | 13 +- .../Traces/TraceTreeJObjectConverterTests.cs | 30 +-- 65 files changed, 1437 insertions(+), 1255 deletions(-) create mode 100644 src/SeqCli/Apps/Hosting/SerilogLevelMapping.cs rename src/SeqCli/Ingestion/{ILogEventReader.cs => IEventReader.cs} (79%) create mode 100644 src/SeqCli/Ingestion/JsonEventReader.cs delete mode 100644 src/SeqCli/Ingestion/JsonLogEventReader.cs create mode 100644 src/SeqCli/Ingestion/SerilogEventJson.cs create mode 100644 src/SeqCli/Ingestion/SerilogTracingConventions.cs delete mode 100644 src/SeqCli/Ingestion/TraceConstants.cs create mode 100644 src/SeqCli/Mapping/EventEntityJson.cs delete mode 100644 src/SeqCli/Mapping/MetricsMapping.cs delete mode 100644 src/SeqCli/Output/StripStructureTypeEnricher.cs delete mode 100644 src/SeqCli/Output/TracingFunctions.cs create mode 100644 src/SeqCli/PlainText/LogEvents/EventJsonBuilder.cs delete mode 100644 src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs delete mode 100644 src/SeqCli/PlainText/LogEvents/TextOnlyException.cs rename src/SeqCli/PlainText/{PlainTextLogEventReader.cs => PlainTextEventReader.cs} (86%) rename src/SeqCli/{ => Sample}/Ingestion/BufferingSink.cs (50%) create mode 100644 src/SeqCli/Sample/Ingestion/MetricsMapping.cs create mode 100644 src/SeqCli/Syntax/EventJson.cs rename src/SeqCli/{Util/TextException.cs => Syntax/IEventEnricher.cs} (60%) rename src/SeqCli/{Output/RedundantEventTypeRemovalEnricher.cs => Syntax/LevelEnricher.cs} (64%) rename src/SeqCli/{Ingestion => Syntax}/ScalarPropertyEnricher.cs (59%) delete mode 100644 src/SeqCli/Syntax/SeqCliNameResolver.cs create mode 100644 src/SeqCli/Syntax/V1/TracingFunctions.cs delete mode 100644 src/SeqCli/Util/JsonNetDestructuringPolicy.cs create mode 100644 src/SeqCli/Util/JsonNodes.cs delete mode 100644 src/SeqCli/Util/LogEventPropertyFactory.cs create mode 100644 test/SeqCli.Tests/Ingestion/SerilogEventJsonTests.cs create mode 100644 test/SeqCli.Tests/PlainText/EventJsonBuilderTests.cs delete mode 100644 test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs rename test/SeqCli.Tests/Support/{FixedLogEventReader.cs => FixedEventReader.cs} (73%) diff --git a/src/SeqCli/Apps/AppLoader.cs b/src/SeqCli/Apps/AppLoader.cs index c0a03ff5..143eb91a 100644 --- a/src/SeqCli/Apps/AppLoader.cs +++ b/src/SeqCli/Apps/AppLoader.cs @@ -29,12 +29,14 @@ class AppLoader : IDisposable readonly string _packageBinaryPath; // These are used for interop between the host process and the app. The - // app _must_ be able to load on the unified version. + // app _must_ be able to load on the unified version. Apps built against Seq.Syntax v1 + // bundle their own `Seq.Syntax.dll`, which loads side-by-side with the host's + // `Seq.Syntax.V2.dll`. readonly Assembly[] _contracts = [ typeof(SeqApp).Assembly, typeof(Log).Assembly, - typeof(SerilogExpression).Assembly + typeof(SeqExpression).Assembly ]; public AppLoader(string packageBinaryPath) diff --git a/src/SeqCli/Apps/Hosting/AppContainer.cs b/src/SeqCli/Apps/Hosting/AppContainer.cs index 8a58c2bd..2ba93fde 100644 --- a/src/SeqCli/Apps/Hosting/AppContainer.cs +++ b/src/SeqCli/Apps/Hosting/AppContainer.cs @@ -21,7 +21,6 @@ using Newtonsoft.Json.Linq; using Seq.Apps; using Seq.Apps.LogEvents; -using SeqCli.Mapping; using Serilog; using Serilog.Events; using Serilog.Formatting.Compact.Reader; @@ -143,7 +142,7 @@ LogEvent ReadSerilogEvent(string clef, out string eventId, out uint eventType) if (jobject.TryGetValue("@l", out var levelToken)) { jobject.Remove("@l"); - jobject.Add("@l", new JValue(LevelMapping.ToSerilogLevel(levelToken.Value()!).ToString())); + jobject.Add("@l", new JValue(SerilogLevelMapping.ToSerilogLevel(levelToken.Value()!).ToString())); } SanitizeTraceIdentifiers(jobject); diff --git a/src/SeqCli/Apps/Hosting/SerilogLevelMapping.cs b/src/SeqCli/Apps/Hosting/SerilogLevelMapping.cs new file mode 100644 index 00000000..f37a9f92 --- /dev/null +++ b/src/SeqCli/Apps/Hosting/SerilogLevelMapping.cs @@ -0,0 +1,41 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using SeqCli.Mapping; +using Serilog.Events; + +namespace SeqCli.Apps.Hosting; + +/// +/// Maps level names onto Serilog's level enum for hosted Seq apps relying on the older Serilog `LogEvent`-based +/// interface (newer apps should generally use raw JSON directly). +/// +static class SerilogLevelMapping +{ + public static LogEventLevel ToSerilogLevel(string level) + { + if (string.IsNullOrEmpty(level)) + return LogEventLevel.Information; + + return LevelMapping.ToFullLevelName(level) switch + { + "Trace" or "Verbose" => LogEventLevel.Verbose, + "Debug" => LogEventLevel.Debug, + "Warning" => LogEventLevel.Warning, + "Error" => LogEventLevel.Error, + "Fatal" or "Critical" or "Emergency" or "Alert" or "Panic" => LogEventLevel.Fatal, + _ => LogEventLevel.Information + }; + } +} diff --git a/src/SeqCli/Cli/Commands/IngestCommand.cs b/src/SeqCli/Cli/Commands/IngestCommand.cs index b965ddb3..ba81fa0a 100644 --- a/src/SeqCli/Cli/Commands/IngestCommand.cs +++ b/src/SeqCli/Cli/Commands/IngestCommand.cs @@ -14,18 +14,16 @@ using System; using System.Collections.Generic; +using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; using SeqCli.Api; using SeqCli.Cli.Features; using SeqCli.Config; using SeqCli.Ingestion; -using SeqCli.Mapping; using SeqCli.PlainText; using SeqCli.Syntax; using Serilog; -using Serilog.Core; -using Serilog.Events; namespace SeqCli.Cli.Commands; @@ -84,19 +82,19 @@ protected override async Task Run() { try { - var enrichers = new List(); - + var enrichers = new List(); + if (_level != null) - enrichers.Add(new ScalarPropertyEnricher(LevelMapping.SurrogateLevelProperty, _level)); - + enrichers.Add(new LevelEnricher(_level)); + foreach (var (name, value) in _properties.FlatProperties) enrichers.Add(new ScalarPropertyEnricher(name, value)); - Func? filter = null; + Func? filter = null; if (_filter != null) { var eval = SeqSyntax.CompileExpression(_filter); - filter = evt => Seq.Syntax.Expressions.ExpressionResult.IsTrue(eval(evt)); + filter = evt => eval(evt).IsTrue(); } var config = RuntimeConfigurationLoader.Load(_storagePath); @@ -112,9 +110,9 @@ protected override async Task Run() { using (input) { - ILogEventReader reader = _json - ? new JsonLogEventReader(input) - : new PlainTextLogEventReader(input, _pattern); + IEventReader reader = _json + ? new JsonEventReader(input) + : new PlainTextEventReader(input, _pattern); reader = new EnrichingReader(reader, enrichers); diff --git a/src/SeqCli/Cli/Commands/PrintCommand.cs b/src/SeqCli/Cli/Commands/PrintCommand.cs index 2740297f..0bc67f59 100644 --- a/src/SeqCli/Cli/Commands/PrintCommand.cs +++ b/src/SeqCli/Cli/Commands/PrintCommand.cs @@ -14,16 +14,16 @@ using System; using System.IO; +using System.Text.Json; +using System.Text.Json.Nodes; using System.Threading.Tasks; -using Newtonsoft.Json; -using Seq.Syntax.Expressions; using SeqCli.Cli.Features; using SeqCli.Config; using SeqCli.Ingestion; using SeqCli.Output; +using SeqCli.Syntax; using SeqCli.Util; using Serilog; -using Serilog.Events; namespace SeqCli.Cli.Commands; @@ -61,16 +61,16 @@ protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storage); - Func? filter = null; + Func? filter = null; if (_filter != null) { - if (!SerilogExpression.TryCompile(_filter, out var compiled, out var error)) + if (!SeqSyntax.TryCompileExpression(_filter, out var compiled, out var error)) { Log.Error("The specified filter could not be compiled: {Error}", error); return 1; } - filter = evt => ExpressionResult.IsTrue(compiled(evt)); + filter = evt => compiled(evt).IsTrue(); } var template = _template == null ? null : PrintTemplate.InterpretEscapeChars(_template); @@ -80,7 +80,7 @@ protected override async Task Run() { using (input) { - var reader = new JsonLogEventReader(input); + var reader = new JsonEventReader(input); var isAtEnd = false; do @@ -90,12 +90,12 @@ protected override async Task Run() var result = await reader.TryReadAsync(); isAtEnd = result.IsAtEnd; - if (result.LogEvent != null && (filter == null || filter(result.LogEvent))) - output.WriteLogEvent(result.LogEvent); + if (result.Document != null && (filter == null || filter(result.Document))) + output.WriteEvent(result.Document); } catch (Exception ex) { - if (ex is not JsonReaderException && ex is not InvalidDataException || + if (ex is not JsonException && ex is not InvalidDataException || _invalidDataHandlingFeature.InvalidDataHandling != InvalidDataHandling.Ignore) throw; } diff --git a/src/SeqCli/Cli/Commands/TraceCommand.cs b/src/SeqCli/Cli/Commands/TraceCommand.cs index 6e255069..67541f3b 100644 --- a/src/SeqCli/Cli/Commands/TraceCommand.cs +++ b/src/SeqCli/Cli/Commands/TraceCommand.cs @@ -151,8 +151,8 @@ protected override async Task Run() } else { - foreach (var logEvent in TraceFormatter.ToLogEvents(subtreeRoot != null ? [subtreeRoot] : roots)) - output.WriteLogEvent(logEvent); + foreach (var eventJson in TraceFormatter.ToEventJson(subtreeRoot != null ? [subtreeRoot] : roots)) + output.WriteEvent(eventJson); } return 0; diff --git a/src/SeqCli/Csv/CsvWriter.cs b/src/SeqCli/Csv/CsvWriter.cs index 75f6553a..f87ae2a3 100644 --- a/src/SeqCli/Csv/CsvWriter.cs +++ b/src/SeqCli/Csv/CsvWriter.cs @@ -1,22 +1,34 @@ using System; -using System.Collections.Generic; using System.IO; using Seq.Api.Model.Data; +using Seq.Syntax.Templates.Themes; using SeqCli.Mcp.Data; -using SeqCli.Output; -using Serilog.Templates.Themes; namespace SeqCli.Csv; static class CsvWriter { + // Delimited output is written directly rather than rendered through a template, so styled + // runs are opened and closed here. + static void SetStyle(TextWriter output, TemplateTheme? theme, TemplateThemeStyle style) + { + if (theme?.Open(style) is { } open) + output.Write(open); + } + + static void ResetStyle(TextWriter output, TemplateTheme? theme, TemplateThemeStyle style) + { + if (theme?.Close(style) is { } close) + output.Write(close); + } + public static void WriteQueryResult(QueryResultPart result, Func stringify, TemplateTheme? theme, TextWriter output) { if (!string.IsNullOrWhiteSpace(result.Error)) { - theme?.Set(output, TemplateThemeStyle.Text); + SetStyle(output, theme, TemplateThemeStyle.Text); QueryResultHelper.WriteErrorResult(output, result); - theme?.Reset(output); + ResetStyle(output, theme, TemplateThemeStyle.Text); } var first = true; @@ -40,39 +52,39 @@ static void WriteCell(TextWriter output, TemplateTheme? theme, object? value, Fu } else { - theme?.Set(output, TemplateThemeStyle.TertiaryText); + SetStyle(output, theme, TemplateThemeStyle.TertiaryText); output.Write(','); - theme?.Reset(output); + ResetStyle(output, theme, TemplateThemeStyle.TertiaryText); } - - theme?.Set(output, TemplateThemeStyle.TertiaryText); + + SetStyle(output, theme, TemplateThemeStyle.TertiaryText); output.Write('"'); - theme?.Reset(output); + ResetStyle(output, theme, TemplateThemeStyle.TertiaryText); var valueAsString = stringify(value); - + var dataStyle = isHeadingRow ? TemplateThemeStyle.Name : TemplateThemeStyle.Text; var doubleQuote = valueAsString.IndexOf('"'); while (doubleQuote != -1) { - theme?.Set(output, dataStyle); + SetStyle(output, theme, dataStyle); output.Write(valueAsString[..doubleQuote]); - theme?.Reset(output); - - theme?.Set(output, TemplateThemeStyle.Scalar); + ResetStyle(output, theme, dataStyle); + + SetStyle(output, theme, TemplateThemeStyle.Scalar); output.Write("\"\""); - theme?.Reset(output); + ResetStyle(output, theme, TemplateThemeStyle.Scalar); valueAsString = valueAsString[(doubleQuote + 1)..]; doubleQuote = valueAsString.IndexOf('"'); } - - theme?.Set(output, dataStyle); + + SetStyle(output, theme, dataStyle); output.Write(valueAsString); - theme?.Reset(output); - - theme?.Set(output, TemplateThemeStyle.TertiaryText); + ResetStyle(output, theme, dataStyle); + + SetStyle(output, theme, TemplateThemeStyle.TertiaryText); output.Write('"'); - theme?.Reset(output); + ResetStyle(output, theme, TemplateThemeStyle.TertiaryText); } } \ No newline at end of file diff --git a/src/SeqCli/Forwarder/ForwarderModule.cs b/src/SeqCli/Forwarder/ForwarderModule.cs index 6bb7ef67..3980787d 100644 --- a/src/SeqCli/Forwarder/ForwarderModule.cs +++ b/src/SeqCli/Forwarder/ForwarderModule.cs @@ -21,9 +21,8 @@ using SeqCli.Forwarder.Channel; using SeqCli.Forwarder.Web.Api; using SeqCli.Forwarder.Web.Host; +using SeqCli.Syntax; using Serilog; -using Serilog.Formatting; -using Serilog.Templates; namespace SeqCli.Forwarder; @@ -68,17 +67,17 @@ protected override void Load(ContainerBuilder builder) Log.ForContext().Warning("Configured to expose ingestion log via HTTP API"); builder.RegisterType().As(); - var ingestionLogTemplate = $"[{{@t:o}} {{@l:u3}}] {{@m}}{Environment.NewLine}"; + var ingestionLogTemplate = $"[{{@Timestamp:o}} {{@Level:u3}}] {{@Message}}{Environment.NewLine}"; if (_config.Forwarder.Diagnostics.IngestionLogShowDetail) { Log.ForContext().Warning("Including full client, payload, and error detail in the ingestion log"); ingestionLogTemplate += $"{{#if ClientHostIP is not null}}Client IP address: {{ClientHostIP}}{Environment.NewLine}{{#end}}" + $"{{#if DocumentStart is not null}}First {{StartToLog}} characters of payload: {{DocumentStart:l}}{Environment.NewLine}{{#end}}" + - "{@x}"; + "{@Exception}"; } - builder.Register(_ => new ExpressionTemplate(ingestionLogTemplate)).As(); + builder.Register(_ => SeqSyntax.ParseTemplate(ingestionLogTemplate)); } builder.Register(c => diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs index cf30acfb..eb98cc58 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs @@ -16,17 +16,18 @@ using System.Text; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; +using Seq.Syntax.Templates; using SeqCli.Forwarder.Diagnostics; -using Serilog.Formatting; +using SeqCli.Ingestion; namespace SeqCli.Forwarder.Web.Api; class IngestionLogEndpoints : IMapEndpoints { - readonly ITextFormatter _formatter; + readonly ExpressionTemplate _formatter; readonly Encoding _utf8 = new UTF8Encoding(false); - public IngestionLogEndpoints(ITextFormatter formatter) + public IngestionLogEndpoints(ExpressionTemplate formatter) { _formatter = formatter; } @@ -45,7 +46,7 @@ public void MapEndpoints(WebApplication app) using var log = new StringWriter(); foreach (var logEvent in events) { - _formatter.Format(logEvent, log); + _formatter.Format(SerilogEventJson.ToEventJson(logEvent), log); } return Results.Content(log.ToString(), "text/plain", _utf8); diff --git a/src/SeqCli/Ingestion/BatchResult.cs b/src/SeqCli/Ingestion/BatchResult.cs index 0c4b52ec..daef0697 100644 --- a/src/SeqCli/Ingestion/BatchResult.cs +++ b/src/SeqCli/Ingestion/BatchResult.cs @@ -1,15 +1,15 @@ -using Serilog.Events; +using System.Text.Json.Nodes; namespace SeqCli.Ingestion; struct BatchResult { - public LogEvent[] LogEvents { get; } + public JsonObject[] Documents { get; } public bool IsLast { get; } - public BatchResult(LogEvent[] logEvents, bool isLast) + public BatchResult(JsonObject[] documents, bool isLast) { - LogEvents = logEvents; + Documents = documents; IsLast = isLast; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Ingestion/EnrichingReader.cs b/src/SeqCli/Ingestion/EnrichingReader.cs index 198ab234..63a25ca3 100644 --- a/src/SeqCli/Ingestion/EnrichingReader.cs +++ b/src/SeqCli/Ingestion/EnrichingReader.cs @@ -1,18 +1,18 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; -using Serilog.Core; +using SeqCli.Syntax; namespace SeqCli.Ingestion; -class EnrichingReader : ILogEventReader +class EnrichingReader : IEventReader { - readonly ILogEventReader _inner; - readonly IReadOnlyCollection _enrichers; + readonly IEventReader _inner; + readonly IReadOnlyCollection _enrichers; public EnrichingReader( - ILogEventReader inner, - IReadOnlyCollection enrichers) + IEventReader inner, + IReadOnlyCollection enrichers) { _inner = inner ?? throw new ArgumentNullException(nameof(inner)); _enrichers = enrichers ?? throw new ArgumentNullException(nameof(enrichers)); @@ -22,13 +22,12 @@ public async Task TryReadAsync() { var result = await _inner.TryReadAsync(); - if (result.LogEvent != null) + if (result.Document != null) { foreach (var enricher in _enrichers) - // We're breaking the nullability contract of `ILogEventEnricher.Enrich()`, here. - enricher.Enrich(result.LogEvent, null!); + enricher.Enrich(result.Document); } return result; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Ingestion/ILogEventReader.cs b/src/SeqCli/Ingestion/IEventReader.cs similarity index 79% rename from src/SeqCli/Ingestion/ILogEventReader.cs rename to src/SeqCli/Ingestion/IEventReader.cs index a92b09b9..0ca24530 100644 --- a/src/SeqCli/Ingestion/ILogEventReader.cs +++ b/src/SeqCli/Ingestion/IEventReader.cs @@ -2,7 +2,7 @@ namespace SeqCli.Ingestion; -interface ILogEventReader +interface IEventReader { Task TryReadAsync(); } \ No newline at end of file diff --git a/src/SeqCli/Ingestion/JsonEventReader.cs b/src/SeqCli/Ingestion/JsonEventReader.cs new file mode 100644 index 00000000..b7a102f6 --- /dev/null +++ b/src/SeqCli/Ingestion/JsonEventReader.cs @@ -0,0 +1,64 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Globalization; +using System.IO; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using SeqCli.PlainText.Framing; +using Superpower; +using Superpower.Model; + +namespace SeqCli.Ingestion; + +class JsonEventReader : IEventReader +{ + static readonly TimeSpan TrailingLineArrivalDeadline = TimeSpan.FromMilliseconds(10); + + readonly FrameReader _reader; + + public JsonEventReader(TextReader input) + { + _reader = new FrameReader( + input ?? throw new ArgumentNullException(nameof(input)), + Parse.Return(TextSpan.None), + TrailingLineArrivalDeadline); + } + + public async Task TryReadAsync() + { + var frame = await _reader.TryReadAsync(); + if (!frame.HasValue) + return new ReadResult(null, frame.IsAtEnd); + + if (frame.IsOrphan) + throw new InvalidDataException($"A line arrived late or could not be parsed: `{frame.Value.Trim()}`."); + + return new ReadResult(ReadFromJson(frame.Value), frame.IsAtEnd); + } + + public static JsonObject ReadFromJson(string json) + { + if (JsonNode.Parse(json) is not JsonObject eventJson) + throw new InvalidDataException($"The line is not a JSON object: `{json.Trim()}`."); + + if (!eventJson.ContainsKey("@t")) + eventJson["@t"] = DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture); + + SerilogTracingConventions.LiftSpanProperties(eventJson); + + return eventJson; + } +} diff --git a/src/SeqCli/Ingestion/JsonLogEventReader.cs b/src/SeqCli/Ingestion/JsonLogEventReader.cs deleted file mode 100644 index 719da10c..00000000 --- a/src/SeqCli/Ingestion/JsonLogEventReader.cs +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright © Datalust and contributors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Globalization; -using System.IO; -using System.Threading.Tasks; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using SeqCli.Mapping; -using SeqCli.PlainText.Framing; -using Serilog.Events; -using Serilog.Formatting.Compact.Reader; -using Superpower; -using Superpower.Model; - -namespace SeqCli.Ingestion; - -class JsonLogEventReader : ILogEventReader -{ - static readonly TimeSpan TrailingLineArrivalDeadline = TimeSpan.FromMilliseconds(10); - static readonly JsonSerializer _serializer = JsonSerializer.Create(new JsonSerializerSettings - { - DateParseHandling = DateParseHandling.None, - Culture = CultureInfo.InvariantCulture - }); - - readonly FrameReader _reader; - - public JsonLogEventReader(TextReader input) - { - _reader = new FrameReader( - input ?? throw new ArgumentNullException(nameof(input)), - Parse.Return(TextSpan.None), - TrailingLineArrivalDeadline); - } - - public async Task TryReadAsync() - { - var frame = await _reader.TryReadAsync(); - if (!frame.HasValue) - return new ReadResult(null, frame.IsAtEnd); - - if (frame.IsOrphan) - throw new InvalidDataException($"A line arrived late or could not be parsed: `{frame.Value.Trim()}`."); - - var frameValue = new JsonTextReader(new StringReader(frame.Value)); - if (!(_serializer.Deserialize(frameValue) is JObject jobject)) - throw new InvalidDataException($"The line is not a JSON object: `{frame.Value.Trim()}`."); - - var evt = ReadFromJObject(jobject); - return new ReadResult(evt, frame.IsAtEnd); - } - - public static LogEvent ReadFromJson(string json) - { - var frameValue = new JsonTextReader(new StringReader(json)); - if (_serializer.Deserialize(frameValue) is not JObject jObject) - throw new InvalidDataException($"The line is not a JSON object: `{json.Trim()}`."); - - return ReadFromJObject(jObject); - } - - static LogEvent ReadFromJObject(JObject jObject) - { - if (!jObject.TryGetValue("@t", out _)) - jObject.Add("@t", new JValue(DateTime.UtcNow.ToString("O"))); - - if (jObject.TryGetValue("@l", out var levelToken)) - { - var originalLevel = levelToken.Value()!; - jObject.Remove("@l"); - - var serilogLevel = LevelMapping.ToSerilogLevel(originalLevel); - if (serilogLevel != LogEventLevel.Information) - jObject.Add("@l", new JValue(serilogLevel.ToString())); - - jObject.Add(LevelMapping.SurrogateLevelProperty, originalLevel); - } - - return LogEventReader.ReadFromJObject(jObject); - } -} \ No newline at end of file diff --git a/src/SeqCli/Ingestion/LogShipper.cs b/src/SeqCli/Ingestion/LogShipper.cs index f0a19741..68674fee 100644 --- a/src/SeqCli/Ingestion/LogShipper.cs +++ b/src/SeqCli/Ingestion/LogShipper.cs @@ -19,22 +19,18 @@ using System.Net.Http; using System.Net.Http.Headers; using System.Text; +using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; using Newtonsoft.Json; using Seq.Api; using SeqCli.Api; -using SeqCli.Output; using Serilog; -using Serilog.Events; -using Serilog.Formatting; namespace SeqCli.Ingestion; static class LogShipper { - static readonly ITextFormatter JsonFormatter = TextFormatters.Json(null); - public static async Task ShipBufferAsync( SeqConnection connection, string? apiKey, @@ -49,7 +45,7 @@ public static async Task ShipBufferAsync( ContentType = new MediaTypeHeaderValue(ApiConstants.ClefMediaType, "utf-8") } }; - + var retries = 0; while (true) { @@ -87,22 +83,22 @@ public static async Task ShipBufferAsync( { sendFailureLog.Error(ex, "Failed to ship a batch"); } - + var millisecondsDelay = (int)Math.Min(Math.Pow(2, retries) * 2000, 60000); sendFailureLog.Information("Backing off connection schedule; will retry in {MillisecondsDelay}", millisecondsDelay); await Task.Delay(millisecondsDelay, cancellationToken); retries += 1; } } - + public static async Task ShipEventsAsync( SeqConnection connection, string? apiKey, - ILogEventReader reader, + IEventReader reader, InvalidDataHandling invalidDataHandling, SendFailureHandling sendFailureHandling, int batchSize, - Func? filter, + Func? filter, CancellationToken cancellationToken) { const int maxEmptyBatchWaitMS = 2000; @@ -116,10 +112,10 @@ public static async Task ShipEventsAsync( var statusCode = await SendBatchAsync( connection, apiKey, - batch.LogEvents, + batch.Documents, sendFailureHandling != SendFailureHandling.Ignore ? Log.Logger : null, cancellationToken); - + sendSucceeded = (int)statusCode is >= 200 and < 300; } catch (Exception ex) @@ -146,7 +142,7 @@ public static async Task ShipEventsAsync( if (batch.IsLast) break; - + batch = await ReadBatchAsync(reader, filter, batchSize, invalidDataHandling, maxEmptyBatchWaitMS); } @@ -154,15 +150,15 @@ public static async Task ShipEventsAsync( } static async Task ReadBatchAsync( - ILogEventReader reader, - Func? filter, + IEventReader reader, + Func? filter, int count, InvalidDataHandling invalidDataHandling, int maxWaitMS) { - var batch = new List(); + var batch = new List(); var isLast = false; - + // Avoid consuming stacks of CPU unnecessarily when there's no work to do. We do eventually yield // an empty batch, because level switching relies on this. var totalWaitMS = 0; @@ -175,7 +171,7 @@ static async Task ReadBatchAsync( { var rr = await reader.TryReadAsync(); isLast = rr.IsAtEnd; - var evt = rr.LogEvent; + var evt = rr.Document; if (evt == null) { if (isLast || batch.Count != 0 || totalWaitMS > maxWaitMS) @@ -195,7 +191,7 @@ static async Task ReadBatchAsync( } catch (Exception ex) { - if (ex is JsonReaderException || ex is InvalidDataException) + if (ex is System.Text.Json.JsonException || ex is InvalidDataException) { if (invalidDataHandling == InvalidDataHandling.Ignore) continue; @@ -211,7 +207,7 @@ static async Task ReadBatchAsync( static async Task SendBatchAsync( SeqConnection connection, string? apiKey, - IReadOnlyCollection batch, + IReadOnlyCollection batch, ILogger? sendFailureLog, CancellationToken cancellationToken) { @@ -223,7 +219,7 @@ static async Task SendBatchAsync( using (var builder = new StringWriter()) { foreach (var evt in batch) - JsonFormatter.Format(evt, builder); + builder.WriteLine(evt.ToJsonString()); content = new StringContent(builder.ToString(), Encoding.UTF8, ApiConstants.ClefMediaType); } @@ -264,4 +260,4 @@ static async Task SendAsync(SeqConnection connection, string? ap sendFailureLog.Error("Shipping failed with status code {StatusCode} ({ReasonPhrase})", result.StatusCode, result.ReasonPhrase); return result.StatusCode; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Ingestion/ReadResult.cs b/src/SeqCli/Ingestion/ReadResult.cs index 87e10076..a44d30f4 100644 --- a/src/SeqCli/Ingestion/ReadResult.cs +++ b/src/SeqCli/Ingestion/ReadResult.cs @@ -1,15 +1,20 @@ -using Serilog.Events; +using System.Text.Json.Nodes; namespace SeqCli.Ingestion; readonly struct ReadResult { - public LogEvent? LogEvent { get; } + /// + /// The event, as a JSON document in Seq's emission schema, or null if no event + /// is available. + /// + public JsonObject? Document { get; } + public bool IsAtEnd { get; } - public ReadResult(LogEvent? logEvent, bool isAtEnd) + public ReadResult(JsonObject? document, bool isAtEnd) { - LogEvent = logEvent; + Document = document; IsAtEnd = isAtEnd; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Ingestion/SerilogEventJson.cs b/src/SeqCli/Ingestion/SerilogEventJson.cs new file mode 100644 index 00000000..3a166419 --- /dev/null +++ b/src/SeqCli/Ingestion/SerilogEventJson.cs @@ -0,0 +1,91 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Globalization; +using System.Linq; +using System.Text.Json.Nodes; +using SeqCli.Syntax; +using Serilog.Events; + +namespace SeqCli.Ingestion; + +/// +/// Converts Serilog events produced within seqcli itself — the sample ingest simulation +/// and the forwarder's diagnostic ingestion log — into event JSON documents in Seq's emission +/// schema. Externally-supplied event data never passes through here: it's read directly into +/// JSON documents. +/// +static class SerilogEventJson +{ + public static JsonObject ToEventJson(LogEvent logEvent) + { + var eventJson = new JsonObject + { + ["@t"] = logEvent.Timestamp.ToString("o", CultureInfo.InvariantCulture), + ["@mt"] = logEvent.MessageTemplate.Text + }; + + if (logEvent.Level != LogEventLevel.Information) + eventJson["@l"] = logEvent.Level.ToString(); + + if (logEvent.Exception != null) + eventJson["@x"] = logEvent.Exception.ToString(); + + if (logEvent.TraceId is { } traceId) + eventJson["@tr"] = traceId.ToHexString(); + + if (logEvent.SpanId is { } spanId) + eventJson["@sp"] = spanId.ToHexString(); + + foreach (var (name, value) in logEvent.Properties) + EventJson.SetUserProperty(eventJson, name, ToJsonNode(value)); + + SerilogTracingConventions.LiftSpanProperties(eventJson); + + return eventJson; + } + + public static JsonNode? ToJsonNode(LogEventPropertyValue value) + { + switch (value) + { + case ScalarValue scalar: + return EventJson.CreateScalar(scalar.Value); + + case SequenceValue sequence: + return new JsonArray(sequence.Elements.Select(ToJsonNode).ToArray()); + + case StructureValue structure: + { + var result = new JsonObject(); + foreach (var property in structure.Properties) + result[property.Name] = ToJsonNode(property.Value); + if (structure.TypeTag != null) + result["$type"] = structure.TypeTag; + return result; + } + + case DictionaryValue dictionary: + { + var result = new JsonObject(); + foreach (var (key, element) in dictionary.Elements) + result[key.Value?.ToString() ?? "null"] = ToJsonNode(element); + return result; + } + + default: + return EventJson.CreateScalar(value.ToString()); + } + } +} diff --git a/src/SeqCli/Ingestion/SerilogTracingConventions.cs b/src/SeqCli/Ingestion/SerilogTracingConventions.cs new file mode 100644 index 00000000..e6eb029e --- /dev/null +++ b/src/SeqCli/Ingestion/SerilogTracingConventions.cs @@ -0,0 +1,46 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Text.Json.Nodes; + +namespace SeqCli.Ingestion; + +/// +/// SerilogTracing emits span fields as regular event properties, because Serilog's data model +/// has nowhere else to put them. Events passing through seqcli lift these into the reified +/// @st and @ps fields so that they're recognized as spans by Seq and by seqcli's +/// own output formatting. +/// +static class SerilogTracingConventions +{ + internal const string ParentSpanIdProperty = "ParentSpanId"; + + internal const string SpanStartTimestampProperty = "SpanStartTimestamp"; + + public static void LiftSpanProperties(JsonObject eventJson) + { + LiftProperty(eventJson, SpanStartTimestampProperty, "@st"); + LiftProperty(eventJson, ParentSpanIdProperty, "@ps"); + } + + static void LiftProperty(JsonObject eventJson, string propertyName, string reifiedName) + { + if (eventJson.TryGetPropertyValue(propertyName, out var value)) + { + eventJson.Remove(propertyName); + if (!eventJson.ContainsKey(reifiedName)) + eventJson[reifiedName] = value; + } + } +} diff --git a/src/SeqCli/Ingestion/StaticMessageTemplateReader.cs b/src/SeqCli/Ingestion/StaticMessageTemplateReader.cs index 973bff60..d5d62591 100644 --- a/src/SeqCli/Ingestion/StaticMessageTemplateReader.cs +++ b/src/SeqCli/Ingestion/StaticMessageTemplateReader.cs @@ -1,37 +1,29 @@ using System; -using System.Linq; using System.Threading.Tasks; -using SeqCli.Util; -using Serilog.Events; -using Serilog.Parsing; namespace SeqCli.Ingestion; -class StaticMessageTemplateReader : ILogEventReader +class StaticMessageTemplateReader : IEventReader { - readonly ILogEventReader _inner; - readonly MessageTemplate _messageTemplate; + readonly IEventReader _inner; + readonly string _messageTemplate; - public StaticMessageTemplateReader(ILogEventReader inner, string messageTemplate) + public StaticMessageTemplateReader(IEventReader inner, string messageTemplate) { _inner = inner ?? throw new ArgumentNullException(nameof(inner)); - _messageTemplate = new MessageTemplateParser().Parse(messageTemplate); + _messageTemplate = messageTemplate ?? throw new ArgumentNullException(nameof(messageTemplate)); } public async Task TryReadAsync() { var result = await _inner.TryReadAsync(); - if (result.LogEvent == null) - return result; + if (result.Document != null) + { + result.Document.Remove("@m"); + result.Document["@mt"] = _messageTemplate; + } - var evt = new LogEvent( - result.LogEvent.Timestamp, - result.LogEvent.Level, - result.LogEvent.Exception, - _messageTemplate, - result.LogEvent.Properties.Select(kv => LogEventPropertyFactory.SafeCreate(kv.Key, kv.Value))); - - return new ReadResult(evt, result.IsAtEnd); + return result; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Ingestion/TraceConstants.cs b/src/SeqCli/Ingestion/TraceConstants.cs deleted file mode 100644 index 55fe7f34..00000000 --- a/src/SeqCli/Ingestion/TraceConstants.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace SeqCli.Ingestion; - -static class TraceConstants -{ - internal const string ParentSpanIdProperty = "ParentSpanId"; - - internal const string SpanStartTimestampProperty = "SpanStartTimestamp"; -} diff --git a/src/SeqCli/Mapping/EventEntityJson.cs b/src/SeqCli/Mapping/EventEntityJson.cs new file mode 100644 index 00000000..e84bb3c9 --- /dev/null +++ b/src/SeqCli/Mapping/EventEntityJson.cs @@ -0,0 +1,104 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; +using System.Text.Json.Nodes; +using Seq.Api.Model.Events; +using Seq.Api.Model.Shared; +using SeqCli.Syntax; +using SeqCli.Util; + +namespace SeqCli.Mapping; + +/// +/// Converts events retrieved from the Seq API into event JSON documents in Seq's emission +/// (CLEF) schema, ready for filtering and formatting with Seq.Syntax. +/// +static class EventEntityJson +{ + public static JsonObject ToEventJson(EventEntity evt) + { + // Timestamps are shown in local time, matching earlier seqcli versions. + var eventJson = new JsonObject + { + ["@t"] = DateTimeOffset.ParseExact(evt.Timestamp, "o", CultureInfo.InvariantCulture) + .ToLocalTime().ToString("o", CultureInfo.InvariantCulture) + }; + + if (evt.MessageTemplateTokens != null) + eventJson["@mt"] = ToMessageTemplateText(evt.MessageTemplateTokens); + + // By the emission convention, `Information` levels are omitted; any other level keeps + // the spelling it was ingested with. + if (!string.IsNullOrWhiteSpace(evt.Level) && evt.Level != "Information") + eventJson["@l"] = evt.Level; + + if (!string.IsNullOrWhiteSpace(evt.Exception)) + eventJson["@x"] = evt.Exception; + + if (!string.IsNullOrWhiteSpace(evt.TraceId)) + eventJson["@tr"] = evt.TraceId; + + if (!string.IsNullOrWhiteSpace(evt.SpanId)) + eventJson["@sp"] = evt.SpanId; + + if (!string.IsNullOrWhiteSpace(evt.ParentId)) + eventJson["@ps"] = evt.ParentId; + + if (!string.IsNullOrWhiteSpace(evt.Start)) + eventJson["@st"] = evt.Start; + + if (!string.IsNullOrWhiteSpace(evt.SpanKind)) + eventJson["@sk"] = evt.SpanKind; + + if (evt.Resource?.Count > 0) + eventJson["@ra"] = ToPropertiesObject(evt.Resource); + + if (evt.Scope?.Count > 0) + eventJson["@sa"] = ToPropertiesObject(evt.Scope); + + if (evt.Properties != null) + { + foreach (var property in evt.Properties) + EventJson.SetUserProperty(eventJson, property.Name, JsonNodes.FromApiValue(property.Value)); + } + + return eventJson; + } + + static string ToMessageTemplateText(List tokens) + { + var text = new StringBuilder(); + foreach (var token in tokens) + { + if (token.Text != null) + text.Append(token.Text.Replace("{", "{{").Replace("}", "}}")); + else + text.Append(token.RawText ?? $"{{{token.PropertyName}}}"); + } + + return text.ToString(); + } + + static JsonObject ToPropertiesObject(List properties) + { + var result = new JsonObject(); + foreach (var property in properties) + result[property.Name] = JsonNodes.FromApiValue(property.Value); + return result; + } +} diff --git a/src/SeqCli/Mapping/LevelMapping.cs b/src/SeqCli/Mapping/LevelMapping.cs index ff79087b..7faa47b4 100644 --- a/src/SeqCli/Mapping/LevelMapping.cs +++ b/src/SeqCli/Mapping/LevelMapping.cs @@ -1,4 +1,4 @@ -// Copyright © Datalust and contributors. +// Copyright © Datalust and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,80 +14,74 @@ using System; using System.Collections.Generic; -using Serilog.Events; namespace SeqCli.Mapping; +/// +/// Recognizes the level spellings found in event data from various sources (info, +/// WARN, trce, …) and maps them to canonical Seq level names. Level values +/// themselves are preserved verbatim throughout the pipeline; the canonical name is used +/// where a normalized form is needed. +/// public static class LevelMapping { - // Use a "hygienic" name for the original level value to avoid collisions - internal static readonly string SurrogateLevelProperty = $"_SeqcliOriginalLevel_{Guid.NewGuid():N}"; - - static readonly Dictionary LevelsByName = + static readonly Dictionary LevelsByName = new(StringComparer.OrdinalIgnoreCase) { - ["t"] = ("Trace", LogEventLevel.Verbose), - ["tr"] = ("Trace", LogEventLevel.Verbose), - ["trc"] = ("Trace", LogEventLevel.Verbose), - ["trce"] = ("Trace", LogEventLevel.Verbose), - ["trace"] = ("Trace", LogEventLevel.Verbose), - ["v"] = ("Verbose", LogEventLevel.Verbose), - ["ver"] = ("Verbose", LogEventLevel.Verbose), - ["vrb"] = ("Verbose", LogEventLevel.Verbose), - ["verb"] = ("Verbose", LogEventLevel.Verbose), - ["verbose"] = ("Verbose", LogEventLevel.Verbose), - ["d"] = ("Debug", LogEventLevel.Debug), - ["de"] = ("Debug", LogEventLevel.Debug), - ["dbg"] = ("Debug", LogEventLevel.Debug), - ["deb"] = ("Debug", LogEventLevel.Debug), - ["dbug"] = ("Debug", LogEventLevel.Debug), - ["debu"] = ("Debug", LogEventLevel.Debug), - ["debug"] = ("Debug", LogEventLevel.Debug), - ["i"] = ("Information", LogEventLevel.Information), - ["in"] = ("Information", LogEventLevel.Information), - ["inf"] = ("Information", LogEventLevel.Information), - ["info"] = ("Information", LogEventLevel.Information), - ["information"] = ("Information", LogEventLevel.Information), - ["notice"] = ("Notice", LogEventLevel.Information), - ["w"] = ("Warning", LogEventLevel.Warning), - ["wa"] = ("Warning", LogEventLevel.Warning), - ["war"] = ("Warning", LogEventLevel.Warning), - ["wrn"] = ("Warning", LogEventLevel.Warning), - ["warn"] = ("Warning", LogEventLevel.Warning), - ["warning"] = ("Warning", LogEventLevel.Warning), - ["e"] = ("Error", LogEventLevel.Error), - ["er"] = ("Error", LogEventLevel.Error), - ["err"] = ("Error", LogEventLevel.Error), - ["erro"] = ("Error", LogEventLevel.Error), - ["eror"] = ("Error", LogEventLevel.Error), - ["error"] = ("Error", LogEventLevel.Error), - ["f"] = ("Fatal", LogEventLevel.Fatal), - ["fa"] = ("Fatal", LogEventLevel.Fatal), - ["ftl"] = ("Fatal", LogEventLevel.Fatal), - ["fat"] = ("Fatal", LogEventLevel.Fatal), - ["fatl"] = ("Fatal", LogEventLevel.Fatal), - ["fatal"] = ("Fatal", LogEventLevel.Fatal), - ["c"] = ("Critical", LogEventLevel.Fatal), - ["cr"] = ("Critical", LogEventLevel.Fatal), - ["crt"] = ("Critical", LogEventLevel.Fatal), - ["cri"] = ("Critical", LogEventLevel.Fatal), - ["crit"] = ("Critical", LogEventLevel.Fatal), - ["critical"] = ("Critical", LogEventLevel.Fatal), - ["emerg"] = ("Emergency", LogEventLevel.Fatal), - ["alert"] = ("Alert", LogEventLevel.Fatal), - ["panic"] = ("Panic", LogEventLevel.Fatal) + ["t"] = "Trace", + ["tr"] = "Trace", + ["trc"] = "Trace", + ["trce"] = "Trace", + ["trace"] = "Trace", + ["v"] = "Verbose", + ["ver"] = "Verbose", + ["vrb"] = "Verbose", + ["verb"] = "Verbose", + ["verbose"] = "Verbose", + ["d"] = "Debug", + ["de"] = "Debug", + ["dbg"] = "Debug", + ["deb"] = "Debug", + ["dbug"] = "Debug", + ["debu"] = "Debug", + ["debug"] = "Debug", + ["i"] = "Information", + ["in"] = "Information", + ["inf"] = "Information", + ["info"] = "Information", + ["information"] = "Information", + ["notice"] = "Notice", + ["w"] = "Warning", + ["wa"] = "Warning", + ["war"] = "Warning", + ["wrn"] = "Warning", + ["warn"] = "Warning", + ["warning"] = "Warning", + ["e"] = "Error", + ["er"] = "Error", + ["err"] = "Error", + ["erro"] = "Error", + ["eror"] = "Error", + ["error"] = "Error", + ["f"] = "Fatal", + ["fa"] = "Fatal", + ["ftl"] = "Fatal", + ["fat"] = "Fatal", + ["fatl"] = "Fatal", + ["fatal"] = "Fatal", + ["c"] = "Critical", + ["cr"] = "Critical", + ["crt"] = "Critical", + ["cri"] = "Critical", + ["crit"] = "Critical", + ["critical"] = "Critical", + ["emerg"] = "Emergency", + ["alert"] = "Alert", + ["panic"] = "Panic" }; - public static LogEventLevel ToSerilogLevel(string level) - { - if (string.IsNullOrEmpty(level)) - return LogEventLevel.Information; - - return LevelsByName.TryGetValue(level, out var m) ? m.Item2 : LogEventLevel.Information; - } - public static string ToFullLevelName(string level) { - return LevelsByName.TryGetValue(level, out var m) ? m.Item1 : level; + return LevelsByName.TryGetValue(level, out var m) ? m : level; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Mapping/MetricsMapping.cs b/src/SeqCli/Mapping/MetricsMapping.cs deleted file mode 100644 index fe104666..00000000 --- a/src/SeqCli/Mapping/MetricsMapping.cs +++ /dev/null @@ -1,8 +0,0 @@ -using System; - -namespace SeqCli.Mapping; - -public static class MetricsMapping -{ - internal static readonly string SurrogateDefinitionsProperty = $"_SeqcliMetricDefinitions_{Guid.NewGuid():N}"; -} diff --git a/src/SeqCli/Mcp/Tools/Search/SearchTools.cs b/src/SeqCli/Mcp/Tools/Search/SearchTools.cs index 7dc9671d..58a3b663 100644 --- a/src/SeqCli/Mcp/Tools/Search/SearchTools.cs +++ b/src/SeqCli/Mcp/Tools/Search/SearchTools.cs @@ -30,8 +30,8 @@ using SeqCli.Mapping; using SeqCli.Output; using SeqCli.Signals; +using SeqCli.Syntax; using Serilog; -using Serilog.Events; using NativeFormatter = SeqCli.Output.NativeFormatter; // ReSharper disable UnusedMember.Global @@ -42,8 +42,9 @@ namespace SeqCli.Mcp.Tools.Search; class SearchTools(McpSession session, SeqConnection connection) { const string ResultIdPropertyName = "__seqcli_ResultId"; - static readonly ExpressionTemplate SearchResultFormatter = new ( - $"{{{ResultIdPropertyName}}} [{{UtcDateTime(@t)}} {{{LevelMapping.SurrogateLevelProperty}}}] {{@m}}{Environment.NewLine}{{#if @x is not null}}{{Substring(ToString(@x), 0, 512)}}...{Environment.NewLine}{{#end}}" + static readonly ExpressionTemplate SearchResultFormatter = SeqSyntax.ParseTemplate( + $"{{{ResultIdPropertyName}}} [{{UtcDateTime(@Timestamp)}} {{@Level}}] {{@Message}}{Environment.NewLine}" + + $"{{#if @Exception is not null}}{{Substring(ToString(@Exception), 0, 512)}}...{Environment.NewLine}{{#end}}" ); [McpServerTool(Name = "seq_new_session", ReadOnly = true, Title = "Begin a new Search/Query Session")] @@ -181,12 +182,10 @@ public async Task SearchEventsAsync( foreach (var result in takenResults) { var resultId = session.ImportSearchResult(result); - - var serilogEvent = OutputFormat.ToSerilogEvent(result); - OutputFormat.FlattenPropertiesUsedWithDottedNames(result, serilogEvent); - serilogEvent.AddOrUpdateProperty(new LogEventProperty(ResultIdPropertyName, new ScalarValue(resultId))); - serilogEvent.AddOrUpdateProperty(new LogEventProperty(LevelMapping.SurrogateLevelProperty, new ScalarValue(result.Level ?? "Information"))); - SearchResultFormatter.Format(serilogEvent, responseText); + + var eventJson = EventEntityJson.ToEventJson(result); + eventJson[ResultIdPropertyName] = resultId; + SearchResultFormatter.Format(eventJson, responseText); } return new CallToolResult diff --git a/src/SeqCli/Output/FlareTheme.cs b/src/SeqCli/Output/FlareTheme.cs index 38d1e578..a1026fee 100644 --- a/src/SeqCli/Output/FlareTheme.cs +++ b/src/SeqCli/Output/FlareTheme.cs @@ -13,13 +13,12 @@ // limitations under the License. using System.Collections.Generic; -using System.IO; -using Serilog.Templates.Themes; +using Seq.Syntax.Templates.Themes; namespace SeqCli.Output; /// -/// Flare is Seq's embedded stream/columnar database. This theme is derived from one build originally +/// Flare is Seq's embedded stream/columnar database. This theme is derived from one built originally /// for the flaretl command-line tooling used there. /// static class FlareTheme @@ -45,26 +44,5 @@ static class FlareTheme [TemplateThemeStyle.LevelFatal] = "\e[38;5;0197m\e[48;5;0238m" }; - public static readonly TemplateTheme SeqCli = new(FlareThemeStyles); - - // `CsvWriter` implements its own theming behavior because the required APIs are not public in Serilog.Expressions. - // The best way forward for this is likely to be porting theming to Seq.Syntax, and exposing the required APIs there. - - const string AnsiStyleResetSequence = "\e[0m"; - - // The passed-in theme is ignored because SerilogExpressions themes are opaque. All formatting uses the SeqCli theme. - // ReSharper disable once UnusedParameter.Global - extension(TemplateTheme theme) - { - public void Set(TextWriter output, TemplateThemeStyle style) - { - if (FlareThemeStyles.TryGetValue(style, out var styleSequence)) - output.Write(styleSequence); - } - - public void Reset(TextWriter output) - { - output.Write(AnsiStyleResetSequence); - } - } -} \ No newline at end of file + public static readonly TemplateTheme SeqCli = new AnsiTheme(FlareThemeStyles); +} diff --git a/src/SeqCli/Output/OutputFormat.cs b/src/SeqCli/Output/OutputFormat.cs index 7b9c5ea7..af137838 100644 --- a/src/SeqCli/Output/OutputFormat.cs +++ b/src/SeqCli/Output/OutputFormat.cs @@ -15,24 +15,21 @@ using System; using System.Collections; using System.Collections.Generic; -using System.Diagnostics; using System.Globalization; -using System.Linq; +using System.Text.Json.Nodes; using Newtonsoft.Json; using Newtonsoft.Json.Converters; using Newtonsoft.Json.Linq; using Seq.Api.Model; using Seq.Api.Model.Data; using Seq.Api.Model.Events; +using Seq.Syntax.Templates; +using Seq.Syntax.Templates.Encoding; +using Seq.Syntax.Templates.Themes; using SeqCli.Config; using SeqCli.Csv; using SeqCli.Mapping; using SeqCli.Util; -using Serilog; -using Serilog.Core; -using Serilog.Events; -using Serilog.Parsing; -using Serilog.Templates.Themes; namespace SeqCli.Output; @@ -40,10 +37,10 @@ sealed class OutputFormat { // See https://no-color.org for semantics. const string NoColorEnvironmentVariable = "NO_COLOR"; - + readonly OutputSyntax _syntax; - readonly string? _plainTextTemplate; - readonly Logger _formatter; + readonly ExpressionTemplate? _eventFormatter; + readonly ExpressionTemplate _jsonValueFormatter; readonly JsonSerializer _serializer = JsonSerializer.CreateDefault(new JsonSerializerSettings { @@ -92,7 +89,6 @@ internal OutputFormat( bool allowAnsiEscapes) { _syntax = syntax; - _plainTextTemplate = plainTextTemplate; var resolvedNoColor = ResolveNoColor(noColor, forceColor, outputConfig, noColorSetInEnvironment, allowAnsiEscapes); var applyThemeToRedirectedOutput = !resolvedNoColor && (forceColor ?? outputConfig.ForceColor); @@ -102,12 +98,20 @@ internal OutputFormat( ? FlareTheme.SeqCli : null; - _formatter = CreateOutputLogger(); + _eventFormatter = Json + ? TextFormatters.Json(TemplateTheme) + : Text + ? TextFormatters.Plain(TemplateTheme, plainTextTemplate) + : null; + + _jsonValueFormatter = new ExpressionTemplate( + "{Value}" + Environment.NewLine, + encoder: TemplateTheme != null ? TemplateOutputEncoder.Ansi(TemplateTheme) : null); } static bool NoColorSetInEnvironment() => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable(NoColorEnvironmentVariable)); - + internal static bool ResolveNoColor( bool? noColorFlag, bool? forceColorFlag, @@ -135,27 +139,6 @@ internal static bool ResolveNoColor( public bool RequiresRender => Native; - Logger CreateOutputLogger() - { - var outputConfiguration = new LoggerConfiguration() - .MinimumLevel.Is(LevelAlias.Minimum) - .Enrich.With(); - - if (Json) - { - outputConfiguration.WriteTo.Console(TextFormatters.Json(TemplateTheme)); - } - else if (Text) - { - outputConfiguration.WriteTo.Console(TextFormatters.Plain(TemplateTheme, _plainTextTemplate)); - } - - // The logger is not configured for Native output, which avoids it. Ideally we'll shift away from using - // Serilog here, and move Text/Json over to EventEntity-driven formatters, too. - - return outputConfiguration.CreateLogger(); - } - public void WriteEntity(Entity entity) { if (entity == null) throw new ArgumentNullException(nameof(entity)); @@ -163,18 +146,11 @@ public void WriteEntity(Entity entity) var jo = JObject.FromObject( entity, _serializer); - + if (Json) { jo.Remove("Links"); - - var writer = new LoggerConfiguration() - .Destructure.With() - .Destructure.ToMaximumDepth(10000) - .Enrich.With() - .WriteTo.Console(TextFormatters.Plain(TemplateTheme, "{@m}" + Environment.NewLine)) - .CreateLogger(); - writer.Information("{@Entity}", jo); + WriteJsonValue(JsonNodes.FromNewtonsoft(jo)); } else if (Text) { @@ -190,22 +166,14 @@ public void WriteEntity(Entity entity) public void WriteObject(object value) { if (value == null) throw new ArgumentNullException(nameof(value)); - + if (Json) { var jo = value is ICollection and not (IDictionary or JToken) ? (JToken)JArray.FromObject(value, _serializer) : JObject.FromObject(value, _serializer); - // Using the same method of JSON colorization as above - - var writer = new LoggerConfiguration() - .Destructure.With() - .Destructure.ToMaximumDepth(10000) - .Enrich.With() - .WriteTo.Console(TextFormatters.Plain(TemplateTheme, "{@m}" + Environment.NewLine)) - .CreateLogger(); - writer.Information("{@Entity}", jo); + WriteJsonValue(JsonNodes.FromNewtonsoft(jo)); } else if (Text) { @@ -218,6 +186,11 @@ public void WriteObject(object value) } } + void WriteJsonValue(JsonNode? value) + { + _jsonValueFormatter.Format(new JsonObject { ["Value"] = value }, Console.Out); + } + public void ListEntities(IEnumerable list) { foreach (var entity in list) @@ -225,7 +198,7 @@ public void ListEntities(IEnumerable list) WriteEntity(entity); } } - + // ReSharper disable once MemberCanBeMadeStatic.Global #pragma warning disable CA1822 public void WriteText(string? text) @@ -259,125 +232,15 @@ public void WriteEventEntity(EventEntity evt) } else { - var serilogEvent = ToSerilogEvent(evt); - - if (Text) - { - // Add flattened versions of structured properties that are referenced using dotted-name syntax in - // message templates, e.g. {user.name}. Serilog.Expressions template rendering doesn't otherwise - // support these. In text output mode, these aren't usually observable, though - // seqcli print --template="{@p}" will make them visible. - FlattenPropertiesUsedWithDottedNames(evt, serilogEvent); - } - - WriteLogEvent(serilogEvent); - } - } - - public void WriteLogEvent(LogEvent logEvent) - { - _formatter.Write(logEvent); - } - - public static LogEvent ToSerilogEvent(EventEntity evt) - { - ActivityTraceId traceId = default; - if (!string.IsNullOrWhiteSpace(evt.TraceId)) - traceId = ActivityTraceId.CreateFromString(evt.TraceId); - - ActivitySpanId spanId = default; - if (!string.IsNullOrWhiteSpace(evt.SpanId)) - spanId = ActivitySpanId.CreateFromString(evt.SpanId); - - var serilogEvent = new LogEvent( - DateTimeOffset.ParseExact(evt.Timestamp, "o", CultureInfo.InvariantCulture).ToLocalTime(), - LevelMapping.ToSerilogLevel(evt.Level), - string.IsNullOrWhiteSpace(evt.Exception) ? null : new TextException(evt.Exception), - new MessageTemplate(evt.MessageTemplateTokens.Select(ToMessageTemplateToken)), - evt.Properties - .Select(p => CreateProperty(p.Name, p.Value)), - traceId, - spanId - ); - - if (evt.Scope?.Count > 0) - serilogEvent.AddOrUpdateProperty(new("@sa", new StructureValue(evt.Scope.Select(p => CreateProperty(p.Name, p.Value))))); - - if (evt.Resource?.Count > 0) - serilogEvent.AddOrUpdateProperty(new("@ra", new StructureValue(evt.Resource.Select(p => CreateProperty(p.Name, p.Value))))); - - if (!string.IsNullOrWhiteSpace(evt.ParentId)) - serilogEvent.AddOrUpdateProperty(new("@ps", new ScalarValue(evt.ParentId))); - - if (!string.IsNullOrWhiteSpace(evt.Start)) - serilogEvent.AddOrUpdateProperty(new("@st", new ScalarValue(evt.Start))); - - if (!string.IsNullOrWhiteSpace(evt.SpanKind)) - serilogEvent.AddOrUpdateProperty(new("@sk", new ScalarValue(evt.SpanKind))); - - return serilogEvent; - } - - public static void FlattenPropertiesUsedWithDottedNames(EventEntity evt, LogEvent serilogEvent) - { - foreach (var token in evt.MessageTemplateTokens) - { - if (token.Text != null || token.PropertyName is not { } name || !name.Contains('.') || - serilogEvent.Properties.ContainsKey(name)) - { - continue; - } - - var steps = name.Split('.'); - var value = evt.Properties.FirstOrDefault(p => p.Name == steps[0])?.Value; - for (var i = 1; i < steps.Length; ++i) - { - value = (value as JObject)?.GetValue(steps[i]); - } - - if (value is JToken resolved) - { - // Existing flat-named properties, where present, win. - serilogEvent.AddPropertyIfAbsent(LogEventPropertyFactory.SafeCreate( - name, resolved is JValue scalar ? new ScalarValue(scalar.Value) : CreatePropertyValue(resolved))); - } + WriteEvent(EventEntityJson.ToEventJson(evt)); } } - static MessageTemplateToken ToMessageTemplateToken(MessageTemplateTokenPart token) - { - // Not ideal, we lose renderings, alignment etc. here. - - if (token.Text != null) - return new TextToken(token.Text); - return new PropertyToken(token.PropertyName, token.RawText ?? $"{{{token.PropertyName}}}"); - } - - static LogEventProperty CreateProperty(string name, object value) + public void WriteEvent(JsonObject eventJson) { - return LogEventPropertyFactory.SafeCreate(name, CreatePropertyValue(value)); + _eventFormatter?.Format(eventJson, Console.Out); } - internal static LogEventPropertyValue CreatePropertyValue(object value) - { - switch (value) - { - case JObject jo: - jo.TryGetValue("$typeTag", out var tt); - return new StructureValue( - jo.Properties() - .Where(kvp => kvp.Name != "$typeTag") - .Select(kvp => CreateProperty(kvp.Name, kvp.Value)), - (tt as JValue)?.Value as string); - - case JArray ja: - return new SequenceValue(ja.Select(CreatePropertyValue)); - - default: - return new ScalarValue(value); - } - } - static string Stringify(object? value) { return value switch diff --git a/src/SeqCli/Output/StripStructureTypeEnricher.cs b/src/SeqCli/Output/StripStructureTypeEnricher.cs deleted file mode 100644 index 352cd1bf..00000000 --- a/src/SeqCli/Output/StripStructureTypeEnricher.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System.Linq; -using SeqCli.Util; -using Serilog.Core; -using Serilog.Data; -using Serilog.Events; - -namespace SeqCli.Output; - -public class StripStructureTypeEnricher : LogEventPropertyValueRewriter, ILogEventEnricher -{ - public void Enrich(LogEvent logEvent, ILogEventPropertyFactory propertyFactory) - { - foreach (var property in logEvent.Properties) - { - var updated = LogEventPropertyFactory.SafeCreate(property.Key, Visit(null, property.Value)); - logEvent.AddOrUpdateProperty(updated); - } - } - - protected override LogEventPropertyValue VisitStructureValue(object? state, StructureValue structure) - { - return new StructureValue(structure.Properties.Select(p => - LogEventPropertyFactory.SafeCreate(p.Name, Visit(null, p.Value)))); - } -} \ No newline at end of file diff --git a/src/SeqCli/Output/TextFormatters.cs b/src/SeqCli/Output/TextFormatters.cs index 86cfbee7..88025fdf 100644 --- a/src/SeqCli/Output/TextFormatters.cs +++ b/src/SeqCli/Output/TextFormatters.cs @@ -13,42 +13,32 @@ // limitations under the License. using System; -using SeqCli.Ingestion; -using SeqCli.Mapping; -using Serilog.Expressions; -using Serilog.Formatting; -using Serilog.Templates; -using Serilog.Templates.Themes; +using Seq.Syntax.Templates; +using Seq.Syntax.Templates.Encoding; +using Seq.Syntax.Templates.Themes; +using SeqCli.Syntax; namespace SeqCli.Output; -// This is the only usage of Serilog.Expressions remaining in seqcli; the upstream Seq.Syntax doesn't yet support -// tracing properties or theming. static class TextFormatters { - public static ITextFormatter Json(TemplateTheme? theme) => new ExpressionTemplate( - $"{{ " + - $"if {MetricsMapping.SurrogateDefinitionsProperty} is not null then " + - // Emit a metric sample - $"{{@t, @l: undefined(), @d: {MetricsMapping.SurrogateDefinitionsProperty}, ..rest()}} " + - $"else " + - // Emit a log or span - $"{{@t, @mt, @l: coalesce({LevelMapping.SurrogateLevelProperty}, if @l = 'Information' then undefined() else @l), @x, @sp, @tr, @ps: coalesce({TraceConstants.ParentSpanIdProperty}, @ps), @st: coalesce({TraceConstants.SpanStartTimestampProperty}, @st), ..rest()}} " + - $"}}" + - Environment.NewLine, - theme: theme, - // The `OutputFormat` constructor has already decided whether to colorize. - applyThemeWhenOutputIsRedirected: true - ); + /// + /// Newline-delimited CLEF output: the event JSON document is written verbatim, with theming + /// when a theme is supplied. + /// + public static ExpressionTemplate Json(TemplateTheme? theme) => new( + "{@Data}" + Environment.NewLine, + encoder: Encoder(theme)); + // Guarding on `@Elapsed` rather than the built-in `IsSpan()` shows elapsed time for any + // event carrying a span start timestamp, whether or not trace and span ids accompany it. static readonly string DefaultPlainTextOutputTemplate = - "[{@t:o} {@l:u3}] {@m}{#if IsSpan()} ({Milliseconds(Elapsed()):0.###} ms){#end}" + Environment.NewLine + "{@x}"; + "[{@Timestamp:o} {@Level:u3}] {@Message}{#if @Elapsed is not null} ({TotalMilliseconds(@Elapsed):0.###} ms){#end}" + + Environment.NewLine + "{@Exception}"; - public static ITextFormatter Plain(TemplateTheme? theme, string? outputTemplate) => new ExpressionTemplate( - outputTemplate ?? DefaultPlainTextOutputTemplate, - theme: theme, - nameResolver: new StaticMemberNameResolver(typeof(TracingFunctions)), - // The `OutputFormat` constructor has already decided whether to colorize. - applyThemeWhenOutputIsRedirected: true - ); -} \ No newline at end of file + public static ExpressionTemplate Plain(TemplateTheme? theme, string? outputTemplate) => + SeqSyntax.ParseTemplate(outputTemplate ?? DefaultPlainTextOutputTemplate, Encoder(theme)); + + static TemplateOutputEncoder? Encoder(TemplateTheme? theme) => + theme != null ? TemplateOutputEncoder.Ansi(theme) : null; +} diff --git a/src/SeqCli/Output/TraceFormatter.cs b/src/SeqCli/Output/TraceFormatter.cs index d2256e3d..ab8238b2 100644 --- a/src/SeqCli/Output/TraceFormatter.cs +++ b/src/SeqCli/Output/TraceFormatter.cs @@ -14,11 +14,11 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Text; -using SeqCli.Mapping; +using System.Text.Json.Nodes; using SeqCli.Traces; using SeqCli.Util; -using Serilog.Events; namespace SeqCli.Output; @@ -35,7 +35,7 @@ static class TraceFormatter public static string OutputTemplate(int columnCount) { - var template = new StringBuilder($"[{{@t:o}} {{@l:u3}}] {{{TreePrefixProperty}}}"); + var template = new StringBuilder($"[{{@Timestamp:o}} {{@Level:u3}}] {{{TreePrefixProperty}}}"); // `<> ''` is undefined, and hence falsy, when the property is missing; the guard thus // drops the column, and its trailing space, for both missing and empty values. @@ -45,22 +45,22 @@ public static string OutputTemplate(int columnCount) template.Append($"{{#if {column} <> ''}}{{{column}}} {{#end}}"); } - template.Append($"{{@m}}{{#if {ElapsedProperty} is not null}} ({{Milliseconds({ElapsedProperty}):0.###}} ms){{#end}}"); - template.Append(Environment.NewLine).Append("{@x}"); + template.Append($"{{@Message}}{{#if {ElapsedProperty} is not null}} ({{TotalMilliseconds({ElapsedProperty}):0.###}} ms){{#end}}"); + template.Append(Environment.NewLine).Append("{@Exception}"); return template.ToString(); } - public static IEnumerable ToLogEvents(IReadOnlyList roots) + public static IEnumerable ToEventJson(IReadOnlyList roots) { foreach (var root in roots) { - yield return ToLogEvent(root, root.Element.IsSpan ? "" : LogConnector); + yield return ToEventJson(root, root.Element.IsSpan ? "" : LogConnector); foreach (var descendant in WalkChildren(root, "")) yield return descendant; } } - static IEnumerable WalkChildren(TraceTreeNode parent, string indent) + static IEnumerable WalkChildren(TraceTreeNode parent, string indent) { for (var i = 0; i < parent.Children.Count; ++i) { @@ -71,40 +71,43 @@ static IEnumerable WalkChildren(TraceTreeNode parent, string indent) isLast ? LastSpanConnector : SpanConnector : LogConnector; - yield return ToLogEvent(child, indent + connector); + yield return ToEventJson(child, indent + connector); foreach (var descendant in WalkChildren(child, indent + (isLast ? Gap : Continuation))) yield return descendant; } } - static LogEvent ToLogEvent(TraceTreeNode treeNode, string treePrefix) + static JsonObject ToEventJson(TraceTreeNode treeNode, string treePrefix) { var evt = treeNode.Element; - var properties = new List + // Spans are positioned and shown at their start time. + var eventJson = new JsonObject { - new(TreePrefixProperty, new ScalarValue(treePrefix)) + ["@t"] = evt.SortKey.ToLocalTime().ToString("o", CultureInfo.InvariantCulture), + ["@mt"] = evt.MessageTemplate, + [TreePrefixProperty] = treePrefix }; - properties.AddRange(evt.TemplateProperties); + if (!string.IsNullOrEmpty(evt.Level)) + eventJson["@l"] = evt.Level; + + if (!string.IsNullOrWhiteSpace(evt.Exception)) + eventJson["@x"] = evt.Exception; + + foreach (var (name, value) in evt.TemplateProperties) + eventJson[name] = value?.DeepClone(); if (evt.Elapsed is { } elapsed) - properties.Add(new(ElapsedProperty, new ScalarValue(elapsed))); + eventJson[ElapsedProperty] = elapsed.ToString("c", CultureInfo.InvariantCulture); for (var i = 0; i < evt.Columns.Count; ++i) { if (evt.Columns[i] is { } value) - properties.Add(LogEventPropertyFactory.SafeCreate( - ColumnPropertyName(i), OutputFormat.CreatePropertyValue(value))); + eventJson[ColumnPropertyName(i)] = JsonNodes.FromApiValue(value); } - // Spans are positioned and shown at their start time. - return new LogEvent( - evt.SortKey.ToLocalTime(), - LevelMapping.ToSerilogLevel(evt.Level ?? ""), - string.IsNullOrWhiteSpace(evt.Exception) ? null : new TextException(evt.Exception), - evt.MessageTemplate, - properties); + return eventJson; } } diff --git a/src/SeqCli/Output/TracingFunctions.cs b/src/SeqCli/Output/TracingFunctions.cs deleted file mode 100644 index 5e2ba112..00000000 --- a/src/SeqCli/Output/TracingFunctions.cs +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright © Datalust Pty Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Globalization; -using SeqCli.Ingestion; -using Serilog.Events; - -namespace SeqCli.Output; - -static class TracingFunctions -{ - public static LogEventPropertyValue? Elapsed(LogEvent logEvent) - { - if (logEvent.Properties.TryGetValue(TraceConstants.SpanStartTimestampProperty, out var sst) && - sst is ScalarValue { Value: DateTime spanStart }) - { - return new ScalarValue(logEvent.Timestamp - spanStart); - } - - if (logEvent.Properties.TryGetValue("@st", out var st) && - st is ScalarValue { Value: string spanStartIso } && - DateTimeOffset.TryParse(spanStartIso, CultureInfo.InvariantCulture, out var spanStartDto)) - { - return new ScalarValue(logEvent.Timestamp - spanStartDto); - } - - return null; - } - - public static LogEventPropertyValue? IsSpan(LogEvent logEvent) - { - return new ScalarValue(Elapsed(logEvent) != null); - } - - public static LogEventPropertyValue? Milliseconds(LogEventPropertyValue? timeSpan) - { - // Truncates instead of rounding. - if (timeSpan is ScalarValue { Value: TimeSpan ts }) - return new ScalarValue((decimal)ts.Ticks / TimeSpan.TicksPerMillisecond); - - return null; - } -} \ No newline at end of file diff --git a/src/SeqCli/PlainText/LogEvents/EventJsonBuilder.cs b/src/SeqCli/PlainText/LogEvents/EventJsonBuilder.cs new file mode 100644 index 00000000..caf4476e --- /dev/null +++ b/src/SeqCli/PlainText/LogEvents/EventJsonBuilder.cs @@ -0,0 +1,100 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text.Json.Nodes; +using SeqCli.Syntax; +using Superpower.Model; + +namespace SeqCli.PlainText.LogEvents; + +/// +/// Assembles the values captured by a plain-text extraction pattern into an event JSON +/// document in Seq's emission schema. +/// +static class EventJsonBuilder +{ + public static JsonObject FromProperties(IDictionary properties, string? remainder) + { + var eventJson = new JsonObject + { + ["@t"] = GetTimestamp(properties).ToString("o", CultureInfo.InvariantCulture) + }; + + if (TryGetText(properties, ReifiedProperties.Level, out var level)) + eventJson["@l"] = level; + + if (TryGetText(properties, ReifiedProperties.Message, out var message)) + eventJson["@m"] = message; + + if (TryGetText(properties, ReifiedProperties.Exception, out var exception)) + eventJson["@x"] = exception; + + if (TryGetText(properties, ReifiedProperties.TraceId, out var traceId)) + eventJson["@tr"] = traceId; + + if (TryGetText(properties, ReifiedProperties.SpanId, out var spanId)) + eventJson["@sp"] = spanId; + + if (TryGetText(properties, ReifiedProperties.StartTimestamp, out var start)) + eventJson["@st"] = start; + + foreach (var (name, value) in properties) + { + if (!ReifiedProperties.IsReifiedProperty(name)) + EventJson.SetUserProperty(eventJson, name, CreateValue(value)); + } + + if (remainder != null) + EventJson.SetUserProperty(eventJson, "@unmatched", remainder); + + return eventJson; + } + + static JsonNode? CreateValue(object? value) + { + return value is TextSpan span + ? JsonValue.Create(span.ToStringValue()) + : EventJson.CreateScalar(value); + } + + static bool TryGetText(IDictionary properties, string name, out string text) + { + if (properties.TryGetValue(name, out var value) && value is TextSpan span) + { + text = span.ToStringValue(); + return true; + } + + text = ""; + return false; + } + + static DateTimeOffset GetTimestamp(IDictionary properties) + { + if (properties.TryGetValue(ReifiedProperties.Timestamp, out var t)) + { + if (t is TextSpan span && DateTimeOffset.TryParse(span.ToStringValue(), + CultureInfo.InvariantCulture, DateTimeStyles.AssumeLocal, out var ts)) + return ts; + + if (t is DateTimeOffset dto) + return dto; + } + + return DateTimeOffset.Now; + } +} diff --git a/src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs b/src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs deleted file mode 100644 index 1362716f..00000000 --- a/src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright © Datalust and contributors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Globalization; -using System.Linq; -using SeqCli.Mapping; -using SeqCli.Util; -using Serilog.Events; -using Serilog.Parsing; -using Superpower.Model; - -namespace SeqCli.PlainText.LogEvents; - -static class LogEventBuilder -{ - public static LogEvent FromProperties(IDictionary properties, string? remainder) - { - var timestamp = GetTimestamp(properties); - var level = GetLevel(properties); - var exception = TryGetException(properties); - var messageTemplate = GetMessageTemplate(properties); - var traceId = GetTraceId(properties); - var spanId = GetSpanId(properties); - var props = GetLogEventProperties(properties, remainder); - - var fallbackMappedLevel = level != null ? LevelMapping.ToSerilogLevel(level) : LogEventLevel.Information; - properties[LevelMapping.SurrogateLevelProperty] = level; - - return new LogEvent( - timestamp, - fallbackMappedLevel, - exception, - messageTemplate, - props, - traceId ?? default, - spanId ?? default - ); - } - - static readonly MessageTemplate NoMessage = new MessageTemplateParser().Parse(""); - - static MessageTemplate GetMessageTemplate(IDictionary properties) - { - if (properties.TryGetValue(ReifiedProperties.Message, out var m) && - m is TextSpan ts) - { - var text = ts.ToStringValue(); - return new MessageTemplate([new TextToken(text)]); - } - - return NoMessage; - } - - static string? GetLevel(IDictionary properties) - { - if (properties.TryGetValue(ReifiedProperties.Level, out var l) && - l is TextSpan ts) - return ts.ToStringValue(); - - return null; - } - - static ActivityTraceId? GetTraceId(IDictionary properties) - { - if (properties.TryGetValue(ReifiedProperties.TraceId, out var tr) && - tr is TextSpan ts) - return ActivityTraceId.CreateFromString(ts.ToStringValue()); - - return null; - } - - static ActivitySpanId? GetSpanId(IDictionary properties) - { - if (properties.TryGetValue(ReifiedProperties.SpanId, out var sp) && - sp is TextSpan ts) - return ActivitySpanId.CreateFromString(ts.ToStringValue()); - - return null; - } - - static Exception? TryGetException(IDictionary properties) - { - if (properties.TryGetValue(ReifiedProperties.Exception, out var x) && - x is TextSpan ts) - return new TextOnlyException(ts.ToStringValue()); - return null; - } - - static IEnumerable GetLogEventProperties(IDictionary properties, string? remainder) - { - var payload = properties - .Where(p => !ReifiedProperties.IsReifiedProperty(p.Key)) - .Select(p => LogEventPropertyFactory.SafeCreate(p.Key, new ScalarValue(p.Value))); - - if (remainder != null) - payload = payload.Concat(new[] - { - LogEventPropertyFactory.SafeCreate("@unmatched", new ScalarValue(remainder)) - }); - return payload; - } - - static DateTimeOffset GetTimestamp(IDictionary properties) - { - if (properties.TryGetValue(ReifiedProperties.Timestamp, out var t)) - { - if (t is TextSpan span && DateTimeOffset.TryParse(span.ToStringValue(), - CultureInfo.InvariantCulture, DateTimeStyles.AssumeLocal, out var ts)) - return ts; - - if (t is DateTimeOffset dto) - return dto; - } - - return DateTimeOffset.Now; - } -} \ No newline at end of file diff --git a/src/SeqCli/PlainText/LogEvents/TextOnlyException.cs b/src/SeqCli/PlainText/LogEvents/TextOnlyException.cs deleted file mode 100644 index 614c927f..00000000 --- a/src/SeqCli/PlainText/LogEvents/TextOnlyException.cs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright © Datalust and contributors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; - -namespace SeqCli.PlainText.LogEvents; - -class TextOnlyException : Exception -{ - readonly string _toStringValue; - - public TextOnlyException(string toStringValue) - { - _toStringValue = toStringValue ?? throw new ArgumentNullException(nameof(toStringValue)); - } - - public override string ToString() - { - return _toStringValue; - } -} \ No newline at end of file diff --git a/src/SeqCli/PlainText/PlainTextLogEventReader.cs b/src/SeqCli/PlainText/PlainTextEventReader.cs similarity index 86% rename from src/SeqCli/PlainText/PlainTextLogEventReader.cs rename to src/SeqCli/PlainText/PlainTextEventReader.cs index fae2df86..fbead08b 100644 --- a/src/SeqCli/PlainText/PlainTextLogEventReader.cs +++ b/src/SeqCli/PlainText/PlainTextEventReader.cs @@ -10,14 +10,14 @@ namespace SeqCli.PlainText; -class PlainTextLogEventReader : ILogEventReader +class PlainTextEventReader : IEventReader { static readonly TimeSpan TrailingLineArrivalDeadline = TimeSpan.FromMilliseconds(10); readonly NameValueExtractor _nameValueExtractor; readonly FrameReader _reader; - public PlainTextLogEventReader(TextReader input, string extractionPattern) + public PlainTextEventReader(TextReader input, string extractionPattern) { if (extractionPattern == null) throw new ArgumentNullException(nameof(extractionPattern)); _nameValueExtractor = ExtractionPatternInterpreter.CreateNameValueExtractor(ExtractionPatternParser.Parse(extractionPattern)); @@ -36,7 +36,7 @@ public async Task TryReadAsync() var (properties, remainder) = _nameValueExtractor.ExtractValues(frame.Value); - var evt = LogEventBuilder.FromProperties(properties, remainder); + var evt = EventJsonBuilder.FromProperties(properties, remainder); return new ReadResult(evt, frame.IsAtEnd); } } \ No newline at end of file diff --git a/src/SeqCli/Ingestion/BufferingSink.cs b/src/SeqCli/Sample/Ingestion/BufferingSink.cs similarity index 50% rename from src/SeqCli/Ingestion/BufferingSink.cs rename to src/SeqCli/Sample/Ingestion/BufferingSink.cs index 879b930c..cab3a4d3 100644 --- a/src/SeqCli/Ingestion/BufferingSink.cs +++ b/src/SeqCli/Sample/Ingestion/BufferingSink.cs @@ -1,31 +1,41 @@ -using System; +using System; using System.Collections.Concurrent; +using System.Text.Json.Nodes; using System.Threading.Tasks; +using SeqCli.Ingestion; using Serilog.Core; using Serilog.Events; -namespace SeqCli.Ingestion; +namespace SeqCli.Sample.Ingestion; -class BufferingSink: ILogEventSink, ILogEventReader, IDisposable +/// +/// Bridges the sample simulation's Serilog-based event generation into the +/// JSON-document-based shipping pipeline. +/// +class BufferingSink: ILogEventSink, IEventReader, IDisposable { - readonly ConcurrentQueue _queue = new(); + readonly ConcurrentQueue _queue = new(); const int QueueCapacity = 10000; volatile bool _disposed; - + public void Emit(LogEvent logEvent) { // No problem if this is racy - we can afford a bit of extra queue space. if (_disposed || _queue.Count > QueueCapacity) return; - - _queue.Enqueue(logEvent); + + var document = MetricsMapping.TryGetMetricSampleJson(logEvent, out var sample) + ? sample + : SerilogEventJson.ToEventJson(logEvent); + + _queue.Enqueue(document); } public Task TryReadAsync() { - if (!_queue.TryDequeue(out var logEvent)) + if (!_queue.TryDequeue(out var document)) return Task.FromResult(new ReadResult(null, _disposed)); - return Task.FromResult(new ReadResult(logEvent, _disposed)); + return Task.FromResult(new ReadResult(document, _disposed)); } public void Dispose() @@ -34,4 +44,4 @@ public void Dispose() _disposed = true; _queue.Clear(); } -} \ No newline at end of file +} diff --git a/src/SeqCli/Sample/Ingestion/MetricsMapping.cs b/src/SeqCli/Sample/Ingestion/MetricsMapping.cs new file mode 100644 index 00000000..8ba3771f --- /dev/null +++ b/src/SeqCli/Sample/Ingestion/MetricsMapping.cs @@ -0,0 +1,59 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Text.Json.Nodes; +using SeqCli.Ingestion; +using SeqCli.Syntax; +using Serilog.Events; + +namespace SeqCli.Sample.Ingestion; + +/// +/// The sample simulation generates metric samples as Serilog events carrying their metric +/// definitions in a surrogate property, because Serilog's data model has no @d +/// equivalent. Events marked this way ship as metric samples rather than logs. +/// +static class MetricsMapping +{ + // Use a "hygienic" name for the definitions property to avoid collisions. + internal static readonly string SurrogateDefinitionsProperty = $"_SeqcliMetricDefinitions_{Guid.NewGuid():N}"; + + public static bool TryGetMetricSampleJson(LogEvent logEvent, [NotNullWhen(true)] out JsonObject? sample) + { + if (!logEvent.Properties.TryGetValue(SurrogateDefinitionsProperty, out var definitions)) + { + sample = null; + return false; + } + + // Metric samples carry only a timestamp, definitions, and their dimension/value + // properties; no message or level. + sample = new JsonObject + { + ["@t"] = logEvent.Timestamp.ToString("o", CultureInfo.InvariantCulture), + ["@d"] = SerilogEventJson.ToJsonNode(definitions) + }; + + foreach (var (name, value) in logEvent.Properties) + { + if (name != SurrogateDefinitionsProperty) + EventJson.SetUserProperty(sample, name, SerilogEventJson.ToJsonNode(value)); + } + + return true; + } +} diff --git a/src/SeqCli/Sample/Loader/Simulation.cs b/src/SeqCli/Sample/Loader/Simulation.cs index a54f0a28..23828632 100644 --- a/src/SeqCli/Sample/Loader/Simulation.cs +++ b/src/SeqCli/Sample/Loader/Simulation.cs @@ -17,7 +17,7 @@ using Roastery.Metrics; using Seq.Api; using SeqCli.Ingestion; -using SeqCli.Mapping; +using SeqCli.Sample.Ingestion; using Serilog; namespace SeqCli.Sample.Loader; diff --git a/src/SeqCli/SeqCli.csproj b/src/SeqCli/SeqCli.csproj index f666ccdd..0e97213a 100644 --- a/src/SeqCli/SeqCli.csproj +++ b/src/SeqCli/SeqCli.csproj @@ -31,7 +31,6 @@ - @@ -43,13 +42,10 @@ - + - - - diff --git a/src/SeqCli/Syntax/EventJson.cs b/src/SeqCli/Syntax/EventJson.cs new file mode 100644 index 00000000..282a3cbf --- /dev/null +++ b/src/SeqCli/Syntax/EventJson.cs @@ -0,0 +1,66 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Globalization; +using System.Text.Json.Nodes; + +namespace SeqCli.Syntax; + +/// +/// Helpers for constructing event JSON documents in Seq's emission (CLEF) schema, where +/// reified fields carry @-prefixed names and user-defined property names beginning +/// with @ are escaped with a second @. +/// +static class EventJson +{ + const string InvalidPropertyNameSubstitute = "(unnamed)"; + + public static string EscapeUserPropertyName(string name) + { + if (string.IsNullOrEmpty(name)) + return InvalidPropertyNameSubstitute; + + return name.StartsWith('@') ? $"@{name}" : name; + } + + public static void SetUserProperty(JsonObject eventJson, string name, JsonNode? value) + { + eventJson[EscapeUserPropertyName(name)] = value; + } + + public static JsonNode? CreateScalar(object? value) + { + return value switch + { + null => null, + string s => JsonValue.Create(s), + bool b => JsonValue.Create(b), + byte n => JsonValue.Create(n), + sbyte n => JsonValue.Create(n), + short n => JsonValue.Create(n), + ushort n => JsonValue.Create(n), + int n => JsonValue.Create(n), + uint n => JsonValue.Create(n), + long n => JsonValue.Create(n), + ulong n => JsonValue.Create(n), + float n => JsonValue.Create(n), + double n => JsonValue.Create(n), + decimal n => JsonValue.Create(n), + DateTime dt => JsonValue.Create(dt.ToString("o", CultureInfo.InvariantCulture)), + DateTimeOffset dto => JsonValue.Create(dto.ToString("o", CultureInfo.InvariantCulture)), + _ => JsonValue.Create(value.ToString()) + }; + } +} diff --git a/src/SeqCli/Util/TextException.cs b/src/SeqCli/Syntax/IEventEnricher.cs similarity index 60% rename from src/SeqCli/Util/TextException.cs rename to src/SeqCli/Syntax/IEventEnricher.cs index 2017129d..9d10d8e7 100644 --- a/src/SeqCli/Util/TextException.cs +++ b/src/SeqCli/Syntax/IEventEnricher.cs @@ -1,4 +1,4 @@ -// Copyright 2013-2015 Serilog Contributors +// Copyright © Datalust and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,22 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; +using System.Text.Json.Nodes; -namespace SeqCli.Util; +namespace SeqCli.Syntax; -class TextException : Exception +/// +/// Adds or updates fields on an event JSON document; the equivalent, in Seq's data model, of a +/// Serilog enricher. +/// +interface IEventEnricher { - readonly string _text; - - public TextException(string text) - : base("This exception type provides ToString() access to details only.") - { - _text = text; - } - - public override string ToString() - { - return _text; - } -} \ No newline at end of file + void Enrich(JsonObject eventJson); +} diff --git a/src/SeqCli/Output/RedundantEventTypeRemovalEnricher.cs b/src/SeqCli/Syntax/LevelEnricher.cs similarity index 64% rename from src/SeqCli/Output/RedundantEventTypeRemovalEnricher.cs rename to src/SeqCli/Syntax/LevelEnricher.cs index d32e6666..c09fdb35 100644 --- a/src/SeqCli/Output/RedundantEventTypeRemovalEnricher.cs +++ b/src/SeqCli/Syntax/LevelEnricher.cs @@ -1,4 +1,4 @@ -// Copyright © Datalust and contributors. +// Copyright © Datalust and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,15 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -using Serilog.Core; -using Serilog.Events; +using System.Text.Json.Nodes; -namespace SeqCli.Output; +namespace SeqCli.Syntax; -public class RedundantEventTypeRemovalEnricher : ILogEventEnricher +/// +/// Overrides the event's @l level with a fixed value. +/// +class LevelEnricher(string level) : IEventEnricher { - public void Enrich(LogEvent logEvent, ILogEventPropertyFactory propertyFactory) + public void Enrich(JsonObject eventJson) { - logEvent.RemovePropertyIfPresent("@i"); + eventJson["@l"] = level; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Ingestion/ScalarPropertyEnricher.cs b/src/SeqCli/Syntax/ScalarPropertyEnricher.cs similarity index 59% rename from src/SeqCli/Ingestion/ScalarPropertyEnricher.cs rename to src/SeqCli/Syntax/ScalarPropertyEnricher.cs index 7146c7e7..95490a3b 100644 --- a/src/SeqCli/Ingestion/ScalarPropertyEnricher.cs +++ b/src/SeqCli/Syntax/ScalarPropertyEnricher.cs @@ -1,4 +1,4 @@ -// Copyright © Datalust and contributors. +// Copyright © Datalust and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,23 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. -using SeqCli.Util; -using Serilog.Core; -using Serilog.Events; +using System.Text.Json.Nodes; -namespace SeqCli.Ingestion; +namespace SeqCli.Syntax; -class ScalarPropertyEnricher : ILogEventEnricher +class ScalarPropertyEnricher : IEventEnricher { - readonly LogEventProperty _property; + readonly string _name; + readonly object? _scalarValue; public ScalarPropertyEnricher(string name, object? scalarValue) { - _property = LogEventPropertyFactory.SafeCreate(name, new ScalarValue(scalarValue)); + _name = EventJson.EscapeUserPropertyName(name); + _scalarValue = scalarValue; } - public void Enrich(LogEvent logEvent, ILogEventPropertyFactory propertyFactory) + public void Enrich(JsonObject eventJson) { - logEvent.AddOrUpdateProperty(_property); + eventJson[_name] = EventJson.CreateScalar(_scalarValue); } -} \ No newline at end of file +} diff --git a/src/SeqCli/Syntax/SeqCliNameResolver.cs b/src/SeqCli/Syntax/SeqCliNameResolver.cs deleted file mode 100644 index 91b4abab..00000000 --- a/src/SeqCli/Syntax/SeqCliNameResolver.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System.Diagnostics.CodeAnalysis; -using Seq.Syntax.Expressions; - -namespace SeqCli.Syntax; - -class SeqCliNameResolver: NameResolver -{ - public override bool TryResolveBuiltInPropertyName(string alias, [MaybeNullWhen(false)] out string target) - { - switch (alias) - { - case "@l": - target = "coalesce(SeqCliOriginalLevel, @l)"; - return true; - default: - target = null; - return false; - } - } -} diff --git a/src/SeqCli/Syntax/SeqSyntax.cs b/src/SeqCli/Syntax/SeqSyntax.cs index 3974b4ad..acbeab8c 100644 --- a/src/SeqCli/Syntax/SeqSyntax.cs +++ b/src/SeqCli/Syntax/SeqSyntax.cs @@ -1,11 +1,55 @@ -using Seq.Syntax.Expressions; +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Diagnostics.CodeAnalysis; +using Seq.Syntax.Expressions; +using Seq.Syntax.Templates; +using Seq.Syntax.Templates.Encoding; +using SeqCli.Syntax.V1; +using V1Compatibility = Seq.Syntax.Compatibility.V1; namespace SeqCli.Syntax; +/// +/// Compiles the expressions and templates accepted on the command line. Uses the Seq.Syntax v1 +/// compatibility shim so that established seqcli syntax — abbreviated built-in names like +/// @l, and the Elapsed()/Milliseconds() functions — keeps working. +/// static class SeqSyntax { public static CompiledExpression CompileExpression(string expression) { - return SerilogExpression.Compile(expression, nameResolver: new SeqCliNameResolver()); + if (!TryCompileExpression(expression, out var compiled, out var error)) + throw new ArgumentException(error); + + return compiled; + } + + public static bool TryCompileExpression( + string expression, + [MaybeNullWhen(false)] out CompiledExpression result, + [MaybeNullWhen(true)] out string error) + { + return V1Compatibility.TryCompileExpression(expression, formatProvider: null, TracingFunctions.Resolver, out result, out error); + } + + public static ExpressionTemplate ParseTemplate(string template, TemplateOutputEncoder? encoder = null) + { + if (!V1Compatibility.TryParseTemplate(template, culture: null, TracingFunctions.Resolver, encoder, out var parsed, out var error)) + throw new ArgumentException(error); + + return parsed; } -} \ No newline at end of file +} diff --git a/src/SeqCli/Syntax/V1/TracingFunctions.cs b/src/SeqCli/Syntax/V1/TracingFunctions.cs new file mode 100644 index 00000000..59193168 --- /dev/null +++ b/src/SeqCli/Syntax/V1/TracingFunctions.cs @@ -0,0 +1,58 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Globalization; +using System.Text.Json.Nodes; +using Seq.Syntax.Expressions; + +namespace SeqCli.Syntax.V1; + +/// +/// Functions carried over from earlier seqcli versions, where Seq.Syntax had no tracing +/// support of its own. Elapsed() and Milliseconds() remain only so that existing +/// user-supplied expressions and output templates keep working; the built-in @Elapsed +/// and TotalMilliseconds() replace them. +/// +static class TracingFunctions +{ + public static readonly NameResolver Resolver = new StaticMemberNameResolver(typeof(TracingFunctions)); + + public static EvaluationResult Elapsed(JsonObject eventJson) + { + if (GetTimestampField(eventJson, "@t") is { } timestamp && + GetTimestampField(eventJson, "@st") is { } start) + { + return JsonValue.Create(timestamp - start)!; + } + + return EvaluationResult.Undefined; + } + + public static EvaluationResult Milliseconds(TimeSpan timeSpan) + { + // Truncates instead of rounding. + return JsonValue.Create(timeSpan.Ticks / (decimal)TimeSpan.TicksPerMillisecond); + } + + static DateTimeOffset? GetTimestampField(JsonObject eventJson, string field) + { + return eventJson.TryGetPropertyValue(field, out var node) && + node is JsonValue value && + value.TryGetValue(out string? text) && + DateTimeOffset.TryParse(text, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var dto) + ? dto + : null; + } +} diff --git a/src/SeqCli/Traces/StructuredMessage.cs b/src/SeqCli/Traces/StructuredMessage.cs index 73654f04..38cae501 100644 --- a/src/SeqCli/Traces/StructuredMessage.cs +++ b/src/SeqCli/Traces/StructuredMessage.cs @@ -14,30 +14,30 @@ using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Text.Json.Nodes; using Newtonsoft.Json.Linq; -using SeqCli.Output; using SeqCli.Util; -using Serilog.Events; -using Serilog.Parsing; namespace SeqCli.Traces; static class StructuredMessage { /// - /// Reads the token array produced by the Seq `@StructuredMessage` property - /// into a Serilog message template, along with the property values needed to render it. + /// Reads the token array produced by the Seq `@StructuredMessage` property into message + /// template text, along with the property values needed to render it. Dotted hole names + /// are stored as nested structures, matching how message rendering resolves them. /// - public static (MessageTemplate Message, IReadOnlyList Properties) Read(object? structuredMessage) + public static (string MessageTemplate, JsonObject Properties) Read(object? structuredMessage) { if (structuredMessage is null or JValue { Type: JTokenType.Null }) - return (new MessageTemplate([]), []); + return ("", new JsonObject()); if (structuredMessage is not JArray tokens) throw new InvalidDataException($"Expected a structured message but found `{structuredMessage}`."); - var templateTokens = new List(); - var properties = new List(); + var templateTokens = new List<(bool IsText, string Text)>(); + var properties = new JsonObject(); var propertyNames = new HashSet(); foreach (var token in tokens) @@ -48,14 +48,14 @@ public static (MessageTemplate Message, IReadOnlyList Properti throw new InvalidDataException("A message template hole is missing its `name`."); // Currently ignores `formatted`. - templateTokens.Add(new PropertyToken(name, (hole["raw"] as JValue)?.Value as string ?? $"{{{name}}}")); + templateTokens.Add((false, (hole["raw"] as JValue)?.Value as string ?? $"{{{name}}}")); if (hole.TryGetValue("value", out var value) && propertyNames.Add(name)) - properties.Add(LogEventPropertyFactory.SafeCreate(name, CreatePropertyValue(value))); + SetPathProperty(properties, name, JsonNodes.FromNewtonsoft(value)); } else if (token is JValue { Type: JTokenType.String } text) { - templateTokens.Add(new TextToken((string)text.Value!)); + templateTokens.Add((true, (string)text.Value!)); } else { @@ -65,27 +65,53 @@ public static (MessageTemplate Message, IReadOnlyList Properti TrimEnd(templateTokens); - return (new MessageTemplate(templateTokens), properties); + var templateText = string.Concat(templateTokens.Select(t => + t.IsText ? t.Text.Replace("{", "{{").Replace("}", "}}") : t.Text)); + + return (templateText, properties); + } + + // Message rendering resolves dotted hole names as paths into nested objects, so `a.b` + // becomes member `b` of object `a`. If placing a value along the path would collide with a + // non-object value, the hole is left unresolvable and renders as raw text. + static void SetPathProperty(JsonObject properties, string name, JsonNode? value) + { + var steps = name.Split('.'); + var target = properties; + for (var i = 0; i < steps.Length - 1; ++i) + { + if (target.TryGetPropertyValue(steps[i], out var next)) + { + if (next is not JsonObject nextObject) + return; + + target = nextObject; + } + else + { + var nextObject = new JsonObject(); + target[steps[i]] = nextObject; + target = nextObject; + } + } + + target[steps[^1]] = value; } - static void TrimEnd(List templateTokens) + static void TrimEnd(List<(bool IsText, string Text)> templateTokens) { - while (templateTokens.Count > 0 && templateTokens[^1] is TextToken text) + while (templateTokens.Count > 0 && templateTokens[^1] is (true, var text)) { - var trimmed = text.Text.TrimEnd(); - if (trimmed.Length == text.Text.Length) + var trimmed = text.TrimEnd(); + if (trimmed.Length == text.Length) break; templateTokens.RemoveAt(templateTokens.Count - 1); if (trimmed.Length > 0) { - templateTokens.Add(new TextToken(trimmed)); + templateTokens.Add((true, trimmed)); break; } } } - - static LogEventPropertyValue CreatePropertyValue(JToken value) => value is JValue scalar ? - new ScalarValue(scalar.Value) : - OutputFormat.CreatePropertyValue(value); } diff --git a/src/SeqCli/Traces/TraceTreeElement.cs b/src/SeqCli/Traces/TraceTreeElement.cs index 52a25a8f..df4a6756 100644 --- a/src/SeqCli/Traces/TraceTreeElement.cs +++ b/src/SeqCli/Traces/TraceTreeElement.cs @@ -14,7 +14,7 @@ using System; using System.Collections.Generic; -using Serilog.Events; +using System.Text.Json.Nodes; namespace SeqCli.Traces; @@ -22,8 +22,8 @@ record TraceTreeElement( string Id, DateTimeOffset Timestamp, string? Level, - MessageTemplate MessageTemplate, - IReadOnlyList TemplateProperties, + string MessageTemplate, + JsonObject TemplateProperties, string? Exception, string? SpanId, string? ParentId, diff --git a/src/SeqCli/Traces/TraceTreeJObjectConverter.cs b/src/SeqCli/Traces/TraceTreeJObjectConverter.cs index 4399c15b..5d170f83 100644 --- a/src/SeqCli/Traces/TraceTreeJObjectConverter.cs +++ b/src/SeqCli/Traces/TraceTreeJObjectConverter.cs @@ -15,17 +15,16 @@ using System.Collections.Generic; using System.Globalization; using System.IO; +using System.Text.Json.Nodes; using Newtonsoft.Json.Linq; -using SeqCli.Mapping; +using Seq.Syntax.Templates; using SeqCli.Output; -using Serilog.Events; -using Serilog.Formatting; namespace SeqCli.Traces; static class TraceTreeJObjectConverter { - static readonly ITextFormatter MessageFormatter = TextFormatters.Plain(theme: null, "{@m}"); + static readonly ExpressionTemplate MessageFormatter = TextFormatters.Plain(theme: null, "{@Message}"); public static JObject FromRoots(string traceId, IReadOnlyList roots, bool complete, bool includeTypeMarker, IReadOnlyList columns) { @@ -82,7 +81,7 @@ static JObject ToJson(TraceTreeNode node, bool includeTypeMarker, IReadOnlyList< json["parentSpanId"] = evt.ParentId; if (!string.IsNullOrEmpty(evt.Level)) - json["level"] = LevelMapping.ToFullLevelName(evt.Level); + json["level"] = evt.Level; if (evt.IsSpan) { @@ -131,15 +130,16 @@ static JObject ToJson(TraceTreeNode node, bool includeTypeMarker, IReadOnlyList< static string RenderMessage(TraceTreeElement evt) { - var logEvent = new LogEvent( - evt.SortKey, - LevelMapping.ToSerilogLevel(evt.Level ?? ""), - exception: null, - evt.MessageTemplate, - evt.TemplateProperties); + var eventJson = new JsonObject + { + ["@mt"] = evt.MessageTemplate + }; + + foreach (var (name, value) in evt.TemplateProperties) + eventJson[name] = value?.DeepClone(); var message = new StringWriter(); - MessageFormatter.Format(logEvent, message); + MessageFormatter.Format(eventJson, message); return message.ToString(); } } diff --git a/src/SeqCli/Util/JsonNetDestructuringPolicy.cs b/src/SeqCli/Util/JsonNetDestructuringPolicy.cs deleted file mode 100644 index 8d9bf7bc..00000000 --- a/src/SeqCli/Util/JsonNetDestructuringPolicy.cs +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2015 Destructurama Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using Newtonsoft.Json.Linq; -using Serilog.Core; -using Serilog.Events; - -namespace SeqCli.Util; - -sealed class JsonNetDestructuringPolicy : IDestructuringPolicy -{ - public bool TryDestructure(object value, ILogEventPropertyValueFactory propertyValueFactory, [NotNullWhen(true)] out LogEventPropertyValue? result) - { - switch (value) - { - case JObject jo: - result = Destructure(jo, propertyValueFactory); - return true; - case JArray ja: - result = Destructure(ja, propertyValueFactory); - return true; - case JValue jv: - result = Destructure(jv, propertyValueFactory); - return true; - } - - result = null; - return false; - } - - static LogEventPropertyValue Destructure(JValue jv, ILogEventPropertyValueFactory propertyValueFactory) - { - return propertyValueFactory.CreatePropertyValue(jv.Value!, destructureObjects: true); - } - - static SequenceValue Destructure(JArray ja, ILogEventPropertyValueFactory propertyValueFactory) - { - var elems = ja.Select(t => propertyValueFactory.CreatePropertyValue(t, destructureObjects: true)); - return new SequenceValue(elems); - } - - static LogEventPropertyValue Destructure(JObject jo, ILogEventPropertyValueFactory propertyValueFactory) - { - string? typeTag = null; - var props = new List(jo.Count); - - foreach (var prop in jo.Properties()) - { - if (prop.Name == "$type") - { - if (prop.Value is JValue typeVal && typeVal.Value is string v) - { - typeTag = v; - continue; - } - } - else if (!LogEventProperty.IsValidName(prop.Name)) - { - return DestructureToDictionaryValue(jo, propertyValueFactory); - } - - props.Add(new LogEventProperty(prop.Name, propertyValueFactory.CreatePropertyValue(prop.Value, destructureObjects: true))); - } - - return new StructureValue(props, typeTag); - } - - static DictionaryValue DestructureToDictionaryValue(JObject jo, ILogEventPropertyValueFactory propertyValueFactory) - { - var elements = jo.Properties().Select( - prop => new KeyValuePair( - new ScalarValue(prop.Name), - propertyValueFactory.CreatePropertyValue(prop.Value, destructureObjects: true)) - ); - return new DictionaryValue(elements); - } -} \ No newline at end of file diff --git a/src/SeqCli/Util/JsonNodes.cs b/src/SeqCli/Util/JsonNodes.cs new file mode 100644 index 00000000..7129be90 --- /dev/null +++ b/src/SeqCli/Util/JsonNodes.cs @@ -0,0 +1,45 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Text.Json.Nodes; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using SeqCli.Syntax; + +namespace SeqCli.Util; + +static class JsonNodes +{ + public static JsonNode? FromNewtonsoft(JToken token) + { + if (token is JValue { Value: null }) + return null; + + return JsonNode.Parse(token.ToString(Formatting.None)); + } + + /// + /// Convert a value deserialized by the Seq API client — a Newtonsoft LINQ-to-JSON token, or + /// a plain CLR scalar — into its System.Text.Json equivalent. + /// + public static JsonNode? FromApiValue(object? value) + { + return value switch + { + null => null, + JToken token => FromNewtonsoft(token), + _ => EventJson.CreateScalar(value) + }; + } +} diff --git a/src/SeqCli/Util/LogEventPropertyFactory.cs b/src/SeqCli/Util/LogEventPropertyFactory.cs deleted file mode 100644 index 89c23987..00000000 --- a/src/SeqCli/Util/LogEventPropertyFactory.cs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright © Datalust Pty Ltd and Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using Serilog.Events; - -namespace SeqCli.Util; - -static class LogEventPropertyFactory -{ - const string InvalidPropertyNameSubstitute = "(unnamed)"; - - public static LogEventProperty SafeCreate(string name, LogEventPropertyValue value) - { - if (value == null) throw new ArgumentNullException(nameof(value)); - - if (!LogEventProperty.IsValidName(name)) - name = InvalidPropertyNameSubstitute; - - return new LogEventProperty(name, value); - } -} \ No newline at end of file diff --git a/test/SeqCli.Tests/Csv/CsvWriterTests.cs b/test/SeqCli.Tests/Csv/CsvWriterTests.cs index cf4dbeb9..8d46d098 100644 --- a/test/SeqCli.Tests/Csv/CsvWriterTests.cs +++ b/test/SeqCli.Tests/Csv/CsvWriterTests.cs @@ -3,7 +3,7 @@ using System.IO; using Seq.Api.Model.Data; using SeqCli.Csv; -using Serilog.Templates.Themes; +using Seq.Syntax.Templates.Themes; using Xunit; namespace SeqCli.Tests.Csv; @@ -12,8 +12,8 @@ public class CsvWriterTests { const char Escape = '\x1b'; - // `CsvWriter` writes to the console without going through Serilog's console sink, so unlike the other - // output paths it has no opportunity to suppress the theme itself. + // `CsvWriter` writes delimited output directly rather than rendering a template, so unlike the + // other output paths it applies (or omits) the theme itself. [Fact] public void QueryResultsAreNotColorizedWhenOutputIsRedirected() { diff --git a/test/SeqCli.Tests/Ingestion/SerilogEventJsonTests.cs b/test/SeqCli.Tests/Ingestion/SerilogEventJsonTests.cs new file mode 100644 index 00000000..f27aaf1f --- /dev/null +++ b/test/SeqCli.Tests/Ingestion/SerilogEventJsonTests.cs @@ -0,0 +1,109 @@ +#nullable enable +using System; +using System.Linq; +using SeqCli.Ingestion; +using SeqCli.Sample.Ingestion; +using Serilog; +using Serilog.Events; +using Xunit; + +namespace SeqCli.Tests.Ingestion; + +public class SerilogEventJsonTests +{ + static LogEvent CaptureEvent(Action log) + { + LogEvent? captured = null; + var logger = new LoggerConfiguration() + .MinimumLevel.Verbose() + .WriteTo.Sink(new CapturingSink(evt => captured = evt)) + .CreateLogger(); + log(logger); + return captured ?? throw new InvalidOperationException("No event was captured."); + } + + class CapturingSink(Action capture) : Serilog.Core.ILogEventSink + { + public void Emit(LogEvent logEvent) => capture(logEvent); + } + + [Fact] + public void EventFieldsMapToTheEmissionSchema() + { + var evt = CaptureEvent(log => log.Warning(new Exception("Boom!"), "Hello, {Name}!", "world")); + var eventJson = SerilogEventJson.ToEventJson(evt); + + Assert.Equal(evt.Timestamp.ToString("o"), (string?)eventJson["@t"]); + Assert.Equal("Hello, {Name}!", (string?)eventJson["@mt"]); + Assert.Equal("Warning", (string?)eventJson["@l"]); + Assert.StartsWith("System.Exception: Boom!", (string?)eventJson["@x"]); + Assert.Equal("world", (string?)eventJson["Name"]); + } + + [Fact] + public void InformationLevelsAreOmitted() + { + var evt = CaptureEvent(log => log.Information("Hello")); + + Assert.False(SerilogEventJson.ToEventJson(evt).ContainsKey("@l")); + } + + [Fact] + public void StructuredValuesSerializeAsJson() + { + var evt = CaptureEvent(log => log.Information("{@Order} {Items}", + new { Id = 7, Total = 4.5 }, new[] { "a", "b" })); + var eventJson = SerilogEventJson.ToEventJson(evt); + + Assert.Equal(7, (int?)eventJson["Order"]!["Id"]); + Assert.Equal(4.5, (double?)eventJson["Order"]!["Total"]); + Assert.Equal(new[] { "a", "b" }, eventJson["Items"]!.AsArray().Select(i => (string?)i).ToArray()); + } + + [Fact] + public void SerilogTracingSpanPropertiesAreLifted() + { + var start = DateTime.UtcNow.AddMilliseconds(-25); + var evt = CaptureEvent(log => log + .ForContext("SpanStartTimestamp", start) + .ForContext("ParentSpanId", "8899aabbccddeeff") + .Information("GET /orders")); + var eventJson = SerilogEventJson.ToEventJson(evt); + + Assert.Equal(start.ToString("o"), (string?)eventJson["@st"]); + Assert.Equal("8899aabbccddeeff", (string?)eventJson["@ps"]); + Assert.False(eventJson.ContainsKey("SpanStartTimestamp")); + Assert.False(eventJson.ContainsKey("ParentSpanId")); + } + + [Fact] + public void MetricDefinitionsProduceMetricSamples() + { + var evt = CaptureEvent(log => log + .ForContext(MetricsMapping.SurrogateDefinitionsProperty, new { roasted_kg = new { unit = "kg" } }, destructureObjects: true) + .ForContext("roasted_kg", 42.5) + .Information("Metrics sampled")); + + Assert.True(MetricsMapping.TryGetMetricSampleJson(evt, out var eventJson)); + Assert.Equal("kg", (string?)eventJson["@d"]!["roasted_kg"]!["unit"]); + Assert.Equal(42.5, (double?)eventJson["roasted_kg"]); + Assert.False(eventJson.ContainsKey("@mt")); + Assert.False(eventJson.ContainsKey("@l")); + } + + [Fact] + public void PlainEventsAreNotMetricSamples() + { + var evt = CaptureEvent(log => log.Information("Hello")); + + Assert.False(MetricsMapping.TryGetMetricSampleJson(evt, out _)); + } + + [Fact] + public void PropertyNamesBeginningWithAtAreEscaped() + { + var evt = CaptureEvent(log => log.ForContext("@evil", "value").Information("Hello")); + + Assert.Equal("value", (string?)SerilogEventJson.ToEventJson(evt)["@@evil"]); + } +} diff --git a/test/SeqCli.Tests/Output/OutputFormatTests.cs b/test/SeqCli.Tests/Output/OutputFormatTests.cs index 4c08a64e..cb1cf55b 100644 --- a/test/SeqCli.Tests/Output/OutputFormatTests.cs +++ b/test/SeqCli.Tests/Output/OutputFormatTests.cs @@ -2,9 +2,9 @@ using Newtonsoft.Json.Linq; using Seq.Api.Model.Events; using SeqCli.Config; +using SeqCli.Mapping; using SeqCli.Output; using SeqCli.Tests.Support; -using Serilog.Events; using Xunit; #nullable enable @@ -128,11 +128,10 @@ static EventEntity MakeDottedHoleEvent(params (string Name, object? Value)[] pro static string RenderMessage(EventEntity evt) { - var serilogEvent = OutputFormat.ToSerilogEvent(evt); - OutputFormat.FlattenPropertiesUsedWithDottedNames(evt, serilogEvent); + var eventJson = EventEntityJson.ToEventJson(evt); var output = new StringWriter(); - TextFormatters.Plain(theme: null, "{@m}").Format(serilogEvent, output); + TextFormatters.Plain(theme: null, "{@m}").Format(eventJson, output); return output.ToString(); } @@ -146,36 +145,53 @@ public void DottedHoleNamesResolveThroughNestedStructures() } [Fact] - public void FlatPropertiesWinOverStructureTraversal() + public void UnresolvableDottedHolesRenderAsRawText() { - var evt = MakeDottedHoleEvent( - ("user.greeting.first", "G'day"), - ("user", JObject.Parse("""{"greeting": {"first": "Hello"}, "name": "Barney"}"""))); + var evt = MakeDottedHoleEvent(("user", JObject.Parse("""{"greeting": 42}"""))); - Assert.Equal("G'day Barney!", RenderMessage(evt)); + Assert.Equal("{user.greeting.first} {user.name}!", RenderMessage(evt)); + } + + static string CaptureConsoleOut(System.Action write) + { + var output = new StringWriter(); + var saved = System.Console.Out; + System.Console.SetOut(output); + try + { + write(); + } + finally + { + System.Console.SetOut(saved); + } + + return output.ToString(); } [Fact] - public void UnresolvableDottedHolesRenderAsRawText() + public void ObjectsAreWrittenAsSingleLineJson() { - var evt = MakeDottedHoleEvent(("user", JObject.Parse("""{"greeting": 42}"""))); + var format = Create(syntax: OutputSyntax.Json); - Assert.Equal("{user.greeting.first} {user.name}!", RenderMessage(evt)); + var written = CaptureConsoleOut(() => format.WriteObject( + new JObject(new JProperty("Title", "Errors"), new JProperty("Count", 42)))); + + Assert.Equal("""{"Title":"Errors","Count":42}""" + System.Environment.NewLine, written); } [Fact] - public void ResolvedScalarsAreUnwrappedFromTheirJsonRepresentation() + public void EntitiesAreWrittenAsJsonWithoutLinks() { - var evt = Some.MakeEvent(e => - { - e.MessageTemplateTokens = [new MessageTemplateTokenPart { PropertyName = "order.total" }]; - e.Properties = Some.MakeProperties(("order", JObject.Parse("""{"total": 42}"""))); - }); + var entity = new Seq.Api.Model.Signals.SignalEntity { Id = "signal-1", Title = "Errors" }; - var serilogEvent = OutputFormat.ToSerilogEvent(evt); - OutputFormat.FlattenPropertiesUsedWithDottedNames(evt, serilogEvent); + var format = Create(syntax: OutputSyntax.Json); + var written = CaptureConsoleOut(() => format.WriteEntity(entity)); - var scalar = Assert.IsType(serilogEvent.Properties["order.total"]); - Assert.Equal(42L, scalar.Value); + Assert.Contains("\"Id\":\"signal-1\"", written); + Assert.Contains("\"Title\":\"Errors\"", written); + Assert.DoesNotContain("Links", written); + Assert.EndsWith(System.Environment.NewLine, written); + Assert.Equal(written.TrimEnd(), written.TrimEnd().ReplaceLineEndings("")); } } diff --git a/test/SeqCli.Tests/Output/TextFormattersTests.cs b/test/SeqCli.Tests/Output/TextFormattersTests.cs index f3548e0d..7675926e 100644 --- a/test/SeqCli.Tests/Output/TextFormattersTests.cs +++ b/test/SeqCli.Tests/Output/TextFormattersTests.cs @@ -1,11 +1,11 @@ #nullable enable using System; using System.IO; +using System.Text.Json.Nodes; +using Seq.Syntax.Templates.Themes; +using SeqCli.Mapping; using SeqCli.Output; using SeqCli.Tests.Support; -using Serilog.Events; -using Serilog.Parsing; -using Serilog.Templates.Themes; using Xunit; namespace SeqCli.Tests.Output; @@ -13,7 +13,7 @@ namespace SeqCli.Tests.Output; public class TextFormattersTests { const char Escape = '\x1b'; - static readonly DateTimeOffset FixedTimestamp = new(2024, 1, 1, 10, 0, 1, 250, TimeSpan.Zero); + const string FixedTimestamp = "2024-01-01T10:00:01.2500000+00:00"; [Fact] public void ThemedJsonOutputIsColorizedRegardlessOfRedirection() @@ -27,12 +27,18 @@ public void UnthemedJsonOutputIsNotColorized() Assert.DoesNotContain(Escape, RenderJson(theme: null)); } + [Fact] + public void UnthemedJsonOutputIsTheEventDocumentVerbatim() + { + Assert.Equal( + """{"@t":"2024-01-01T10:00:01.2500000+00:00","@mt":"Hello, {Name}!","Name":"world"}""" + Environment.NewLine, + RenderJson(theme: null, SomeEventJson())); + } + [Fact] public void LogEventsAreFormattedWithTheDefaultTextTemplate() { - var evt = SomeLogEvent( - level: LogEventLevel.Warning, - properties: new LogEventProperty("Name", new ScalarValue("world"))); + var evt = SomeEventJson(level: "Warning"); Assert.Equal( $"[2024-01-01T10:00:01.2500000+00:00 WRN] Hello, world!{Environment.NewLine}", @@ -42,11 +48,7 @@ public void LogEventsAreFormattedWithTheDefaultTextTemplate() [Fact] public void ExceptionsAreIncludedInTextOutput() { - var evt = SomeLogEvent( - FixedTimestamp, - LogEventLevel.Error, - new Exception("Boom!"), - new LogEventProperty("Name", new ScalarValue("world"))); + var evt = SomeEventJson(level: "Error", exception: "System.Exception: Boom!"); Assert.Equal( $"[2024-01-01T10:00:01.2500000+00:00 ERR] Hello, world!{Environment.NewLine}System.Exception: Boom!{Environment.NewLine}", @@ -54,14 +56,10 @@ public void ExceptionsAreIncludedInTextOutput() } [Fact] - public void SpanElapsedTimeIsComputedFromTheStartTimestampProperty() + public void SpanElapsedTimeIsComputedFromTheStartTimestamp() { - // Events retrieved from the Seq API carry span start timestamps in ISO-8601 `@st` properties. - var evt = SomeLogEvent(FixedTimestamp, properties: - [ - new LogEventProperty("Name", new ScalarValue("world")), - new LogEventProperty("@st", new ScalarValue("2024-01-01T10:00:00.0000000Z")) - ]); + var evt = SomeEventJson(); + evt["@st"] = "2024-01-01T10:00:00.0000000Z"; Assert.Equal( $"[2024-01-01T10:00:01.2500000+00:00 INF] Hello, world! (1250 ms){Environment.NewLine}", @@ -69,53 +67,41 @@ public void SpanElapsedTimeIsComputedFromTheStartTimestampProperty() } [Fact] - public void SpanElapsedTimeIsComputedFromTheSurrogateStartTimestampProperty() + public void ACustomOutputTemplateReplacesTheDefault() { - // Ingested spans carry a surrogate `SpanStartTimestamp` property with a `DateTime` value. - var evt = SomeLogEvent(FixedTimestamp, properties: - [ - new LogEventProperty("Name", new ScalarValue("world")), - new LogEventProperty("SpanStartTimestamp", new ScalarValue( - FixedTimestamp.UtcDateTime.AddMilliseconds(-1.5))) - ]); - Assert.Equal( - $"[2024-01-01T10:00:01.2500000+00:00 INF] Hello, world! (1.5 ms){Environment.NewLine}", - RenderText(evt)); + $"INF Hello, world!{Environment.NewLine}", + RenderText(SomeEventJson(), $"{{@l:u3}} {{@m}}{Environment.NewLine}")); } - [Fact] - public void ACustomOutputTemplateReplacesTheDefault() + static JsonObject SomeEventJson(string? level = null, string? exception = null) { - var evt = SomeLogEvent(properties: new LogEventProperty("Name", new ScalarValue("world"))); + var evt = new JsonObject + { + ["@t"] = FixedTimestamp, + ["@mt"] = "Hello, {Name}!", + ["Name"] = "world" + }; - Assert.Equal($"INF Hello, world!{Environment.NewLine}", RenderText(evt, $"{{@l:u3}} {{@m}}{Environment.NewLine}")); - } + if (level != null) + evt["@l"] = level; - static LogEvent SomeLogEvent( - DateTimeOffset? timestamp = null, - LogEventLevel level = LogEventLevel.Information, - Exception? exception = null, - params LogEventProperty[] properties) - { - return new LogEvent( - timestamp ?? FixedTimestamp, - level, - exception, - new MessageTemplateParser().Parse("Hello, {Name}!"), - properties); + if (exception != null) + evt["@x"] = exception; + + return evt; } - static string RenderText(LogEvent evt, string? outputTemplate = null) + static string RenderText(JsonObject evt, string? outputTemplate = null) { var output = new StringWriter(); TextFormatters.Plain(theme: null, outputTemplate).Format(evt, output); return output.ToString(); } - static string RenderJson(TemplateTheme? theme) + static string RenderJson(TemplateTheme? theme, JsonObject? evt = null) { - var evt = OutputFormat.ToSerilogEvent(Some.MakeEvent(e => e.Properties = [])); + evt ??= EventEntityJson.ToEventJson(Some.MakeEvent(e => e.Properties = [])); var output = new StringWriter(); TextFormatters.Json(theme).Format(evt, output); diff --git a/test/SeqCli.Tests/Output/TraceFormatterTests.cs b/test/SeqCli.Tests/Output/TraceFormatterTests.cs index fce99356..9dc3a694 100644 --- a/test/SeqCli.Tests/Output/TraceFormatterTests.cs +++ b/test/SeqCli.Tests/Output/TraceFormatterTests.cs @@ -3,10 +3,9 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text.Json.Nodes; using SeqCli.Output; using SeqCli.Traces; -using Serilog.Events; -using Serilog.Parsing; using Xunit; namespace SeqCli.Tests.Output; @@ -18,14 +17,14 @@ public class TraceFormatterTests static TraceTreeElement Span(string spanId, string? parentId, double startMs = 0, double elapsedMs = 1, string? message = null, IReadOnlyList? columns = null) => new($"event-span-{spanId}", T0.AddMilliseconds(startMs + elapsedMs), null, - new MessageTemplate([new TextToken(message ?? $"span {spanId}")]), [], + message ?? $"span {spanId}", new JsonObject(), null, spanId, parentId, T0.AddMilliseconds(startMs), TimeSpan.FromMilliseconds(elapsedMs), columns ?? []); static TraceTreeElement Log(string? spanId, double timestampMs, string message = "log", string? level = null, string? exception = null) => new($"event-log-{timestampMs}-{message}", T0.AddMilliseconds(timestampMs), level, - new MessageTemplate([new TextToken(message)]), [], exception, + message, new JsonObject(), exception, spanId, null, null, null, []); static string Render(params TraceTreeElement[] events) @@ -33,8 +32,8 @@ static string Render(params TraceTreeElement[] events) var output = new StringWriter(); var formatter = TextFormatters.Plain(theme: null, TraceFormatter.OutputTemplate(events.Max(e => e.Columns.Count))); - foreach (var logEvent in TraceFormatter.ToLogEvents(TraceTreeBuilder.Build(events))) - formatter.Format(logEvent, output); + foreach (var eventJson in TraceFormatter.ToEventJson(TraceTreeBuilder.Build(events))) + formatter.Format(eventJson, output); return output.ToString(); } @@ -115,12 +114,8 @@ public void MissingAndEmptyColumnValuesLeaveNoRedundantSpace(object? first) public void TemplateHolesAreFilledFromMessageProperties() { var evt = new TraceTreeElement("event-1", T0.AddMilliseconds(1.5), null, - new MessageTemplate([ - new TextToken("GET "), - new PropertyToken("Route", "{Route}"), - new TextToken(" as "), - new PropertyToken("User", "{User}")]), - [new LogEventProperty("Route", new ScalarValue("/orders"))], + "GET {Route} as {User}", + new JsonObject { ["Route"] = "/orders" }, null, "a", null, T0, TimeSpan.FromMilliseconds(1.5), []); Assert.Equal( diff --git a/test/SeqCli.Tests/PlainText/EventJsonBuilderTests.cs b/test/SeqCli.Tests/PlainText/EventJsonBuilderTests.cs new file mode 100644 index 00000000..0df85264 --- /dev/null +++ b/test/SeqCli.Tests/PlainText/EventJsonBuilderTests.cs @@ -0,0 +1,60 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Globalization; +using SeqCli.PlainText.LogEvents; +using Superpower.Model; +using Xunit; + +namespace SeqCli.Tests.PlainText; + +public class EventJsonBuilderTests +{ + [Fact] + public void SuppliedValuesAreUsed() + { + var properties = new Dictionary + { + ["@t"] = new TextSpan("2018-02-01T13:00:00.123Z"), + ["@l"] = new TextSpan("WRN"), + ["@m"] = new TextSpan("Hello, world"), + ["@x"] = new TextSpan("EverythingFailedException"), + ["MachineName"] = new TextSpan("TP"), + ["Count"] = 42 + }; + + var remainder = "rem"; + var evt = EventJsonBuilder.FromProperties(properties, remainder); + + Assert.Equal("2018-02-01T13:00:00.1230000+00:00", + DateTimeOffset.Parse((string)evt["@t"]!, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind) + .ToUniversalTime().ToString("o")); + Assert.Equal("Hello, world", (string?)evt["@m"]); + Assert.Equal("WRN", (string?)evt["@l"]); + Assert.Equal("EverythingFailedException", (string?)evt["@x"]); + Assert.Equal(42, (int?)evt["Count"]); + Assert.Equal("TP", (string?)evt["MachineName"]); + Assert.Equal("rem", (string?)evt["@@unmatched"]); + } + + [Fact] + public void MissingValuesAreDefaulted() + { + var evt = EventJsonBuilder.FromProperties(new Dictionary(), null); + + var timestamp = DateTimeOffset.Parse((string)evt["@t"]!, CultureInfo.InvariantCulture, + DateTimeStyles.RoundtripKind); + Assert.True(timestamp > DateTimeOffset.Now.AddSeconds(-5)); + Assert.False(evt.ContainsKey("@m")); + Assert.False(evt.ContainsKey("@l")); + Assert.False(evt.ContainsKey("@x")); + } + + [Fact] + public void DateTimeOffsetTimestampsAreAccepted() + { + var then = DateTimeOffset.Now.AddDays(-5); + var evt = EventJsonBuilder.FromProperties(new Dictionary{["@t"] = then}, null); + Assert.Equal(then.ToString("o", CultureInfo.InvariantCulture), (string?)evt["@t"]); + } +} diff --git a/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs b/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs deleted file mode 100644 index 75eaf8d5..00000000 --- a/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs +++ /dev/null @@ -1,56 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using SeqCli.PlainText.LogEvents; -using Serilog.Events; -using Superpower.Model; -using Xunit; - -namespace SeqCli.Tests.PlainText; - -public class LogEventBuilderTests -{ - [Fact] - public void SuppliedValuesAreUsed() - { - var properties = new Dictionary - { - ["@t"] = new TextSpan("2018-02-01T13:00:00.123Z"), - ["@l"] = new TextSpan("WRN"), - ["@m"] = new TextSpan("Hello, world"), - ["@x"] = new TextSpan("EverythingFailedException"), - ["MachineName"] = new TextSpan("TP"), - ["Count"] = 42 - }; - - var remainder = "rem"; - var evt = LogEventBuilder.FromProperties(properties, remainder); - - Assert.Equal("2018-02-01T13:00:00.1230000+00:00", evt.Timestamp.ToString("o")); - Assert.Equal("Hello, world", evt.RenderMessage()); - Assert.Equal(LogEventLevel.Warning, evt.Level); - Assert.Equal("EverythingFailedException", evt.Exception?.ToString()); - Assert.Equal(42, ((ScalarValue)evt.Properties["Count"]).Value); - Assert.Equal("TP", ((ScalarValue)evt.Properties["MachineName"]).Value!.ToString()); - Assert.Equal("rem", ((ScalarValue)evt.Properties["@unmatched"]).Value!.ToString()); - } - - [Fact] - public void MissingValuesAreDefaulted() - { - var evt = LogEventBuilder.FromProperties(new Dictionary(), null); - - Assert.True(evt.Timestamp > DateTimeOffset.Now.AddSeconds(-5)); - Assert.Equal("", evt.RenderMessage()); - Assert.Equal(LogEventLevel.Information, evt.Level); - Assert.Null(evt.Exception); - } - - [Fact] - public void DateTimeOffsetTimestampsAreAccepted() - { - var then = DateTimeOffset.Now.AddDays(-5); - var evt = LogEventBuilder.FromProperties(new Dictionary{["@t"] = then}, null); - Assert.Equal(then, evt.Timestamp); - } -} \ No newline at end of file diff --git a/test/SeqCli.Tests/PlainText/StaticMessageTemplateReaderTests.cs b/test/SeqCli.Tests/PlainText/StaticMessageTemplateReaderTests.cs index 29f99471..9c68a1d5 100644 --- a/test/SeqCli.Tests/PlainText/StaticMessageTemplateReaderTests.cs +++ b/test/SeqCli.Tests/PlainText/StaticMessageTemplateReaderTests.cs @@ -1,4 +1,5 @@ -using System.Threading.Tasks; +#nullable enable +using System.Threading.Tasks; using SeqCli.Ingestion; using SeqCli.Tests.Support; using Xunit; @@ -10,11 +11,13 @@ public class StaticMessageTemplateReaderTests [Fact] public async Task ReaderSubstitutesMessageTemplate() { - var evt = Some.LogEvent(); + var evt = Some.EventJson(); + evt["@m"] = "A pre-rendered message"; const string mt = "This is a message template"; - var reader = new FixedLogEventReader(new ReadResult(evt, false)); + var reader = new FixedEventReader(new ReadResult(evt, false)); var wrapper = new StaticMessageTemplateReader(reader, mt); var result = await wrapper.TryReadAsync(); - Assert.Equal(mt, result.LogEvent.MessageTemplate.Text); + Assert.Equal(mt, (string?)result.Document!["@mt"]); + Assert.False(result.Document.ContainsKey("@m")); } -} \ No newline at end of file +} diff --git a/test/SeqCli.Tests/Support/FixedLogEventReader.cs b/test/SeqCli.Tests/Support/FixedEventReader.cs similarity index 73% rename from test/SeqCli.Tests/Support/FixedLogEventReader.cs rename to test/SeqCli.Tests/Support/FixedEventReader.cs index 538c5b65..2c29398f 100644 --- a/test/SeqCli.Tests/Support/FixedLogEventReader.cs +++ b/test/SeqCli.Tests/Support/FixedEventReader.cs @@ -3,11 +3,11 @@ namespace SeqCli.Tests.Support; -class FixedLogEventReader : ILogEventReader +class FixedEventReader : IEventReader { readonly ReadResult _result; - public FixedLogEventReader(ReadResult result) + public FixedEventReader(ReadResult result) { _result = result; } diff --git a/test/SeqCli.Tests/Support/Some.cs b/test/SeqCli.Tests/Support/Some.cs index 7ba27d31..0a2aa24e 100644 --- a/test/SeqCli.Tests/Support/Some.cs +++ b/test/SeqCli.Tests/Support/Some.cs @@ -1,11 +1,10 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Security.Cryptography; +using System.Text.Json.Nodes; using Seq.Api.Model.Events; using Seq.Api.Model.Shared; -using Serilog.Events; -using Serilog.Parsing; namespace SeqCli.Tests.Support; @@ -15,14 +14,13 @@ static class Some { static readonly RandomNumberGenerator Rng = RandomNumberGenerator.Create(); - public static LogEvent LogEvent() + public static JsonObject EventJson() { - return new LogEvent( - DateTimeOffset.UtcNow, - LogEventLevel.Information, - null, - new MessageTemplateParser().Parse("Test"), - Enumerable.Empty()); + return new JsonObject + { + ["@t"] = DateTimeOffset.UtcNow.ToString("o"), + ["@mt"] = "Test" + }; } public static string String() @@ -41,7 +39,7 @@ public static byte[] Bytes(int count) Rng.GetBytes(bytes); return bytes; } - + public static EventEntity MakeEvent(Action? configure = null) { var evt = new EventEntity @@ -58,4 +56,4 @@ public static EventEntity MakeEvent(Action? configure = null) public static List MakeProperties(params (string Name, object? Value)[] items) => items.Select(i => new EventPropertyPart(i.Name, i.Value)).ToList(); -} \ No newline at end of file +} diff --git a/test/SeqCli.Tests/Traces/StructuredMessageTests.cs b/test/SeqCli.Tests/Traces/StructuredMessageTests.cs index 731a0bb8..4180126e 100644 --- a/test/SeqCli.Tests/Traces/StructuredMessageTests.cs +++ b/test/SeqCli.Tests/Traces/StructuredMessageTests.cs @@ -1,10 +1,8 @@ #nullable enable using System.IO; -using System.Linq; +using System.Text.Json.Nodes; using Newtonsoft.Json.Linq; using SeqCli.Traces; -using Serilog.Events; -using Serilog.Parsing; using Xunit; namespace SeqCli.Tests.Traces; @@ -25,7 +23,7 @@ public void MissingStructuredMessagesReadAsEmpty() foreach (var cell in new object?[] { null, JValue.CreateNull() }) { var (message, properties) = StructuredMessage.Read(cell); - Assert.Empty(message.Tokens); + Assert.Equal("", message); Assert.Empty(properties); } } @@ -35,24 +33,29 @@ public void TextTokensAreRead() { var (message, properties) = StructuredMessage.Read(new JArray("Hello", ", ", "world")); - Assert.Equal("Hello, world", message.Text); - Assert.All(message.Tokens, token => Assert.IsType(token)); + Assert.Equal("Hello, world", message); Assert.Empty(properties); } + [Fact] + public void LiteralBracesAreEscapedInTemplateText() + { + var (message, _) = StructuredMessage.Read(new JArray("a {not-a-hole} b")); + + Assert.Equal("a {{not-a-hole}} b", message); + } + [Fact] public void HolesCarryRawTextAndValues() { var (message, properties) = StructuredMessage.Read(new JArray( "Hello, ", Hole("Name", "{Name:x}", "World"), "!")); - Assert.Equal("Hello, {Name:x}!", message.Text); - var hole = Assert.IsType(message.Tokens.ElementAt(1)); - Assert.Equal("Name", hole.PropertyName); + Assert.Equal("Hello, {Name:x}!", message); var property = Assert.Single(properties); - Assert.Equal("Name", property.Name); - Assert.Equal(new ScalarValue("World"), property.Value); + Assert.Equal("Name", property.Key); + Assert.Equal("World", (string?)property.Value); } [Fact] @@ -60,7 +63,7 @@ public void HolesWithoutValuesContributeNoProperties() { var (message, properties) = StructuredMessage.Read(new JArray(Hole("Name"))); - Assert.Equal("{Name}", message.Text); + Assert.Equal("{Name}", message); Assert.Empty(properties); } @@ -74,22 +77,32 @@ public void DuplicateHolesContributeASingleProperty() } [Fact] - public void ScalarHoleValuesAreUnwrapped() + public void ScalarHoleValuesAreRead() { var (_, properties) = StructuredMessage.Read(new JArray(Hole("Count", value: 42L))); - var scalar = Assert.IsType(Assert.Single(properties).Value); - Assert.Equal(42L, scalar.Value); + Assert.Equal(42L, (long?)Assert.Single(properties).Value); } [Fact] - public void StructuredHoleValuesBecomeStructures() + public void StructuredHoleValuesBecomeObjects() { var (_, properties) = StructuredMessage.Read(new JArray( Hole("Order", value: new JObject(new JProperty("Id", 7))))); - var structure = Assert.IsType(Assert.Single(properties).Value); - Assert.Equal("Id", Assert.Single(structure.Properties).Name); + var structure = Assert.IsType(Assert.Single(properties).Value); + Assert.Equal(7, (int?)structure["Id"]); + } + + [Fact] + public void DottedHoleNamesBecomeNestedObjects() + { + var (message, properties) = StructuredMessage.Read(new JArray( + Hole("user.name", value: "Barney"))); + + Assert.Equal("{user.name}", message); + var user = Assert.IsType(properties["user"]); + Assert.Equal("Barney", (string?)user["name"]); } [Fact] @@ -97,7 +110,7 @@ public void TrailingWhitespaceIsTrimmed() { var (message, _) = StructuredMessage.Read(new JArray("Hi ", "}", " \n")); - Assert.Equal("Hi }", message.Text); + Assert.Equal("Hi }}", message); } [Fact] @@ -105,7 +118,7 @@ public void WhitespaceOnlyMessagesReadAsEmpty() { var (message, _) = StructuredMessage.Read(new JArray(" ")); - Assert.Empty(message.Tokens); + Assert.Equal("", message); } [Fact] @@ -113,7 +126,7 @@ public void TrailingHolesAreNotTrimmed() { var (message, _) = StructuredMessage.Read(new JArray("Took ", Hole("Elapsed"))); - Assert.Equal("Took {Elapsed}", message.Text); + Assert.Equal("Took {Elapsed}", message); } [Fact] diff --git a/test/SeqCli.Tests/Traces/TraceQueryTests.cs b/test/SeqCli.Tests/Traces/TraceQueryTests.cs index fb282be5..84a5baab 100644 --- a/test/SeqCli.Tests/Traces/TraceQueryTests.cs +++ b/test/SeqCli.Tests/Traces/TraceQueryTests.cs @@ -3,7 +3,6 @@ using Newtonsoft.Json.Linq; using Seq.Api.Model.Data; using SeqCli.Traces; -using Serilog.Events; using Xunit; namespace SeqCli.Tests.Traces; @@ -85,7 +84,7 @@ public void SpanRowsAreRead() Assert.Equal("event-1", evt.Id); Assert.Equal(timestamp, evt.Timestamp); Assert.Equal("INFO", evt.Level); - Assert.Equal("Hello!", evt.MessageTemplate.Text); + Assert.Equal("Hello!", evt.MessageTemplate); Assert.Empty(evt.TemplateProperties); Assert.Null(evt.Exception); Assert.Equal("0011223344556677", evt.SpanId); @@ -157,10 +156,10 @@ public void StructuredMessageHolesBecomeTemplatePropertiesAndValues() var evt = Assert.Single(TraceQuery.ReadEvents(result, includeExceptions: true, [])); - Assert.Equal("Hello, {Name}!", evt.MessageTemplate.Text); + Assert.Equal("Hello, {Name}!", evt.MessageTemplate); var property = Assert.Single(evt.TemplateProperties); - Assert.Equal("Name", property.Name); - Assert.Equal(new ScalarValue("World"), property.Value); + Assert.Equal("Name", property.Key); + Assert.Equal("World", (string?)property.Value); } [Fact] diff --git a/test/SeqCli.Tests/Traces/TraceTreeBuilderTests.cs b/test/SeqCli.Tests/Traces/TraceTreeBuilderTests.cs index 2511109f..2f16c2f9 100644 --- a/test/SeqCli.Tests/Traces/TraceTreeBuilderTests.cs +++ b/test/SeqCli.Tests/Traces/TraceTreeBuilderTests.cs @@ -1,9 +1,8 @@ #nullable enable using System; using System.Linq; +using System.Text.Json.Nodes; using SeqCli.Traces; -using Serilog.Events; -using Serilog.Parsing; using Xunit; namespace SeqCli.Tests.Traces; @@ -14,12 +13,12 @@ public class TraceTreeBuilderTests static TraceTreeElement Span(string spanId, string? parentId, double startMs = 0, double elapsedMs = 1) => new($"event-span-{spanId}", T0.AddMilliseconds(startMs + elapsedMs), null, - new MessageTemplate([new TextToken($"span {spanId}")]), [], null, + $"span {spanId}", new JsonObject(), null, spanId, parentId, T0.AddMilliseconds(startMs), TimeSpan.FromMilliseconds(elapsedMs), []); static TraceTreeElement Log(string? spanId, double timestampMs, string message = "log") => new($"event-log-{timestampMs}-{message}", T0.AddMilliseconds(timestampMs), null, - new MessageTemplate([new TextToken(message)]), [], null, + message, new JsonObject(), null, spanId, null, null, null, []); [Fact] @@ -76,7 +75,7 @@ public void SiblingSpansAndLogsInterleaveChronologically() var root = Assert.Single(roots); Assert.Equal( ["first", "span c", "span b", "last"], - root.Children.Select(c => c.Element.MessageTemplate.Text).ToArray()); + root.Children.Select(c => c.Element.MessageTemplate).ToArray()); } [Fact] @@ -104,7 +103,7 @@ public void OrphanLogsBecomeRootsAlongsideSpans() Assert.Equal( ["span a", "first", "second", "span b"], - roots.Select(r => r.Element.MessageTemplate.Text).ToArray()); + roots.Select(r => r.Element.MessageTemplate).ToArray()); Assert.All(roots, r => Assert.Empty(r.Children)); } @@ -117,7 +116,7 @@ public void OrphanLogsWithNoRootSpanRemainAtRootLevel() ]); Assert.Equal(2, roots.Count); - Assert.Equal(["first", "second"], roots.Select(r => r.Element.MessageTemplate.Text).ToArray()); + Assert.Equal(["first", "second"], roots.Select(r => r.Element.MessageTemplate).ToArray()); } [Fact] diff --git a/test/SeqCli.Tests/Traces/TraceTreeJObjectConverterTests.cs b/test/SeqCli.Tests/Traces/TraceTreeJObjectConverterTests.cs index b0dfa1c9..6eb90582 100644 --- a/test/SeqCli.Tests/Traces/TraceTreeJObjectConverterTests.cs +++ b/test/SeqCli.Tests/Traces/TraceTreeJObjectConverterTests.cs @@ -2,10 +2,9 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text.Json.Nodes; using Newtonsoft.Json.Linq; using SeqCli.Traces; -using Serilog.Events; -using Serilog.Parsing; using Xunit; namespace SeqCli.Tests.Traces; @@ -20,14 +19,14 @@ static TraceTreeElement Span(string spanId, string? parentId, double startMs = 0 string? message = null, string? level = null, string? exception = null, IReadOnlyList? columns = null) => new($"event-span-{spanId}", T0.AddMilliseconds(startMs + elapsedMs), level, - new MessageTemplate([new TextToken(message ?? $"span {spanId}")]), [], + message ?? $"span {spanId}", new JsonObject(), exception, spanId, parentId, T0.AddMilliseconds(startMs), TimeSpan.FromMilliseconds(elapsedMs), columns ?? []); static TraceTreeElement Log(string? spanId, double timestampMs, string message = "log", string? level = null, string? exception = null, IReadOnlyList? columns = null) => new($"event-log-{timestampMs}-{message}", T0.AddMilliseconds(timestampMs), level, - new MessageTemplate([new TextToken(message)]), [], exception, + message, new JsonObject(), exception, spanId, null, null, null, columns ?? []); static JObject ToJson(params TraceTreeElement[] events) => ToJson([], events); @@ -142,30 +141,13 @@ public void LogsWithNoCapturedEnclosingSpanBecomeOrphans() Assert.Equal("uncaptured", (string?)orphans[0]["spanId"]); Assert.Null(orphans[2]["spanId"]); } - - [Fact] - public void LevelsAreNormalizedToFullNames() - { - var document = ToJson( - Span("a", null), - Log("a", 1, level: "warn"), - Log("a", 2, level: "Nonstandard")); - - var children = (JArray)document["root"]!["children"]!; - Assert.Equal("Warning", (string?)children[0]["level"]); - Assert.Equal("Nonstandard", (string?)children[1]["level"]); - } - + [Fact] public void TemplateHolesAreFilledFromMessageProperties() { var evt = new TraceTreeElement("event-1", T0.AddMilliseconds(1.5), null, - new MessageTemplate([ - new TextToken("GET "), - new PropertyToken("Route", "{Route}"), - new TextToken(" as "), - new PropertyToken("User", "{User}")]), - [new LogEventProperty("Route", new ScalarValue("/orders"))], + "GET {Route} as {User}", + new JsonObject { ["Route"] = "/orders" }, null, "a", null, T0, TimeSpan.FromMilliseconds(1.5), []); var document = ToJson(evt); From 96b40197bb9daea2083d237e28d735524df0f5e3 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Tue, 1 Sep 2026 15:55:07 +1000 Subject: [PATCH 07/32] Localize Serilog event conversion to the sample application - we don't want easy or obvious conversions into Serilog types, the set of scenarios that require this should be and stay vanishingly small --- src/SeqCli/Forwarder/ForwarderModule.cs | 21 ++----- .../Web/Api/IngestionLogEndpoints.cs | 58 +++++++++++++++++-- src/SeqCli/Ingestion/JsonEventReader.cs | 6 +- .../Ingestion/SerilogEventJson.cs | 2 +- .../Ingestion/SerilogTracingConventions.cs | 8 +-- .../SerilogEventJsonTests.cs | 3 +- 6 files changed, 63 insertions(+), 35 deletions(-) rename src/SeqCli/{ => Sample}/Ingestion/SerilogEventJson.cs (98%) rename src/SeqCli/{ => Sample}/Ingestion/SerilogTracingConventions.cs (78%) rename test/SeqCli.Tests/{Ingestion => Sample}/SerilogEventJsonTests.cs (98%) diff --git a/src/SeqCli/Forwarder/ForwarderModule.cs b/src/SeqCli/Forwarder/ForwarderModule.cs index 3980787d..9ed614b4 100644 --- a/src/SeqCli/Forwarder/ForwarderModule.cs +++ b/src/SeqCli/Forwarder/ForwarderModule.cs @@ -21,7 +21,6 @@ using SeqCli.Forwarder.Channel; using SeqCli.Forwarder.Web.Api; using SeqCli.Forwarder.Web.Host; -using SeqCli.Syntax; using Serilog; namespace SeqCli.Forwarder; @@ -65,25 +64,17 @@ protected override void Load(ContainerBuilder builder) if (_config.Forwarder.Diagnostics.ExposeIngestionLog) { Log.ForContext().Warning("Configured to expose ingestion log via HTTP API"); - builder.RegisterType().As(); - - var ingestionLogTemplate = $"[{{@Timestamp:o}} {{@Level:u3}}] {{@Message}}{Environment.NewLine}"; if (_config.Forwarder.Diagnostics.IngestionLogShowDetail) { Log.ForContext().Warning("Including full client, payload, and error detail in the ingestion log"); - ingestionLogTemplate += - $"{{#if ClientHostIP is not null}}Client IP address: {{ClientHostIP}}{Environment.NewLine}{{#end}}" + - $"{{#if DocumentStart is not null}}First {{StartToLog}} characters of payload: {{DocumentStart:l}}{Environment.NewLine}{{#end}}" + - "{@Exception}"; } - - builder.Register(_ => SeqSyntax.ParseTemplate(ingestionLogTemplate)); + + builder.Register(_ => new IngestionLogEndpoints(_config.Forwarder.Diagnostics.IngestionLogShowDetail)).As(); } - builder.Register(c => + builder.Register(_ => { - var config = c.Resolve(); - var baseUri = config.Connection.ServerUrl; + var baseUri = _config.Connection.ServerUrl; if (string.IsNullOrWhiteSpace(baseUri)) throw new ArgumentException("The destination Seq server URL must be configured in `SeqCli.json`."); @@ -94,13 +85,13 @@ protected override void Load(ContainerBuilder builder) // this expression, using an "or" operator. var hasSocketHandlerOption = - config.Connection.PooledConnectionLifetimeMilliseconds.HasValue; + _config.Connection.PooledConnectionLifetimeMilliseconds.HasValue; if (hasSocketHandlerOption) { var httpMessageHandler = new SocketsHttpHandler { - PooledConnectionLifetime = config.Connection.PooledConnectionLifetimeMilliseconds.HasValue ? TimeSpan.FromMilliseconds(config.Connection.PooledConnectionLifetimeMilliseconds.Value) : Timeout.InfiniteTimeSpan, + PooledConnectionLifetime = _config.Connection.PooledConnectionLifetimeMilliseconds.HasValue ? TimeSpan.FromMilliseconds(_config.Connection.PooledConnectionLifetimeMilliseconds.Value) : Timeout.InfiniteTimeSpan, }; return new HttpClient(httpMessageHandler) { BaseAddress = new Uri(baseUri) }; diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs index eb98cc58..eba1c743 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionLogEndpoints.cs @@ -12,24 +12,25 @@ // See the License for the specific language governing permissions and // limitations under the License. +using System; +using System.Globalization; using System.IO; using System.Text; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; -using Seq.Syntax.Templates; using SeqCli.Forwarder.Diagnostics; -using SeqCli.Ingestion; +using Serilog.Events; namespace SeqCli.Forwarder.Web.Api; class IngestionLogEndpoints : IMapEndpoints { - readonly ExpressionTemplate _formatter; + readonly bool _showDetail; readonly Encoding _utf8 = new UTF8Encoding(false); - public IngestionLogEndpoints(ExpressionTemplate formatter) + public IngestionLogEndpoints(bool showDetail) { - _formatter = formatter; + _showDetail = showDetail; } public void MapEndpoints(WebApplication app) @@ -46,10 +47,55 @@ public void MapEndpoints(WebApplication app) using var log = new StringWriter(); foreach (var logEvent in events) { - _formatter.Format(SerilogEventJson.ToEventJson(logEvent), log); + Format(logEvent, log); } return Results.Content(log.ToString(), "text/plain", _utf8); }); } + + void Format(LogEvent logEvent, TextWriter log) + { + log.Write($"[{logEvent.Timestamp:o} {Abbreviate(logEvent.Level)}] "); + + static string Abbreviate(LogEventLevel logEventLevel) + { + // Here because we don't want Serilog level conversion routines, or any other Serilog model conversion + // routines, to propagate. + return logEventLevel switch + { + LogEventLevel.Verbose => "VRB", + LogEventLevel.Debug => "DBG", + LogEventLevel.Information => "INF", + LogEventLevel.Warning => "WAR", + LogEventLevel.Error => "ERR", + LogEventLevel.Fatal => "FTL", + _ => throw new ArgumentOutOfRangeException(nameof(logEventLevel), logEventLevel, null) + }; + } + + logEvent.RenderMessage(log, CultureInfo.InvariantCulture); + log.WriteLine(); + if (_showDetail) + { + if (logEvent.Properties.TryGetValue("ClientHostIP", out var clientHostIPProperty) && + clientHostIPProperty is ScalarValue { Value: string clientHostIP}) + { + log.WriteLine($"Client IP address: {clientHostIP}"); + } + + if (logEvent.Properties.TryGetValue("DocumentStart", out var documentStartProperty) && + documentStartProperty is ScalarValue { Value: string documentStart} && + logEvent.Properties.TryGetValue("StartToLog", out var startToLogProperty) && + startToLogProperty is ScalarValue { Value: {} startToLog }) + { + log.WriteLine($"First {startToLog} characters of payload: {documentStart}"); + } + + if (logEvent.Exception is { } exception) + { + log.WriteLine(exception); + } + } + } } diff --git a/src/SeqCli/Ingestion/JsonEventReader.cs b/src/SeqCli/Ingestion/JsonEventReader.cs index b7a102f6..a8d4be8c 100644 --- a/src/SeqCli/Ingestion/JsonEventReader.cs +++ b/src/SeqCli/Ingestion/JsonEventReader.cs @@ -49,16 +49,14 @@ public async Task TryReadAsync() return new ReadResult(ReadFromJson(frame.Value), frame.IsAtEnd); } - public static JsonObject ReadFromJson(string json) + static JsonObject ReadFromJson(string json) { if (JsonNode.Parse(json) is not JsonObject eventJson) throw new InvalidDataException($"The line is not a JSON object: `{json.Trim()}`."); if (!eventJson.ContainsKey("@t")) eventJson["@t"] = DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture); - - SerilogTracingConventions.LiftSpanProperties(eventJson); - + return eventJson; } } diff --git a/src/SeqCli/Ingestion/SerilogEventJson.cs b/src/SeqCli/Sample/Ingestion/SerilogEventJson.cs similarity index 98% rename from src/SeqCli/Ingestion/SerilogEventJson.cs rename to src/SeqCli/Sample/Ingestion/SerilogEventJson.cs index 3a166419..4750e756 100644 --- a/src/SeqCli/Ingestion/SerilogEventJson.cs +++ b/src/SeqCli/Sample/Ingestion/SerilogEventJson.cs @@ -18,7 +18,7 @@ using SeqCli.Syntax; using Serilog.Events; -namespace SeqCli.Ingestion; +namespace SeqCli.Sample.Ingestion; /// /// Converts Serilog events produced within seqcli itself — the sample ingest simulation diff --git a/src/SeqCli/Ingestion/SerilogTracingConventions.cs b/src/SeqCli/Sample/Ingestion/SerilogTracingConventions.cs similarity index 78% rename from src/SeqCli/Ingestion/SerilogTracingConventions.cs rename to src/SeqCli/Sample/Ingestion/SerilogTracingConventions.cs index e6eb029e..f08bc184 100644 --- a/src/SeqCli/Ingestion/SerilogTracingConventions.cs +++ b/src/SeqCli/Sample/Ingestion/SerilogTracingConventions.cs @@ -14,14 +14,8 @@ using System.Text.Json.Nodes; -namespace SeqCli.Ingestion; +namespace SeqCli.Sample.Ingestion; -/// -/// SerilogTracing emits span fields as regular event properties, because Serilog's data model -/// has nowhere else to put them. Events passing through seqcli lift these into the reified -/// @st and @ps fields so that they're recognized as spans by Seq and by seqcli's -/// own output formatting. -/// static class SerilogTracingConventions { internal const string ParentSpanIdProperty = "ParentSpanId"; diff --git a/test/SeqCli.Tests/Ingestion/SerilogEventJsonTests.cs b/test/SeqCli.Tests/Sample/SerilogEventJsonTests.cs similarity index 98% rename from test/SeqCli.Tests/Ingestion/SerilogEventJsonTests.cs rename to test/SeqCli.Tests/Sample/SerilogEventJsonTests.cs index f27aaf1f..256d7a3f 100644 --- a/test/SeqCli.Tests/Ingestion/SerilogEventJsonTests.cs +++ b/test/SeqCli.Tests/Sample/SerilogEventJsonTests.cs @@ -1,13 +1,12 @@ #nullable enable using System; using System.Linq; -using SeqCli.Ingestion; using SeqCli.Sample.Ingestion; using Serilog; using Serilog.Events; using Xunit; -namespace SeqCli.Tests.Ingestion; +namespace SeqCli.Tests.Sample; public class SerilogEventJsonTests { From d53501719d21c2b915399618a2940a0c204e6ee8 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Tue, 1 Sep 2026 16:16:19 +1000 Subject: [PATCH 08/32] Trim down more --- src/SeqCli/Apps/AppLoader.cs | 5 +-- src/SeqCli/Sample/Ingestion/BufferingSink.cs | 2 +- src/SeqCli/Sample/Ingestion/MetricsMapping.cs | 6 +-- .../Ingestion/SerilogTracingConventions.cs | 40 ------------------- ...SerilogEventJson.cs => SimulationEvent.cs} | 32 ++++++++++----- ...ntJsonTests.cs => SimulationEventTests.cs} | 12 +++--- 6 files changed, 35 insertions(+), 62 deletions(-) delete mode 100644 src/SeqCli/Sample/Ingestion/SerilogTracingConventions.cs rename src/SeqCli/Sample/Ingestion/{SerilogEventJson.cs => SimulationEvent.cs} (74%) rename test/SeqCli.Tests/Sample/{SerilogEventJsonTests.cs => SimulationEventTests.cs} (90%) diff --git a/src/SeqCli/Apps/AppLoader.cs b/src/SeqCli/Apps/AppLoader.cs index 143eb91a..e46cdfed 100644 --- a/src/SeqCli/Apps/AppLoader.cs +++ b/src/SeqCli/Apps/AppLoader.cs @@ -29,13 +29,12 @@ class AppLoader : IDisposable readonly string _packageBinaryPath; // These are used for interop between the host process and the app. The - // app _must_ be able to load on the unified version. Apps built against Seq.Syntax v1 - // bundle their own `Seq.Syntax.dll`, which loads side-by-side with the host's - // `Seq.Syntax.V2.dll`. + // app _must_ be able to load on the unified version. readonly Assembly[] _contracts = [ typeof(SeqApp).Assembly, typeof(Log).Assembly, + // Seq.Syntax uses version-specific assembly names to improve our chances of successful loading. typeof(SeqExpression).Assembly ]; diff --git a/src/SeqCli/Sample/Ingestion/BufferingSink.cs b/src/SeqCli/Sample/Ingestion/BufferingSink.cs index cab3a4d3..59b0225d 100644 --- a/src/SeqCli/Sample/Ingestion/BufferingSink.cs +++ b/src/SeqCli/Sample/Ingestion/BufferingSink.cs @@ -25,7 +25,7 @@ public void Emit(LogEvent logEvent) var document = MetricsMapping.TryGetMetricSampleJson(logEvent, out var sample) ? sample - : SerilogEventJson.ToEventJson(logEvent); + : SimulationEvent.ToJsonObject(logEvent); _queue.Enqueue(document); } diff --git a/src/SeqCli/Sample/Ingestion/MetricsMapping.cs b/src/SeqCli/Sample/Ingestion/MetricsMapping.cs index 8ba3771f..12fbb91f 100644 --- a/src/SeqCli/Sample/Ingestion/MetricsMapping.cs +++ b/src/SeqCli/Sample/Ingestion/MetricsMapping.cs @@ -1,4 +1,4 @@ -// Copyright © Datalust and contributors. +// Copyright © Datalust and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -45,13 +45,13 @@ public static bool TryGetMetricSampleJson(LogEvent logEvent, [NotNullWhen(true)] sample = new JsonObject { ["@t"] = logEvent.Timestamp.ToString("o", CultureInfo.InvariantCulture), - ["@d"] = SerilogEventJson.ToJsonNode(definitions) + ["@d"] = SimulationEvent.ToJsonNode(definitions) }; foreach (var (name, value) in logEvent.Properties) { if (name != SurrogateDefinitionsProperty) - EventJson.SetUserProperty(sample, name, SerilogEventJson.ToJsonNode(value)); + EventJson.SetUserProperty(sample, name, SimulationEvent.ToJsonNode(value)); } return true; diff --git a/src/SeqCli/Sample/Ingestion/SerilogTracingConventions.cs b/src/SeqCli/Sample/Ingestion/SerilogTracingConventions.cs deleted file mode 100644 index f08bc184..00000000 --- a/src/SeqCli/Sample/Ingestion/SerilogTracingConventions.cs +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright © Datalust and contributors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System.Text.Json.Nodes; - -namespace SeqCli.Sample.Ingestion; - -static class SerilogTracingConventions -{ - internal const string ParentSpanIdProperty = "ParentSpanId"; - - internal const string SpanStartTimestampProperty = "SpanStartTimestamp"; - - public static void LiftSpanProperties(JsonObject eventJson) - { - LiftProperty(eventJson, SpanStartTimestampProperty, "@st"); - LiftProperty(eventJson, ParentSpanIdProperty, "@ps"); - } - - static void LiftProperty(JsonObject eventJson, string propertyName, string reifiedName) - { - if (eventJson.TryGetPropertyValue(propertyName, out var value)) - { - eventJson.Remove(propertyName); - if (!eventJson.ContainsKey(reifiedName)) - eventJson[reifiedName] = value; - } - } -} diff --git a/src/SeqCli/Sample/Ingestion/SerilogEventJson.cs b/src/SeqCli/Sample/Ingestion/SimulationEvent.cs similarity index 74% rename from src/SeqCli/Sample/Ingestion/SerilogEventJson.cs rename to src/SeqCli/Sample/Ingestion/SimulationEvent.cs index 4750e756..bc48e5cc 100644 --- a/src/SeqCli/Sample/Ingestion/SerilogEventJson.cs +++ b/src/SeqCli/Sample/Ingestion/SimulationEvent.cs @@ -20,15 +20,13 @@ namespace SeqCli.Sample.Ingestion; -/// -/// Converts Serilog events produced within seqcli itself — the sample ingest simulation -/// and the forwarder's diagnostic ingestion log — into event JSON documents in Seq's emission -/// schema. Externally-supplied event data never passes through here: it's read directly into -/// JSON documents. -/// -static class SerilogEventJson +/// Used only in the Roastery simulation; no other event data should ever be processed using this type. +static class SimulationEvent { - public static JsonObject ToEventJson(LogEvent logEvent) + const string ParentSpanIdProperty = "ParentSpanId", + SpanStartTimestampProperty = "SpanStartTimestamp"; + + public static JsonObject ToJsonObject(LogEvent logEvent) { var eventJson = new JsonObject { @@ -51,7 +49,7 @@ public static JsonObject ToEventJson(LogEvent logEvent) foreach (var (name, value) in logEvent.Properties) EventJson.SetUserProperty(eventJson, name, ToJsonNode(value)); - SerilogTracingConventions.LiftSpanProperties(eventJson); + LiftSpanProperties(eventJson); return eventJson; } @@ -88,4 +86,20 @@ public static JsonObject ToEventJson(LogEvent logEvent) return EventJson.CreateScalar(value.ToString()); } } + + static void LiftSpanProperties(JsonObject eventJson) + { + LiftProperty(eventJson, SpanStartTimestampProperty, "@st"); + LiftProperty(eventJson, ParentSpanIdProperty, "@ps"); + } + + static void LiftProperty(JsonObject eventJson, string propertyName, string reifiedName) + { + if (eventJson.TryGetPropertyValue(propertyName, out var value)) + { + eventJson.Remove(propertyName); + if (!eventJson.ContainsKey(reifiedName)) + eventJson[reifiedName] = value; + } + } } diff --git a/test/SeqCli.Tests/Sample/SerilogEventJsonTests.cs b/test/SeqCli.Tests/Sample/SimulationEventTests.cs similarity index 90% rename from test/SeqCli.Tests/Sample/SerilogEventJsonTests.cs rename to test/SeqCli.Tests/Sample/SimulationEventTests.cs index 256d7a3f..0a71da14 100644 --- a/test/SeqCli.Tests/Sample/SerilogEventJsonTests.cs +++ b/test/SeqCli.Tests/Sample/SimulationEventTests.cs @@ -8,7 +8,7 @@ namespace SeqCli.Tests.Sample; -public class SerilogEventJsonTests +public class SimulationEventTests { static LogEvent CaptureEvent(Action log) { @@ -30,7 +30,7 @@ class CapturingSink(Action capture) : Serilog.Core.ILogEventSink public void EventFieldsMapToTheEmissionSchema() { var evt = CaptureEvent(log => log.Warning(new Exception("Boom!"), "Hello, {Name}!", "world")); - var eventJson = SerilogEventJson.ToEventJson(evt); + var eventJson = SimulationEvent.ToJsonObject(evt); Assert.Equal(evt.Timestamp.ToString("o"), (string?)eventJson["@t"]); Assert.Equal("Hello, {Name}!", (string?)eventJson["@mt"]); @@ -44,7 +44,7 @@ public void InformationLevelsAreOmitted() { var evt = CaptureEvent(log => log.Information("Hello")); - Assert.False(SerilogEventJson.ToEventJson(evt).ContainsKey("@l")); + Assert.False(SimulationEvent.ToJsonObject(evt).ContainsKey("@l")); } [Fact] @@ -52,7 +52,7 @@ public void StructuredValuesSerializeAsJson() { var evt = CaptureEvent(log => log.Information("{@Order} {Items}", new { Id = 7, Total = 4.5 }, new[] { "a", "b" })); - var eventJson = SerilogEventJson.ToEventJson(evt); + var eventJson = SimulationEvent.ToJsonObject(evt); Assert.Equal(7, (int?)eventJson["Order"]!["Id"]); Assert.Equal(4.5, (double?)eventJson["Order"]!["Total"]); @@ -67,7 +67,7 @@ public void SerilogTracingSpanPropertiesAreLifted() .ForContext("SpanStartTimestamp", start) .ForContext("ParentSpanId", "8899aabbccddeeff") .Information("GET /orders")); - var eventJson = SerilogEventJson.ToEventJson(evt); + var eventJson = SimulationEvent.ToJsonObject(evt); Assert.Equal(start.ToString("o"), (string?)eventJson["@st"]); Assert.Equal("8899aabbccddeeff", (string?)eventJson["@ps"]); @@ -103,6 +103,6 @@ public void PropertyNamesBeginningWithAtAreEscaped() { var evt = CaptureEvent(log => log.ForContext("@evil", "value").Information("Hello")); - Assert.Equal("value", (string?)SerilogEventJson.ToEventJson(evt)["@@evil"]); + Assert.Equal("value", (string?)SimulationEvent.ToJsonObject(evt)["@@evil"]); } } From 33b115d46b2735bda88308eda08dfea2a346e7ac Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Tue, 1 Sep 2026 16:30:52 +1000 Subject: [PATCH 09/32] More tidy-up --- src/SeqCli/Cli/Commands/TailCommand.cs | 8 ++++++-- src/SeqCli/Mapping/EventEntityJson.cs | 11 ++++++----- src/SeqCli/Mcp/Tools/Search/SearchTools.cs | 1 - 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/SeqCli/Cli/Commands/TailCommand.cs b/src/SeqCli/Cli/Commands/TailCommand.cs index 9d4d4957..291433ba 100644 --- a/src/SeqCli/Cli/Commands/TailCommand.cs +++ b/src/SeqCli/Cli/Commands/TailCommand.cs @@ -13,6 +13,8 @@ // limitations under the License. using System; +using System.IO; +using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; using SeqCli.Api; @@ -63,13 +65,15 @@ protected override async Task Run() try { - await foreach (var evt in connection.Events.StreamAsync( + await foreach (var evt in connection.Events.StreamDocumentsAsync( filter: strict, signal: _signal.Signal, render: true, + clef: true, cancellationToken: cancel.Token)) { - output.WriteEventEntity(evt); + var eventJson = JsonNode.Parse(evt)?.AsObject() ?? throw new InvalidDataException("Non-JSON document received."); + output.WriteEvent(eventJson); } } catch (OperationCanceledException) diff --git a/src/SeqCli/Mapping/EventEntityJson.cs b/src/SeqCli/Mapping/EventEntityJson.cs index e84bb3c9..68a5a485 100644 --- a/src/SeqCli/Mapping/EventEntityJson.cs +++ b/src/SeqCli/Mapping/EventEntityJson.cs @@ -19,22 +19,25 @@ using System.Text.Json.Nodes; using Seq.Api.Model.Events; using Seq.Api.Model.Shared; +using SeqCli.Output; using SeqCli.Syntax; using SeqCli.Util; namespace SeqCli.Mapping; /// -/// Converts events retrieved from the Seq API into event JSON documents in Seq's emission -/// (CLEF) schema, ready for filtering and formatting with Seq.Syntax. +/// Converts event entities into compact JSON format for further processing. This class is only necessary because +/// Seq.Api doesn't yet provide a simple compact-JSON based result format for searches. Once we've filled +/// that gap, this class, and can be removed. /// static class EventEntityJson { public static JsonObject ToEventJson(EventEntity evt) { - // Timestamps are shown in local time, matching earlier seqcli versions. var eventJson = new JsonObject { + // Earlier versions relied on Serilog output formatting to show timestamps in local time; we'll need + // to consider adding some compensating mechanism to `Seq.Syntax`. ["@t"] = DateTimeOffset.ParseExact(evt.Timestamp, "o", CultureInfo.InvariantCulture) .ToLocalTime().ToString("o", CultureInfo.InvariantCulture) }; @@ -42,8 +45,6 @@ public static JsonObject ToEventJson(EventEntity evt) if (evt.MessageTemplateTokens != null) eventJson["@mt"] = ToMessageTemplateText(evt.MessageTemplateTokens); - // By the emission convention, `Information` levels are omitted; any other level keeps - // the spelling it was ingested with. if (!string.IsNullOrWhiteSpace(evt.Level) && evt.Level != "Information") eventJson["@l"] = evt.Level; diff --git a/src/SeqCli/Mcp/Tools/Search/SearchTools.cs b/src/SeqCli/Mcp/Tools/Search/SearchTools.cs index 58a3b663..9ceda5e5 100644 --- a/src/SeqCli/Mcp/Tools/Search/SearchTools.cs +++ b/src/SeqCli/Mcp/Tools/Search/SearchTools.cs @@ -28,7 +28,6 @@ using Seq.Api.Model.Signals; using Seq.Syntax.Templates; using SeqCli.Mapping; -using SeqCli.Output; using SeqCli.Signals; using SeqCli.Syntax; using Serilog; From 8b828b33a78d4739f1ba8a44e9bbffebba70121c Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Tue, 1 Sep 2026 16:41:32 +1000 Subject: [PATCH 10/32] More cleanup --- src/SeqCli/Apps/Hosting/AppContainer.cs | 8 ++-- src/SeqCli/Apps/Hosting/EventFormat.cs | 2 +- .../Apps/Hosting/SerilogLevelMapping.cs | 41 ------------------- .../Cli/Commands/Alert/CreateCommand.cs | 2 +- .../Cli/Commands/ApiKey/CreateCommand.cs | 2 +- src/SeqCli/Mapping/LevelMapping.cs | 23 +++++++++-- src/SeqCli/PlainText/Extraction/Matchers.cs | 3 +- 7 files changed, 29 insertions(+), 52 deletions(-) delete mode 100644 src/SeqCli/Apps/Hosting/SerilogLevelMapping.cs diff --git a/src/SeqCli/Apps/Hosting/AppContainer.cs b/src/SeqCli/Apps/Hosting/AppContainer.cs index 2ba93fde..b65029f5 100644 --- a/src/SeqCli/Apps/Hosting/AppContainer.cs +++ b/src/SeqCli/Apps/Hosting/AppContainer.cs @@ -21,6 +21,7 @@ using Newtonsoft.Json.Linq; using Seq.Apps; using Seq.Apps.LogEvents; +using SeqCli.Mapping; using Serilog; using Serilog.Events; using Serilog.Formatting.Compact.Reader; @@ -108,11 +109,11 @@ async Task SendTypedEventAsync(string clef) { if (_seqApp is ISubscribeTo led) { - led.On(EventFormat.FromRaw(eventId, eventType, serilogEvent)); + led.On(EventFormat.FromSerilogLogEvent(eventId, eventType, serilogEvent)); } else if (_seqApp is ISubscribeToAsync leda) { - await leda.OnAsync(EventFormat.FromRaw(eventId, eventType, serilogEvent)); + await leda.OnAsync(EventFormat.FromSerilogLogEvent(eventId, eventType, serilogEvent)); } else if (_seqApp is ISubscribeTo sled) { @@ -142,7 +143,8 @@ LogEvent ReadSerilogEvent(string clef, out string eventId, out uint eventType) if (jobject.TryGetValue("@l", out var levelToken)) { jobject.Remove("@l"); - jobject.Add("@l", new JValue(SerilogLevelMapping.ToSerilogLevel(levelToken.Value()!).ToString())); + // The Seq.Api `LogEventLevel` enum intentionally matches the Serilog one. + jobject.Add("@l", new JValue(LevelMapping.ToSeqApiLogEventLevel(levelToken.Value()!).ToString())); } SanitizeTraceIdentifiers(jobject); diff --git a/src/SeqCli/Apps/Hosting/EventFormat.cs b/src/SeqCli/Apps/Hosting/EventFormat.cs index 1ff23253..67a37b4d 100644 --- a/src/SeqCli/Apps/Hosting/EventFormat.cs +++ b/src/SeqCli/Apps/Hosting/EventFormat.cs @@ -24,7 +24,7 @@ namespace SeqCli.Apps.Hosting; static class EventFormat { - public static Event FromRaw(string eventId, uint eventType, LogEvent raw) + public static Event FromSerilogLogEvent(string eventId, uint eventType, LogEvent raw) { var properties = new Dictionary(); foreach (var prop in raw.Properties) diff --git a/src/SeqCli/Apps/Hosting/SerilogLevelMapping.cs b/src/SeqCli/Apps/Hosting/SerilogLevelMapping.cs deleted file mode 100644 index f37a9f92..00000000 --- a/src/SeqCli/Apps/Hosting/SerilogLevelMapping.cs +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright © Datalust and contributors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using SeqCli.Mapping; -using Serilog.Events; - -namespace SeqCli.Apps.Hosting; - -/// -/// Maps level names onto Serilog's level enum for hosted Seq apps relying on the older Serilog `LogEvent`-based -/// interface (newer apps should generally use raw JSON directly). -/// -static class SerilogLevelMapping -{ - public static LogEventLevel ToSerilogLevel(string level) - { - if (string.IsNullOrEmpty(level)) - return LogEventLevel.Information; - - return LevelMapping.ToFullLevelName(level) switch - { - "Trace" or "Verbose" => LogEventLevel.Verbose, - "Debug" => LogEventLevel.Debug, - "Warning" => LogEventLevel.Warning, - "Error" => LogEventLevel.Error, - "Fatal" or "Critical" or "Emergency" or "Alert" or "Panic" => LogEventLevel.Fatal, - _ => LogEventLevel.Information - }; - } -} diff --git a/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs b/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs index 49ceba1d..e7de0520 100644 --- a/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs @@ -178,7 +178,7 @@ protected override async Task Run() alert.Having = _having; if (_notificationLevel != null) - alert.NotificationLevel = Enum.Parse(LevelMapping.ToFullLevelName(_notificationLevel)); + alert.NotificationLevel = LevelMapping.ToSeqApiLogEventLevel(_notificationLevel); if (_suppressionTime != null) alert.SuppressionTime = DurationMoniker.ToTimeSpan(_suppressionTime); diff --git a/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs b/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs index fe514434..03c3cf25 100644 --- a/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs @@ -125,7 +125,7 @@ protected override async Task Run() if (_level != null) { - apiKey.InputSettings.MinimumLevel = Enum.Parse(LevelMapping.ToFullLevelName(_level)); + apiKey.InputSettings.MinimumLevel = LevelMapping.ToSeqApiLogEventLevel(_level); } apiKey.AssignedPermissions.Clear(); diff --git a/src/SeqCli/Mapping/LevelMapping.cs b/src/SeqCli/Mapping/LevelMapping.cs index 7faa47b4..2d866366 100644 --- a/src/SeqCli/Mapping/LevelMapping.cs +++ b/src/SeqCli/Mapping/LevelMapping.cs @@ -14,14 +14,12 @@ using System; using System.Collections.Generic; +using Seq.Api.Model.LogEvents; namespace SeqCli.Mapping; /// -/// Recognizes the level spellings found in event data from various sources (info, -/// WARN, trce, …) and maps them to canonical Seq level names. Level values -/// themselves are preserved verbatim throughout the pipeline; the canonical name is used -/// where a normalized form is needed. +/// Some Seq API /// public static class LevelMapping { @@ -80,8 +78,25 @@ public static class LevelMapping ["panic"] = "Panic" }; + // Intended only for use by ingest extraction patterns. public static string ToFullLevelName(string level) { return LevelsByName.TryGetValue(level, out var m) ? m : level; } + + public static LogEventLevel ToSeqApiLogEventLevel(string level) + { + if (string.IsNullOrEmpty(level)) + return LogEventLevel.Information; + + return ToFullLevelName(level) switch + { + "Trace" or "Verbose" => LogEventLevel.Verbose, + "Debug" => LogEventLevel.Debug, + "Warning" => LogEventLevel.Warning, + "Error" => LogEventLevel.Error, + "Fatal" or "Critical" or "Emergency" or "Alert" or "Panic" => LogEventLevel.Fatal, + _ => LogEventLevel.Information + }; + } } diff --git a/src/SeqCli/PlainText/Extraction/Matchers.cs b/src/SeqCli/PlainText/Extraction/Matchers.cs index c2326f01..5ae49e6e 100644 --- a/src/SeqCli/PlainText/Extraction/Matchers.cs +++ b/src/SeqCli/PlainText/Extraction/Matchers.cs @@ -8,6 +8,7 @@ using Superpower; using Superpower.Model; using Superpower.Parsers; +// ReSharper disable MemberCanBePrivate.Global namespace SeqCli.PlainText.Extraction; @@ -122,7 +123,7 @@ static class Matchers // Equivalent to :* at end-of-pattern public static TextParser MultiLineContent { get; } = - Span.WithAll(ch => true) + Span.WithAll(_ => true) .Select(span => (object?)span); [Matcher("n")] From 7ebd2ad63a324eb979b040de772964ac579b7d4d Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Tue, 1 Sep 2026 16:48:19 +1000 Subject: [PATCH 11/32] More clean-up --- .../JsonNodes.cs => Api/ToSystemTextJson.cs} | 24 +++++++++---------- src/SeqCli/Mapping/EventEntityJson.cs | 5 ++-- src/SeqCli/Output/OutputFormat.cs | 5 ++-- src/SeqCli/Output/TraceFormatter.cs | 3 ++- src/SeqCli/Traces/StructuredMessage.cs | 3 ++- 5 files changed, 22 insertions(+), 18 deletions(-) rename src/SeqCli/{Util/JsonNodes.cs => Api/ToSystemTextJson.cs} (83%) diff --git a/src/SeqCli/Util/JsonNodes.cs b/src/SeqCli/Api/ToSystemTextJson.cs similarity index 83% rename from src/SeqCli/Util/JsonNodes.cs rename to src/SeqCli/Api/ToSystemTextJson.cs index 7129be90..56d92eda 100644 --- a/src/SeqCli/Util/JsonNodes.cs +++ b/src/SeqCli/Api/ToSystemTextJson.cs @@ -17,21 +17,12 @@ using Newtonsoft.Json.Linq; using SeqCli.Syntax; -namespace SeqCli.Util; +namespace SeqCli.Api; -static class JsonNodes +static class ToSystemTextJson { - public static JsonNode? FromNewtonsoft(JToken token) - { - if (token is JValue { Value: null }) - return null; - - return JsonNode.Parse(token.ToString(Formatting.None)); - } - /// - /// Convert a value deserialized by the Seq API client — a Newtonsoft LINQ-to-JSON token, or - /// a plain CLR scalar — into its System.Text.Json equivalent. + /// Convert a value deserialized by the Seq API client into its `System.Text.Json` equivalent. /// public static JsonNode? FromApiValue(object? value) { @@ -42,4 +33,13 @@ static class JsonNodes _ => EventJson.CreateScalar(value) }; } + + /// Conversion helper for values retrieved through the Seq API client. + public static JsonNode? FromNewtonsoft(JToken token) + { + if (token is JValue { Value: null }) + return null; + + return JsonNode.Parse(token.ToString(Formatting.None)); + } } diff --git a/src/SeqCli/Mapping/EventEntityJson.cs b/src/SeqCli/Mapping/EventEntityJson.cs index 68a5a485..05433965 100644 --- a/src/SeqCli/Mapping/EventEntityJson.cs +++ b/src/SeqCli/Mapping/EventEntityJson.cs @@ -19,6 +19,7 @@ using System.Text.Json.Nodes; using Seq.Api.Model.Events; using Seq.Api.Model.Shared; +using SeqCli.Api; using SeqCli.Output; using SeqCli.Syntax; using SeqCli.Util; @@ -75,7 +76,7 @@ public static JsonObject ToEventJson(EventEntity evt) if (evt.Properties != null) { foreach (var property in evt.Properties) - EventJson.SetUserProperty(eventJson, property.Name, JsonNodes.FromApiValue(property.Value)); + EventJson.SetUserProperty(eventJson, property.Name, ToSystemTextJson.FromApiValue(property.Value)); } return eventJson; @@ -99,7 +100,7 @@ static JsonObject ToPropertiesObject(List properties) { var result = new JsonObject(); foreach (var property in properties) - result[property.Name] = JsonNodes.FromApiValue(property.Value); + result[property.Name] = ToSystemTextJson.FromApiValue(property.Value); return result; } } diff --git a/src/SeqCli/Output/OutputFormat.cs b/src/SeqCli/Output/OutputFormat.cs index af137838..c57bd175 100644 --- a/src/SeqCli/Output/OutputFormat.cs +++ b/src/SeqCli/Output/OutputFormat.cs @@ -26,6 +26,7 @@ using Seq.Syntax.Templates; using Seq.Syntax.Templates.Encoding; using Seq.Syntax.Templates.Themes; +using SeqCli.Api; using SeqCli.Config; using SeqCli.Csv; using SeqCli.Mapping; @@ -150,7 +151,7 @@ public void WriteEntity(Entity entity) if (Json) { jo.Remove("Links"); - WriteJsonValue(JsonNodes.FromNewtonsoft(jo)); + WriteJsonValue(ToSystemTextJson.FromNewtonsoft(jo)); } else if (Text) { @@ -173,7 +174,7 @@ public void WriteObject(object value) (JToken)JArray.FromObject(value, _serializer) : JObject.FromObject(value, _serializer); - WriteJsonValue(JsonNodes.FromNewtonsoft(jo)); + WriteJsonValue(ToSystemTextJson.FromNewtonsoft(jo)); } else if (Text) { diff --git a/src/SeqCli/Output/TraceFormatter.cs b/src/SeqCli/Output/TraceFormatter.cs index ab8238b2..9f11263c 100644 --- a/src/SeqCli/Output/TraceFormatter.cs +++ b/src/SeqCli/Output/TraceFormatter.cs @@ -17,6 +17,7 @@ using System.Globalization; using System.Text; using System.Text.Json.Nodes; +using SeqCli.Api; using SeqCli.Traces; using SeqCli.Util; @@ -105,7 +106,7 @@ static JsonObject ToEventJson(TraceTreeNode treeNode, string treePrefix) for (var i = 0; i < evt.Columns.Count; ++i) { if (evt.Columns[i] is { } value) - eventJson[ColumnPropertyName(i)] = JsonNodes.FromApiValue(value); + eventJson[ColumnPropertyName(i)] = ToSystemTextJson.FromApiValue(value); } return eventJson; diff --git a/src/SeqCli/Traces/StructuredMessage.cs b/src/SeqCli/Traces/StructuredMessage.cs index 38cae501..a4a5a281 100644 --- a/src/SeqCli/Traces/StructuredMessage.cs +++ b/src/SeqCli/Traces/StructuredMessage.cs @@ -17,6 +17,7 @@ using System.Linq; using System.Text.Json.Nodes; using Newtonsoft.Json.Linq; +using SeqCli.Api; using SeqCli.Util; namespace SeqCli.Traces; @@ -51,7 +52,7 @@ public static (string MessageTemplate, JsonObject Properties) Read(object? struc templateTokens.Add((false, (hole["raw"] as JValue)?.Value as string ?? $"{{{name}}}")); if (hole.TryGetValue("value", out var value) && propertyNames.Add(name)) - SetPathProperty(properties, name, JsonNodes.FromNewtonsoft(value)); + SetPathProperty(properties, name, ToSystemTextJson.FromNewtonsoft(value)); } else if (token is JValue { Type: JTokenType.String } text) { From 5aadf5788e00c5fcca6a1cc5db71212b885f2fcf Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Tue, 1 Sep 2026 16:56:21 +1000 Subject: [PATCH 12/32] No need to maintain compat with undocumented internal functions we previously used for trace formatting --- src/SeqCli/Syntax/SeqSyntax.cs | 7 ++- src/SeqCli/Syntax/V1/TracingFunctions.cs | 58 ------------------------ 2 files changed, 3 insertions(+), 62 deletions(-) delete mode 100644 src/SeqCli/Syntax/V1/TracingFunctions.cs diff --git a/src/SeqCli/Syntax/SeqSyntax.cs b/src/SeqCli/Syntax/SeqSyntax.cs index acbeab8c..4d05d078 100644 --- a/src/SeqCli/Syntax/SeqSyntax.cs +++ b/src/SeqCli/Syntax/SeqSyntax.cs @@ -17,8 +17,7 @@ using Seq.Syntax.Expressions; using Seq.Syntax.Templates; using Seq.Syntax.Templates.Encoding; -using SeqCli.Syntax.V1; -using V1Compatibility = Seq.Syntax.Compatibility.V1; +using Seq.Syntax.Compatibility; namespace SeqCli.Syntax; @@ -42,12 +41,12 @@ public static bool TryCompileExpression( [MaybeNullWhen(false)] out CompiledExpression result, [MaybeNullWhen(true)] out string error) { - return V1Compatibility.TryCompileExpression(expression, formatProvider: null, TracingFunctions.Resolver, out result, out error); + return V1.TryCompileExpression(expression, formatProvider: null, null, out result, out error); } public static ExpressionTemplate ParseTemplate(string template, TemplateOutputEncoder? encoder = null) { - if (!V1Compatibility.TryParseTemplate(template, culture: null, TracingFunctions.Resolver, encoder, out var parsed, out var error)) + if (!V1.TryParseTemplate(template, culture: null, null, encoder, out var parsed, out var error)) throw new ArgumentException(error); return parsed; diff --git a/src/SeqCli/Syntax/V1/TracingFunctions.cs b/src/SeqCli/Syntax/V1/TracingFunctions.cs deleted file mode 100644 index 59193168..00000000 --- a/src/SeqCli/Syntax/V1/TracingFunctions.cs +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright © Datalust Pty Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Globalization; -using System.Text.Json.Nodes; -using Seq.Syntax.Expressions; - -namespace SeqCli.Syntax.V1; - -/// -/// Functions carried over from earlier seqcli versions, where Seq.Syntax had no tracing -/// support of its own. Elapsed() and Milliseconds() remain only so that existing -/// user-supplied expressions and output templates keep working; the built-in @Elapsed -/// and TotalMilliseconds() replace them. -/// -static class TracingFunctions -{ - public static readonly NameResolver Resolver = new StaticMemberNameResolver(typeof(TracingFunctions)); - - public static EvaluationResult Elapsed(JsonObject eventJson) - { - if (GetTimestampField(eventJson, "@t") is { } timestamp && - GetTimestampField(eventJson, "@st") is { } start) - { - return JsonValue.Create(timestamp - start)!; - } - - return EvaluationResult.Undefined; - } - - public static EvaluationResult Milliseconds(TimeSpan timeSpan) - { - // Truncates instead of rounding. - return JsonValue.Create(timeSpan.Ticks / (decimal)TimeSpan.TicksPerMillisecond); - } - - static DateTimeOffset? GetTimestampField(JsonObject eventJson, string field) - { - return eventJson.TryGetPropertyValue(field, out var node) && - node is JsonValue value && - value.TryGetValue(out string? text) && - DateTimeOffset.TryParse(text, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var dto) - ? dto - : null; - } -} From 8d9fad05844dc6df3d6f8eaae360785471a22ec1 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Tue, 1 Sep 2026 17:05:10 +1000 Subject: [PATCH 13/32] Separate Data/ from Syntax/ - still not particularly cohesive, but should point us in the right direction --- src/SeqCli/Api/ToSystemTextJson.cs | 3 ++- src/SeqCli/Cli/Commands/IngestCommand.cs | 1 + .../{Syntax/EventJson.cs => Data/EventJsonDocument.cs} | 9 ++------- src/SeqCli/{Syntax => Data}/IEventEnricher.cs | 2 +- src/SeqCli/{Syntax => Data}/ScalarPropertyEnricher.cs | 6 +++--- src/SeqCli/Ingestion/EnrichingReader.cs | 1 + src/SeqCli/Mapping/EventEntityJson.cs | 3 ++- src/SeqCli/PlainText/LogEvents/EventJsonBuilder.cs | 7 ++++--- src/SeqCli/Sample/Ingestion/MetricsMapping.cs | 3 ++- src/SeqCli/Sample/Ingestion/SimulationEvent.cs | 7 ++++--- src/SeqCli/Syntax/LevelEnricher.cs | 1 + 11 files changed, 23 insertions(+), 20 deletions(-) rename src/SeqCli/{Syntax/EventJson.cs => Data/EventJsonDocument.cs} (87%) rename src/SeqCli/{Syntax => Data}/IEventEnricher.cs (97%) rename src/SeqCli/{Syntax => Data}/ScalarPropertyEnricher.cs (85%) diff --git a/src/SeqCli/Api/ToSystemTextJson.cs b/src/SeqCli/Api/ToSystemTextJson.cs index 56d92eda..35716f03 100644 --- a/src/SeqCli/Api/ToSystemTextJson.cs +++ b/src/SeqCli/Api/ToSystemTextJson.cs @@ -15,6 +15,7 @@ using System.Text.Json.Nodes; using Newtonsoft.Json; using Newtonsoft.Json.Linq; +using SeqCli.Data; using SeqCli.Syntax; namespace SeqCli.Api; @@ -30,7 +31,7 @@ static class ToSystemTextJson { null => null, JToken token => FromNewtonsoft(token), - _ => EventJson.CreateScalar(value) + _ => EventJsonDocument.CreateScalar(value) }; } diff --git a/src/SeqCli/Cli/Commands/IngestCommand.cs b/src/SeqCli/Cli/Commands/IngestCommand.cs index ba81fa0a..e0ad35a6 100644 --- a/src/SeqCli/Cli/Commands/IngestCommand.cs +++ b/src/SeqCli/Cli/Commands/IngestCommand.cs @@ -20,6 +20,7 @@ using SeqCli.Api; using SeqCli.Cli.Features; using SeqCli.Config; +using SeqCli.Data; using SeqCli.Ingestion; using SeqCli.PlainText; using SeqCli.Syntax; diff --git a/src/SeqCli/Syntax/EventJson.cs b/src/SeqCli/Data/EventJsonDocument.cs similarity index 87% rename from src/SeqCli/Syntax/EventJson.cs rename to src/SeqCli/Data/EventJsonDocument.cs index 282a3cbf..d4ac9c08 100644 --- a/src/SeqCli/Syntax/EventJson.cs +++ b/src/SeqCli/Data/EventJsonDocument.cs @@ -16,14 +16,9 @@ using System.Globalization; using System.Text.Json.Nodes; -namespace SeqCli.Syntax; +namespace SeqCli.Data; -/// -/// Helpers for constructing event JSON documents in Seq's emission (CLEF) schema, where -/// reified fields carry @-prefixed names and user-defined property names beginning -/// with @ are escaped with a second @. -/// -static class EventJson +static class EventJsonDocument { const string InvalidPropertyNameSubstitute = "(unnamed)"; diff --git a/src/SeqCli/Syntax/IEventEnricher.cs b/src/SeqCli/Data/IEventEnricher.cs similarity index 97% rename from src/SeqCli/Syntax/IEventEnricher.cs rename to src/SeqCli/Data/IEventEnricher.cs index 9d10d8e7..55c8584b 100644 --- a/src/SeqCli/Syntax/IEventEnricher.cs +++ b/src/SeqCli/Data/IEventEnricher.cs @@ -14,7 +14,7 @@ using System.Text.Json.Nodes; -namespace SeqCli.Syntax; +namespace SeqCli.Data; /// /// Adds or updates fields on an event JSON document; the equivalent, in Seq's data model, of a diff --git a/src/SeqCli/Syntax/ScalarPropertyEnricher.cs b/src/SeqCli/Data/ScalarPropertyEnricher.cs similarity index 85% rename from src/SeqCli/Syntax/ScalarPropertyEnricher.cs rename to src/SeqCli/Data/ScalarPropertyEnricher.cs index 95490a3b..6e9193d3 100644 --- a/src/SeqCli/Syntax/ScalarPropertyEnricher.cs +++ b/src/SeqCli/Data/ScalarPropertyEnricher.cs @@ -14,7 +14,7 @@ using System.Text.Json.Nodes; -namespace SeqCli.Syntax; +namespace SeqCli.Data; class ScalarPropertyEnricher : IEventEnricher { @@ -23,12 +23,12 @@ class ScalarPropertyEnricher : IEventEnricher public ScalarPropertyEnricher(string name, object? scalarValue) { - _name = EventJson.EscapeUserPropertyName(name); + _name = EventJsonDocument.EscapeUserPropertyName(name); _scalarValue = scalarValue; } public void Enrich(JsonObject eventJson) { - eventJson[_name] = EventJson.CreateScalar(_scalarValue); + eventJson[_name] = EventJsonDocument.CreateScalar(_scalarValue); } } diff --git a/src/SeqCli/Ingestion/EnrichingReader.cs b/src/SeqCli/Ingestion/EnrichingReader.cs index 63a25ca3..dcbbebae 100644 --- a/src/SeqCli/Ingestion/EnrichingReader.cs +++ b/src/SeqCli/Ingestion/EnrichingReader.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; +using SeqCli.Data; using SeqCli.Syntax; namespace SeqCli.Ingestion; diff --git a/src/SeqCli/Mapping/EventEntityJson.cs b/src/SeqCli/Mapping/EventEntityJson.cs index 05433965..093046dc 100644 --- a/src/SeqCli/Mapping/EventEntityJson.cs +++ b/src/SeqCli/Mapping/EventEntityJson.cs @@ -20,6 +20,7 @@ using Seq.Api.Model.Events; using Seq.Api.Model.Shared; using SeqCli.Api; +using SeqCli.Data; using SeqCli.Output; using SeqCli.Syntax; using SeqCli.Util; @@ -76,7 +77,7 @@ public static JsonObject ToEventJson(EventEntity evt) if (evt.Properties != null) { foreach (var property in evt.Properties) - EventJson.SetUserProperty(eventJson, property.Name, ToSystemTextJson.FromApiValue(property.Value)); + EventJsonDocument.SetUserProperty(eventJson, property.Name, ToSystemTextJson.FromApiValue(property.Value)); } return eventJson; diff --git a/src/SeqCli/PlainText/LogEvents/EventJsonBuilder.cs b/src/SeqCli/PlainText/LogEvents/EventJsonBuilder.cs index caf4476e..caf55b5e 100644 --- a/src/SeqCli/PlainText/LogEvents/EventJsonBuilder.cs +++ b/src/SeqCli/PlainText/LogEvents/EventJsonBuilder.cs @@ -16,6 +16,7 @@ using System.Collections.Generic; using System.Globalization; using System.Text.Json.Nodes; +using SeqCli.Data; using SeqCli.Syntax; using Superpower.Model; @@ -55,11 +56,11 @@ public static JsonObject FromProperties(IDictionary properties, foreach (var (name, value) in properties) { if (!ReifiedProperties.IsReifiedProperty(name)) - EventJson.SetUserProperty(eventJson, name, CreateValue(value)); + EventJsonDocument.SetUserProperty(eventJson, name, CreateValue(value)); } if (remainder != null) - EventJson.SetUserProperty(eventJson, "@unmatched", remainder); + EventJsonDocument.SetUserProperty(eventJson, "@unmatched", remainder); return eventJson; } @@ -68,7 +69,7 @@ public static JsonObject FromProperties(IDictionary properties, { return value is TextSpan span ? JsonValue.Create(span.ToStringValue()) - : EventJson.CreateScalar(value); + : EventJsonDocument.CreateScalar(value); } static bool TryGetText(IDictionary properties, string name, out string text) diff --git a/src/SeqCli/Sample/Ingestion/MetricsMapping.cs b/src/SeqCli/Sample/Ingestion/MetricsMapping.cs index 12fbb91f..0271b2f2 100644 --- a/src/SeqCli/Sample/Ingestion/MetricsMapping.cs +++ b/src/SeqCli/Sample/Ingestion/MetricsMapping.cs @@ -16,6 +16,7 @@ using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Text.Json.Nodes; +using SeqCli.Data; using SeqCli.Ingestion; using SeqCli.Syntax; using Serilog.Events; @@ -51,7 +52,7 @@ public static bool TryGetMetricSampleJson(LogEvent logEvent, [NotNullWhen(true)] foreach (var (name, value) in logEvent.Properties) { if (name != SurrogateDefinitionsProperty) - EventJson.SetUserProperty(sample, name, SimulationEvent.ToJsonNode(value)); + EventJsonDocument.SetUserProperty(sample, name, SimulationEvent.ToJsonNode(value)); } return true; diff --git a/src/SeqCli/Sample/Ingestion/SimulationEvent.cs b/src/SeqCli/Sample/Ingestion/SimulationEvent.cs index bc48e5cc..b6a3656e 100644 --- a/src/SeqCli/Sample/Ingestion/SimulationEvent.cs +++ b/src/SeqCli/Sample/Ingestion/SimulationEvent.cs @@ -15,6 +15,7 @@ using System.Globalization; using System.Linq; using System.Text.Json.Nodes; +using SeqCli.Data; using SeqCli.Syntax; using Serilog.Events; @@ -47,7 +48,7 @@ public static JsonObject ToJsonObject(LogEvent logEvent) eventJson["@sp"] = spanId.ToHexString(); foreach (var (name, value) in logEvent.Properties) - EventJson.SetUserProperty(eventJson, name, ToJsonNode(value)); + EventJsonDocument.SetUserProperty(eventJson, name, ToJsonNode(value)); LiftSpanProperties(eventJson); @@ -59,7 +60,7 @@ public static JsonObject ToJsonObject(LogEvent logEvent) switch (value) { case ScalarValue scalar: - return EventJson.CreateScalar(scalar.Value); + return EventJsonDocument.CreateScalar(scalar.Value); case SequenceValue sequence: return new JsonArray(sequence.Elements.Select(ToJsonNode).ToArray()); @@ -83,7 +84,7 @@ public static JsonObject ToJsonObject(LogEvent logEvent) } default: - return EventJson.CreateScalar(value.ToString()); + return EventJsonDocument.CreateScalar(value.ToString()); } } diff --git a/src/SeqCli/Syntax/LevelEnricher.cs b/src/SeqCli/Syntax/LevelEnricher.cs index c09fdb35..8a1606bb 100644 --- a/src/SeqCli/Syntax/LevelEnricher.cs +++ b/src/SeqCli/Syntax/LevelEnricher.cs @@ -13,6 +13,7 @@ // limitations under the License. using System.Text.Json.Nodes; +using SeqCli.Data; namespace SeqCli.Syntax; From 45551a20d3cf595c15be31d362a4b0e565552f0a Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Tue, 1 Sep 2026 17:27:58 +1000 Subject: [PATCH 14/32] Drop some brittle test scaffolding --- test/SeqCli.Tests/Output/OutputFormatTests.cs | 43 ------------------- 1 file changed, 43 deletions(-) diff --git a/test/SeqCli.Tests/Output/OutputFormatTests.cs b/test/SeqCli.Tests/Output/OutputFormatTests.cs index cb1cf55b..a6777bef 100644 --- a/test/SeqCli.Tests/Output/OutputFormatTests.cs +++ b/test/SeqCli.Tests/Output/OutputFormatTests.cs @@ -151,47 +151,4 @@ public void UnresolvableDottedHolesRenderAsRawText() Assert.Equal("{user.greeting.first} {user.name}!", RenderMessage(evt)); } - - static string CaptureConsoleOut(System.Action write) - { - var output = new StringWriter(); - var saved = System.Console.Out; - System.Console.SetOut(output); - try - { - write(); - } - finally - { - System.Console.SetOut(saved); - } - - return output.ToString(); - } - - [Fact] - public void ObjectsAreWrittenAsSingleLineJson() - { - var format = Create(syntax: OutputSyntax.Json); - - var written = CaptureConsoleOut(() => format.WriteObject( - new JObject(new JProperty("Title", "Errors"), new JProperty("Count", 42)))); - - Assert.Equal("""{"Title":"Errors","Count":42}""" + System.Environment.NewLine, written); - } - - [Fact] - public void EntitiesAreWrittenAsJsonWithoutLinks() - { - var entity = new Seq.Api.Model.Signals.SignalEntity { Id = "signal-1", Title = "Errors" }; - - var format = Create(syntax: OutputSyntax.Json); - var written = CaptureConsoleOut(() => format.WriteEntity(entity)); - - Assert.Contains("\"Id\":\"signal-1\"", written); - Assert.Contains("\"Title\":\"Errors\"", written); - Assert.DoesNotContain("Links", written); - Assert.EndsWith(System.Environment.NewLine, written); - Assert.Equal(written.TrimEnd(), written.TrimEnd().ReplaceLineEndings("")); - } } From 575be722f383d0823d90003391fe0b5a6ae48c7c Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 2 Sep 2026 07:17:26 +1000 Subject: [PATCH 15/32] Fix namespace --- src/SeqCli/{Syntax => Data}/LevelEnricher.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) rename src/SeqCli/{Syntax => Data}/LevelEnricher.cs (95%) diff --git a/src/SeqCli/Syntax/LevelEnricher.cs b/src/SeqCli/Data/LevelEnricher.cs similarity index 95% rename from src/SeqCli/Syntax/LevelEnricher.cs rename to src/SeqCli/Data/LevelEnricher.cs index 8a1606bb..b50df51c 100644 --- a/src/SeqCli/Syntax/LevelEnricher.cs +++ b/src/SeqCli/Data/LevelEnricher.cs @@ -13,9 +13,8 @@ // limitations under the License. using System.Text.Json.Nodes; -using SeqCli.Data; -namespace SeqCli.Syntax; +namespace SeqCli.Data; /// /// Overrides the event's @l level with a fixed value. From 8d598c623029ed733cd71e915493990d8938c7ba Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 2 Sep 2026 07:19:09 +1000 Subject: [PATCH 16/32] Remove half comment --- src/SeqCli/Mapping/LevelMapping.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/SeqCli/Mapping/LevelMapping.cs b/src/SeqCli/Mapping/LevelMapping.cs index 2d866366..33639c3a 100644 --- a/src/SeqCli/Mapping/LevelMapping.cs +++ b/src/SeqCli/Mapping/LevelMapping.cs @@ -18,9 +18,6 @@ namespace SeqCli.Mapping; -/// -/// Some Seq API -/// public static class LevelMapping { static readonly Dictionary LevelsByName = From 725b97710e7232747a3602a2810254c4970974a6 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 2 Sep 2026 07:22:37 +1000 Subject: [PATCH 17/32] Namespaces, using --- src/Roastery/Data/Database.cs | 1 - src/Roastery/Util/Distribution.cs | 1 - src/Roastery/Web/RequestLoggingMiddleware.cs | 1 - src/SeqCli/Api/ToSystemTextJson.cs | 1 - src/SeqCli/Cli/Commands/Alert/CreateCommand.cs | 1 - src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs | 1 - src/SeqCli/Ingestion/EnrichingReader.cs | 1 - src/SeqCli/Mapping/EventEntityJson.cs | 2 -- src/SeqCli/Output/OutputFormat.cs | 1 - src/SeqCli/Output/TraceFormatter.cs | 1 - src/SeqCli/PlainText/{LogEvents => }/EventJsonBuilder.cs | 3 +-- src/SeqCli/PlainText/PlainTextEventReader.cs | 2 +- src/SeqCli/Sample/Ingestion/MetricsMapping.cs | 2 -- src/SeqCli/Sample/Ingestion/SimulationEvent.cs | 1 - src/SeqCli/Traces/StructuredMessage.cs | 1 - test/SeqCli.EndToEnd/Events/EventsDeleteTestCase.cs | 1 - .../Forwarder/ForwarderSimpleIngestionTestCase.cs | 1 - test/SeqCli.EndToEnd/Mcp/McpMetricsBasicsTestCase.cs | 1 - test/SeqCli.EndToEnd/Settings/SettingBasicsTestCase.cs | 3 +-- test/SeqCli.EndToEnd/Skills/SkillsInstallTestCase.cs | 1 - test/SeqCli.EndToEnd/User/UserCreateRemoveTestCase.cs | 1 - test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs | 1 - test/SeqCli.Tests/PlainText/EventJsonBuilderTests.cs | 3 ++- .../PlainText/ExtractionPatternInterpreterTests.cs | 1 - test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs | 3 +-- test/SeqCli.Tests/Syntax/AliasedExpressionParserTests.cs | 1 - 26 files changed, 6 insertions(+), 31 deletions(-) rename src/SeqCli/PlainText/{LogEvents => }/EventJsonBuilder.cs (98%) diff --git a/src/Roastery/Data/Database.cs b/src/Roastery/Data/Database.cs index 9234f1ce..251b67f1 100644 --- a/src/Roastery/Data/Database.cs +++ b/src/Roastery/Data/Database.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.Globalization; using System.Linq; using System.Reflection; diff --git a/src/Roastery/Util/Distribution.cs b/src/Roastery/Util/Distribution.cs index b20ffb49..5264cd14 100644 --- a/src/Roastery/Util/Distribution.cs +++ b/src/Roastery/Util/Distribution.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Runtime.CompilerServices; -using System.Threading; namespace Roastery.Util; diff --git a/src/Roastery/Web/RequestLoggingMiddleware.cs b/src/Roastery/Web/RequestLoggingMiddleware.cs index c75a5002..92d50bf3 100644 --- a/src/Roastery/Web/RequestLoggingMiddleware.cs +++ b/src/Roastery/Web/RequestLoggingMiddleware.cs @@ -1,6 +1,5 @@ using System; using System.Diagnostics; -using System.Diagnostics.Metrics; using System.Net; using System.Threading.Tasks; using Roastery.Metrics; diff --git a/src/SeqCli/Api/ToSystemTextJson.cs b/src/SeqCli/Api/ToSystemTextJson.cs index 35716f03..52773250 100644 --- a/src/SeqCli/Api/ToSystemTextJson.cs +++ b/src/SeqCli/Api/ToSystemTextJson.cs @@ -16,7 +16,6 @@ using Newtonsoft.Json; using Newtonsoft.Json.Linq; using SeqCli.Data; -using SeqCli.Syntax; namespace SeqCli.Api; diff --git a/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs b/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs index e7de0520..e74d833a 100644 --- a/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs @@ -17,7 +17,6 @@ using System.Linq; using System.Threading.Tasks; using Seq.Api.Model.Alerting; -using Seq.Api.Model.LogEvents; using Seq.Api.Model.Shared; using SeqCli.Api; using SeqCli.Cli.Features; diff --git a/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs b/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs index 03c3cf25..cf928d8c 100644 --- a/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs @@ -16,7 +16,6 @@ using System.Linq; using System.Threading.Tasks; using Seq.Api; -using Seq.Api.Model.LogEvents; using Seq.Api.Model.Security; using Seq.Api.Model.Shared; using SeqCli.Api; diff --git a/src/SeqCli/Ingestion/EnrichingReader.cs b/src/SeqCli/Ingestion/EnrichingReader.cs index dcbbebae..6207bf4c 100644 --- a/src/SeqCli/Ingestion/EnrichingReader.cs +++ b/src/SeqCli/Ingestion/EnrichingReader.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Threading.Tasks; using SeqCli.Data; -using SeqCli.Syntax; namespace SeqCli.Ingestion; diff --git a/src/SeqCli/Mapping/EventEntityJson.cs b/src/SeqCli/Mapping/EventEntityJson.cs index 093046dc..169d7a49 100644 --- a/src/SeqCli/Mapping/EventEntityJson.cs +++ b/src/SeqCli/Mapping/EventEntityJson.cs @@ -22,8 +22,6 @@ using SeqCli.Api; using SeqCli.Data; using SeqCli.Output; -using SeqCli.Syntax; -using SeqCli.Util; namespace SeqCli.Mapping; diff --git a/src/SeqCli/Output/OutputFormat.cs b/src/SeqCli/Output/OutputFormat.cs index c57bd175..9efb3b97 100644 --- a/src/SeqCli/Output/OutputFormat.cs +++ b/src/SeqCli/Output/OutputFormat.cs @@ -30,7 +30,6 @@ using SeqCli.Config; using SeqCli.Csv; using SeqCli.Mapping; -using SeqCli.Util; namespace SeqCli.Output; diff --git a/src/SeqCli/Output/TraceFormatter.cs b/src/SeqCli/Output/TraceFormatter.cs index 9f11263c..c0a575ef 100644 --- a/src/SeqCli/Output/TraceFormatter.cs +++ b/src/SeqCli/Output/TraceFormatter.cs @@ -19,7 +19,6 @@ using System.Text.Json.Nodes; using SeqCli.Api; using SeqCli.Traces; -using SeqCli.Util; namespace SeqCli.Output; diff --git a/src/SeqCli/PlainText/LogEvents/EventJsonBuilder.cs b/src/SeqCli/PlainText/EventJsonBuilder.cs similarity index 98% rename from src/SeqCli/PlainText/LogEvents/EventJsonBuilder.cs rename to src/SeqCli/PlainText/EventJsonBuilder.cs index caf55b5e..bff33836 100644 --- a/src/SeqCli/PlainText/LogEvents/EventJsonBuilder.cs +++ b/src/SeqCli/PlainText/EventJsonBuilder.cs @@ -17,10 +17,9 @@ using System.Globalization; using System.Text.Json.Nodes; using SeqCli.Data; -using SeqCli.Syntax; using Superpower.Model; -namespace SeqCli.PlainText.LogEvents; +namespace SeqCli.PlainText; /// /// Assembles the values captured by a plain-text extraction pattern into an event JSON diff --git a/src/SeqCli/PlainText/PlainTextEventReader.cs b/src/SeqCli/PlainText/PlainTextEventReader.cs index fbead08b..21da4e89 100644 --- a/src/SeqCli/PlainText/PlainTextEventReader.cs +++ b/src/SeqCli/PlainText/PlainTextEventReader.cs @@ -1,10 +1,10 @@ using System; using System.IO; using System.Threading.Tasks; +using SeqCli.Data; using SeqCli.Ingestion; using SeqCli.PlainText.Extraction; using SeqCli.PlainText.Framing; -using SeqCli.PlainText.LogEvents; using SeqCli.PlainText.Parsers; using SeqCli.PlainText.Patterns; diff --git a/src/SeqCli/Sample/Ingestion/MetricsMapping.cs b/src/SeqCli/Sample/Ingestion/MetricsMapping.cs index 0271b2f2..6782e6c5 100644 --- a/src/SeqCli/Sample/Ingestion/MetricsMapping.cs +++ b/src/SeqCli/Sample/Ingestion/MetricsMapping.cs @@ -17,8 +17,6 @@ using System.Globalization; using System.Text.Json.Nodes; using SeqCli.Data; -using SeqCli.Ingestion; -using SeqCli.Syntax; using Serilog.Events; namespace SeqCli.Sample.Ingestion; diff --git a/src/SeqCli/Sample/Ingestion/SimulationEvent.cs b/src/SeqCli/Sample/Ingestion/SimulationEvent.cs index b6a3656e..89bc110c 100644 --- a/src/SeqCli/Sample/Ingestion/SimulationEvent.cs +++ b/src/SeqCli/Sample/Ingestion/SimulationEvent.cs @@ -16,7 +16,6 @@ using System.Linq; using System.Text.Json.Nodes; using SeqCli.Data; -using SeqCli.Syntax; using Serilog.Events; namespace SeqCli.Sample.Ingestion; diff --git a/src/SeqCli/Traces/StructuredMessage.cs b/src/SeqCli/Traces/StructuredMessage.cs index a4a5a281..702c9006 100644 --- a/src/SeqCli/Traces/StructuredMessage.cs +++ b/src/SeqCli/Traces/StructuredMessage.cs @@ -18,7 +18,6 @@ using System.Text.Json.Nodes; using Newtonsoft.Json.Linq; using SeqCli.Api; -using SeqCli.Util; namespace SeqCli.Traces; diff --git a/test/SeqCli.EndToEnd/Events/EventsDeleteTestCase.cs b/test/SeqCli.EndToEnd/Events/EventsDeleteTestCase.cs index 29b585c9..61d8281b 100644 --- a/test/SeqCli.EndToEnd/Events/EventsDeleteTestCase.cs +++ b/test/SeqCli.EndToEnd/Events/EventsDeleteTestCase.cs @@ -1,4 +1,3 @@ -using System; using System.IO; using System.Threading.Tasks; using Seq.Api; diff --git a/test/SeqCli.EndToEnd/Forwarder/ForwarderSimpleIngestionTestCase.cs b/test/SeqCli.EndToEnd/Forwarder/ForwarderSimpleIngestionTestCase.cs index bb199942..d90065e5 100644 --- a/test/SeqCli.EndToEnd/Forwarder/ForwarderSimpleIngestionTestCase.cs +++ b/test/SeqCli.EndToEnd/Forwarder/ForwarderSimpleIngestionTestCase.cs @@ -1,5 +1,4 @@ using System; -using System.Globalization; using System.Threading.Tasks; using Seq.Api; using SeqCli.EndToEnd.Support; diff --git a/test/SeqCli.EndToEnd/Mcp/McpMetricsBasicsTestCase.cs b/test/SeqCli.EndToEnd/Mcp/McpMetricsBasicsTestCase.cs index 77fc5f69..f1fa88fc 100644 --- a/test/SeqCli.EndToEnd/Mcp/McpMetricsBasicsTestCase.cs +++ b/test/SeqCli.EndToEnd/Mcp/McpMetricsBasicsTestCase.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Text.Json.Nodes; using System.Threading.Tasks; using JetBrains.Annotations; using ModelContextProtocol.Client; diff --git a/test/SeqCli.EndToEnd/Settings/SettingBasicsTestCase.cs b/test/SeqCli.EndToEnd/Settings/SettingBasicsTestCase.cs index f387a400..1de07801 100644 --- a/test/SeqCli.EndToEnd/Settings/SettingBasicsTestCase.cs +++ b/test/SeqCli.EndToEnd/Settings/SettingBasicsTestCase.cs @@ -1,5 +1,4 @@ -using System; -using System.Threading.Tasks; +using System.Threading.Tasks; using Seq.Api; using SeqCli.EndToEnd.Support; using Serilog; diff --git a/test/SeqCli.EndToEnd/Skills/SkillsInstallTestCase.cs b/test/SeqCli.EndToEnd/Skills/SkillsInstallTestCase.cs index 88871bc1..5bca2a32 100644 --- a/test/SeqCli.EndToEnd/Skills/SkillsInstallTestCase.cs +++ b/test/SeqCli.EndToEnd/Skills/SkillsInstallTestCase.cs @@ -1,6 +1,5 @@ using System.IO; using System.Threading.Tasks; -using JetBrains.Annotations; using Seq.Api; using SeqCli.EndToEnd.Support; using Serilog; diff --git a/test/SeqCli.EndToEnd/User/UserCreateRemoveTestCase.cs b/test/SeqCli.EndToEnd/User/UserCreateRemoveTestCase.cs index ddbf95d2..25e3fb99 100644 --- a/test/SeqCli.EndToEnd/User/UserCreateRemoveTestCase.cs +++ b/test/SeqCli.EndToEnd/User/UserCreateRemoveTestCase.cs @@ -4,7 +4,6 @@ using SeqCli.EndToEnd.Support; using Serilog; using Xunit; -using System.IO; using System.Linq; namespace SeqCli.EndToEnd.User; diff --git a/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs b/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs index 60dee141..7e6bf24e 100644 --- a/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs +++ b/test/SeqCli.Tests/Forwarder/Storage/BufferTests.cs @@ -1,5 +1,4 @@ using System.Linq; -using SeqCli.Forwarder.Filesystem.System; using SeqCli.Forwarder.Storage; using SeqCli.Tests.Forwarder.Filesystem; using Xunit; diff --git a/test/SeqCli.Tests/PlainText/EventJsonBuilderTests.cs b/test/SeqCli.Tests/PlainText/EventJsonBuilderTests.cs index 0df85264..0e392eb9 100644 --- a/test/SeqCli.Tests/PlainText/EventJsonBuilderTests.cs +++ b/test/SeqCli.Tests/PlainText/EventJsonBuilderTests.cs @@ -2,7 +2,8 @@ using System; using System.Collections.Generic; using System.Globalization; -using SeqCli.PlainText.LogEvents; +using SeqCli.Data; +using SeqCli.PlainText; using Superpower.Model; using Xunit; diff --git a/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs b/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs index 0993251f..fa251cb9 100644 --- a/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs +++ b/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Globalization; -using SeqCli.PlainText; using SeqCli.PlainText.Extraction; using SeqCli.PlainText.Patterns; using Xunit; diff --git a/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs b/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs index 171d6aba..fac72167 100644 --- a/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs +++ b/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs @@ -1,5 +1,4 @@ -using System; -using System.Linq; +using System.Linq; using SeqCli.PlainText.Patterns; using Superpower; using Xunit; diff --git a/test/SeqCli.Tests/Syntax/AliasedExpressionParserTests.cs b/test/SeqCli.Tests/Syntax/AliasedExpressionParserTests.cs index 43cceeca..7699f513 100644 --- a/test/SeqCli.Tests/Syntax/AliasedExpressionParserTests.cs +++ b/test/SeqCli.Tests/Syntax/AliasedExpressionParserTests.cs @@ -1,4 +1,3 @@ -using System; using SeqCli.Syntax; using Xunit; From c230f57a21e6878127edb0f28895890c3f327619 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 2 Sep 2026 11:09:30 +1000 Subject: [PATCH 18/32] More tidy-up --- src/SeqCli/Api/ToSystemTextJson.cs | 2 +- .../{EventJsonDocument.cs => EventJsonFormat.cs} | 14 ++------------ src/SeqCli/Data/ScalarPropertyEnricher.cs | 4 ++-- src/SeqCli/Mapping/EventEntityJson.cs | 2 +- src/SeqCli/PlainText/EventJsonBuilder.cs | 10 ++++++---- src/SeqCli/Sample/Ingestion/MetricsMapping.cs | 2 +- src/SeqCli/Sample/Ingestion/SimulationEvent.cs | 6 +++--- src/SeqCli/SeqCli.csproj | 2 +- 8 files changed, 17 insertions(+), 25 deletions(-) rename src/SeqCli/Data/{EventJsonDocument.cs => EventJsonFormat.cs} (83%) diff --git a/src/SeqCli/Api/ToSystemTextJson.cs b/src/SeqCli/Api/ToSystemTextJson.cs index 52773250..92428c06 100644 --- a/src/SeqCli/Api/ToSystemTextJson.cs +++ b/src/SeqCli/Api/ToSystemTextJson.cs @@ -30,7 +30,7 @@ static class ToSystemTextJson { null => null, JToken token => FromNewtonsoft(token), - _ => EventJsonDocument.CreateScalar(value) + _ => EventJsonFormat.CreateScalar(value) }; } diff --git a/src/SeqCli/Data/EventJsonDocument.cs b/src/SeqCli/Data/EventJsonFormat.cs similarity index 83% rename from src/SeqCli/Data/EventJsonDocument.cs rename to src/SeqCli/Data/EventJsonFormat.cs index d4ac9c08..1f0a83dd 100644 --- a/src/SeqCli/Data/EventJsonDocument.cs +++ b/src/SeqCli/Data/EventJsonFormat.cs @@ -18,23 +18,13 @@ namespace SeqCli.Data; -static class EventJsonDocument +static class EventJsonFormat { - const string InvalidPropertyNameSubstitute = "(unnamed)"; - public static string EscapeUserPropertyName(string name) { - if (string.IsNullOrEmpty(name)) - return InvalidPropertyNameSubstitute; - return name.StartsWith('@') ? $"@{name}" : name; } - - public static void SetUserProperty(JsonObject eventJson, string name, JsonNode? value) - { - eventJson[EscapeUserPropertyName(name)] = value; - } - + public static JsonNode? CreateScalar(object? value) { return value switch diff --git a/src/SeqCli/Data/ScalarPropertyEnricher.cs b/src/SeqCli/Data/ScalarPropertyEnricher.cs index 6e9193d3..0d5f2a4c 100644 --- a/src/SeqCli/Data/ScalarPropertyEnricher.cs +++ b/src/SeqCli/Data/ScalarPropertyEnricher.cs @@ -23,12 +23,12 @@ class ScalarPropertyEnricher : IEventEnricher public ScalarPropertyEnricher(string name, object? scalarValue) { - _name = EventJsonDocument.EscapeUserPropertyName(name); + _name = EventJsonFormat.EscapeUserPropertyName(name); _scalarValue = scalarValue; } public void Enrich(JsonObject eventJson) { - eventJson[_name] = EventJsonDocument.CreateScalar(_scalarValue); + eventJson[_name] = EventJsonFormat.CreateScalar(_scalarValue); } } diff --git a/src/SeqCli/Mapping/EventEntityJson.cs b/src/SeqCli/Mapping/EventEntityJson.cs index 169d7a49..ac8fc6e4 100644 --- a/src/SeqCli/Mapping/EventEntityJson.cs +++ b/src/SeqCli/Mapping/EventEntityJson.cs @@ -75,7 +75,7 @@ public static JsonObject ToEventJson(EventEntity evt) if (evt.Properties != null) { foreach (var property in evt.Properties) - EventJsonDocument.SetUserProperty(eventJson, property.Name, ToSystemTextJson.FromApiValue(property.Value)); + eventJson[EventJsonFormat.EscapeUserPropertyName(property.Name)] = ToSystemTextJson.FromApiValue(property.Value); } return eventJson; diff --git a/src/SeqCli/PlainText/EventJsonBuilder.cs b/src/SeqCli/PlainText/EventJsonBuilder.cs index bff33836..7acddcb2 100644 --- a/src/SeqCli/PlainText/EventJsonBuilder.cs +++ b/src/SeqCli/PlainText/EventJsonBuilder.cs @@ -55,20 +55,22 @@ public static JsonObject FromProperties(IDictionary properties, foreach (var (name, value) in properties) { if (!ReifiedProperties.IsReifiedProperty(name)) - EventJsonDocument.SetUserProperty(eventJson, name, CreateValue(value)); + eventJson[EventJsonFormat.EscapeUserPropertyName(name)] = UnwrapTextSpans(value); } if (remainder != null) - EventJsonDocument.SetUserProperty(eventJson, "@unmatched", remainder); + eventJson[EventJsonFormat.EscapeUserPropertyName("@unmatched")] = UnwrapTextSpans(remainder); return eventJson; } - static JsonNode? CreateValue(object? value) + static JsonNode? UnwrapTextSpans(object? value) { + // We should consider whether text spans might also end up in extracted dictionary or array elements, though + // I don't think they will, currently. return value is TextSpan span ? JsonValue.Create(span.ToStringValue()) - : EventJsonDocument.CreateScalar(value); + : EventJsonFormat.CreateScalar(value); } static bool TryGetText(IDictionary properties, string name, out string text) diff --git a/src/SeqCli/Sample/Ingestion/MetricsMapping.cs b/src/SeqCli/Sample/Ingestion/MetricsMapping.cs index 6782e6c5..f50da4d7 100644 --- a/src/SeqCli/Sample/Ingestion/MetricsMapping.cs +++ b/src/SeqCli/Sample/Ingestion/MetricsMapping.cs @@ -50,7 +50,7 @@ public static bool TryGetMetricSampleJson(LogEvent logEvent, [NotNullWhen(true)] foreach (var (name, value) in logEvent.Properties) { if (name != SurrogateDefinitionsProperty) - EventJsonDocument.SetUserProperty(sample, name, SimulationEvent.ToJsonNode(value)); + sample[EventJsonFormat.EscapeUserPropertyName(name)] = SimulationEvent.ToJsonNode(value); } return true; diff --git a/src/SeqCli/Sample/Ingestion/SimulationEvent.cs b/src/SeqCli/Sample/Ingestion/SimulationEvent.cs index 89bc110c..988b4952 100644 --- a/src/SeqCli/Sample/Ingestion/SimulationEvent.cs +++ b/src/SeqCli/Sample/Ingestion/SimulationEvent.cs @@ -47,7 +47,7 @@ public static JsonObject ToJsonObject(LogEvent logEvent) eventJson["@sp"] = spanId.ToHexString(); foreach (var (name, value) in logEvent.Properties) - EventJsonDocument.SetUserProperty(eventJson, name, ToJsonNode(value)); + eventJson[EventJsonFormat.EscapeUserPropertyName(name)] = ToJsonNode(value); LiftSpanProperties(eventJson); @@ -59,7 +59,7 @@ public static JsonObject ToJsonObject(LogEvent logEvent) switch (value) { case ScalarValue scalar: - return EventJsonDocument.CreateScalar(scalar.Value); + return EventJsonFormat.CreateScalar(scalar.Value); case SequenceValue sequence: return new JsonArray(sequence.Elements.Select(ToJsonNode).ToArray()); @@ -83,7 +83,7 @@ public static JsonObject ToJsonObject(LogEvent logEvent) } default: - return EventJsonDocument.CreateScalar(value.ToString()); + return EventJsonFormat.CreateScalar(value.ToString()); } } diff --git a/src/SeqCli/SeqCli.csproj b/src/SeqCli/SeqCli.csproj index 0e97213a..16f10fb8 100644 --- a/src/SeqCli/SeqCli.csproj +++ b/src/SeqCli/SeqCli.csproj @@ -42,7 +42,7 @@ - + From edf85055c72daee3c5ac8cc06724d6ab10d2b7d2 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 2 Sep 2026 11:24:26 +1000 Subject: [PATCH 19/32] Add coverage for `search --filter`. Assisted-by: Claude:claude-fable-5-1 --- .../Search/SearchWithFilterTestCase.cs | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 test/SeqCli.EndToEnd/Search/SearchWithFilterTestCase.cs diff --git a/test/SeqCli.EndToEnd/Search/SearchWithFilterTestCase.cs b/test/SeqCli.EndToEnd/Search/SearchWithFilterTestCase.cs new file mode 100644 index 00000000..0e3a2f1d --- /dev/null +++ b/test/SeqCli.EndToEnd/Search/SearchWithFilterTestCase.cs @@ -0,0 +1,35 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using Seq.Api; +using SeqCli.EndToEnd.Support; +using Serilog; +using Xunit; + +namespace SeqCli.EndToEnd.Search; + +public class SearchWithFilterTestCase : ICliTestCase +{ + public async Task ExecuteAsync( + SeqConnection connection, + ILogger logger, + CliCommandRunner runner) + { + await DirectIngestion.IngestClef(connection, "'@mt': 'Event {N}', 'N': 1, 'Host': 'xmpweb-01.example.com'"); + await DirectIngestion.IngestClef(connection, "'@mt': 'Event {N}', 'N': 2, 'Host': 'xmpweb-02.example.com'"); + await DirectIngestion.IngestClef(connection, "'@mt': 'Event {N}', 'N': 3, 'Host': 'xmpweb-02.example.com'"); + + var exit = runner.Exec("search", "--filter=\"Host = 'xmpweb-02.example.com' and N > 2\" --count=10 --json"); + Assert.Equal(0, exit); + + var results = runner.LastRunProcess!.Output + .Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries) + .Select(JObject.Parse) + .ToList(); + + var evt = Assert.Single(results); + Assert.Equal(3, evt["N"]!.Value()); + Assert.Equal("xmpweb-02.example.com", evt["Host"]!.Value()); + } +} From 2675a1d0334b7c119e47ea77c884b07b5b8f49ca Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 2 Sep 2026 17:08:15 +1000 Subject: [PATCH 20/32] Feedback --- .../{Mapping => Api}/EventEntityJson.cs | 3 +- src/SeqCli/{Mapping => Api}/LevelMapping.cs | 2 +- src/SeqCli/Apps/Hosting/AppContainer.cs | 2 +- .../Cli/Commands/Alert/CreateCommand.cs | 1 - .../Cli/Commands/ApiKey/CreateCommand.cs | 1 - src/SeqCli/Data/EventJsonFormat.cs | 10 +++-- src/SeqCli/Ingestion/JsonEventReader.cs | 3 +- src/SeqCli/Ingestion/LogShipper.cs | 10 ++--- src/SeqCli/Ingestion/ReadResult.cs | 18 +++++++-- src/SeqCli/Mcp/Tools/Search/SearchTools.cs | 2 +- src/SeqCli/Output/OutputFormat.cs | 1 - src/SeqCli/Output/TraceFormatter.cs | 3 +- src/SeqCli/PlainText/Extraction/Matchers.cs | 2 +- test/SeqCli.Tests/Output/OutputFormatTests.cs | 2 +- .../Output/TextFormattersTests.cs | 2 +- .../Sample/SimulationEventTests.cs | 2 +- .../Traces/StructuredMessageTests.cs | 38 +++++++++---------- 17 files changed, 55 insertions(+), 47 deletions(-) rename src/SeqCli/{Mapping => Api}/EventEntityJson.cs (98%) rename src/SeqCli/{Mapping => Api}/LevelMapping.cs (99%) diff --git a/src/SeqCli/Mapping/EventEntityJson.cs b/src/SeqCli/Api/EventEntityJson.cs similarity index 98% rename from src/SeqCli/Mapping/EventEntityJson.cs rename to src/SeqCli/Api/EventEntityJson.cs index ac8fc6e4..308c0735 100644 --- a/src/SeqCli/Mapping/EventEntityJson.cs +++ b/src/SeqCli/Api/EventEntityJson.cs @@ -19,11 +19,10 @@ using System.Text.Json.Nodes; using Seq.Api.Model.Events; using Seq.Api.Model.Shared; -using SeqCli.Api; using SeqCli.Data; using SeqCli.Output; -namespace SeqCli.Mapping; +namespace SeqCli.Api; /// /// Converts event entities into compact JSON format for further processing. This class is only necessary because diff --git a/src/SeqCli/Mapping/LevelMapping.cs b/src/SeqCli/Api/LevelMapping.cs similarity index 99% rename from src/SeqCli/Mapping/LevelMapping.cs rename to src/SeqCli/Api/LevelMapping.cs index 33639c3a..bc987977 100644 --- a/src/SeqCli/Mapping/LevelMapping.cs +++ b/src/SeqCli/Api/LevelMapping.cs @@ -16,7 +16,7 @@ using System.Collections.Generic; using Seq.Api.Model.LogEvents; -namespace SeqCli.Mapping; +namespace SeqCli.Api; public static class LevelMapping { diff --git a/src/SeqCli/Apps/Hosting/AppContainer.cs b/src/SeqCli/Apps/Hosting/AppContainer.cs index b65029f5..6949921d 100644 --- a/src/SeqCli/Apps/Hosting/AppContainer.cs +++ b/src/SeqCli/Apps/Hosting/AppContainer.cs @@ -21,7 +21,7 @@ using Newtonsoft.Json.Linq; using Seq.Apps; using Seq.Apps.LogEvents; -using SeqCli.Mapping; +using SeqCli.Api; using Serilog; using Serilog.Events; using Serilog.Formatting.Compact.Reader; diff --git a/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs b/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs index e74d833a..d8a2182d 100644 --- a/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/Alert/CreateCommand.cs @@ -21,7 +21,6 @@ using SeqCli.Api; using SeqCli.Cli.Features; using SeqCli.Config; -using SeqCli.Mapping; using SeqCli.Signals; using SeqCli.Syntax; using SeqCli.Util; diff --git a/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs b/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs index cf928d8c..25375b1e 100644 --- a/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs +++ b/src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs @@ -21,7 +21,6 @@ using SeqCli.Api; using SeqCli.Cli.Features; using SeqCli.Config; -using SeqCli.Mapping; using SeqCli.Util; using Serilog; diff --git a/src/SeqCli/Data/EventJsonFormat.cs b/src/SeqCli/Data/EventJsonFormat.cs index 1f0a83dd..a862e4ff 100644 --- a/src/SeqCli/Data/EventJsonFormat.cs +++ b/src/SeqCli/Data/EventJsonFormat.cs @@ -13,7 +13,6 @@ // limitations under the License. using System; -using System.Globalization; using System.Text.Json.Nodes; namespace SeqCli.Data; @@ -25,6 +24,10 @@ public static string EscapeUserPropertyName(string name) return name.StartsWith('@') ? $"@{name}" : name; } + /// + /// Use this function when converting a value of uncertain or non-primitive type into a . It's + /// okay to use for strongly-typed primitives. + /// public static JsonNode? CreateScalar(object? value) { return value switch @@ -43,8 +46,9 @@ public static string EscapeUserPropertyName(string name) float n => JsonValue.Create(n), double n => JsonValue.Create(n), decimal n => JsonValue.Create(n), - DateTime dt => JsonValue.Create(dt.ToString("o", CultureInfo.InvariantCulture)), - DateTimeOffset dto => JsonValue.Create(dto.ToString("o", CultureInfo.InvariantCulture)), + TimeSpan ts => JsonValue.Create(ts.ToString("c")), + DateTime dt => JsonValue.Create(dt), + DateTimeOffset dto => JsonValue.Create(dto), _ => JsonValue.Create(value.ToString()) }; } diff --git a/src/SeqCli/Ingestion/JsonEventReader.cs b/src/SeqCli/Ingestion/JsonEventReader.cs index a8d4be8c..689710bd 100644 --- a/src/SeqCli/Ingestion/JsonEventReader.cs +++ b/src/SeqCli/Ingestion/JsonEventReader.cs @@ -13,7 +13,6 @@ // limitations under the License. using System; -using System.Globalization; using System.IO; using System.Text.Json.Nodes; using System.Threading.Tasks; @@ -55,7 +54,7 @@ static JsonObject ReadFromJson(string json) throw new InvalidDataException($"The line is not a JSON object: `{json.Trim()}`."); if (!eventJson.ContainsKey("@t")) - eventJson["@t"] = DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture); + eventJson["@t"] = DateTime.UtcNow; return eventJson; } diff --git a/src/SeqCli/Ingestion/LogShipper.cs b/src/SeqCli/Ingestion/LogShipper.cs index 68674fee..313d6ceb 100644 --- a/src/SeqCli/Ingestion/LogShipper.cs +++ b/src/SeqCli/Ingestion/LogShipper.cs @@ -22,7 +22,6 @@ using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; -using Newtonsoft.Json; using Seq.Api; using SeqCli.Api; using Serilog; @@ -132,7 +131,7 @@ public static async Task ShipEventsAsync( if (sendFailureHandling == SendFailureHandling.Retry) { var millisecondsDelay = (int)Math.Min(Math.Pow(2, retries) * 2000, 60000); - await Task.Delay(millisecondsDelay); + await Task.Delay(millisecondsDelay, cancellationToken); retries += 1; continue; } @@ -191,7 +190,7 @@ static async Task ReadBatchAsync( } catch (Exception ex) { - if (ex is System.Text.Json.JsonException || ex is InvalidDataException) + if (ex is System.Text.Json.JsonException or InvalidDataException) { if (invalidDataHandling == InvalidDataHandling.Ignore) continue; @@ -200,7 +199,7 @@ static async Task ReadBatchAsync( throw; } - return new BatchResult(batch.ToArray(), isLast); + return new BatchResult([.. batch], isLast); } while (true); } @@ -219,6 +218,7 @@ static async Task SendBatchAsync( using (var builder = new StringWriter()) { foreach (var evt in batch) + // ReSharper disable once MethodHasAsyncOverload builder.WriteLine(evt.ToJsonString()); content = new StringContent(builder.ToString(), Encoding.UTF8, ApiConstants.ClefMediaType); @@ -243,7 +243,7 @@ static async Task SendAsync(SeqConnection connection, string? ap { try { - var error = JsonConvert.DeserializeObject(resultJson)!; + var error = Newtonsoft.Json.JsonConvert.DeserializeObject(resultJson)!; sendFailureLog.Error("Shipping failed with status code {StatusCode}: {ErrorMessage}", result.StatusCode, diff --git a/src/SeqCli/Ingestion/ReadResult.cs b/src/SeqCli/Ingestion/ReadResult.cs index a44d30f4..0e074de5 100644 --- a/src/SeqCli/Ingestion/ReadResult.cs +++ b/src/SeqCli/Ingestion/ReadResult.cs @@ -1,13 +1,23 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + using System.Text.Json.Nodes; namespace SeqCli.Ingestion; readonly struct ReadResult { - /// - /// The event, as a JSON document in Seq's emission schema, or null if no event - /// is available. - /// public JsonObject? Document { get; } public bool IsAtEnd { get; } diff --git a/src/SeqCli/Mcp/Tools/Search/SearchTools.cs b/src/SeqCli/Mcp/Tools/Search/SearchTools.cs index 9ceda5e5..7d62d9ef 100644 --- a/src/SeqCli/Mcp/Tools/Search/SearchTools.cs +++ b/src/SeqCli/Mcp/Tools/Search/SearchTools.cs @@ -27,7 +27,7 @@ using Seq.Api.Model.Events; using Seq.Api.Model.Signals; using Seq.Syntax.Templates; -using SeqCli.Mapping; +using SeqCli.Api; using SeqCli.Signals; using SeqCli.Syntax; using Serilog; diff --git a/src/SeqCli/Output/OutputFormat.cs b/src/SeqCli/Output/OutputFormat.cs index 9efb3b97..ae2492fb 100644 --- a/src/SeqCli/Output/OutputFormat.cs +++ b/src/SeqCli/Output/OutputFormat.cs @@ -29,7 +29,6 @@ using SeqCli.Api; using SeqCli.Config; using SeqCli.Csv; -using SeqCli.Mapping; namespace SeqCli.Output; diff --git a/src/SeqCli/Output/TraceFormatter.cs b/src/SeqCli/Output/TraceFormatter.cs index c0a575ef..a69a1324 100644 --- a/src/SeqCli/Output/TraceFormatter.cs +++ b/src/SeqCli/Output/TraceFormatter.cs @@ -18,6 +18,7 @@ using System.Text; using System.Text.Json.Nodes; using SeqCli.Api; +using SeqCli.Data; using SeqCli.Traces; namespace SeqCli.Output; @@ -100,7 +101,7 @@ static JsonObject ToEventJson(TraceTreeNode treeNode, string treePrefix) eventJson[name] = value?.DeepClone(); if (evt.Elapsed is { } elapsed) - eventJson[ElapsedProperty] = elapsed.ToString("c", CultureInfo.InvariantCulture); + eventJson[ElapsedProperty] = EventJsonFormat.CreateScalar(elapsed); for (var i = 0; i < evt.Columns.Count; ++i) { diff --git a/src/SeqCli/PlainText/Extraction/Matchers.cs b/src/SeqCli/PlainText/Extraction/Matchers.cs index 5ae49e6e..fef3c2cc 100644 --- a/src/SeqCli/PlainText/Extraction/Matchers.cs +++ b/src/SeqCli/PlainText/Extraction/Matchers.cs @@ -3,7 +3,7 @@ using System.Globalization; using System.Linq; using System.Reflection; -using SeqCli.Mapping; +using SeqCli.Api; using SeqCli.PlainText.Parsers; using Superpower; using Superpower.Model; diff --git a/test/SeqCli.Tests/Output/OutputFormatTests.cs b/test/SeqCli.Tests/Output/OutputFormatTests.cs index a6777bef..e323aa60 100644 --- a/test/SeqCli.Tests/Output/OutputFormatTests.cs +++ b/test/SeqCli.Tests/Output/OutputFormatTests.cs @@ -1,8 +1,8 @@ using System.IO; using Newtonsoft.Json.Linq; using Seq.Api.Model.Events; +using SeqCli.Api; using SeqCli.Config; -using SeqCli.Mapping; using SeqCli.Output; using SeqCli.Tests.Support; using Xunit; diff --git a/test/SeqCli.Tests/Output/TextFormattersTests.cs b/test/SeqCli.Tests/Output/TextFormattersTests.cs index 7675926e..340b18fc 100644 --- a/test/SeqCli.Tests/Output/TextFormattersTests.cs +++ b/test/SeqCli.Tests/Output/TextFormattersTests.cs @@ -3,7 +3,7 @@ using System.IO; using System.Text.Json.Nodes; using Seq.Syntax.Templates.Themes; -using SeqCli.Mapping; +using SeqCli.Api; using SeqCli.Output; using SeqCli.Tests.Support; using Xunit; diff --git a/test/SeqCli.Tests/Sample/SimulationEventTests.cs b/test/SeqCli.Tests/Sample/SimulationEventTests.cs index 0a71da14..0c69a6e4 100644 --- a/test/SeqCli.Tests/Sample/SimulationEventTests.cs +++ b/test/SeqCli.Tests/Sample/SimulationEventTests.cs @@ -69,7 +69,7 @@ public void SerilogTracingSpanPropertiesAreLifted() .Information("GET /orders")); var eventJson = SimulationEvent.ToJsonObject(evt); - Assert.Equal(start.ToString("o"), (string?)eventJson["@st"]); + Assert.Equal(start, eventJson["@st"]!.GetValue()); Assert.Equal("8899aabbccddeeff", (string?)eventJson["@ps"]); Assert.False(eventJson.ContainsKey("SpanStartTimestamp")); Assert.False(eventJson.ContainsKey("ParentSpanId")); diff --git a/test/SeqCli.Tests/Traces/StructuredMessageTests.cs b/test/SeqCli.Tests/Traces/StructuredMessageTests.cs index 4180126e..a89ed699 100644 --- a/test/SeqCli.Tests/Traces/StructuredMessageTests.cs +++ b/test/SeqCli.Tests/Traces/StructuredMessageTests.cs @@ -22,8 +22,8 @@ public void MissingStructuredMessagesReadAsEmpty() { foreach (var cell in new object?[] { null, JValue.CreateNull() }) { - var (message, properties) = StructuredMessage.Read(cell); - Assert.Equal("", message); + var (mt, properties) = StructuredMessage.Read(cell); + Assert.Equal("", mt); Assert.Empty(properties); } } @@ -31,27 +31,26 @@ public void MissingStructuredMessagesReadAsEmpty() [Fact] public void TextTokensAreRead() { - var (message, properties) = StructuredMessage.Read(new JArray("Hello", ", ", "world")); + var (mt, properties) = StructuredMessage.Read(new JArray("Hello", ", ", "world")); - Assert.Equal("Hello, world", message); + Assert.Equal("Hello, world", mt); Assert.Empty(properties); } [Fact] public void LiteralBracesAreEscapedInTemplateText() { - var (message, _) = StructuredMessage.Read(new JArray("a {not-a-hole} b")); - - Assert.Equal("a {{not-a-hole}} b", message); + var (mt, _) = StructuredMessage.Read(new JArray("a {not-a-hole} b")); + Assert.Equal("a {{not-a-hole}} b", mt); } [Fact] public void HolesCarryRawTextAndValues() { - var (message, properties) = StructuredMessage.Read(new JArray( + var (mt, properties) = StructuredMessage.Read(new JArray( "Hello, ", Hole("Name", "{Name:x}", "World"), "!")); - Assert.Equal("Hello, {Name:x}!", message); + Assert.Equal("Hello, {Name:x}!", mt); var property = Assert.Single(properties); Assert.Equal("Name", property.Key); @@ -61,9 +60,9 @@ public void HolesCarryRawTextAndValues() [Fact] public void HolesWithoutValuesContributeNoProperties() { - var (message, properties) = StructuredMessage.Read(new JArray(Hole("Name"))); + var (mt, properties) = StructuredMessage.Read(new JArray(Hole("Name"))); - Assert.Equal("{Name}", message); + Assert.Equal("{Name}", mt); Assert.Empty(properties); } @@ -97,10 +96,10 @@ public void StructuredHoleValuesBecomeObjects() [Fact] public void DottedHoleNamesBecomeNestedObjects() { - var (message, properties) = StructuredMessage.Read(new JArray( + var (mt, properties) = StructuredMessage.Read(new JArray( Hole("user.name", value: "Barney"))); - Assert.Equal("{user.name}", message); + Assert.Equal("{user.name}", mt); var user = Assert.IsType(properties["user"]); Assert.Equal("Barney", (string?)user["name"]); } @@ -108,25 +107,24 @@ public void DottedHoleNamesBecomeNestedObjects() [Fact] public void TrailingWhitespaceIsTrimmed() { - var (message, _) = StructuredMessage.Read(new JArray("Hi ", "}", " \n")); + var (mt, _) = StructuredMessage.Read(new JArray("Hi ", "}", " \n")); - Assert.Equal("Hi }}", message); + Assert.Equal("Hi }}", mt); } [Fact] public void WhitespaceOnlyMessagesReadAsEmpty() { - var (message, _) = StructuredMessage.Read(new JArray(" ")); + var (mt, _) = StructuredMessage.Read(new JArray(" ")); - Assert.Equal("", message); + Assert.Equal("", mt); } [Fact] public void TrailingHolesAreNotTrimmed() { - var (message, _) = StructuredMessage.Read(new JArray("Took ", Hole("Elapsed"))); - - Assert.Equal("Took {Elapsed}", message); + var (mt, _) = StructuredMessage.Read(new JArray("Took ", Hole("Elapsed"))); + Assert.Equal("Took {Elapsed}", mt); } [Fact] From 479c2a19cdeabc33eb40ac09d21b4db1d1a7f105 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Fri, 21 Aug 2026 15:04:02 +1000 Subject: [PATCH 21/32] Replicate the `trace` command's `--column` feature for `search`. Assisted-by: Claude:claude-fable-5 --- README.md | 1 + src/SeqCli/Cli/Commands/SearchCommand.cs | 21 ++++- .../Cli/Features/OutputFormatFeature.cs | 5 +- src/SeqCli/Output/EventColumns.cs | 91 +++++++++++++++++++ src/SeqCli/Output/OutputFormat.cs | 12 ++- src/SeqCli/Output/TextFormatters.cs | 7 +- src/SeqCli/Output/TraceFormatter.cs | 15 +-- .../Events/SearchColumnsTestCase.cs | 45 +++++++++ test/SeqCli.Tests/Output/EventColumnsTests.cs | 89 ++++++++++++++++++ test/SeqCli.Tests/Output/OutputFormatTests.cs | 1 + 10 files changed, 267 insertions(+), 20 deletions(-) create mode 100644 src/SeqCli/Output/EventColumns.cs create mode 100644 test/SeqCli.EndToEnd/Events/SearchColumnsTestCase.cs create mode 100644 test/SeqCli.Tests/Output/EventColumnsTests.cs diff --git a/README.md b/README.md index 1213183d..a5da031f 100644 --- a/README.md +++ b/README.md @@ -1464,6 +1464,7 @@ seqcli search -f "@Exception like '%TimeoutException%'" -c 30 | ------ | ----------- | | `-f`, `--filter=VALUE` | A filter to apply to the search, for example `Host = 'xmpweb-01.example.com'` | | `-c`, `--count=VALUE` | The maximum number of events to retrieve; the default is 1 | +| `--column=VALUE` | A column to display preceding each event's message; any Seq expression can be supplied, for example `OrderId`, `@SpanKind`, or `@Resource['service.name']`; this argument can be used multiple times, adding columns in order; applies to plain-text output only | | `--start=VALUE` | ISO 8601 date/time to query from | | `--end=VALUE` | ISO 8601 date/time to query to | | `--json` | Print output in newline-delimited JSON (the default is plain text) | diff --git a/src/SeqCli/Cli/Commands/SearchCommand.cs b/src/SeqCli/Cli/Commands/SearchCommand.cs index 365725b7..6604892d 100644 --- a/src/SeqCli/Cli/Commands/SearchCommand.cs +++ b/src/SeqCli/Cli/Commands/SearchCommand.cs @@ -13,11 +13,14 @@ // limitations under the License. using System; +using System.Collections.Generic; using System.Globalization; using System.Threading.Tasks; using SeqCli.Api; using SeqCli.Cli.Features; using SeqCli.Config; +using SeqCli.Output; +using SeqCli.Util; using Serilog; // ReSharper disable UnusedType.Global @@ -33,6 +36,7 @@ class SearchCommand : Command readonly DateRangeFeature _range; readonly SignalExpressionFeature _signal; readonly StoragePathFeature _storagePath; + readonly List _columns = []; string? _filter; int _count = 1; int _httpClientTimeout = 100000; @@ -49,6 +53,13 @@ public SearchCommand() $"The maximum number of events to retrieve; the default is {_count}", v => _count = int.Parse(v, CultureInfo.InvariantCulture)); + Options.Add( + "column=", + "A column to display preceding each event's message; any Seq expression can be supplied, for " + + "example `OrderId`, `@SpanKind`, or `@Resource['service.name']`; this argument can be used multiple " + + "times, adding columns in order; applies to plain-text output only", + c => _columns.Add(ArgumentString.Normalize(c) ?? throw new ArgumentException("Columns require a value."))); + _range = Enable(); _output = Enable(new OutputFormatFeature(supportNative: true, supportJson: true)); _storagePath = Enable(); @@ -71,7 +82,15 @@ protected override async Task Run() try { var config = RuntimeConfigurationLoader.Load(_storagePath); - var output = _output.GetOutputFormat(config); + + EventColumns? columns = null; + if (_columns.Count > 0 && !EventColumns.TryCreate(_columns, out columns, out var error)) + { + Log.Error("The column expression could not be compiled: {Error}", error); + return 1; + } + + var output = _output.GetOutputFormat(config, columns?.OutputTemplate(), columns); var connection = SeqConnectionFactory.Connect(_connection, config); connection.Client.HttpClient.Timeout = TimeSpan.FromMilliseconds(_httpClientTimeout); diff --git a/src/SeqCli/Cli/Features/OutputFormatFeature.cs b/src/SeqCli/Cli/Features/OutputFormatFeature.cs index 792766b2..05190b7b 100644 --- a/src/SeqCli/Cli/Features/OutputFormatFeature.cs +++ b/src/SeqCli/Cli/Features/OutputFormatFeature.cs @@ -13,6 +13,7 @@ // limitations under the License. using SeqCli.Config; +using SeqCli.Data; using SeqCli.Output; namespace SeqCli.Cli.Features; @@ -26,9 +27,9 @@ class OutputFormatFeature(bool supportNative, bool supportJson) : CommandFeature public OutputFormatFeature() : this(supportNative: false, supportJson: true) { } - public OutputFormat GetOutputFormat(SeqCliConfig config, string? outputTemplate = null) + public OutputFormat GetOutputFormat(SeqCliConfig config, string? outputTemplate = null, IEventEnricher? textEnricher = null) { - return new OutputFormat(_syntax, _noColor, _forceColor, config.Output, outputTemplate); + return new OutputFormat(_syntax, _noColor, _forceColor, config.Output, outputTemplate, textEnricher); } public string JsonArgumentHelp { get; init; } = "Print output in newline-delimited JSON (the default is plain text)"; diff --git a/src/SeqCli/Output/EventColumns.cs b/src/SeqCli/Output/EventColumns.cs new file mode 100644 index 00000000..e26c97f3 --- /dev/null +++ b/src/SeqCli/Output/EventColumns.cs @@ -0,0 +1,91 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Text; +using System.Text.Json.Nodes; +using Seq.Syntax.Expressions; +using SeqCli.Data; +using SeqCli.Syntax; + +namespace SeqCli.Output; + +/// +/// Evaluates a list of column expressions against each event, storing the results in synthetic properties that +/// the plain-text output template shows ahead of the message. +/// +class EventColumns : IEventEnricher +{ + static readonly string ColumnPrefixProperty = $"_SeqcliColumn_{Guid.NewGuid():N}"; + + internal static string ColumnPropertyName(int index) => $"{ColumnPrefixProperty}_{index}"; + + internal static string TemplateColumnsFragment(int columnCount) + { + // `<> ''` is undefined, and hence falsy, when the property is missing; the guard thus + // drops the column, and its trailing space, for both missing and empty values. + var fragment = new StringBuilder(); + for (var i = 0; i < columnCount; ++i) + { + var column = ColumnPropertyName(i); + fragment.Append($"{{#if {column} <> ''}}{{{column}}} {{#end}}"); + } + + return fragment.ToString(); + } + + readonly CompiledExpression[] _columns; + + EventColumns(CompiledExpression[] columns) + { + _columns = columns; + } + + public static bool TryCreate( + IReadOnlyList expressions, + [NotNullWhen(true)] out EventColumns? columns, + [NotNullWhen(false)] out string? error) + { + var compiled = new CompiledExpression[expressions.Count]; + for (var i = 0; i < expressions.Count; ++i) + { + if (!SeqSyntax.TryCompileExpression(expressions[i], out var expression, out error)) + { + columns = null; + return false; + } + + compiled[i] = expression; + } + + columns = new EventColumns(compiled); + error = null; + return true; + } + + public string OutputTemplate() => TextFormatters.PlainOutputTemplate(_columns.Length); + + public void Enrich(JsonObject eventJson) + { + for (var i = 0; i < _columns.Length; ++i) + { + // Property accessors return nodes still attached to the event, so they're cloned before being + // re-parented. + if (_columns[i](eventJson).TryGetValue(out var value)) + eventJson[ColumnPropertyName(i)] = value?.DeepClone(); + } + } +} diff --git a/src/SeqCli/Output/OutputFormat.cs b/src/SeqCli/Output/OutputFormat.cs index ae2492fb..741b5750 100644 --- a/src/SeqCli/Output/OutputFormat.cs +++ b/src/SeqCli/Output/OutputFormat.cs @@ -29,6 +29,7 @@ using SeqCli.Api; using SeqCli.Config; using SeqCli.Csv; +using SeqCli.Data; namespace SeqCli.Output; @@ -39,6 +40,7 @@ sealed class OutputFormat readonly OutputSyntax _syntax; readonly ExpressionTemplate? _eventFormatter; + readonly IEventEnricher? _textEnricher; readonly ExpressionTemplate _jsonValueFormatter; readonly JsonSerializer _serializer = JsonSerializer.CreateDefault(new JsonSerializerSettings @@ -55,13 +57,15 @@ public OutputFormat( bool? noColor, bool? forceColor, SeqCliOutputConfig outputConfig, - string? plainTextTemplate = null) + string? plainTextTemplate = null, + IEventEnricher? textEnricher = null) : this( syntax, noColor, forceColor, outputConfig, plainTextTemplate, + textEnricher, noColorSetInEnvironment: NoColorSetInEnvironment(), outputIsRedirected: Console.IsOutputRedirected, allowAnsiEscapes: TerminalFeatures.TryEnableAnsiEscapes()) @@ -73,6 +77,7 @@ public OutputFormat( /// The value of --force-color, if specified. /// Configured output defaults. /// The template controlling plain-text formatting, or null for the default. + /// An enricher applied to events written as plain text, or null. /// Whether NO_COLOR is set; see . /// Whether STDOUT is redirected, i.e. not attached to a terminal. /// Whether ANSI escape sequences are allowed; generally false for interactive @@ -83,12 +88,16 @@ internal OutputFormat( bool? forceColor, SeqCliOutputConfig outputConfig, string? plainTextTemplate, + IEventEnricher? textEnricher, bool noColorSetInEnvironment, bool outputIsRedirected, bool allowAnsiEscapes) { _syntax = syntax; + // Enrichment supports plain-text templates, so JSON output shows events verbatim. + _textEnricher = Text ? textEnricher : null; + var resolvedNoColor = ResolveNoColor(noColor, forceColor, outputConfig, noColorSetInEnvironment, allowAnsiEscapes); var applyThemeToRedirectedOutput = !resolvedNoColor && (forceColor ?? outputConfig.ForceColor); var colorize = !resolvedNoColor && (applyThemeToRedirectedOutput || !outputIsRedirected); @@ -237,6 +246,7 @@ public void WriteEventEntity(EventEntity evt) public void WriteEvent(JsonObject eventJson) { + _textEnricher?.Enrich(eventJson); _eventFormatter?.Format(eventJson, Console.Out); } diff --git a/src/SeqCli/Output/TextFormatters.cs b/src/SeqCli/Output/TextFormatters.cs index 88025fdf..c4c3e7cc 100644 --- a/src/SeqCli/Output/TextFormatters.cs +++ b/src/SeqCli/Output/TextFormatters.cs @@ -32,12 +32,13 @@ static class TextFormatters // Guarding on `@Elapsed` rather than the built-in `IsSpan()` shows elapsed time for any // event carrying a span start timestamp, whether or not trace and span ids accompany it. - static readonly string DefaultPlainTextOutputTemplate = - "[{@Timestamp:o} {@Level:u3}] {@Message}{#if @Elapsed is not null} ({TotalMilliseconds(@Elapsed):0.###} ms){#end}" + + internal static string PlainOutputTemplate(int columnCount = 0) => + "[{@Timestamp:o} {@Level:u3}] " + EventColumns.TemplateColumnsFragment(columnCount) + + "{@Message}{#if @Elapsed is not null} ({TotalMilliseconds(@Elapsed):0.###} ms){#end}" + Environment.NewLine + "{@Exception}"; public static ExpressionTemplate Plain(TemplateTheme? theme, string? outputTemplate) => - SeqSyntax.ParseTemplate(outputTemplate ?? DefaultPlainTextOutputTemplate, Encoder(theme)); + SeqSyntax.ParseTemplate(outputTemplate ?? PlainOutputTemplate(), Encoder(theme)); static TemplateOutputEncoder? Encoder(TemplateTheme? theme) => theme != null ? TemplateOutputEncoder.Ansi(theme) : null; diff --git a/src/SeqCli/Output/TraceFormatter.cs b/src/SeqCli/Output/TraceFormatter.cs index a69a1324..0f770604 100644 --- a/src/SeqCli/Output/TraceFormatter.cs +++ b/src/SeqCli/Output/TraceFormatter.cs @@ -27,25 +27,14 @@ static class TraceFormatter { static readonly string TreePrefixProperty = $"_SeqcliTraceTreePrefix_{Guid.NewGuid():N}"; static readonly string ElapsedProperty = $"_SeqcliTraceElapsed_{Guid.NewGuid():N}"; - static readonly string ColumnPrefixProperty = $"_SeqcliTraceColumn_{Guid.NewGuid():N}"; const string SpanConnector = "├─ ", LastSpanConnector = "└─ ", LogConnector = "┊ ", Continuation = "│ ", Gap = " "; - static string ColumnPropertyName(int index) => $"{ColumnPrefixProperty}_{index}"; - public static string OutputTemplate(int columnCount) { var template = new StringBuilder($"[{{@Timestamp:o}} {{@Level:u3}}] {{{TreePrefixProperty}}}"); - - // `<> ''` is undefined, and hence falsy, when the property is missing; the guard thus - // drops the column, and its trailing space, for both missing and empty values. - for (var i = 0; i < columnCount; ++i) - { - var column = ColumnPropertyName(i); - template.Append($"{{#if {column} <> ''}}{{{column}}} {{#end}}"); - } - + template.Append(EventColumns.TemplateColumnsFragment(columnCount)); template.Append($"{{@Message}}{{#if {ElapsedProperty} is not null}} ({{TotalMilliseconds({ElapsedProperty}):0.###}} ms){{#end}}"); template.Append(Environment.NewLine).Append("{@Exception}"); return template.ToString(); @@ -106,7 +95,7 @@ static JsonObject ToEventJson(TraceTreeNode treeNode, string treePrefix) for (var i = 0; i < evt.Columns.Count; ++i) { if (evt.Columns[i] is { } value) - eventJson[ColumnPropertyName(i)] = ToSystemTextJson.FromApiValue(value); + eventJson[EventColumns.ColumnPropertyName(i)] = ToSystemTextJson.FromApiValue(value); } return eventJson; diff --git a/test/SeqCli.EndToEnd/Events/SearchColumnsTestCase.cs b/test/SeqCli.EndToEnd/Events/SearchColumnsTestCase.cs new file mode 100644 index 00000000..036bec56 --- /dev/null +++ b/test/SeqCli.EndToEnd/Events/SearchColumnsTestCase.cs @@ -0,0 +1,45 @@ +using System.IO; +using System.Threading.Tasks; +using Seq.Api; +using SeqCli.EndToEnd.Support; +using Serilog; +using Xunit; + +#nullable enable + +namespace SeqCli.EndToEnd.Events; + +public class SearchColumnsTestCase : ICliTestCase +{ + const string TraceId = "7d4dedcc73b18e449e0e4ea08cbe346d"; + + public Task ExecuteAsync( + SeqConnection connection, + ILogger logger, + CliCommandRunner runner) + { + var inputFile = Path.Combine("Data", "trace-tree.clef"); + Assert.True(File.Exists(inputFile)); + + var exit = runner.Exec("ingest", $"--json -i {inputFile}"); + Assert.Equal(0, exit); + + var filter = $"--filter=\"@TraceId = '{TraceId}' and Customer is not null\""; + + exit = runner.Exec("search", $"{filter} -c 10 --column Customer --column RowCount"); + Assert.Equal(0, exit); + Assert.Contains("] scott GET /orders", runner.LastRunProcess!.Output); + + // Columns apply to plain-text output only. + exit = runner.Exec("search", $"{filter} -c 10 --column Customer --json"); + Assert.Equal(0, exit); + Assert.Contains("GET {Route}", runner.LastRunProcess!.Output); + Assert.DoesNotContain("_SeqcliColumn", runner.LastRunProcess!.Output); + + exit = runner.Exec("search", $"{filter} -c 10 --column \"not a valid (\""); + Assert.Equal(1, exit); + Assert.Contains("could not be compiled", runner.LastRunProcess!.Output); + + return Task.CompletedTask; + } +} diff --git a/test/SeqCli.Tests/Output/EventColumnsTests.cs b/test/SeqCli.Tests/Output/EventColumnsTests.cs new file mode 100644 index 00000000..80a46551 --- /dev/null +++ b/test/SeqCli.Tests/Output/EventColumnsTests.cs @@ -0,0 +1,89 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using Seq.Api.Model.Events; +using SeqCli.Api; +using SeqCli.Output; +using SeqCli.Tests.Support; +using Xunit; + +namespace SeqCli.Tests.Output; + +public class EventColumnsTests +{ + static EventColumns Create(params string[] expressions) + { + Assert.True(EventColumns.TryCreate(expressions, out var columns, out var error), error); + return columns; + } + + static string Render(EventEntity evt, EventColumns columns) + { + var eventJson = EventEntityJson.ToEventJson(evt); + columns.Enrich(eventJson); + + var output = new StringWriter(); + TextFormatters.Plain(theme: null, columns.OutputTemplate()).Format(eventJson, output); + return output.ToString(); + } + + static string At(EventEntity evt) => + DateTimeOffset.ParseExact(evt.Timestamp, "o", CultureInfo.InvariantCulture).ToLocalTime().ToString("o"); + + [Fact] + public void ColumnsPrecedeTheMessageInOrder() + { + var evt = Some.MakeEvent(e => e.Properties = Some.MakeProperties(("Customer", "scott"), ("OrderId", 42))); + + Assert.Equal( + $"[{At(evt)} INF] scott 42 Hello{Environment.NewLine}", + Render(evt, Create("Customer", "OrderId"))); + } + + [Fact] + public void MissingAndEmptyColumnValuesLeaveNoRedundantSpace() + { + var evt = Some.MakeEvent(e => e.Properties = Some.MakeProperties(("Empty", ""), ("OrderId", 42))); + + Assert.Equal( + $"[{At(evt)} INF] 42 Hello{Environment.NewLine}", + Render(evt, Create("Missing", "Empty", "OrderId"))); + } + + [Fact] + public void SeqStyleNamesResolveAgainstApiEvents() + { + var evt = Some.MakeEvent(e => + { + e.Properties = []; + e.Level = "Warning"; + e.SpanKind = "Server"; + e.Resource = Some.MakeProperties(("service.name", "frontend")); + }); + + Assert.Equal( + $"[{At(evt)} WRN] frontend Server Hello{Environment.NewLine}", + Render(evt, Create("@Resource['service.name']", "@SpanKind"))); + } + + [Fact] + public void ComputedColumnValuesAreRendered() + { + var evt = Some.MakeEvent(e => e.Properties = Some.MakeProperties(("OrderId", 42))); + + Assert.Equal( + $"[{At(evt)} INF] order-42 Hello{Environment.NewLine}", + Render(evt, Create("concat('order-', tostring(OrderId))"))); + } + + [Fact] + public void InvalidExpressionsAreReportedByTryCreate() + { + Assert.False(EventColumns.TryCreate(new List {"OrderId", "not a valid ("}, out var columns, out var error)); + Assert.Null(columns); + Assert.NotNull(error); + Assert.NotEmpty(error); + } +} diff --git a/test/SeqCli.Tests/Output/OutputFormatTests.cs b/test/SeqCli.Tests/Output/OutputFormatTests.cs index e323aa60..9361a8f2 100644 --- a/test/SeqCli.Tests/Output/OutputFormatTests.cs +++ b/test/SeqCli.Tests/Output/OutputFormatTests.cs @@ -27,6 +27,7 @@ static OutputFormat Create( forceColor, new SeqCliOutputConfig { DisableColor = disableColor }, plainTextTemplate: null, + textEnricher: null, noColorSetInEnvironment, outputIsRedirected, supportsAnsiEscapes); From c1ec28be13f1476fec3f2fd63b08e7c52b17c27b Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Fri, 21 Aug 2026 16:24:16 +1000 Subject: [PATCH 22/32] Include signal columns in search command output --- src/SeqCli/Cli/Commands/SearchCommand.cs | 31 ++++++++++++++--- .../Signals/SignalExpressionPartExtensions.cs | 34 +++++++++++++++++++ 2 files changed, 60 insertions(+), 5 deletions(-) create mode 100644 src/SeqCli/Signals/SignalExpressionPartExtensions.cs diff --git a/src/SeqCli/Cli/Commands/SearchCommand.cs b/src/SeqCli/Cli/Commands/SearchCommand.cs index 6604892d..2ae85f4a 100644 --- a/src/SeqCli/Cli/Commands/SearchCommand.cs +++ b/src/SeqCli/Cli/Commands/SearchCommand.cs @@ -16,10 +16,12 @@ using System.Collections.Generic; using System.Globalization; using System.Threading.Tasks; +using Seq.Api.Model.Signals; using SeqCli.Api; using SeqCli.Cli.Features; using SeqCli.Config; using SeqCli.Output; +using SeqCli.Signals; using SeqCli.Util; using Serilog; @@ -40,7 +42,7 @@ class SearchCommand : Command string? _filter; int _count = 1; int _httpClientTimeout = 100000; - bool _trace, _noWebSockets; + bool _trace, _noWebSockets, _noSignalColumns; public SearchCommand() { @@ -48,6 +50,7 @@ public SearchCommand() "f=|filter=", "A filter to apply to the search, for example `Host = 'xmpweb-01.example.com'`", v => _filter = v); + Options.Add( "c=|count=", $"The maximum number of events to retrieve; the default is {_count}", @@ -56,7 +59,7 @@ public SearchCommand() Options.Add( "column=", "A column to display preceding each event's message; any Seq expression can be supplied, for " + - "example `OrderId`, `@SpanKind`, or `@Resource['service.name']`; this argument can be used multiple " + + "example `OrderId`, `@SpanKind`, or `@Resource.service.name`; this argument can be used multiple " + "times, adding columns in order; applies to plain-text output only", c => _columns.Add(ArgumentString.Normalize(c) ?? throw new ArgumentException("Columns require a value."))); @@ -74,6 +77,8 @@ public SearchCommand() Options.Add("no-websockets", "Do not use WebSocket-driven streaming searches", _ => _noWebSockets = true); + Options.Add("no-signal-columns", "Do not show columns associated with the specified signal expression", _ => _noSignalColumns = true); + _connection = Enable(); } @@ -83,16 +88,32 @@ protected override async Task Run() { var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = SeqConnectionFactory.Connect(_connection, config); + connection.Client.HttpClient.Timeout = TimeSpan.FromMilliseconds(_httpClientTimeout); + + var collectedColumns = new List(); + if (!_noSignalColumns && _signal.Signal is { } signalExpression) + { + foreach (var signalId in signalExpression.ReferencedSignalIds()) + { + var signal = await connection.Signals.FindAsync(signalId); + foreach (var column in signal.Columns) + { + collectedColumns.Add(column.Expression); + } + } + } + + collectedColumns.AddRange(_columns); + EventColumns? columns = null; - if (_columns.Count > 0 && !EventColumns.TryCreate(_columns, out columns, out var error)) + if (collectedColumns.Count > 0 && !EventColumns.TryCreate(collectedColumns, out columns, out var error)) { Log.Error("The column expression could not be compiled: {Error}", error); return 1; } var output = _output.GetOutputFormat(config, columns?.OutputTemplate(), columns); - var connection = SeqConnectionFactory.Connect(_connection, config); - connection.Client.HttpClient.Timeout = TimeSpan.FromMilliseconds(_httpClientTimeout); string? filter = null; if (!string.IsNullOrWhiteSpace(_filter)) diff --git a/src/SeqCli/Signals/SignalExpressionPartExtensions.cs b/src/SeqCli/Signals/SignalExpressionPartExtensions.cs new file mode 100644 index 00000000..d99ddb39 --- /dev/null +++ b/src/SeqCli/Signals/SignalExpressionPartExtensions.cs @@ -0,0 +1,34 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Seq.Api.Model.Signals; + +namespace SeqCli.Signals; + +static class SignalExpressionPartExtensions +{ + public static IEnumerable ReferencedSignalIds(this SignalExpressionPart expr) + { + return expr.Kind switch + { + SignalExpressionKind.Signal => [expr.SignalId], + SignalExpressionKind.Intersection or SignalExpressionKind.Union => expr.Left.ReferencedSignalIds() + .Concat(expr.Right.ReferencedSignalIds()), + _ => throw new ArgumentOutOfRangeException(nameof(expr)) + }; + } +} \ No newline at end of file From f1bcdf1db99b0c99bdf0425b1bba10e7e50d9397 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Fri, 21 Aug 2026 16:35:00 +1000 Subject: [PATCH 23/32] Signal columns tests. Assisted-by: Claude:claude-opus-5 --- .../Cli/Features/SignalExpressionFeature.cs | 5 +- .../Events/SearchSignalColumnsTestCase.cs | 101 ++++++++++++++++++ 2 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 test/SeqCli.EndToEnd/Events/SearchSignalColumnsTestCase.cs diff --git a/src/SeqCli/Cli/Features/SignalExpressionFeature.cs b/src/SeqCli/Cli/Features/SignalExpressionFeature.cs index 54522c15..f7c45467 100644 --- a/src/SeqCli/Cli/Features/SignalExpressionFeature.cs +++ b/src/SeqCli/Cli/Features/SignalExpressionFeature.cs @@ -13,6 +13,7 @@ // limitations under the License. using Seq.Api.Model.Signals; +using SeqCli.Signals; namespace SeqCli.Cli.Features; @@ -27,9 +28,7 @@ public SignalExpressionPart? Signal if (string.IsNullOrWhiteSpace(_signalExpression)) return null; - // This is a hack that just happens to work because of the way - // signal ids are passed through ToString() as literals - return SignalExpressionPart.Signal(_signalExpression.Trim()); + return SignalExpressionParser.ParseExpression(_signalExpression); } } diff --git a/test/SeqCli.EndToEnd/Events/SearchSignalColumnsTestCase.cs b/test/SeqCli.EndToEnd/Events/SearchSignalColumnsTestCase.cs new file mode 100644 index 00000000..dabfb028 --- /dev/null +++ b/test/SeqCli.EndToEnd/Events/SearchSignalColumnsTestCase.cs @@ -0,0 +1,101 @@ +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Seq.Api; +using SeqCli.EndToEnd.Support; +using Serilog; +using Xunit; + +#nullable enable + +namespace SeqCli.EndToEnd.Events; + +public class SearchSignalColumnsTestCase : ICliTestCase +{ + const string TraceId = "7d4dedcc73b18e449e0e4ea08cbe346d"; + + public async Task ExecuteAsync( + SeqConnection connection, + ILogger logger, + CliCommandRunner runner) + { + var inputFile = Path.Combine("Data", "trace-tree.clef"); + Assert.True(File.Exists(inputFile)); + + var exit = runner.Exec("ingest", $"--json -i {inputFile}"); + Assert.Equal(0, exit); + + exit = runner.Exec("signal create", "-t Orders -f \"@TraceId is not null\" -c Customer -c RowCount"); + Assert.Equal(0, exit); + + exit = runner.Exec("signal create", "-t Rows -f \"RowCount is not null\" -c \"RowCount * 2\""); + Assert.Equal(0, exit); + + exit = runner.Exec("signal create", "-t Unadorned -f \"@TraceId is not null\""); + Assert.Equal(0, exit); + + var signals = await connection.Signals.ListAsync(shared: true); + var orders = signals.Single(s => s.Title == "Orders").Id; + var rows = signals.Single(s => s.Title == "Rows").Id; + var unadorned = signals.Single(s => s.Title == "Unadorned").Id; + + var filter = $"--filter=\"@TraceId = '{TraceId}'\""; + + // The signal's columns are displayed, in the order the signal declares them. + exit = runner.Exec("search", $"--signal {orders} {filter} -c 10"); + Assert.Equal(0, exit); + var output = runner.LastRunProcess!.Output; + Assert.Contains("] scott GET /orders", output); + Assert.Contains("] 42 42 rows retrieved", output); + + // Signal columns precede any specified with `--column`. + exit = runner.Exec("search", $"--signal {orders} {filter} -c 10 --column \"@Level\""); + Assert.Equal(0, exit); + output = runner.LastRunProcess!.Output; + Assert.Contains("] scott Information GET /orders", output); + Assert.Contains("] 42 Warning 42 rows retrieved", output); + + // `--no-signal-columns` drops the signal's columns, but not those specified with `--column`. + exit = runner.Exec("search", $"--signal {orders} {filter} -c 10 --no-signal-columns"); + Assert.Equal(0, exit); + output = runner.LastRunProcess!.Output; + Assert.Contains("] GET /orders", output); + Assert.DoesNotContain("scott", output); + + exit = runner.Exec("search", $"--signal {orders} {filter} -c 10 --no-signal-columns --column \"@Level\""); + Assert.Equal(0, exit); + output = runner.LastRunProcess!.Output; + Assert.Contains("] Information GET /orders", output); + Assert.DoesNotContain("scott", output); + + // Signal columns apply to plain-text output only. + exit = runner.Exec("search", $"--signal {orders} {filter} -c 10 --json"); + Assert.Equal(0, exit); + output = runner.LastRunProcess!.Output; + Assert.Contains("GET {Route}", output); + Assert.DoesNotContain("_SeqcliColumn", output); + + // Columns are collected from every signal referenced by the expression. + exit = runner.Exec("search", $"--signal {orders},{rows} {filter} -c 10"); + Assert.Equal(0, exit); + Assert.Contains("] 42 84 42 rows retrieved", runner.LastRunProcess!.Output); + + exit = runner.Exec("search", $"--signal \"{orders}~{rows}\" {filter} -c 10"); + Assert.Equal(0, exit); + output = runner.LastRunProcess!.Output; + Assert.Contains("] scott GET /orders", output); + Assert.Contains("] 42 84 42 rows retrieved", output); + + // A signal without columns contributes none. + exit = runner.Exec("search", $"--signal {unadorned} {filter} -c 10"); + Assert.Equal(0, exit); + output = runner.LastRunProcess!.Output; + Assert.Contains("] GET /orders", output); + Assert.DoesNotContain("scott", output); + + // A signal that can't be found is reported, rather than silently ignored. + exit = runner.Exec("search", $"--signal signal-999999 {filter} -c 10"); + Assert.Equal(1, exit); + Assert.Contains("Could not retrieve search result", runner.LastRunProcess!.Output); + } +} From 142f019f88ab720d8bbfa3d773b652163c33b96c Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Tue, 25 Aug 2026 16:44:35 +1000 Subject: [PATCH 24/32] Add `--column` support to `seqcli tail`. Assisted-by: Claude:claude-fable-5 --- README.md | 3 + src/SeqCli/Cli/Commands/SearchCommand.cs | 42 ++--------- src/SeqCli/Cli/Commands/TailCommand.cs | 5 +- .../Cli/Features/EventColumnsFeature.cs | 71 +++++++++++++++++++ .../Events/TailColumnsTestCase.cs | 67 +++++++++++++++++ 5 files changed, 149 insertions(+), 39 deletions(-) create mode 100644 src/SeqCli/Cli/Features/EventColumnsFeature.cs create mode 100644 test/SeqCli.EndToEnd/Events/TailColumnsTestCase.cs diff --git a/README.md b/README.md index a5da031f..3faeda58 100644 --- a/README.md +++ b/README.md @@ -1465,6 +1465,7 @@ seqcli search -f "@Exception like '%TimeoutException%'" -c 30 | `-f`, `--filter=VALUE` | A filter to apply to the search, for example `Host = 'xmpweb-01.example.com'` | | `-c`, `--count=VALUE` | The maximum number of events to retrieve; the default is 1 | | `--column=VALUE` | A column to display preceding each event's message; any Seq expression can be supplied, for example `OrderId`, `@SpanKind`, or `@Resource['service.name']`; this argument can be used multiple times, adding columns in order; applies to plain-text output only | +| `--no-signal-columns` | Do not show columns associated with the specified signal expression | | `--start=VALUE` | ISO 8601 date/time to query from | | `--end=VALUE` | ISO 8601 date/time to query to | | `--json` | Print output in newline-delimited JSON (the default is plain text) | @@ -1637,6 +1638,8 @@ Stream log events matching a filter. | Option | Description | | ------ | ----------- | | `-f`, `--filter=VALUE` | An optional server-side filter to apply to the stream, for example `@Level = 'Error'` | +| `--column=VALUE` | A column to display preceding each event's message; any Seq expression can be supplied, for example `OrderId`, `@SpanKind`, or `@Resource['service.name']`; this argument can be used multiple times, adding columns in order; applies to plain-text output only | +| `--no-signal-columns` | Do not show columns associated with the specified signal expression | | `--json` | Print output in newline-delimited JSON (the default is plain text) | | `--no-color` | Don't colorize text output | | `--force-color` | Force redirected output to have ANSI color (unless `--no-color` is also specified) | diff --git a/src/SeqCli/Cli/Commands/SearchCommand.cs b/src/SeqCli/Cli/Commands/SearchCommand.cs index 2ae85f4a..d7b2898c 100644 --- a/src/SeqCli/Cli/Commands/SearchCommand.cs +++ b/src/SeqCli/Cli/Commands/SearchCommand.cs @@ -13,16 +13,11 @@ // limitations under the License. using System; -using System.Collections.Generic; using System.Globalization; using System.Threading.Tasks; -using Seq.Api.Model.Signals; using SeqCli.Api; using SeqCli.Cli.Features; using SeqCli.Config; -using SeqCli.Output; -using SeqCli.Signals; -using SeqCli.Util; using Serilog; // ReSharper disable UnusedType.Global @@ -38,11 +33,11 @@ class SearchCommand : Command readonly DateRangeFeature _range; readonly SignalExpressionFeature _signal; readonly StoragePathFeature _storagePath; - readonly List _columns = []; + readonly EventColumnsFeature _eventColumns; string? _filter; int _count = 1; int _httpClientTimeout = 100000; - bool _trace, _noWebSockets, _noSignalColumns; + bool _trace, _noWebSockets; public SearchCommand() { @@ -56,13 +51,7 @@ public SearchCommand() $"The maximum number of events to retrieve; the default is {_count}", v => _count = int.Parse(v, CultureInfo.InvariantCulture)); - Options.Add( - "column=", - "A column to display preceding each event's message; any Seq expression can be supplied, for " + - "example `OrderId`, `@SpanKind`, or `@Resource.service.name`; this argument can be used multiple " + - "times, adding columns in order; applies to plain-text output only", - c => _columns.Add(ArgumentString.Normalize(c) ?? throw new ArgumentException("Columns require a value."))); - + _eventColumns = Enable(); _range = Enable(); _output = Enable(new OutputFormatFeature(supportNative: true, supportJson: true)); _storagePath = Enable(); @@ -77,8 +66,6 @@ public SearchCommand() Options.Add("no-websockets", "Do not use WebSocket-driven streaming searches", _ => _noWebSockets = true); - Options.Add("no-signal-columns", "Do not show columns associated with the specified signal expression", _ => _noSignalColumns = true); - _connection = Enable(); } @@ -91,28 +78,7 @@ protected override async Task Run() var connection = SeqConnectionFactory.Connect(_connection, config); connection.Client.HttpClient.Timeout = TimeSpan.FromMilliseconds(_httpClientTimeout); - var collectedColumns = new List(); - if (!_noSignalColumns && _signal.Signal is { } signalExpression) - { - foreach (var signalId in signalExpression.ReferencedSignalIds()) - { - var signal = await connection.Signals.FindAsync(signalId); - foreach (var column in signal.Columns) - { - collectedColumns.Add(column.Expression); - } - } - } - - collectedColumns.AddRange(_columns); - - EventColumns? columns = null; - if (collectedColumns.Count > 0 && !EventColumns.TryCreate(collectedColumns, out columns, out var error)) - { - Log.Error("The column expression could not be compiled: {Error}", error); - return 1; - } - + var columns = await _eventColumns.GetEventColumns(connection, _signal.Signal); var output = _output.GetOutputFormat(config, columns?.OutputTemplate(), columns); string? filter = null; diff --git a/src/SeqCli/Cli/Commands/TailCommand.cs b/src/SeqCli/Cli/Commands/TailCommand.cs index 291433ba..88253a70 100644 --- a/src/SeqCli/Cli/Commands/TailCommand.cs +++ b/src/SeqCli/Cli/Commands/TailCommand.cs @@ -31,6 +31,7 @@ class TailCommand : Command readonly OutputFormatFeature _output; readonly SignalExpressionFeature _signal; readonly StoragePathFeature _storagePath; + readonly EventColumnsFeature _eventColumns; string? _filter; public TailCommand() @@ -40,6 +41,7 @@ public TailCommand() "An optional server-side filter to apply to the stream, for example `@Level = 'Error'`", v => _filter = v); + _eventColumns = Enable(); _output = Enable(new OutputFormatFeature(supportNative: true, supportJson: true)); _storagePath = Enable(); _signal = Enable(); @@ -61,7 +63,8 @@ protected override async Task Run() strict = converted.StrictExpression; } - var output = _output.GetOutputFormat(config); + var columns = await _eventColumns.GetEventColumns(connection, _signal.Signal); + var output = _output.GetOutputFormat(config, columns?.OutputTemplate(), columns); try { diff --git a/src/SeqCli/Cli/Features/EventColumnsFeature.cs b/src/SeqCli/Cli/Features/EventColumnsFeature.cs new file mode 100644 index 00000000..22346a90 --- /dev/null +++ b/src/SeqCli/Cli/Features/EventColumnsFeature.cs @@ -0,0 +1,71 @@ +// Copyright © Datalust and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Seq.Api; +using Seq.Api.Model.Signals; +using SeqCli.Output; +using SeqCli.Signals; +using SeqCli.Util; + +namespace SeqCli.Cli.Features; + +class EventColumnsFeature : CommandFeature +{ + readonly List _columns = []; + bool _noSignalColumns; + + public override void Enable(OptionSet options) + { + options.Add( + "column=", + "A column to display preceding each event's message; any Seq expression can be supplied, for " + + "example `OrderId`, `@SpanKind`, or `@Resource.service.name`; this argument can be used multiple " + + "times, adding columns in order; applies to plain-text output only", + c => _columns.Add(ArgumentString.Normalize(c) ?? throw new ArgumentException("Columns require a value."))); + + options.Add( + "no-signal-columns", + "Do not show columns associated with the specified signal expression", + _ => _noSignalColumns = true); + } + + public async Task GetEventColumns(SeqConnection connection, SignalExpressionPart? signal) + { + var collectedColumns = new List(); + if (!_noSignalColumns && signal is { } signalExpression) + { + foreach (var signalId in signalExpression.ReferencedSignalIds()) + { + var signalEntity = await connection.Signals.FindAsync(signalId); + foreach (var column in signalEntity.Columns) + { + collectedColumns.Add(column.Expression); + } + } + } + + collectedColumns.AddRange(_columns); + + if (collectedColumns.Count == 0) + return null; + + if (!EventColumns.TryCreate(collectedColumns, out var columns, out var error)) + throw new ArgumentException($"The column expression could not be compiled: {error}"); + + return columns; + } +} diff --git a/test/SeqCli.EndToEnd/Events/TailColumnsTestCase.cs b/test/SeqCli.EndToEnd/Events/TailColumnsTestCase.cs new file mode 100644 index 00000000..02fa26de --- /dev/null +++ b/test/SeqCli.EndToEnd/Events/TailColumnsTestCase.cs @@ -0,0 +1,67 @@ +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Seq.Api; +using SeqCli.EndToEnd.Support; +using Serilog; +using Xunit; + +#nullable enable + +namespace SeqCli.EndToEnd.Events; + +public class TailColumnsTestCase : ICliTestCase +{ + public async Task ExecuteAsync( + SeqConnection connection, + ILogger logger, + CliCommandRunner runner) + { + var inputFile = Path.Combine("Data", "trace-tree.clef"); + Assert.True(File.Exists(inputFile)); + + var exit = runner.Exec("signal create", "-t Orders -f \"@TraceId is not null\" -c Customer -c RowCount"); + Assert.Equal(0, exit); + + var signals = await connection.Signals.ListAsync(shared: true); + var orders = signals.Single(s => s.Title == "Orders").Id; + + var filter = "--filter=\"Customer is not null\""; + + // A column expression that can't be compiled is reported. + exit = runner.Exec("tail", "--column \"not a valid (\""); + Assert.Equal(1, exit); + Assert.Contains("could not be compiled", runner.LastRunProcess!.Output); + + // Signal columns precede those specified with `--column`. + using (var tail = runner.Spawn("tail", $"--signal {orders} {filter} --column \"@Level\"")) + { + await IngestUntilTailWrites(runner, tail, inputFile, "] scott Information GET /orders"); + } + + // `--no-signal-columns` drops the signal's columns, but not those specified with `--column`. + using (var tail = runner.Spawn("tail", $"--signal {orders} {filter} --no-signal-columns --column \"@Level\"")) + { + await IngestUntilTailWrites(runner, tail, inputFile, "] Information GET /orders"); + Assert.DoesNotContain("scott", tail.Output); + } + } + + // Events ingested before the tail command's streaming connection is established won't be + // observed, so ingest the test data repeatedly until the expected line appears. + static async Task IngestUntilTailWrites(CliCommandRunner runner, CaptiveProcess tail, string inputFile, string expected) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (!tail.Output.Contains(expected)) + { + if (DateTime.UtcNow > deadline) + Assert.Fail($"Timed out waiting for `{expected}` in: {tail.Output}"); + + var exit = runner.Exec("ingest", $"--json -i {inputFile}"); + Assert.Equal(0, exit); + + await Task.Delay(TimeSpan.FromSeconds(1)); + } + } +} From bb7627a5050f34cbbf570c25a8ee62f55d8d7c49 Mon Sep 17 00:00:00 2001 From: liammclennan-ro Date: Thu, 3 Sep 2026 14:25:23 +1000 Subject: [PATCH 25/32] Add npm package and publish. --- .github/workflows/ci.yml | 53 ++++ .gitignore | 6 + README.md | 8 + build/Build.Common.ps1 | 9 + build/Build.Npm.ps1 | 250 ++++++++++++++++++ npm/README.md | 33 +++ npm/platform-package.json | 24 ++ npm/seqcli/bin/seqcli.js | 147 ++++++++++ npm/seqcli/package.json | 50 ++++ src/SeqCli/Mcp/McpServerInstaller.cs | 61 ++++- .../Mcp/McpServerInstallerTests.cs | 75 ++++++ 11 files changed, 714 insertions(+), 2 deletions(-) create mode 100644 build/Build.Npm.ps1 create mode 100644 npm/README.md create mode 100644 npm/platform-package.json create mode 100755 npm/seqcli/bin/seqcli.js create mode 100644 npm/seqcli/package.json create mode 100644 test/SeqCli.Tests/Mcp/McpServerInstallerTests.cs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dea0c78e..53907f7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,6 +42,17 @@ jobs: shell: pwsh run: | ./build/Build.Windows.ps1 + - name: Upload archives + # Consumed by the publish-npm job + if: github.event_name != 'pull_request' + uses: actions/upload-artifact@v7 + with: + name: archives + path: | + artifacts/seqcli-*-*.zip + artifacts/seqcli-*-*.tar.gz + if-no-files-found: error + retention-days: 1 build-linux: name: Build (Linux) @@ -69,3 +80,45 @@ jobs: shell: pwsh run: | ./build/Build.Linux.ps1 -SeqDockerTag $env:SEQ_DOCKER_TAG + + publish-npm: + name: Publish (npm) + runs-on: ubuntu-24.04 + needs: build-windows + + # Mirrors NuGet publishing: builds from any branch this workflow targets (dev builds as + # prereleases under the `dev` dist-tag, main builds as `latest`), but never pull requests. + if: github.event_name != 'pull_request' + + permissions: + contents: read + # Required for npm trusted publishing (OIDC) and provenance + id-token: write + + steps: + - uses: actions/checkout@v6 + - name: Setup + uses: actions/setup-node@v7 + with: + node-version: 24.x + # Bootstrap only: together with NODE_AUTH_TOKEN below, authenticates using the NPM_TOKEN + # secret. Once trusted publishing is configured for every @datalust/seqcli* package + # (bound to this workflow file, ci.yml), remove `registry-url` here and `NODE_AUTH_TOKEN` + # below so that npm authenticates with the OIDC token instead. + registry-url: https://registry.npmjs.org/ + - name: Update npm + # Trusted publishing and automatic provenance require npm 11.5.1 or later + run: | + npm install -g npm@latest + npm --version + - name: Download archives + uses: actions/download-artifact@v8 + with: + name: archives + path: npm-archives + - name: Publish + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + shell: pwsh + run: | + ./build/Build.Npm.ps1 -ArchiveDir ./npm-archives diff --git a/.gitignore b/.gitignore index 4e050436..5cbf57dc 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,8 @@ x64/ x86/ bld/ [Bb]in/ +# The npm launcher package keeps its script in bin/ +!npm/seqcli/bin/ [Oo]bj/ [Ll]og/ @@ -296,3 +298,7 @@ global.json .claude/ .qwen/ .agents/ + +# npm packaging staging area (build/Build.Npm.ps1) +npm-staging/ +npm-archives/ diff --git a/README.md b/README.md index 1213183d..5a9eebd7 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,14 @@ The Seq installer for Windows includes `seqcli`. Otherwise, download the [releas dotnet tool install --global seqcli ``` +With Node.js installed, `seqcli` can be installed from npm using: + +``` +npm install -g @datalust/seqcli +``` + +On Windows, if the Seq installation directory is on your `PATH`, the `seqcli` bundled with Seq may take precedence over the npm-installed copy; `where seqcli` shows the resolution order, and `npx @datalust/seqcli ` always runs the npm version. + To set a default server URL and API key, run: ``` diff --git a/build/Build.Common.ps1 b/build/Build.Common.ps1 index 7d9e97a1..7abf7b7d 100644 --- a/build/Build.Common.ps1 +++ b/build/Build.Common.ps1 @@ -12,3 +12,12 @@ function Get-SemVer() $base + "." + $revision } } + +function Get-NpmVersion($version) +{ + # npm requires strict semver, which forbids leading zeros in numeric identifiers; the build number + # is zero-padded (e.g. 2026.1.02616), so strip the padding from the patch component (-> 2026.1.2616). + # Prerelease suffixes are alphanumeric identifiers and are left as-is. + if ($version -notmatch '^(\d+)\.(\d+)\.(\d+)(.*)$') { throw "Unrecognized version: $version" } + "$([int]$Matches[1]).$([int]$Matches[2]).$([int]$Matches[3])$($Matches[4])" +} diff --git a/build/Build.Npm.ps1 b/build/Build.Npm.ps1 new file mode 100644 index 00000000..3c5d3453 --- /dev/null +++ b/build/Build.Npm.ps1 @@ -0,0 +1,250 @@ +# Publishes the npm packages for a seqcli build: one `@datalust/seqcli-` package per release +# archive, then the launcher package `@datalust/seqcli` (from ./npm/seqcli) with its +# optionalDependencies pinned to the same version. +# +# In CI (see publish-npm in .github/workflows/ci.yml) the archives come from the build-windows job's +# artifacts, so dev builds are published as prereleases (dist-tag `dev`) and main builds as `latest`, +# matching NuGet. Packages that already exist on the registry at the target version are skipped, so +# a partially-failed run can simply be re-run. +# +# Usage: +# ./build/Build.Npm.ps1 -ArchiveDir ./npm-archives # CI: version from Get-SemVer +# ./build/Build.Npm.ps1 -Version 2026.1.02616 # (re)publish GitHub release v2026.1.02616 +# ./build/Build.Npm.ps1 -Version 2026.1.02616 -ArchiveDir ./x -DryRun # stage and `npm pack` only +param( + # Build version as it appears in archive names, e.g. 2026.1.02616 or 2026.1.02700-dev-02700. + # Defaults to Get-SemVer, which in CI reproduces the version computed by the build jobs. + [string] $Version, + + # npm dist-tag; defaults to `latest` for release versions and `dev` for prereleases. + [string] $DistTag, + + # Directory containing seqcli--.zip|.tar.gz archives; when omitted, the archives + # are downloaded from GitHub release v with `gh release download`. + [string] $ArchiveDir, + + # GitHub repository to download release assets from. Also identifies the repository whose CI + # publishes via npm trusted publishing (forks without an NPM_TOKEN skip publishing). + [string] $Repo = 'datalust/seqcli', + + # Stage the packages and run `npm pack` instead of `npm publish`. + [switch] $DryRun +) + +Push-Location $PSScriptRoot/../ + +. ./build/Build.Common.ps1 + +$ErrorActionPreference = 'Stop' + +$scope = '@datalust' +$launcherName = "$scope/seqcli" +$staging = './npm-staging' + +if (-not $Version) { + $Version = Get-SemVer +} + +$npmVersion = Get-NpmVersion $Version + +if (-not $DistTag) { + $DistTag = @{ $true = 'dev'; $false = 'latest' }[$npmVersion.Contains('-')] +} + +Write-Host "Release version: $Version" +Write-Host "npm version: $npmVersion" +Write-Host "npm dist-tag: $DistTag" +Write-Host "Dry run: $DryRun" + +if (-not $DryRun -and -not $env:NODE_AUTH_TOKEN -and $env:GITHUB_REPOSITORY -ne $Repo) { + # Forks have neither the NPM_TOKEN secret nor a trusted publisher configuration. + Write-Host "Skipping npm publishing: no npm credentials are available in this environment" + Pop-Location + exit 0 +} + +function Get-Rids +{ + $([xml](Get-Content ./src/SeqCli/SeqCli.csproj)).Project.PropertyGroup.RuntimeIdentifiers.Split(';') +} + +function Get-PlatformSpec($rid) +{ + $os = switch -Wildcard ($rid) { + 'win-*' { 'win32' } + 'osx-*' { 'darwin' } + 'linux-*' { 'linux' } + default { throw "Unrecognized RID: $rid" } + } + + $cpu = ($rid -split '-')[-1] + + $libc = $null + if ($rid -like 'linux-musl-*') { $libc = 'musl' } + elseif ($rid -like 'linux-*') { $libc = 'glibc' } + + return @{ os = $os; cpu = $cpu; libc = $libc; isWindows = ($os -eq 'win32') } +} + +function Get-ReleaseArchive($rid) +{ + $pattern = "seqcli-$Version-$rid.*" + + if ($ArchiveDir) { + $archive = Get-ChildItem -Path $ArchiveDir -Filter $pattern | Select-Object -First 1 + if (-not $archive) { throw "No archive matching $pattern in $ArchiveDir" } + return $archive.FullName + } + + $downloads = "$staging/download" + New-Item -ItemType Directory -Force -Path $downloads | Out-Null + + & gh release download "v$Version" --repo $Repo --dir $downloads --pattern $pattern --clobber + if ($LASTEXITCODE -ne 0) { throw "Downloading $pattern from release v$Version failed" } + + $archive = Get-ChildItem -Path $downloads -Filter $pattern | Select-Object -First 1 + if (-not $archive) { throw "Release v$Version has no asset matching $pattern" } + return $archive.FullName +} + +function Expand-ReleaseArchive($archive, $destination) +{ + if (Test-Path $destination) { Remove-Item -Recurse -Force $destination } + New-Item -ItemType Directory -Force -Path $destination | Out-Null + + if ($archive -like '*.zip') { + Expand-Archive -Path $archive -DestinationPath $destination -Force + } else { + & tar -xzf $archive -C $destination + if ($LASTEXITCODE -ne 0) { throw "Extracting $archive failed" } + } + + # The archives contain a single `seqcli--/` root folder; the package needs the + # binary at its root, so lift the contents up one level. + $entries = @(Get-ChildItem -Force $destination) + if ($entries.Count -eq 1 -and $entries[0].PSIsContainer) { + $root = $entries[0].FullName + Get-ChildItem -Force $root | Move-Item -Destination $destination + Remove-Item -Force $root + } +} + +function Write-PlatformPackageJson($rid, $spec, $destination) +{ + $package = Get-Content ./npm/platform-package.json -Raw | ConvertFrom-Json -AsHashtable + $package.name = "$scope/seqcli-$rid" + $package.version = $npmVersion + $package.description = $package.description.Replace('{{rid}}', $rid) + $package.os = @($spec.os) + $package.cpu = @($spec.cpu) + if ($spec.libc) { $package.libc = @($spec.libc) } + + $package | ConvertTo-Json -Depth 5 | Set-Content -Path "$destination/package.json" -NoNewline +} + +function Test-NpmPublished($name) +{ + $output = & npm view "$name@$npmVersion" version --json 2>$null + return ($LASTEXITCODE -eq 0) -and -not [string]::IsNullOrWhiteSpace(($output -join '')) +} + +function Publish-NpmPackage($name, $directory) +{ + if ($DryRun) { + Write-Host "Packing $name@$npmVersion" + $tarballs = "$staging/tarballs" + New-Item -ItemType Directory -Force -Path $tarballs | Out-Null + & npm pack $directory --pack-destination $tarballs + if ($LASTEXITCODE -ne 0) { throw "Packing $name failed" } + return + } + + if (Test-NpmPublished $name) { + Write-Host "Skipping $name@$npmVersion; already published" + return + } + + Write-Host "Publishing $name@$npmVersion with dist-tag $DistTag" + $arguments = @('publish', $directory, '--access', 'public', '--tag', $DistTag) + if ($env:GITHUB_ACTIONS -eq 'true') { $arguments += '--provenance' } + & npm @arguments + if ($LASTEXITCODE -ne 0) { throw "Publishing $name failed" } +} + +function Assert-NpmPublished($name) +{ + # The registry can take a moment to reflect a new version. + for ($attempt = 1; $attempt -le 6; $attempt++) { + if (Test-NpmPublished $name) { return } + Start-Sleep -Seconds 5 + } + throw "$name@$npmVersion is not visible on the registry" +} + +function Stage-PlatformPackage($rid) +{ + $spec = Get-PlatformSpec $rid + $directory = "$staging/seqcli-$rid" + + $archive = Get-ReleaseArchive $rid + Write-Host "Staging $scope/seqcli-$rid from $archive" + Expand-ReleaseArchive $archive $directory + + $binary = Join-Path $directory $(if ($spec.isWindows) { 'seqcli.exe' } else { 'seqcli' }) + if (-not (Test-Path $binary)) { throw "Expected $binary in $archive" } + + if (-not $spec.isWindows) { + & chmod +x $binary + if ($LASTEXITCODE -ne 0) { throw "chmod failed for $binary" } + } + + Write-PlatformPackageJson $rid $spec $directory + + return $directory +} + +function Stage-LauncherPackage($rids) +{ + $directory = "$staging/seqcli" + if (Test-Path $directory) { Remove-Item -Recurse -Force $directory } + Copy-Item -Recurse ./npm/seqcli $directory + + $package = Get-Content "$directory/package.json" -Raw | ConvertFrom-Json -AsHashtable + $package.version = $npmVersion + $package.optionalDependencies = [ordered]@{} + foreach ($rid in $rids) { + $package.optionalDependencies["$scope/seqcli-$rid"] = $npmVersion + } + + $package | ConvertTo-Json -Depth 5 | Set-Content -Path "$directory/package.json" -NoNewline + + return $directory +} + +if (Test-Path $staging) { Remove-Item -Recurse -Force $staging } +New-Item -ItemType Directory -Force -Path $staging | Out-Null + +$rids = Get-Rids + +foreach ($rid in $rids) { + $directory = Stage-PlatformPackage $rid + Publish-NpmPackage "$scope/seqcli-$rid" $directory +} + +if (-not $DryRun) { + # Never expose a launcher whose optional dependencies can't all be resolved. + foreach ($rid in $rids) { + Assert-NpmPublished "$scope/seqcli-$rid" + } +} + +$launcherDirectory = Stage-LauncherPackage $rids +Publish-NpmPackage $launcherName $launcherDirectory + +if (-not $DryRun) { + Assert-NpmPublished $launcherName + & npm view $launcherName dist-tags + Write-Host "Install with: npm install -g $launcherName@$npmVersion" +} + +Pop-Location diff --git a/npm/README.md b/npm/README.md new file mode 100644 index 00000000..fca21e72 --- /dev/null +++ b/npm/README.md @@ -0,0 +1,33 @@ +## How the package works + +`@datalust/seqcli` is a small launcher. The self-contained `seqcli` binary for your platform is installed alongside it as an optional dependency from one of these packages: + +| Package | Platform | +|---|---| +| `@datalust/seqcli-win-x64` | Windows x64 | +| `@datalust/seqcli-win-arm64` | Windows ARM64 | +| `@datalust/seqcli-osx-x64` | macOS x64 | +| `@datalust/seqcli-osx-arm64` | macOS ARM64 (Apple Silicon) | +| `@datalust/seqcli-linux-x64` | Linux x64 (glibc) | +| `@datalust/seqcli-linux-arm64` | Linux ARM64 (glibc) | +| `@datalust/seqcli-linux-musl-x64` | Linux x64 (musl, e.g. Alpine) | +| `@datalust/seqcli-linux-musl-arm64` | Linux ARM64 (musl, e.g. Alpine) | + +The binaries are byte-for-byte the ones attached to the matching [GitHub release](https://github.com/datalust/seqcli/releases). Because the .NET runtime is bundled, no `dotnet` installation is needed. Each platform package is roughly 45 MB to download and 120 MB on disk. + +Do not install with `--omit=optional` (or `--no-optional`): the platform package would be skipped and `seqcli` would fail to start with a message explaining how to fix it. + +## Alpine Linux + +The musl builds need the ICU globalization libraries, which minimal Alpine images don't include: either `apk add icu-libs`, or set `DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1` to run without them. + +## Versions + +npm versions are the GitHub release versions with leading zeros removed from the last component, because npm requires strict semantic versioning. For example, release `v2026.1.02616` is published to npm as `2026.1.2616`. Prerelease builds are published under the `dev` dist-tag. + +## Notes for Windows users + +The Seq installer for Windows also installs `seqcli.exe`, into `C:\Program Files\Seq`. If that directory is on your `PATH` ahead of npm's global bin directory (`%APPDATA%\npm`), running `seqcli` will use the copy bundled with Seq rather than the one installed by npm. Run `where seqcli` to see which copies are found and in what order; `npx @datalust/seqcli ` always runs the npm-installed version. Both copies share the same `SeqCli.json` configuration. + +Do not use the npm package to host the `seqcli forwarder` Windows service. The service registers the path of the executable that installed it, and npm replaces the installed files on every upgrade. Use the Seq installer or a release archive from GitHub instead. + diff --git a/npm/platform-package.json b/npm/platform-package.json new file mode 100644 index 00000000..eee4c44a --- /dev/null +++ b/npm/platform-package.json @@ -0,0 +1,24 @@ +{ + "name": "{{name}}", + "version": "{{version}}", + "description": "seqcli binaries for {{rid}}. Install @datalust/seqcli instead of depending on this package directly.", + "license": "Apache-2.0", + "homepage": "https://github.com/datalust/seqcli", + "repository": { + "type": "git", + "url": "git+https://github.com/datalust/seqcli.git" + }, + "engines": { + "node": ">=18" + }, + "os": [ + "{{os}}" + ], + "cpu": [ + "{{cpu}}" + ], + "preferUnplugged": true, + "publishConfig": { + "access": "public" + } +} diff --git a/npm/seqcli/bin/seqcli.js b/npm/seqcli/bin/seqcli.js new file mode 100755 index 00000000..d7078783 --- /dev/null +++ b/npm/seqcli/bin/seqcli.js @@ -0,0 +1,147 @@ +#!/usr/bin/env node +'use strict'; + +// Launcher for the platform-specific seqcli binary. The binary itself ships in one of the +// `@datalust/seqcli-` packages, installed as an optional dependency of `@datalust/seqcli` +// and selected by npm using the `os`/`cpu`/`libc` fields in each package. + +const { spawn } = require('child_process'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const SCOPE = '@datalust'; +const RELEASES_URL = 'https://github.com/datalust/seqcli/releases'; + +// `${process.platform}-${process.arch}` (with `-musl` inserted for musl-based Linux) -> .NET RID. +const RIDS = { + 'win32-x64': 'win-x64', + 'win32-arm64': 'win-arm64', + 'darwin-x64': 'osx-x64', + 'darwin-arm64': 'osx-arm64', + 'linux-x64': 'linux-x64', + 'linux-arm64': 'linux-arm64', + 'linux-musl-x64': 'linux-musl-x64', + 'linux-musl-arm64': 'linux-musl-arm64', +}; + +function isMusl() { + try { + return !process.report.getReport().header.glibcVersionRuntime; + } catch { + return false; + } +} + +// Candidate platform packages in order of preference. On Linux the detected libc variant is tried +// first, then the other one, so that package managers that ignore the `libc` field (and therefore +// install both) still run the right binary. +function candidatePackages() { + const { platform, arch } = process; + const keys = []; + if (platform === 'linux') { + const musl = isMusl(); + keys.push(musl ? `linux-musl-${arch}` : `linux-${arch}`); + keys.push(musl ? `linux-${arch}` : `linux-musl-${arch}`); + } else { + keys.push(`${platform}-${arch}`); + } + return keys.filter((k) => RIDS[k]).map((k) => `${SCOPE}/seqcli-${RIDS[k]}`); +} + +function locateBinary() { + const launcherVersion = require('../package.json').version; + const candidates = candidatePackages(); + const problems = []; + + for (const name of candidates) { + let packageJsonPath; + try { + packageJsonPath = require.resolve(`${name}/package.json`); + } catch { + problems.push(`${name} is not installed`); + continue; + } + + const installedVersion = require(packageJsonPath).version; + if (installedVersion !== launcherVersion) { + problems.push(`${name}@${installedVersion} does not match ${SCOPE}/seqcli@${launcherVersion}`); + continue; + } + + const exe = path.join(path.dirname(packageJsonPath), process.platform === 'win32' ? 'seqcli.exe' : 'seqcli'); + if (!fs.existsSync(exe)) { + problems.push(`${name} is installed but ${exe} is missing`); + continue; + } + + return exe; + } + + const lines = []; + if (candidates.length === 0) { + lines.push(`seqcli: ${process.platform}-${process.arch} is not supported by the npm package.`); + } else { + lines.push('seqcli: could not find the platform-specific seqcli package.'); + for (const p of problems) lines.push(` - ${p}`); + lines.push(''); + lines.push(`Reinstall with: npm install -g ${SCOPE}/seqcli@${launcherVersion}`); + lines.push('(optional dependencies must not be omitted; check for --omit=optional / --no-optional)'); + lines.push(`or install the platform package directly: npm install -g ${candidates[0]}@${launcherVersion}`); + } + lines.push(''); + lines.push(`Supported platforms: ${Object.values(RIDS).join(', ')}.`); + lines.push(`Other downloads: ${RELEASES_URL}`); + console.error(lines.join('\n')); + process.exit(1); +} + +function ensureExecutable(exe) { + if (process.platform === 'win32') return; + try { + fs.accessSync(exe, fs.constants.X_OK); + } catch { + try { + fs.chmodSync(exe, 0o755); + } catch { + // Reported by spawn() as EACCES below. + } + } +} + +function run() { + const exe = locateBinary(); + ensureExecutable(exe); + + const child = spawn(exe, process.argv.slice(2), { stdio: 'inherit', windowsHide: true }); + + // Ctrl+C is delivered by the terminal to the whole foreground process group (or console on + // Windows), so the child already receives it. Ignore it here so this process outlives the + // child and can report the child's exit status. + process.on('SIGINT', () => {}); + + // Signals from supervisors (kill, systemd, CI cancellation) target this process only; forward them. + for (const signal of ['SIGTERM', 'SIGHUP']) { + process.on(signal, () => child.kill(signal)); + } + + child.on('error', (err) => { + console.error(`seqcli: failed to start ${exe}: ${err.message}`); + process.exit(1); + }); + + child.on('exit', (code, signal) => { + if (signal) { + process.removeAllListeners(signal); + try { + process.kill(process.pid, signal); + } catch { + // Fall through to a conventional exit code. + } + process.exit(128 + (os.constants.signals[signal] || 0)); + } + process.exit(code === null ? 1 : code); + }); +} + +run(); diff --git a/npm/seqcli/package.json b/npm/seqcli/package.json new file mode 100644 index 00000000..fbd36399 --- /dev/null +++ b/npm/seqcli/package.json @@ -0,0 +1,50 @@ +{ + "name": "@datalust/seqcli", + "//version": "These version are replaced during the deployment process", + "version": "0.0.0", + "description": "The Seq command-line client. Administer, log, ingest, search, from any OS.", + "keywords": [ + "seq", + "seqcli", + "datalust", + "logging", + "structured-logging", + "cli" + ], + "scripts": { + "prepack": "cp ../../README.md ./README.md && cp ../../LICENSE ./LICENSE", + "postpack": "rm ./README.md ./LICENSE" + }, + "license": "Apache-2.0", + "homepage": "https://github.com/datalust/seqcli", + "repository": { + "type": "git", + "url": "git+https://github.com/datalust/seqcli.git" + }, + "bugs": { + "url": "https://github.com/datalust/seqcli/issues" + }, + "bin": { + "seqcli": "bin/seqcli.js" + }, + "files": [ + "bin", + "README.md" + ], + "engines": { + "node": ">=18" + }, + "publishConfig": { + "access": "public" + }, + "optionalDependencies": { + "@datalust/seqcli-win-x64": "0.0.0", + "@datalust/seqcli-win-arm64": "0.0.0", + "@datalust/seqcli-linux-x64": "0.0.0", + "@datalust/seqcli-linux-arm64": "0.0.0", + "@datalust/seqcli-linux-musl-x64": "0.0.0", + "@datalust/seqcli-linux-musl-arm64": "0.0.0", + "@datalust/seqcli-osx-x64": "0.0.0", + "@datalust/seqcli-osx-arm64": "0.0.0" + } +} diff --git a/src/SeqCli/Mcp/McpServerInstaller.cs b/src/SeqCli/Mcp/McpServerInstaller.cs index 1160aa33..8ada8823 100644 --- a/src/SeqCli/Mcp/McpServerInstaller.cs +++ b/src/SeqCli/Mcp/McpServerInstaller.cs @@ -15,6 +15,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using Newtonsoft.Json.Linq; using Serilog; @@ -121,9 +122,11 @@ public static void Install(string? agent, bool global, string? profileName = nul root[target.ServerMapKey] = serverMap; } + var (command, leadingArgs) = ResolveCommand(); + // A connection profile is the only connection setting we propagate; the server URL and // API key are resolved from config at runtime so they're not baked into the agent's file. - var args = new JArray("mcp", "run"); + var args = new JArray(leadingArgs.Concat(["mcp", "run"]).ToArray()); if (profileName != null) { args.Add("--profile"); @@ -132,7 +135,7 @@ public static void Install(string? agent, bool global, string? profileName = nul serverMap[ServerName] = new JObject { - ["command"] = "seqcli", + ["command"] = command, ["args"] = args, }; @@ -146,6 +149,60 @@ public static void Install(string? agent, bool global, string? profileName = nul Log.Information("Installed Seq MCP server for {Agent} to {Path}", agent, path); } + // Agents resolve `seqcli` from PATH when they start the server. On Windows, an npm-installed + // `seqcli` is a `seqcli.cmd` shim, which hosts that spawn processes without a shell can't run + // directly, so in that case the server is launched through `cmd /c` instead. + static (string Command, string[] LeadingArgs) ResolveCommand() => + ResolveCommand( + OperatingSystem.IsWindows(), + Environment.GetEnvironmentVariable("PATH"), + Environment.GetEnvironmentVariable("PATHEXT"), + File.Exists); + + internal static (string Command, string[] LeadingArgs) ResolveCommand( + bool isWindows, + string? path, + string? pathExt, + Func fileExists) + { + if (!isWindows) + return ("seqcli", []); + + var found = FindOnWindowsPath("seqcli", path, pathExt, fileExists); + if (found == null) + return ("seqcli", []); + + var extension = Path.GetExtension(found); + if (extension.Equals(".cmd", StringComparison.OrdinalIgnoreCase) || + extension.Equals(".bat", StringComparison.OrdinalIgnoreCase)) + { + Log.Information("Found `seqcli` on PATH as {ShimPath}; the MCP server will be launched via `cmd /c`", found); + return ("cmd", ["/c", "seqcli"]); + } + + return ("seqcli", []); + } + + // Mirrors how Windows locates a command: each PATH directory in turn, trying the PATHEXT + // extensions in order within it. + static string? FindOnWindowsPath(string name, string? path, string? pathExt, Func fileExists) + { + var extensions = (pathExt is { Length: > 0 } ? pathExt : ".COM;.EXE;.BAT;.CMD") + .Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + + foreach (var directory in (path ?? "").Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + foreach (var extension in extensions) + { + var candidate = Path.Combine(directory, name + extension); + if (fileExists(candidate)) + return candidate; + } + } + + return null; + } + static AgentTarget Unsupported(string message) => new(_ => throw new NotSupportedException(message), "mcpServers"); diff --git a/test/SeqCli.Tests/Mcp/McpServerInstallerTests.cs b/test/SeqCli.Tests/Mcp/McpServerInstallerTests.cs new file mode 100644 index 00000000..b6334c5f --- /dev/null +++ b/test/SeqCli.Tests/Mcp/McpServerInstallerTests.cs @@ -0,0 +1,75 @@ +using System; +using System.Collections.Generic; +using SeqCli.Mcp; +using Xunit; + +namespace SeqCli.Tests.Mcp; + +public class McpServerInstallerTests +{ + // Candidate paths are built with Path.Combine, which uses the host's separator; normalize so the + // Windows-style expectations hold when the tests run on Linux or macOS. + static Func FileSystemWith(params string[] files) + { + var set = new HashSet(files, StringComparer.OrdinalIgnoreCase); + return candidate => set.Contains(candidate.Replace('/', '\\')); + } + + [Fact] + public void OnNonWindowsPlatformsSeqCliIsLaunchedDirectly() + { + var (command, leadingArgs) = McpServerInstaller.ResolveCommand( + false, "/usr/local/bin:/usr/bin", null, _ => true); + + Assert.Equal("seqcli", command); + Assert.Empty(leadingArgs); + } + + [Fact] + public void OnWindowsAnExecutableOnPathIsLaunchedDirectly() + { + var (command, leadingArgs) = McpServerInstaller.ResolveCommand( + true, + @"C:\Program Files\Seq;C:\Users\me\AppData\Roaming\npm", + ".COM;.EXE;.BAT;.CMD", + FileSystemWith(@"C:\Program Files\Seq\seqcli.exe", @"C:\Users\me\AppData\Roaming\npm\seqcli.cmd")); + + Assert.Equal("seqcli", command); + Assert.Empty(leadingArgs); + } + + [Fact] + public void OnWindowsAnNpmShimOnPathIsLaunchedViaCmd() + { + var (command, leadingArgs) = McpServerInstaller.ResolveCommand( + true, + @"C:\Users\me\AppData\Roaming\npm;C:\Program Files\Seq", + ".COM;.EXE;.BAT;.CMD", + FileSystemWith(@"C:\Users\me\AppData\Roaming\npm\seqcli.cmd", @"C:\Program Files\Seq\seqcli.exe")); + + Assert.Equal("cmd", command); + Assert.Equal(["/c", "seqcli"], leadingArgs); + } + + [Fact] + public void OnWindowsWhenSeqCliIsNotOnPathItIsLaunchedDirectly() + { + var (command, leadingArgs) = McpServerInstaller.ResolveCommand( + true, @"C:\Windows\system32", null, _ => false); + + Assert.Equal("seqcli", command); + Assert.Empty(leadingArgs); + } + + [Fact] + public void PathExtDefaultsAreUsedWhenTheVariableIsMissing() + { + var (command, _) = McpServerInstaller.ResolveCommand( + true, + @"C:\Users\me\AppData\Roaming\npm", + null, + FileSystemWith(@"C:\Users\me\AppData\Roaming\npm\seqcli.cmd")); + + Assert.Equal("cmd", command); + } +} From f987446b43cb4a8e2798d14fe6445220b7511577 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Thu, 3 Sep 2026 14:58:44 +1000 Subject: [PATCH 26/32] Include updated Seq.Syntax --- src/SeqCli/SeqCli.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SeqCli/SeqCli.csproj b/src/SeqCli/SeqCli.csproj index 16f10fb8..346d0ed8 100644 --- a/src/SeqCli/SeqCli.csproj +++ b/src/SeqCli/SeqCli.csproj @@ -42,7 +42,7 @@ - + From 2388cc1234a123ba4cb635a1881d487e2c494da6 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Thu, 3 Sep 2026 15:27:54 +1000 Subject: [PATCH 27/32] Rework now that Seq.Syntax can evaluate column expressions directly in the formatter --- src/SeqCli/Cli/Commands/SearchCommand.cs | 5 +- src/SeqCli/Cli/Commands/TailCommand.cs | 5 +- .../Cli/Features/EventColumnsFeature.cs | 21 +++-- .../Cli/Features/OutputFormatFeature.cs | 5 +- src/SeqCli/Output/EventColumns.cs | 91 ------------------- src/SeqCli/Output/OutputFormat.cs | 12 +-- src/SeqCli/Output/TextFormatters.cs | 23 ++++- src/SeqCli/Output/TraceFormatter.cs | 10 +- test/SeqCli.Tests/Output/EventColumnsTests.cs | 89 ------------------ test/SeqCli.Tests/Output/OutputFormatTests.cs | 1 - .../Output/TextFormattersTests.cs | 69 ++++++++++++++ 11 files changed, 116 insertions(+), 215 deletions(-) delete mode 100644 src/SeqCli/Output/EventColumns.cs delete mode 100644 test/SeqCli.Tests/Output/EventColumnsTests.cs diff --git a/src/SeqCli/Cli/Commands/SearchCommand.cs b/src/SeqCli/Cli/Commands/SearchCommand.cs index d7b2898c..37c9c95e 100644 --- a/src/SeqCli/Cli/Commands/SearchCommand.cs +++ b/src/SeqCli/Cli/Commands/SearchCommand.cs @@ -18,6 +18,7 @@ using SeqCli.Api; using SeqCli.Cli.Features; using SeqCli.Config; +using SeqCli.Output; using Serilog; // ReSharper disable UnusedType.Global @@ -78,8 +79,8 @@ protected override async Task Run() var connection = SeqConnectionFactory.Connect(_connection, config); connection.Client.HttpClient.Timeout = TimeSpan.FromMilliseconds(_httpClientTimeout); - var columns = await _eventColumns.GetEventColumns(connection, _signal.Signal); - var output = _output.GetOutputFormat(config, columns?.OutputTemplate(), columns); + var columns = await _eventColumns.GetColumns(connection, _signal.Signal); + var output = _output.GetOutputFormat(config, TextFormatters.PlainOutputTemplate(columns)); string? filter = null; if (!string.IsNullOrWhiteSpace(_filter)) diff --git a/src/SeqCli/Cli/Commands/TailCommand.cs b/src/SeqCli/Cli/Commands/TailCommand.cs index 88253a70..b4670112 100644 --- a/src/SeqCli/Cli/Commands/TailCommand.cs +++ b/src/SeqCli/Cli/Commands/TailCommand.cs @@ -20,6 +20,7 @@ using SeqCli.Api; using SeqCli.Cli.Features; using SeqCli.Config; +using SeqCli.Output; namespace SeqCli.Cli.Commands; @@ -63,8 +64,8 @@ protected override async Task Run() strict = converted.StrictExpression; } - var columns = await _eventColumns.GetEventColumns(connection, _signal.Signal); - var output = _output.GetOutputFormat(config, columns?.OutputTemplate(), columns); + var columns = await _eventColumns.GetColumns(connection, _signal.Signal); + var output = _output.GetOutputFormat(config, TextFormatters.PlainOutputTemplate(columns)); try { diff --git a/src/SeqCli/Cli/Features/EventColumnsFeature.cs b/src/SeqCli/Cli/Features/EventColumnsFeature.cs index 22346a90..68ee7116 100644 --- a/src/SeqCli/Cli/Features/EventColumnsFeature.cs +++ b/src/SeqCli/Cli/Features/EventColumnsFeature.cs @@ -17,8 +17,8 @@ using System.Threading.Tasks; using Seq.Api; using Seq.Api.Model.Signals; -using SeqCli.Output; using SeqCli.Signals; +using SeqCli.Syntax; using SeqCli.Util; namespace SeqCli.Cli.Features; @@ -43,9 +43,9 @@ public override void Enable(OptionSet options) _ => _noSignalColumns = true); } - public async Task GetEventColumns(SeqConnection connection, SignalExpressionPart? signal) + public async Task> GetColumns(SeqConnection connection, SignalExpressionPart? signal) { - var collectedColumns = new List(); + var columns = new List(); if (!_noSignalColumns && signal is { } signalExpression) { foreach (var signalId in signalExpression.ReferencedSignalIds()) @@ -53,18 +53,19 @@ public override void Enable(OptionSet options) var signalEntity = await connection.Signals.FindAsync(signalId); foreach (var column in signalEntity.Columns) { - collectedColumns.Add(column.Expression); + columns.Add(column.Expression); } } } - collectedColumns.AddRange(_columns); + columns.AddRange(_columns); - if (collectedColumns.Count == 0) - return null; - - if (!EventColumns.TryCreate(collectedColumns, out var columns, out var error)) - throw new ArgumentException($"The column expression could not be compiled: {error}"); + foreach (var column in columns) + { + // A better error than a failed output template parse. + if (!SeqSyntax.TryCompileExpression(column, out _, out var error)) + throw new ArgumentException($"The column expression `{column}` could not be compiled: {error}"); + } return columns; } diff --git a/src/SeqCli/Cli/Features/OutputFormatFeature.cs b/src/SeqCli/Cli/Features/OutputFormatFeature.cs index 05190b7b..792766b2 100644 --- a/src/SeqCli/Cli/Features/OutputFormatFeature.cs +++ b/src/SeqCli/Cli/Features/OutputFormatFeature.cs @@ -13,7 +13,6 @@ // limitations under the License. using SeqCli.Config; -using SeqCli.Data; using SeqCli.Output; namespace SeqCli.Cli.Features; @@ -27,9 +26,9 @@ class OutputFormatFeature(bool supportNative, bool supportJson) : CommandFeature public OutputFormatFeature() : this(supportNative: false, supportJson: true) { } - public OutputFormat GetOutputFormat(SeqCliConfig config, string? outputTemplate = null, IEventEnricher? textEnricher = null) + public OutputFormat GetOutputFormat(SeqCliConfig config, string? outputTemplate = null) { - return new OutputFormat(_syntax, _noColor, _forceColor, config.Output, outputTemplate, textEnricher); + return new OutputFormat(_syntax, _noColor, _forceColor, config.Output, outputTemplate); } public string JsonArgumentHelp { get; init; } = "Print output in newline-delimited JSON (the default is plain text)"; diff --git a/src/SeqCli/Output/EventColumns.cs b/src/SeqCli/Output/EventColumns.cs deleted file mode 100644 index e26c97f3..00000000 --- a/src/SeqCli/Output/EventColumns.cs +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright © Datalust and contributors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Text; -using System.Text.Json.Nodes; -using Seq.Syntax.Expressions; -using SeqCli.Data; -using SeqCli.Syntax; - -namespace SeqCli.Output; - -/// -/// Evaluates a list of column expressions against each event, storing the results in synthetic properties that -/// the plain-text output template shows ahead of the message. -/// -class EventColumns : IEventEnricher -{ - static readonly string ColumnPrefixProperty = $"_SeqcliColumn_{Guid.NewGuid():N}"; - - internal static string ColumnPropertyName(int index) => $"{ColumnPrefixProperty}_{index}"; - - internal static string TemplateColumnsFragment(int columnCount) - { - // `<> ''` is undefined, and hence falsy, when the property is missing; the guard thus - // drops the column, and its trailing space, for both missing and empty values. - var fragment = new StringBuilder(); - for (var i = 0; i < columnCount; ++i) - { - var column = ColumnPropertyName(i); - fragment.Append($"{{#if {column} <> ''}}{{{column}}} {{#end}}"); - } - - return fragment.ToString(); - } - - readonly CompiledExpression[] _columns; - - EventColumns(CompiledExpression[] columns) - { - _columns = columns; - } - - public static bool TryCreate( - IReadOnlyList expressions, - [NotNullWhen(true)] out EventColumns? columns, - [NotNullWhen(false)] out string? error) - { - var compiled = new CompiledExpression[expressions.Count]; - for (var i = 0; i < expressions.Count; ++i) - { - if (!SeqSyntax.TryCompileExpression(expressions[i], out var expression, out error)) - { - columns = null; - return false; - } - - compiled[i] = expression; - } - - columns = new EventColumns(compiled); - error = null; - return true; - } - - public string OutputTemplate() => TextFormatters.PlainOutputTemplate(_columns.Length); - - public void Enrich(JsonObject eventJson) - { - for (var i = 0; i < _columns.Length; ++i) - { - // Property accessors return nodes still attached to the event, so they're cloned before being - // re-parented. - if (_columns[i](eventJson).TryGetValue(out var value)) - eventJson[ColumnPropertyName(i)] = value?.DeepClone(); - } - } -} diff --git a/src/SeqCli/Output/OutputFormat.cs b/src/SeqCli/Output/OutputFormat.cs index 741b5750..ae2492fb 100644 --- a/src/SeqCli/Output/OutputFormat.cs +++ b/src/SeqCli/Output/OutputFormat.cs @@ -29,7 +29,6 @@ using SeqCli.Api; using SeqCli.Config; using SeqCli.Csv; -using SeqCli.Data; namespace SeqCli.Output; @@ -40,7 +39,6 @@ sealed class OutputFormat readonly OutputSyntax _syntax; readonly ExpressionTemplate? _eventFormatter; - readonly IEventEnricher? _textEnricher; readonly ExpressionTemplate _jsonValueFormatter; readonly JsonSerializer _serializer = JsonSerializer.CreateDefault(new JsonSerializerSettings @@ -57,15 +55,13 @@ public OutputFormat( bool? noColor, bool? forceColor, SeqCliOutputConfig outputConfig, - string? plainTextTemplate = null, - IEventEnricher? textEnricher = null) + string? plainTextTemplate = null) : this( syntax, noColor, forceColor, outputConfig, plainTextTemplate, - textEnricher, noColorSetInEnvironment: NoColorSetInEnvironment(), outputIsRedirected: Console.IsOutputRedirected, allowAnsiEscapes: TerminalFeatures.TryEnableAnsiEscapes()) @@ -77,7 +73,6 @@ public OutputFormat( /// The value of --force-color, if specified. /// Configured output defaults. /// The template controlling plain-text formatting, or null for the default. - /// An enricher applied to events written as plain text, or null. /// Whether NO_COLOR is set; see . /// Whether STDOUT is redirected, i.e. not attached to a terminal. /// Whether ANSI escape sequences are allowed; generally false for interactive @@ -88,16 +83,12 @@ internal OutputFormat( bool? forceColor, SeqCliOutputConfig outputConfig, string? plainTextTemplate, - IEventEnricher? textEnricher, bool noColorSetInEnvironment, bool outputIsRedirected, bool allowAnsiEscapes) { _syntax = syntax; - // Enrichment supports plain-text templates, so JSON output shows events verbatim. - _textEnricher = Text ? textEnricher : null; - var resolvedNoColor = ResolveNoColor(noColor, forceColor, outputConfig, noColorSetInEnvironment, allowAnsiEscapes); var applyThemeToRedirectedOutput = !resolvedNoColor && (forceColor ?? outputConfig.ForceColor); var colorize = !resolvedNoColor && (applyThemeToRedirectedOutput || !outputIsRedirected); @@ -246,7 +237,6 @@ public void WriteEventEntity(EventEntity evt) public void WriteEvent(JsonObject eventJson) { - _textEnricher?.Enrich(eventJson); _eventFormatter?.Format(eventJson, Console.Out); } diff --git a/src/SeqCli/Output/TextFormatters.cs b/src/SeqCli/Output/TextFormatters.cs index c4c3e7cc..92d07c2f 100644 --- a/src/SeqCli/Output/TextFormatters.cs +++ b/src/SeqCli/Output/TextFormatters.cs @@ -13,6 +13,8 @@ // limitations under the License. using System; +using System.Collections.Generic; +using System.Text; using Seq.Syntax.Templates; using Seq.Syntax.Templates.Encoding; using Seq.Syntax.Templates.Themes; @@ -30,13 +32,26 @@ static class TextFormatters "{@Data}" + Environment.NewLine, encoder: Encoder(theme)); - // Guarding on `@Elapsed` rather than the built-in `IsSpan()` shows elapsed time for any - // event carrying a span start timestamp, whether or not trace and span ids accompany it. - internal static string PlainOutputTemplate(int columnCount = 0) => - "[{@Timestamp:o} {@Level:u3}] " + EventColumns.TemplateColumnsFragment(columnCount) + + /// + /// The default plain-text template, showing ahead of each + /// event's message. + /// + /// Column expressions, evaluated against each event; any Seq expression can be + /// supplied. + internal static string PlainOutputTemplate(IEnumerable? columns = null) => + "[{@Timestamp:o} {@Level:u3}] " + ColumnsFragment(columns ?? []) + "{@Message}{#if @Elapsed is not null} ({TotalMilliseconds(@Elapsed):0.###} ms){#end}" + Environment.NewLine + "{@Exception}"; + internal static string ColumnsFragment(IEnumerable columns) + { + var fragment = new StringBuilder(); + foreach (var column in columns) + fragment.Append($"{{#if ({column}) <> ''}}{{({column})}} {{#end}}"); + + return fragment.ToString(); + } + public static ExpressionTemplate Plain(TemplateTheme? theme, string? outputTemplate) => SeqSyntax.ParseTemplate(outputTemplate ?? PlainOutputTemplate(), Encoder(theme)); diff --git a/src/SeqCli/Output/TraceFormatter.cs b/src/SeqCli/Output/TraceFormatter.cs index 0f770604..b74b2651 100644 --- a/src/SeqCli/Output/TraceFormatter.cs +++ b/src/SeqCli/Output/TraceFormatter.cs @@ -15,6 +15,7 @@ using System; using System.Collections.Generic; using System.Globalization; +using System.Linq; using System.Text; using System.Text.Json.Nodes; using SeqCli.Api; @@ -27,14 +28,19 @@ static class TraceFormatter { static readonly string TreePrefixProperty = $"_SeqcliTraceTreePrefix_{Guid.NewGuid():N}"; static readonly string ElapsedProperty = $"_SeqcliTraceElapsed_{Guid.NewGuid():N}"; + static readonly string ColumnPrefixProperty = $"_SeqcliTraceColumn_{Guid.NewGuid():N}"; const string SpanConnector = "├─ ", LastSpanConnector = "└─ ", LogConnector = "┊ ", Continuation = "│ ", Gap = " "; + // The trace query evaluates column expressions server-side, so their results are carried in + // surrogate properties rather than being recomputed by the output template. + static string ColumnProperty(int index) => $"{ColumnPrefixProperty}_{index}"; + public static string OutputTemplate(int columnCount) { var template = new StringBuilder($"[{{@Timestamp:o}} {{@Level:u3}}] {{{TreePrefixProperty}}}"); - template.Append(EventColumns.TemplateColumnsFragment(columnCount)); + template.Append(TextFormatters.ColumnsFragment(Enumerable.Range(0, columnCount).Select(ColumnProperty))); template.Append($"{{@Message}}{{#if {ElapsedProperty} is not null}} ({{TotalMilliseconds({ElapsedProperty}):0.###}} ms){{#end}}"); template.Append(Environment.NewLine).Append("{@Exception}"); return template.ToString(); @@ -95,7 +101,7 @@ static JsonObject ToEventJson(TraceTreeNode treeNode, string treePrefix) for (var i = 0; i < evt.Columns.Count; ++i) { if (evt.Columns[i] is { } value) - eventJson[EventColumns.ColumnPropertyName(i)] = ToSystemTextJson.FromApiValue(value); + eventJson[ColumnProperty(i)] = ToSystemTextJson.FromApiValue(value); } return eventJson; diff --git a/test/SeqCli.Tests/Output/EventColumnsTests.cs b/test/SeqCli.Tests/Output/EventColumnsTests.cs deleted file mode 100644 index 80a46551..00000000 --- a/test/SeqCli.Tests/Output/EventColumnsTests.cs +++ /dev/null @@ -1,89 +0,0 @@ -#nullable enable -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using Seq.Api.Model.Events; -using SeqCli.Api; -using SeqCli.Output; -using SeqCli.Tests.Support; -using Xunit; - -namespace SeqCli.Tests.Output; - -public class EventColumnsTests -{ - static EventColumns Create(params string[] expressions) - { - Assert.True(EventColumns.TryCreate(expressions, out var columns, out var error), error); - return columns; - } - - static string Render(EventEntity evt, EventColumns columns) - { - var eventJson = EventEntityJson.ToEventJson(evt); - columns.Enrich(eventJson); - - var output = new StringWriter(); - TextFormatters.Plain(theme: null, columns.OutputTemplate()).Format(eventJson, output); - return output.ToString(); - } - - static string At(EventEntity evt) => - DateTimeOffset.ParseExact(evt.Timestamp, "o", CultureInfo.InvariantCulture).ToLocalTime().ToString("o"); - - [Fact] - public void ColumnsPrecedeTheMessageInOrder() - { - var evt = Some.MakeEvent(e => e.Properties = Some.MakeProperties(("Customer", "scott"), ("OrderId", 42))); - - Assert.Equal( - $"[{At(evt)} INF] scott 42 Hello{Environment.NewLine}", - Render(evt, Create("Customer", "OrderId"))); - } - - [Fact] - public void MissingAndEmptyColumnValuesLeaveNoRedundantSpace() - { - var evt = Some.MakeEvent(e => e.Properties = Some.MakeProperties(("Empty", ""), ("OrderId", 42))); - - Assert.Equal( - $"[{At(evt)} INF] 42 Hello{Environment.NewLine}", - Render(evt, Create("Missing", "Empty", "OrderId"))); - } - - [Fact] - public void SeqStyleNamesResolveAgainstApiEvents() - { - var evt = Some.MakeEvent(e => - { - e.Properties = []; - e.Level = "Warning"; - e.SpanKind = "Server"; - e.Resource = Some.MakeProperties(("service.name", "frontend")); - }); - - Assert.Equal( - $"[{At(evt)} WRN] frontend Server Hello{Environment.NewLine}", - Render(evt, Create("@Resource['service.name']", "@SpanKind"))); - } - - [Fact] - public void ComputedColumnValuesAreRendered() - { - var evt = Some.MakeEvent(e => e.Properties = Some.MakeProperties(("OrderId", 42))); - - Assert.Equal( - $"[{At(evt)} INF] order-42 Hello{Environment.NewLine}", - Render(evt, Create("concat('order-', tostring(OrderId))"))); - } - - [Fact] - public void InvalidExpressionsAreReportedByTryCreate() - { - Assert.False(EventColumns.TryCreate(new List {"OrderId", "not a valid ("}, out var columns, out var error)); - Assert.Null(columns); - Assert.NotNull(error); - Assert.NotEmpty(error); - } -} diff --git a/test/SeqCli.Tests/Output/OutputFormatTests.cs b/test/SeqCli.Tests/Output/OutputFormatTests.cs index 9361a8f2..e323aa60 100644 --- a/test/SeqCli.Tests/Output/OutputFormatTests.cs +++ b/test/SeqCli.Tests/Output/OutputFormatTests.cs @@ -27,7 +27,6 @@ static OutputFormat Create( forceColor, new SeqCliOutputConfig { DisableColor = disableColor }, plainTextTemplate: null, - textEnricher: null, noColorSetInEnvironment, outputIsRedirected, supportsAnsiEscapes); diff --git a/test/SeqCli.Tests/Output/TextFormattersTests.cs b/test/SeqCli.Tests/Output/TextFormattersTests.cs index 340b18fc..c3412725 100644 --- a/test/SeqCli.Tests/Output/TextFormattersTests.cs +++ b/test/SeqCli.Tests/Output/TextFormattersTests.cs @@ -1,7 +1,9 @@ #nullable enable using System; +using System.Globalization; using System.IO; using System.Text.Json.Nodes; +using Seq.Api.Model.Events; using Seq.Syntax.Templates.Themes; using SeqCli.Api; using SeqCli.Output; @@ -74,6 +76,73 @@ public void ACustomOutputTemplateReplacesTheDefault() RenderText(SomeEventJson(), $"{{@l:u3}} {{@m}}{Environment.NewLine}")); } + [Fact] + public void ColumnsPrecedeTheMessageInOrder() + { + var evt = Some.MakeEvent(e => e.Properties = Some.MakeProperties(("Customer", "scott"), ("OrderId", 42))); + + Assert.Equal( + $"[{At(evt)} INF] scott 42 Hello{Environment.NewLine}", + RenderText(evt, "Customer", "OrderId")); + } + + [Fact] + public void MissingAndEmptyColumnValuesLeaveNoRedundantSpace() + { + var evt = Some.MakeEvent(e => e.Properties = Some.MakeProperties(("Empty", ""), ("OrderId", 42))); + + Assert.Equal( + $"[{At(evt)} INF] 42 Hello{Environment.NewLine}", + RenderText(evt, "Missing", "Empty", "OrderId")); + } + + [Fact] + public void SeqStyleNamesResolveAgainstApiEvents() + { + var evt = Some.MakeEvent(e => + { + e.Properties = []; + e.Level = "Warning"; + e.SpanKind = "Server"; + e.Resource = Some.MakeProperties(("service.name", "frontend")); + }); + + Assert.Equal( + $"[{At(evt)} WRN] frontend Server Hello{Environment.NewLine}", + RenderText(evt, "@Resource['service.name']", "@SpanKind")); + } + + [Fact] + public void ComputedColumnValuesAreRendered() + { + var evt = Some.MakeEvent(e => e.Properties = Some.MakeProperties(("OrderId", 42))); + + Assert.Equal( + $"[{At(evt)} INF] order-42 Hello{Environment.NewLine}", + RenderText(evt, "concat('order-', tostring(OrderId))")); + } + + [Theory] + [InlineData("if OrderId > 40 then 'big' else 'small'", "big")] + [InlineData("{id: OrderId}.id", "42")] + [InlineData("concat('{', tostring(OrderId), '}')", "{42}")] + [InlineData("Missing or OrderId = 42", "true")] + [InlineData("[OrderId, 'x'][0]", "42")] + public void ColumnExpressionsUsingTemplateDelimitersAreRendered(string column, string expected) + { + var evt = Some.MakeEvent(e => e.Properties = Some.MakeProperties(("OrderId", 42))); + + Assert.Equal( + $"[{At(evt)} INF] {expected} Hello{Environment.NewLine}", + RenderText(evt, column)); + } + + static string At(EventEntity evt) => + DateTimeOffset.ParseExact(evt.Timestamp, "o", CultureInfo.InvariantCulture).ToLocalTime().ToString("o"); + + static string RenderText(EventEntity evt, params string[] columns) => + RenderText(EventEntityJson.ToEventJson(evt), TextFormatters.PlainOutputTemplate(columns)); + static JsonObject SomeEventJson(string? level = null, string? exception = null) { var evt = new JsonObject From f96c733e180ae250971bfdea6669accc813f62dd Mon Sep 17 00:00:00 2001 From: liammclennan-ro Date: Fri, 4 Sep 2026 16:40:35 +1000 Subject: [PATCH 28/32] Add a build README --- build/README.md | 88 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 build/README.md diff --git a/build/README.md b/build/README.md new file mode 100644 index 00000000..6dd06c32 --- /dev/null +++ b/build/README.md @@ -0,0 +1,88 @@ +# Building and publishing `seqcli` + +This directory holds the PowerShell scripts that build, test, package, and publish `seqcli`. They are driven by the GitHub Actions workflow in [`.github/workflows/ci.yml`](../.github/workflows/ci.yml), but can also be run locally with `pwsh`. + +| Script | Runs on | Produces | +|---|---|---| +| `Build.Common.ps1` | (dot-sourced by the others) | Version number helpers | +| `Build.Windows.ps1` | Windows | Release archives for every platform, the `seqcli` .NET tool package, generated docs, a GitHub release, NuGet publish | +| `Build.Linux.ps1` | Linux | `datalust/seqcli` Docker images for `linux/amd64` and `linux/arm64` | +| `Build.Npm.ps1` | Linux (any OS locally) | `@datalust/seqcli` and `@datalust/seqcli-` npm packages | + +`7-zip/` contains a vendored copy of `7za.exe`, used by `Build.Windows.ps1` to produce `.zip` and `.tar.gz` archives with consistent contents on Windows. + +## Versioning + +Every artifact from one CI run shares a single version, computed by `Get-SemVer` in `Build.Common.ps1`: + +``` +.[--] +``` + +* `` is read from [`baseversion`](../baseversion) in the repository root (e.g. `2026.1`). Bump it there when starting a new release line. +* `` is `CI_BUILD_NUMBER_BASE + 2300`, zero-padded to five digits (e.g. `02616`). `CI_BUILD_NUMBER_BASE` is the GitHub Actions `run_number`; the fixed offset keeps build numbers increasing across the move from the previous CI system. Note the comment at the top of `ci.yml`: renaming the workflow file resets `run_number`, which would produce lower version numbers than already-published releases. Locally, where the variable is unset, the build number is the literal string `local`. +* The prerelease suffix is added for every branch except `main`. It is the first ten characters of the branch name, stripped of anything other than letters, digits and hyphens, followed by the build number. A `dev` build is therefore `2026.1.02700-dev-02700`; a `main` build is `2026.1.02616`. + +`CI_TARGET_BRANCH` overrides the branch detected from git. In CI it is set from `github.head_ref` (for pull requests) or `github.ref_name`. + +### npm versions + +npm enforces strict semver, which forbids leading zeros in numeric components, so `Get-NpmVersion` strips the padding from the patch component when publishing to npm: release `v2026.1.02616` becomes `2026.1.2616` on npm, and `2026.1.02700-dev-02700` becomes `2026.1.2700-dev-02700` (prerelease identifiers are left as-is). Archive names and GitHub release tags always use the padded form. + +## The CI workflow + +The workflow runs on pushes and pull requests to `dev` and `main`, and can be triggered manually with `workflow_dispatch`. Three jobs run: + +1. **Build (Windows)** runs `Build.Windows.ps1`. On non-PR builds, it uploads the release archives as a workflow artifact named `archives` for the npm job to consume. +2. **Build (Linux)** runs `Build.Linux.ps1`, after configuring `binfmt` so that ARM64 images can be built on the x64 runner. It runs in parallel with the Windows job. +3. **Publish (npm)** runs `Build.Npm.ps1` after the Windows job succeeds, on every non-PR build. + +Environment variables control what gets published: + +| Variable | Set in CI to | Effect | +|---|---|---| +| `CI_BUILD_NUMBER_BASE` | `github.run_number` | Build number component of the version | +| `CI_TARGET_BRANCH` | `github.head_ref` or `github.ref_name` | Branch component of the version | +| `CI_PUBLISH` | `True` for pushes to `main`, or when the manual `publish` input is set | Whether `Build.Windows.ps1` creates a GitHub release | +| `NUGET_API_KEY` | `secrets.NUGET_API_KEY` | When non-empty, `Build.Windows.ps1` pushes the tool package to NuGet | +| `GH_TOKEN` | `secrets.GITHUB_TOKEN` | Used by `gh release create` | +| `DOCKER_USER`, `DOCKER_TOKEN` | Docker Hub secrets | When `DOCKER_TOKEN` is non-empty, `Build.Linux.ps1` pushes images | +| `NODE_AUTH_TOKEN` | `secrets.NPM_TOKEN` | Authenticates `npm publish` (see below) | +| `SEQ_DOCKER_TAG` | e.g. `2026.1` | Seq image tag used for the Linux end-to-end tests | + +GitHub only supplies repository secrets to branch builds, not to pull requests, so PR builds run the full build and test steps but publish nothing. + +### What each branch publishes + +| Trigger | GitHub release | NuGet | Docker Hub | npm | +|---|---|---|---|---| +| Pull request | no | no | no | no | +| Push to `dev` | no (unless manual `publish`) | prerelease version | `datalust/seqcli-ci:` | prerelease, dist-tag `dev` | +| Push to `main` | yes, `v` | release version | `datalust/seqcli-ci:` | release, dist-tag `latest` | + +Note that `Build.Linux.ps1` always pushes to the `datalust/seqcli-ci` repository (the image name with a `-ci` suffix), never directly to `datalust/seqcli`. Promoting an image to the public `datalust/seqcli` repository, and tagging it `latest`, is a separate step outside this repository. + +To publish a release from `dev` (for example a preview build), run the workflow manually from the Actions tab with the **Publish a GitHub release** input checked. The release is marked as a prerelease because the branch is not `main`. + +## Running the builds locally + +All scripts need PowerShell 7 (`pwsh`) and the .NET 10 SDK, and must be run from anywhere inside the repository (they `Push-Location` to the root themselves). Without the `CI_*` variables the version is `.local`, and nothing is published because the credential variables are unset. + +* `Build.Windows.ps1` requires Windows: it uses `7za.exe`, installs Seq with Chocolatey for the end-to-end tests, and builds `win-*` RIDs. Run it as `./build/Build.Windows.ps1`. +* `Build.Linux.ps1` requires Docker with `buildx`. Building the `arm64` image on an x64 host needs `binfmt` configured, as the workflow does. Run it as `./build/Build.Linux.ps1 -SeqDockerTag 2026.1`. +* `Build.Npm.ps1` runs anywhere with `npm`, `tar`, and `gh`. To exercise it end to end without publishing, point it at a directory of archives (for example a local `artifacts/` from a Windows build, or assets downloaded from a release) and use `-DryRun`: + + ```shell + gh release download v2026.1.02616 --dir ./npm-archives --pattern 'seqcli-*-*.*' + ./build/Build.Npm.ps1 -Version 2026.1.02616 -ArchiveDir ./npm-archives -DryRun + ls npm-staging/tarballs + ``` + +`artifacts/`, `npm-staging/` and `npm-archives/` are all ignored by git. + +## Checklist for a new release line + +1. Update [`baseversion`](../baseversion). +2. Update the SDK version in [`ci.global.json`](../ci.global.json) to match the Seq release. +3. Update `SEQ_DOCKER_TAG` in [`ci.yml`](../.github/workflows/ci.yml) to the Seq image the end-to-end tests should run against. +4. If the target framework changes, update `$framework` in `Build.Windows.ps1` and `Build.Linux.ps1`, and the `COPY` paths in the Dockerfiles. From 35a1580a64b97dde3ee7adaea95f175dc62315fe Mon Sep 17 00:00:00 2001 From: liammclennan-ro Date: Mon, 7 Sep 2026 11:05:56 +1000 Subject: [PATCH 29/32] Use the regular README as the npm README --- build/Build.Npm.ps1 | 5 +++++ npm/README.md | 33 --------------------------------- npm/seqcli/package.json | 4 ---- 3 files changed, 5 insertions(+), 37 deletions(-) delete mode 100644 npm/README.md diff --git a/build/Build.Npm.ps1 b/build/Build.Npm.ps1 index 3c5d3453..afc8d082 100644 --- a/build/Build.Npm.ps1 +++ b/build/Build.Npm.ps1 @@ -209,6 +209,11 @@ function Stage-LauncherPackage($rids) if (Test-Path $directory) { Remove-Item -Recurse -Force $directory } Copy-Item -Recurse ./npm/seqcli $directory + # The launcher package is the one users see on npmjs.com, so it carries the repository README + # and license rather than maintaining separate copies under ./npm. + Copy-Item ./README.md "$directory/README.md" + Copy-Item ./LICENSE "$directory/LICENSE" + $package = Get-Content "$directory/package.json" -Raw | ConvertFrom-Json -AsHashtable $package.version = $npmVersion $package.optionalDependencies = [ordered]@{} diff --git a/npm/README.md b/npm/README.md deleted file mode 100644 index fca21e72..00000000 --- a/npm/README.md +++ /dev/null @@ -1,33 +0,0 @@ -## How the package works - -`@datalust/seqcli` is a small launcher. The self-contained `seqcli` binary for your platform is installed alongside it as an optional dependency from one of these packages: - -| Package | Platform | -|---|---| -| `@datalust/seqcli-win-x64` | Windows x64 | -| `@datalust/seqcli-win-arm64` | Windows ARM64 | -| `@datalust/seqcli-osx-x64` | macOS x64 | -| `@datalust/seqcli-osx-arm64` | macOS ARM64 (Apple Silicon) | -| `@datalust/seqcli-linux-x64` | Linux x64 (glibc) | -| `@datalust/seqcli-linux-arm64` | Linux ARM64 (glibc) | -| `@datalust/seqcli-linux-musl-x64` | Linux x64 (musl, e.g. Alpine) | -| `@datalust/seqcli-linux-musl-arm64` | Linux ARM64 (musl, e.g. Alpine) | - -The binaries are byte-for-byte the ones attached to the matching [GitHub release](https://github.com/datalust/seqcli/releases). Because the .NET runtime is bundled, no `dotnet` installation is needed. Each platform package is roughly 45 MB to download and 120 MB on disk. - -Do not install with `--omit=optional` (or `--no-optional`): the platform package would be skipped and `seqcli` would fail to start with a message explaining how to fix it. - -## Alpine Linux - -The musl builds need the ICU globalization libraries, which minimal Alpine images don't include: either `apk add icu-libs`, or set `DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1` to run without them. - -## Versions - -npm versions are the GitHub release versions with leading zeros removed from the last component, because npm requires strict semantic versioning. For example, release `v2026.1.02616` is published to npm as `2026.1.2616`. Prerelease builds are published under the `dev` dist-tag. - -## Notes for Windows users - -The Seq installer for Windows also installs `seqcli.exe`, into `C:\Program Files\Seq`. If that directory is on your `PATH` ahead of npm's global bin directory (`%APPDATA%\npm`), running `seqcli` will use the copy bundled with Seq rather than the one installed by npm. Run `where seqcli` to see which copies are found and in what order; `npx @datalust/seqcli ` always runs the npm-installed version. Both copies share the same `SeqCli.json` configuration. - -Do not use the npm package to host the `seqcli forwarder` Windows service. The service registers the path of the executable that installed it, and npm replaces the installed files on every upgrade. Use the Seq installer or a release archive from GitHub instead. - diff --git a/npm/seqcli/package.json b/npm/seqcli/package.json index fbd36399..23f767a2 100644 --- a/npm/seqcli/package.json +++ b/npm/seqcli/package.json @@ -11,10 +11,6 @@ "structured-logging", "cli" ], - "scripts": { - "prepack": "cp ../../README.md ./README.md && cp ../../LICENSE ./LICENSE", - "postpack": "rm ./README.md ./LICENSE" - }, "license": "Apache-2.0", "homepage": "https://github.com/datalust/seqcli", "repository": { From f0386a0beebfd8545423d0de59986824d4bdf59c Mon Sep 17 00:00:00 2001 From: liammclennan-ro Date: Tue, 8 Sep 2026 09:46:06 +1000 Subject: [PATCH 30/32] Extend npm publish timeout --- build/Build.Npm.ps1 | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/build/Build.Npm.ps1 b/build/Build.Npm.ps1 index afc8d082..71e2fc37 100644 --- a/build/Build.Npm.ps1 +++ b/build/Build.Npm.ps1 @@ -173,12 +173,24 @@ function Publish-NpmPackage($name, $directory) function Assert-NpmPublished($name) { - # The registry can take a moment to reflect a new version. - for ($attempt = 1; $attempt -le 6; $attempt++) { + # `npm publish` returns as soon as the registry accepts the upload, but the read path (the + # packument served via npm's CDN) is updated asynchronously and can lag by several minutes, + # particularly for the first-ever publish of a package name. Poll for up to ten minutes. + $timeout = [TimeSpan]::FromMinutes(10) + $interval = 15 + $started = Get-Date + + while ($true) { if (Test-NpmPublished $name) { return } - Start-Sleep -Seconds 5 + + $elapsed = (Get-Date) - $started + if ($elapsed -ge $timeout) { break } + + Write-Host ("Waiting for {0}@{1} to become visible on the registry ({2:mm\:ss} elapsed)" -f $name, $npmVersion, $elapsed) + Start-Sleep -Seconds $interval } - throw "$name@$npmVersion is not visible on the registry" + + throw "$name@$npmVersion is not visible on the registry after $($timeout.TotalMinutes) minutes" } function Stage-PlatformPackage($rid) From 22527ed6adf79f71fb9f78bbc09c4aa3cde11f55 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Tue, 8 Sep 2026 12:49:44 +1000 Subject: [PATCH 31/32] Tidy exception reporting up a bit; log full exception details at outer handler when `--verbose` is specified --- src/SeqCli/Cli/Commands/IngestCommand.cs | 98 +++++++------- .../Cli/Commands/Metrics/SearchCommand.cs | 81 +++++------- src/SeqCli/Cli/Commands/SearchCommand.cs | 84 ++++++------ src/SeqCli/Cli/Commands/TraceCommand.cs | 124 ++++++++---------- src/SeqCli/Program.cs | 7 +- src/SeqCli/Util/Presentation.cs | 10 +- 6 files changed, 188 insertions(+), 216 deletions(-) diff --git a/src/SeqCli/Cli/Commands/IngestCommand.cs b/src/SeqCli/Cli/Commands/IngestCommand.cs index e0ad35a6..be413c56 100644 --- a/src/SeqCli/Cli/Commands/IngestCommand.cs +++ b/src/SeqCli/Cli/Commands/IngestCommand.cs @@ -81,66 +81,58 @@ public IngestCommand() protected override async Task Run() { - try - { - var enrichers = new List(); + var enrichers = new List(); - if (_level != null) - enrichers.Add(new LevelEnricher(_level)); + if (_level != null) + enrichers.Add(new LevelEnricher(_level)); - foreach (var (name, value) in _properties.FlatProperties) - enrichers.Add(new ScalarPropertyEnricher(name, value)); + foreach (var (name, value) in _properties.FlatProperties) + enrichers.Add(new ScalarPropertyEnricher(name, value)); - Func? filter = null; - if (_filter != null) - { - var eval = SeqSyntax.CompileExpression(_filter); - filter = evt => eval(evt).IsTrue(); - } + Func? filter = null; + if (_filter != null) + { + var eval = SeqSyntax.CompileExpression(_filter); + filter = evt => eval(evt).IsTrue(); + } - var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = SeqConnectionFactory.Connect(_connection, config); - - // The API key is passed through separately because `SeqConnection` doesn't expose a batched ingestion - // mechanism and so we manually construct `HttpRequestMessage`s deeper in the stack. Nice feature gap to - // close at some point! - var (_, apiKey) = SeqConnectionFactory.GetConnectionDetails(_connection, config); - var batchSize = _batchSize.Value; + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = SeqConnectionFactory.Connect(_connection, config); + + // The API key is passed through separately because `SeqConnection` doesn't expose a batched ingestion + // mechanism and so we manually construct `HttpRequestMessage`s deeper in the stack. Nice feature gap to + // close at some point! + var (_, apiKey) = SeqConnectionFactory.GetConnectionDetails(_connection, config); + var batchSize = _batchSize.Value; - foreach (var input in _fileInputFeature.OpenInputs()) + foreach (var input in _fileInputFeature.OpenInputs()) + { + using (input) { - using (input) - { - IEventReader reader = _json - ? new JsonEventReader(input) - : new PlainTextEventReader(input, _pattern); - - reader = new EnrichingReader(reader, enrichers); - - if (_message != null) - reader = new StaticMessageTemplateReader(reader, _message); - - var exit = await LogShipper.ShipEventsAsync( - connection, - apiKey, - reader, - _invalidDataHandlingFeature.InvalidDataHandling, - _sendFailureHandlingFeature.SendFailureHandling, - batchSize, - filter, - CancellationToken.None); - - if (exit != 0) - return exit; - } + IEventReader reader = _json + ? new JsonEventReader(input) + : new PlainTextEventReader(input, _pattern); + + reader = new EnrichingReader(reader, enrichers); + + if (_message != null) + reader = new StaticMessageTemplateReader(reader, _message); + + var exit = await LogShipper.ShipEventsAsync( + connection, + apiKey, + reader, + _invalidDataHandlingFeature.InvalidDataHandling, + _sendFailureHandlingFeature.SendFailureHandling, + batchSize, + filter, + CancellationToken.None); + + if (exit != 0) + return exit; } - - return 0; - } - catch (Exception ex) - { - Log.Error(ex, "Ingestion failed: {ErrorMessage}", ex.Message); - return 1; } + + return 0; } } \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Metrics/SearchCommand.cs b/src/SeqCli/Cli/Commands/Metrics/SearchCommand.cs index 5cc1c3ee..9b86cd6b 100644 --- a/src/SeqCli/Cli/Commands/Metrics/SearchCommand.cs +++ b/src/SeqCli/Cli/Commands/Metrics/SearchCommand.cs @@ -22,7 +22,6 @@ using SeqCli.Cli.Features; using SeqCli.Config; using SeqCli.Util; -using Serilog; namespace SeqCli.Cli.Commands.Metrics; @@ -69,56 +68,48 @@ public SearchCommand() protected override async Task Run() { - try - { - var config = RuntimeConfigurationLoader.Load(_storagePath); - var output = _output.GetOutputFormat(config); - var connection = SeqConnectionFactory.Connect(_connection, config); + var config = RuntimeConfigurationLoader.Load(_storagePath); + var output = _output.GetOutputFormat(config); + var connection = SeqConnectionFactory.Connect(_connection, config); - string? filter = null; - if (!string.IsNullOrWhiteSpace(_filter)) - filter = (await connection.Expressions.ToStrictAsync(_filter)).StrictExpression; + string? filter = null; + if (!string.IsNullOrWhiteSpace(_filter)) + filter = (await connection.Expressions.ToStrictAsync(_filter)).StrictExpression; - var result = await connection.Metrics.SearchAsync( - _groups, - filter, - _count, - rangeStartUtc: _range.Start, - rangeEndUtc: _range.End, - trace: _trace); - - // We convert the metric into a query result to improve formatting consistency. Room for an abstraction of - // some kind here. - var rows = new List(); - foreach (var metric in result.Metrics) - { - var row = new List - { - metric.Name ?? metric.Accessor, - metric.Kind, - metric.Unit, - metric.Description - }; - - foreach (var value in metric.GroupKey) - row.Add(value); - - rows.Add(row.ToArray()); - } - var asRowset = new QueryResultPart + var result = await connection.Metrics.SearchAsync( + _groups, + filter, + _count, + rangeStartUtc: _range.Start, + rangeEndUtc: _range.End, + trace: _trace); + + // We convert the metric into a query result to improve formatting consistency. Room for an abstraction of + // some kind here. + var rows = new List(); + foreach (var metric in result.Metrics) + { + var row = new List { - Columns = new[] { "Name", "Kind", "Unit", "Description" }.Concat(_groups).ToArray(), - Rows = rows.ToArray() + metric.Name ?? metric.Accessor, + metric.Kind, + metric.Unit, + metric.Description }; - output.WriteQueryResult(asRowset); - - return 0; + foreach (var value in metric.GroupKey) + row.Add(value); + + rows.Add(row.ToArray()); } - catch (Exception ex) + var asRowset = new QueryResultPart { - Log.Error(ex, "Could not retrieve metrics: {ErrorMessage}", ex.Message); - return 1; - } + Columns = new[] { "Name", "Kind", "Unit", "Description" }.Concat(_groups).ToArray(), + Rows = rows.ToArray() + }; + + output.WriteQueryResult(asRowset); + + return 0; } } \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/SearchCommand.cs b/src/SeqCli/Cli/Commands/SearchCommand.cs index 37c9c95e..1ce602e7 100644 --- a/src/SeqCli/Cli/Commands/SearchCommand.cs +++ b/src/SeqCli/Cli/Commands/SearchCommand.cs @@ -72,62 +72,54 @@ public SearchCommand() protected override async Task Run() { - try - { - var config = RuntimeConfigurationLoader.Load(_storagePath); + var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = SeqConnectionFactory.Connect(_connection, config); - connection.Client.HttpClient.Timeout = TimeSpan.FromMilliseconds(_httpClientTimeout); + var connection = SeqConnectionFactory.Connect(_connection, config); + connection.Client.HttpClient.Timeout = TimeSpan.FromMilliseconds(_httpClientTimeout); - var columns = await _eventColumns.GetColumns(connection, _signal.Signal); - var output = _output.GetOutputFormat(config, TextFormatters.PlainOutputTemplate(columns)); + var columns = await _eventColumns.GetColumns(connection, _signal.Signal); + var output = _output.GetOutputFormat(config, TextFormatters.PlainOutputTemplate(columns)); - string? filter = null; - if (!string.IsNullOrWhiteSpace(_filter)) - filter = (await connection.Expressions.ToStrictAsync(_filter)).StrictExpression; + string? filter = null; + if (!string.IsNullOrWhiteSpace(_filter)) + filter = (await connection.Expressions.ToStrictAsync(_filter)).StrictExpression; - try + try + { + if (!_noWebSockets) { - if (!_noWebSockets) + await foreach (var evt in connection.Events.EnumerateAsync(null, + _signal.Signal, + filter, + _count, + fromDateUtc: _range.Start, + toDateUtc: _range.End, + trace: _trace, + render: output.RequiresRender)) { - await foreach (var evt in connection.Events.EnumerateAsync(null, - _signal.Signal, - filter, - _count, - fromDateUtc: _range.Start, - toDateUtc: _range.End, - trace: _trace, - render: output.RequiresRender)) - { - output.WriteEventEntity(evt); - } - - return 0; + output.WriteEventEntity(evt); } - } - catch (NotSupportedException nse) - { - Log.Information(nse, "WebSockets not supported; falling back to paged search"); - } - - await foreach (var evt in connection.Events.PagedEnumerateAsync(null, - _signal.Signal, - filter, - _count, - fromDateUtc: _range.Start, - toDateUtc: _range.End, - trace: _trace, - render: output.RequiresRender)) - { - output.WriteEventEntity(evt); - } - return 0; + return 0; + } } - catch (Exception ex) + catch (NotSupportedException nse) { - Log.Error(ex, "Could not retrieve search result: {ErrorMessage}", ex.Message); - return 1; + Log.Information(nse, "WebSockets not supported; falling back to paged search"); } + + await foreach (var evt in connection.Events.PagedEnumerateAsync(null, + _signal.Signal, + filter, + _count, + fromDateUtc: _range.Start, + toDateUtc: _range.End, + trace: _trace, + render: output.RequiresRender)) + { + output.WriteEventEntity(evt); + } + + return 0; } } \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/TraceCommand.cs b/src/SeqCli/Cli/Commands/TraceCommand.cs index 67541f3b..348e40f9 100644 --- a/src/SeqCli/Cli/Commands/TraceCommand.cs +++ b/src/SeqCli/Cli/Commands/TraceCommand.cs @@ -81,86 +81,78 @@ public TraceCommand() protected override async Task Run() { - try + if (_id == null) { - if (_id == null) - { - Log.Error("A trace id must be specified"); - return 1; - } - - var traceId = _id.ToLowerInvariant(); - if (!TraceQuery.IsValidTraceId(traceId)) - { - Log.Error("The trace id {TraceId} is not valid; trace ids are 32 hexadecimal digits", _id); - return 1; - } + Log.Error("A trace id must be specified"); + return 1; + } - var spanId = _spanId?.ToLowerInvariant(); - if (spanId != null && !TraceQuery.IsValidSpanId(spanId)) - { - Log.Error("The span id {SpanId} is not valid; span ids are 16 hexadecimal digits", _spanId); - return 1; - } + var traceId = _id.ToLowerInvariant(); + if (!TraceQuery.IsValidTraceId(traceId)) + { + Log.Error("The trace id {TraceId} is not valid; trace ids are 32 hexadecimal digits", _id); + return 1; + } - var config = RuntimeConfigurationLoader.Load(_storagePath); - var connection = SeqConnectionFactory.Connect(_connection, config); + var spanId = _spanId?.ToLowerInvariant(); + if (spanId != null && !TraceQuery.IsValidSpanId(spanId)) + { + Log.Error("The span id {SpanId} is not valid; span ids are 16 hexadecimal digits", _spanId); + return 1; + } - var result = await connection.Data.TryQueryAsync(TraceQuery.Build(traceId, _includeLogs, _includeExceptions, _columns)); - if (!string.IsNullOrWhiteSpace(result.Error)) - { - Log.Error("Could not retrieve trace: {ErrorMessage}", result.Error); - foreach (var reason in result.Reasons) - Log.Error("{Reason}", reason); - return 1; - } + var config = RuntimeConfigurationLoader.Load(_storagePath); + var connection = SeqConnectionFactory.Connect(_connection, config); - var traceEvents = TraceQuery.ReadEvents(result, _includeExceptions, _columns); - if (traceEvents.Count == 0) - { - Log.Error("No events found for trace {TraceId}", traceId); - return 1; - } + var result = await connection.Data.TryQueryAsync(TraceQuery.Build(traceId, _includeLogs, _includeExceptions, _columns)); + if (!string.IsNullOrWhiteSpace(result.Error)) + { + Log.Error("Could not retrieve trace: {ErrorMessage}", result.Error); + foreach (var reason in result.Reasons) + Log.Error("{Reason}", reason); + return 1; + } - var complete = traceEvents.Count != TraceQuery.MaxEvents; - if (!complete) - Log.Warning("Only the first {Count} events in the trace were retrieved; the tree may be incomplete", - TraceQuery.MaxEvents); + var traceEvents = TraceQuery.ReadEvents(result, _includeExceptions, _columns); + if (traceEvents.Count == 0) + { + Log.Error("No events found for trace {TraceId}", traceId); + return 1; + } - var roots = TraceTreeBuilder.Build(traceEvents); + var complete = traceEvents.Count != TraceQuery.MaxEvents; + if (!complete) + Log.Warning("Only the first {Count} events in the trace were retrieved; the tree may be incomplete", + TraceQuery.MaxEvents); - TraceTreeNode? subtreeRoot = null; - if (spanId != null) - { - subtreeRoot = TraceTreeBuilder.FindSpan(roots, spanId); - if (subtreeRoot == null) - { - Log.Error("The span {SpanId} does not appear in trace {TraceId}", spanId, traceId); - return 1; - } - } + var roots = TraceTreeBuilder.Build(traceEvents); - var output = _output.GetOutputFormat(config, TraceFormatter.OutputTemplate(_columns.Count)); - if (output.Json) - { - var document = subtreeRoot != null ? - TraceTreeJObjectConverter.FromSubtree(traceId, subtreeRoot, complete, _includeLogs, _columns) : - TraceTreeJObjectConverter.FromRoots(traceId, roots, complete, _includeLogs, _columns); - - output.WriteObject(document); - } - else + TraceTreeNode? subtreeRoot = null; + if (spanId != null) + { + subtreeRoot = TraceTreeBuilder.FindSpan(roots, spanId); + if (subtreeRoot == null) { - foreach (var eventJson in TraceFormatter.ToEventJson(subtreeRoot != null ? [subtreeRoot] : roots)) - output.WriteEvent(eventJson); + Log.Error("The span {SpanId} does not appear in trace {TraceId}", spanId, traceId); + return 1; } + } - return 0; + var output = _output.GetOutputFormat(config, TraceFormatter.OutputTemplate(_columns.Count)); + if (output.Json) + { + var document = subtreeRoot != null ? + TraceTreeJObjectConverter.FromSubtree(traceId, subtreeRoot, complete, _includeLogs, _columns) : + TraceTreeJObjectConverter.FromRoots(traceId, roots, complete, _includeLogs, _columns); + + output.WriteObject(document); } - catch (Exception ex) + else { - Log.Error(ex, "Could not retrieve trace: {ErrorMessage}", ex.Message); - return 1; + foreach (var eventJson in TraceFormatter.ToEventJson(subtreeRoot != null ? [subtreeRoot] : roots)) + output.WriteEvent(eventJson); } + + return 0; } } diff --git a/src/SeqCli/Program.cs b/src/SeqCli/Program.cs index f6a0a11f..0c42c5ff 100644 --- a/src/SeqCli/Program.cs +++ b/src/SeqCli/Program.cs @@ -54,8 +54,11 @@ static async Task Main(string[] args) } catch (Exception ex) { - Log.Debug(ex, "Unhandled command exception"); - Log.Fatal("The command failed: {UnhandledExceptionMessage}", Presentation.FormattedMessage(ex)); + // The `--verbose` flag flips the level switch from `Error` to `Information`; we use that as a signal to + // include full stack traces, it's a bit of a sneaky backchannel but saves adding yet more infrastructure. + var reportedException = levelSwitch.MinimumLevel < LogEventLevel.Error ? ex : null; + + Log.Fatal(reportedException, "The command failed: {UnhandledExceptionMessage}", Presentation.FormattedMessage(ex)); return 1; } finally diff --git a/src/SeqCli/Util/Presentation.cs b/src/SeqCli/Util/Presentation.cs index 125ee15b..7a4df543 100644 --- a/src/SeqCli/Util/Presentation.cs +++ b/src/SeqCli/Util/Presentation.cs @@ -29,7 +29,7 @@ static class Presentation /// and causal chain. public static string FormattedMessage(Exception ex) { - if (ex == null) throw new ArgumentNullException(nameof(ex)); + ArgumentNullException.ThrowIfNull(ex); static Exception Unwrap(Exception outer) { @@ -38,8 +38,10 @@ static Exception Unwrap(Exception outer) static string Describe(Exception toDescribe) { - // :-) - return toDescribe.Message.Replace(", see inner exception", ""); + var described = toDescribe.Message.Replace(", see inner exception", "").Trim(); + if (!described.EndsWith('.')) + described += "."; + return described; } var unwrapped = Unwrap(ex); @@ -49,7 +51,7 @@ static string Describe(Exception toDescribe) { unwrapped = Unwrap(unwrapped.InnerException); - message.Append(' '); + message.Append(" → "); message.Append(Describe(unwrapped)); } From 17a88358df696314f6e33831c7e0614db43860a3 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Tue, 8 Sep 2026 13:05:14 +1000 Subject: [PATCH 32/32] Test case --- test/SeqCli.EndToEnd/Events/EventsDeleteTestCase.cs | 2 +- .../Events/EventsDeleteWithDateRangeAllTestCase.cs | 2 +- test/SeqCli.EndToEnd/Events/SearchSignalColumnsTestCase.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/SeqCli.EndToEnd/Events/EventsDeleteTestCase.cs b/test/SeqCli.EndToEnd/Events/EventsDeleteTestCase.cs index 61d8281b..1a28fd48 100644 --- a/test/SeqCli.EndToEnd/Events/EventsDeleteTestCase.cs +++ b/test/SeqCli.EndToEnd/Events/EventsDeleteTestCase.cs @@ -5,7 +5,7 @@ using Serilog; using Xunit; -namespace SeqCli.EndToEnd.Delete; +namespace SeqCli.EndToEnd.Events; public class EventsDeleteTestCase : ICliTestCase { diff --git a/test/SeqCli.EndToEnd/Events/EventsDeleteWithDateRangeAllTestCase.cs b/test/SeqCli.EndToEnd/Events/EventsDeleteWithDateRangeAllTestCase.cs index ed56ac08..72688e94 100644 --- a/test/SeqCli.EndToEnd/Events/EventsDeleteWithDateRangeAllTestCase.cs +++ b/test/SeqCli.EndToEnd/Events/EventsDeleteWithDateRangeAllTestCase.cs @@ -6,7 +6,7 @@ using Serilog; using Xunit; -namespace SeqCli.EndToEnd.Delete; +namespace SeqCli.EndToEnd.Events; public class EventsDeleteWithDateRangeAllTestCase : ICliTestCase { diff --git a/test/SeqCli.EndToEnd/Events/SearchSignalColumnsTestCase.cs b/test/SeqCli.EndToEnd/Events/SearchSignalColumnsTestCase.cs index dabfb028..2f874c7b 100644 --- a/test/SeqCli.EndToEnd/Events/SearchSignalColumnsTestCase.cs +++ b/test/SeqCli.EndToEnd/Events/SearchSignalColumnsTestCase.cs @@ -96,6 +96,6 @@ public async Task ExecuteAsync( // A signal that can't be found is reported, rather than silently ignored. exit = runner.Exec("search", $"--signal signal-999999 {filter} -c 10"); Assert.Equal(1, exit); - Assert.Contains("Could not retrieve search result", runner.LastRunProcess!.Output); + Assert.Contains("The command failed", runner.LastRunProcess!.Output); } }