Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
161 changes: 161 additions & 0 deletions src/LogExpert.Benchmarks/DirectReaderBenchmarks.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
using System.Buffers;
using System.Text;

using BenchmarkDotNet.Attributes;

using LogExpert.Core.Classes.Log.Streamreaders;
using LogExpert.Core.Entities;

namespace LogExpert.Benchmarks;

/// <summary>
/// Measures the default (Direct) stream reader the way <c>LogfileReader</c> drives it: lines come
/// from <c>TryReadLine</c>, and every <see cref="LINES_PER_BUFFER"/> lines the char blocks are
/// detached and returned to the pool, as a <c>LogBuffer</c> does when it is evicted.
/// The valid-UTF-8 workloads are the regression gate for reader changes; the invalid-bytes
/// workload covers files with binary blocks (issue #639).
/// </summary>
[MemoryDiagnoser]
public class DirectReaderBenchmarks
{
private const int LINES_PER_BUFFER = 500;
private const int MAX_LINE_LENGTH = 20_000;

private byte[] _asciiMedium = null!;
private byte[] _asciiLarge = null!;
private byte[] _unicode = null!;
private byte[] _mixedLineEndings = null!;
private byte[] _longLines = null!;
private byte[] _utf16 = null!;
private byte[] _invalidBytes = null!;

[GlobalSetup]
public void Setup ()
{
_asciiMedium = Encoding.UTF8.GetBytes(GenerateAscii(10_000, 100, "\r\n"));
_asciiLarge = Encoding.UTF8.GetBytes(GenerateAscii(200_000, 200, "\r\n"));
_unicode = Encoding.UTF8.GetBytes(GenerateUnicode(100_000));
_mixedLineEndings = Encoding.UTF8.GetBytes(GenerateMixedLineEndings(100_000));
_longLines = Encoding.UTF8.GetBytes(GenerateAscii(500, 40_000, "\n"));
_utf16 = Encoding.Unicode.GetBytes(GenerateAscii(100_000, 100, "\r\n"));
_invalidBytes = GenerateInvalidBytes(100_000);
}

[Benchmark(Baseline = true)]
public int AsciiMedium () => ReadAll(_asciiMedium, Encoding.UTF8);

[Benchmark]
public int AsciiLarge () => ReadAll(_asciiLarge, Encoding.UTF8);

[Benchmark]
public int Unicode () => ReadAll(_unicode, Encoding.UTF8);

[Benchmark]
public int MixedLineEndings () => ReadAll(_mixedLineEndings, Encoding.UTF8);

[Benchmark]
public int LongLines () => ReadAll(_longLines, Encoding.UTF8);

[Benchmark]
public int Utf16 () => ReadAll(_utf16, Encoding.Unicode);

[Benchmark]
public int InvalidBytes () => ReadAll(_invalidBytes, Encoding.UTF8);

private static int ReadAll (byte[] data, Encoding encoding)
{
using var stream = new MemoryStream(data, writable: false);
using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions { Encoding = encoding }, MAX_LINE_LENGTH);

var lines = 0;
while (reader.TryReadLine(out _))
{
if (++lines % LINES_PER_BUFFER == 0)
{
ReturnBlocks(reader.DetachCharBlocks());
}
}

ReturnBlocks(reader.DetachCharBlocks());
return lines;
}

private static void ReturnBlocks (List<char[]> blocks)
{
foreach (var block in blocks)
{
ArrayPool<char>.Shared.Return(block);
}
}

[System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "Benchmark data generation")]
private static string GenerateAscii (int lineCount, int avgLineLength, string newLine)
{
var sb = new StringBuilder();
var random = new Random(42);

for (var i = 0; i < lineCount; i++)
{
var lineLength = avgLineLength + random.Next(-10, 11);
_ = sb.Append("2026-04-23 12:00:00.").Append(i % 1000).Append(" [INFO] ")
.Append('X', Math.Max(0, lineLength - 32))
.Append(newLine);
}

return sb.ToString();
}

[System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "Benchmark data generation")]
private static string GenerateUnicode (int lineCount)
{
var sb = new StringBuilder();
var random = new Random(42);

for (var i = 0; i < lineCount; i++)
{
_ = random.Next(0, 3) switch
{
0 => sb.Append("Line ").Append(i).Append(": ASCII text only, nothing special here\r\n"),
1 => sb.Append("Line ").Append(i).Append(": Hello 世界, 日志文件 (Chinese)\r\n"),
_ => sb.Append("Line ").Append(i).Append(": Спасибо большое 🙂 (Russian, emoji)\r\n")
};
}

return sb.ToString();
}

private static string GenerateMixedLineEndings (int lineCount)
{
var sb = new StringBuilder();
string[] endings = ["\r\n", "\n", "\r"];

for (var i = 0; i < lineCount; i++)
{
_ = sb.Append("2026-04-23 12:00:00 [WARN] mixed ending line ").Append(i).Append(endings[i % endings.Length]);
}

return sb.ToString();
}

// CSV-like text with a random binary block every 1000 lines, the shape of the issue #639 sample.
[System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "Benchmark data generation")]
private static byte[] GenerateInvalidBytes (int lineCount)
{
var random = new Random(42);
using var stream = new MemoryStream();

for (var i = 0; i < lineCount; i++)
{
if (i % 1000 == 500)
{
var binary = new byte[2048];
random.NextBytes(binary);
stream.Write(binary);
}

stream.Write(Encoding.UTF8.GetBytes($"06/23/2026 00:00:00,MXS SEP26,3Minute,Info,OnVarUpdate(),\"line {i}\"\r\n"));
}

return stream.ToArray();
}
}
10 changes: 5 additions & 5 deletions src/LogExpert.Benchmarks/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ public static void Main (string[] args)
Console.WriteLine("No benchmarks specified. Running all benchmarks...");

// Run all benchmarks if no arguments are provided
_ = BenchmarkRunner.Run<StreamReaderBenchmarks>();
_ = BenchmarkRunner.Run<DirectReaderBenchmarks>();
_ = BenchmarkRunner.Run<BufferIndexBenchmarks>();
_ = BenchmarkRunner.Run<ReadThroughputBenchmarks>();
_ = BenchmarkRunner.Run<BufferIndexContentionBenchmarks>();
Expand All @@ -25,7 +25,7 @@ public static void Main (string[] args)
}

Console.WriteLine("Replace <benchmarkname> with the name of the benchmark you want to run, e.g. ");
Console.WriteLine("StreamReaderBenchmarks: Benchmarks for stream readers");
Console.WriteLine("DirectReaderBenchmarks: Benchmarks for the default (Direct) reader, the regression gate for reader changes");
Console.WriteLine("ReadThroughputBenchmarks: Benchmarks for read throughput");
Console.WriteLine("BufferIndexBenchmarks: Benchmarks for buffer index");
Console.WriteLine("BufferIndexContentionBenchmarks: Benchmarks for buffer index contention");
Expand All @@ -42,11 +42,11 @@ public static void Main (string[] args)
/*
* Comment / Uncommen the benchmark to run, careful some can run longer
* 1.) a dry run
* dotnet run -c Release -- --filter "StreamReaderBenchmarks" --job Dry --noOverwrite
* dotnet run -c Release -- --filter "DirectReaderBenchmarks" --job Dry --noOverwrite
* 2.) a short run
* dotnet run -c Release -- --filter "StreamReaderBenchmarks" --job Short --noOverwrite
* dotnet run -c Release -- --filter "DirectReaderBenchmarks" --job Short --noOverwrite
* 3.) a full baseline run
* dotnet run -c Release -- --filter "StreamReaderBenchmarks" --noOverwrite
* dotnet run -c Release -- --filter "DirectReaderBenchmarks" --noOverwrite
*
* The full baseline run generates a MD file
* BenchmarkDotNet.Artifacts/results/*-report-github.md
Expand Down
51 changes: 0 additions & 51 deletions src/LogExpert.Benchmarks/ReadThroughputBenchmarks.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@

using LogExpert.Core.Classes.Log;
using LogExpert.Core.Entities;
using LogExpert.Core.Enums;

namespace LogExpert.Benchmarks;

Expand Down Expand Up @@ -45,7 +44,6 @@ public int ReadWithNullReporter ()
bufferCount: 500,
linesPerBuffer: 500,
new MultiFileOptions(),
ReaderType.System,
PluginRegistry.PluginRegistry.Instance,
maximumLineLength: 500,
progressReporter: Core.Classes.Log.ProgressReporters.NullProgressReporter.Instance);
Expand All @@ -67,7 +65,6 @@ public int ReadWithPeriodicReporter ()
bufferCount: 500,
linesPerBuffer: 500,
new MultiFileOptions(),
ReaderType.System,
PluginRegistry.PluginRegistry.Instance,
maximumLineLength: 500);
// No progressReporter = default PeriodicProgressReporter
Expand All @@ -76,54 +73,6 @@ public int ReadWithPeriodicReporter ()
return reader.LineCount;
}

/// <summary>
/// Post-change: read with block-based allocation (System reader uses CharBlockAllocator).
/// Compare Gen0/Gen1/Gen2 collections vs baseline to validate allocation reduction.
/// This method is identical to ReadWithNullReporter — it exists solely for explicit
/// before/after naming in benchmark reports.
/// </summary>
[Benchmark]
public int ReadWithBlockAllocation ()
{
using var reader = new LogfileReader(
_tempFile,
new EncodingOptions { Encoding = Encoding.UTF8 },
multiFile: false,
bufferCount: 500,
linesPerBuffer: 500,
new MultiFileOptions(),
ReaderType.System,
PluginRegistry.PluginRegistry.Instance,
maximumLineLength: 500,
progressReporter: Core.Classes.Log.ProgressReporters.NullProgressReporter.Instance);

reader.ReadFiles();
return reader.LineCount;
}

/// <summary>
/// Direct-read: reads decoded chars directly into pooled blocks without per-line string allocation.
/// Compare allocations and GC counts vs ReadWithBlockAllocation.
/// </summary>
[Benchmark]
public int ReadWithDirectRead ()
{
using var reader = new LogfileReader(
_tempFile,
new EncodingOptions { Encoding = Encoding.UTF8 },
multiFile: false,
bufferCount: 500,
linesPerBuffer: 500,
new MultiFileOptions(),
ReaderType.SystemDirect,
PluginRegistry.PluginRegistry.Instance,
maximumLineLength: 500,
progressReporter: Core.Classes.Log.ProgressReporters.NullProgressReporter.Instance);

reader.ReadFiles();
return reader.LineCount;
}

[GlobalCleanup]
public void Cleanup ()
{
Expand Down
Loading
Loading